From 14326d0657cbdc051b0f753242ca440395fde56c Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 29 Jul 2014 10:49:52 +0300 Subject: [PATCH 0001/1247] initial commit --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..e139b335fe --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.* + +# +# git files that we don't want to ignore even it they are dot-files +# +!.gitignore +!.mailmap +patches-* +build/ From 75ca002be8f7aa275e00225a2717d229b72d9796 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 28 Oct 2014 17:50:33 +0200 Subject: [PATCH 0002/1247] add outgoing/ to .gitignore Signed-off-by: Oded Gabbay --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e139b335fe..57f4774867 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ !.mailmap patches-* build/ +outgoing/ From b0b875964fef7d1d40ee20c203f73b8ae2d02c5c Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 29 Jul 2014 10:52:32 +0300 Subject: [PATCH 0003/1247] Add IOCTL set definitions of amdkfd - KFD_IOC_GET_VERSION: Retrieves the interface version of amdkfd - KFD_IOC_CREATE_QUEUE: Creates a usermode queue that runs on a specific GPU device - KFD_IOC_DESTROY_QUEUE: Destroys an existing usermode queue - KFD_IOC_SET_MEMORY_POLICY: Sets the memory policy of the default and alternate aperture of the calling process - KFD_IOC_GET_CLOCK_COUNTERS: Retrieves counters (timestamps) of CPU and GPU - KFD_IOC_GET_PROCESS_APERTURES: Retrieves information about process apertures that were initialized during the open() call of the amdkfd device - KFD_IOC_UPDATE_QUEUE: Updates configuration of an existing usermode queue Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 154 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 include/linux/kfd_ioctl.h diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h new file mode 100644 index 0000000000..7acef41fc2 --- /dev/null +++ b/include/linux/kfd_ioctl.h @@ -0,0 +1,154 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef KFD_IOCTL_H_INCLUDED +#define KFD_IOCTL_H_INCLUDED + +#include +#include + +#define KFD_IOCTL_MAJOR_VERSION 1 +#define KFD_IOCTL_MINOR_VERSION 0 + +struct kfd_ioctl_get_version_args { + uint32_t major_version; /* from KFD */ + uint32_t minor_version; /* from KFD */ +}; + +/* For kfd_ioctl_create_queue_args.queue_type. */ +#define KFD_IOC_QUEUE_TYPE_COMPUTE 0 +#define KFD_IOC_QUEUE_TYPE_SDMA 1 +#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 2 + +#define KFD_MAX_QUEUE_PERCENTAGE 100 +#define KFD_MAX_QUEUE_PRIORITY 15 + +struct kfd_ioctl_create_queue_args { + uint64_t ring_base_address; /* to KFD */ + uint64_t write_pointer_address; /* from KFD */ + uint64_t read_pointer_address; /* from KFD */ + uint64_t doorbell_offset; /* from KFD */ + + uint32_t ring_size; /* to KFD */ + uint32_t gpu_id; /* to KFD */ + uint32_t queue_type; /* to KFD */ + uint32_t queue_percentage; /* to KFD */ + uint32_t queue_priority; /* to KFD */ + uint32_t queue_id; /* from KFD */ + + uint64_t eop_buffer_address; /* to KFD */ + uint64_t eop_buffer_size; /* to KFD */ + uint64_t ctx_save_restore_address; /* to KFD */ + uint64_t ctx_save_restore_size; /* to KFD */ +}; + +struct kfd_ioctl_destroy_queue_args { + uint32_t queue_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_update_queue_args { + uint64_t ring_base_address; /* to KFD */ + + uint32_t queue_id; /* to KFD */ + uint32_t ring_size; /* to KFD */ + uint32_t queue_percentage; /* to KFD */ + uint32_t queue_priority; /* to KFD */ +}; + +/* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ +#define KFD_IOC_CACHE_POLICY_COHERENT 0 +#define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 + +struct kfd_ioctl_set_memory_policy_args { + uint64_t alternate_aperture_base; /* to KFD */ + uint64_t alternate_aperture_size; /* to KFD */ + + uint32_t gpu_id; /* to KFD */ + uint32_t default_policy; /* to KFD */ + uint32_t alternate_policy; /* to KFD */ + uint32_t pad; +}; + +/* + * All counters are monotonic. They are used for profiling of compute jobs. + * The profiling is done by userspace. + * + * In case of GPU reset, the counter should not be affected. + */ + +struct kfd_ioctl_get_clock_counters_args { + uint64_t gpu_clock_counter; /* from KFD */ + uint64_t cpu_clock_counter; /* from KFD */ + uint64_t system_clock_counter; /* from KFD */ + uint64_t system_clock_freq; /* from KFD */ + + uint32_t gpu_id; /* to KFD */ + uint32_t pad; +}; + +#define NUM_OF_SUPPORTED_GPUS 7 + +struct kfd_process_device_apertures { + uint64_t lds_base; /* from KFD */ + uint64_t lds_limit; /* from KFD */ + uint64_t scratch_base; /* from KFD */ + uint64_t scratch_limit; /* from KFD */ + uint64_t gpuvm_base; /* from KFD */ + uint64_t gpuvm_limit; /* from KFD */ + uint32_t gpu_id; /* from KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_get_process_apertures_args { + struct kfd_process_device_apertures + process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */ + + /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */ + uint32_t num_of_nodes; + uint32_t pad; +}; + +#define KFD_IOC_MAGIC 'K' + +#define KFD_IOC_GET_VERSION \ + _IOR(KFD_IOC_MAGIC, 1, struct kfd_ioctl_get_version_args) + +#define KFD_IOC_CREATE_QUEUE \ + _IOWR(KFD_IOC_MAGIC, 2, struct kfd_ioctl_create_queue_args) + +#define KFD_IOC_DESTROY_QUEUE \ + _IOWR(KFD_IOC_MAGIC, 3, struct kfd_ioctl_destroy_queue_args) + +#define KFD_IOC_SET_MEMORY_POLICY \ + _IOW(KFD_IOC_MAGIC, 4, struct kfd_ioctl_set_memory_policy_args) + +#define KFD_IOC_GET_CLOCK_COUNTERS \ + _IOWR(KFD_IOC_MAGIC, 5, struct kfd_ioctl_get_clock_counters_args) + +#define KFD_IOC_GET_PROCESS_APERTURES \ + _IOR(KFD_IOC_MAGIC, 6, struct kfd_ioctl_get_process_apertures_args) + +#define KFD_IOC_UPDATE_QUEUE \ + _IOW(KFD_IOC_MAGIC, 7, struct kfd_ioctl_update_queue_args) + +#endif From 813af2b89aa6d0077bb50d8e3a98d637f219040f Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 29 Jul 2014 10:55:52 +0300 Subject: [PATCH 0004/1247] Add definitions of HSA functions and types Signed-off-by: Oded Gabbay --- include/hsakmt.h | 565 ++++++++++++++++++++++++++++ include/hsakmttypes.h | 851 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1416 insertions(+) create mode 100644 include/hsakmt.h create mode 100644 include/hsakmttypes.h diff --git a/include/hsakmt.h b/include/hsakmt.h new file mode 100644 index 0000000000..3983aa0412 --- /dev/null +++ b/include/hsakmt.h @@ -0,0 +1,565 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _HSAKMT_H_ +#define _HSAKMT_H_ + +#include "hsakmttypes.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +/** + "Opens" the HSA kernel driver for user-kernel mode communication. + + On Windows, this function gets a handle to the KFD's AMDKFDIO device object that + is responsible for user-kernel communication, this handle is used internally by + the thunk library to send device I/O control to the HSA kernel driver. + No other thunk library function may be called unless the user-kernel communication + channel is opened first. + + On Linux this call opens the "/dev/kfd" device file to establish a communication + path to the kernel. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtOpenKFD( void ); + +/** + "Closes" the user-kernel communication path. + + On Windows, the handle obtained by the hsaKmtOpenKFD() function is closed; + no other communication with the kernel driver is possible after the successful + execution of the saKmdCloseKFD() function. Depending on the failure reason, + the user-kernel communication path may or may not be still active. + + On Linux the function closes the "dev/kfd" device file. + No further communication to the kernel driver is allowed until hsaKmtOpenKFD() + function is called again. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtCloseKFD( void ); + + +/** + Returns the user-kernel interface version supported by KFD. + Higher major numbers usually add new features to KFD and may break user-kernel + compatibility; higher minor numbers define additional functionality associated + within a major number. + The calling software should validate that it meets the minimum interface version + as described in the API specification. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetVersion( + HsaVersionInfo* VersionInfo //OUT + ); + +/** + The function takes a "snapshot" of the topology information within the KFD + to avoid any changes during the enumeration process. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtAcquireSystemProperties( + HsaSystemProperties* SystemProperties //OUT + ); + +/** + Releases the topology "snapshot" taken by hsaKmtAcquireSystemProperties() +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtReleaseSystemProperties( void ) ; + +/** + Retrieves the discoverable sub-properties for a given HSA + node. The parameters returned allow the application or runtime to size the + management structures necessary to store the information. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeProperties( + HSAuint32 NodeId, //IN + HsaNodeProperties* NodeProperties //OUT + ); + +/** + Retrieves the memory properties of a specific HSA node. + the memory pointer passed as MemoryProperties is sized as + NumBanks * sizeof(HsaMemoryProperties). NumBanks is retrieved with the + hsaKmtGetNodeProperties() call. + + Some of the data returned is optional. Not all implementations may return all + parameters in the hsaMemoryProperties. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeMemoryProperties( + HSAuint32 NodeId, //IN + HSAuint32 NumBanks, //IN + HsaMemoryProperties* MemoryProperties //OUT + ); + +/** + Retrieves the cache properties of a specific HSA node and processor ID. + ProcessorID refers to either a CPU core or a SIMD unit as enumerated earlier + via the hsaKmtGetNodeProperties() call. + The memory pointer passed as CacheProperties is sized as + NumCaches * sizeof(HsaCacheProperties). NumCaches is retrieved with the + hsaKmtGetNodeProperties() call. + + The data returned is optional. Not all implementations may return all + parameters in the CacheProperties. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeCacheProperties( + HSAuint32 NodeId, //IN + HSAuint32 ProcessorId, //IN + HSAuint32 NumCaches, //IN + HsaCacheProperties* CacheProperties //OUT + ); + +/** + Retrieves the HSA IO affinity properties of a specific HSA node. + the memory pointer passed as Properties is sized as + NumIoLinks * sizeof(HsaIoLinkProperties). NumIoLinks is retrieved with the + hsaKmtGetNodeProperties() call. + + The data returned is optional. Not all implementations may return all + parameters in the IoLinkProperties. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeIoLinkProperties( + HSAuint32 NodeId, //IN + HSAuint32 NumIoLinks, //IN + HsaIoLinkProperties* IoLinkProperties //OUT + ); + + + +/** + Creates an operating system event associated with a HSA event ID +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtCreateEvent( + HsaEventDescriptor* EventDesc, //IN + bool ManualReset, //IN + bool IsSignaled, //IN + HsaEvent** Event //OUT + ); + +/** + Destroys an operating system event associated with a HSA event ID +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDestroyEvent( + HsaEvent* Event //IN + ); + +/** + Sets the specified event object to the signaled state +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetEvent( + HsaEvent* Event //IN + ); + +/** + Sets the specified event object to the non-signaled state +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtResetEvent( + HsaEvent* Event //IN + ); + +/** + Queries the state of the specified event object +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueryEventState( + HsaEvent* Event //IN + ); + +/** + Checks the current state of the event object. If the object's state is + nonsignaled, the calling thread enters the wait state. + + The function returns when one of the following occurs: +- The specified event object is in the signaled state. +- The time-out interval elapses. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtWaitOnEvent( + HsaEvent* Event, //IN + HSAuint32 Milliseconds //IN + ); + +/** + Checks the current state of multiple event objects. + + The function returns when one of the following occurs: +- Either any one or all of the specified objects are in the signaled state + - if "WaitOnAll" is "true" the function returns when the state of all + objects in array is signaled + - if "WaitOnAll" is "false" the function returns when the state of any + one of the objects is set to signaled +- The time-out interval elapses. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtWaitOnMultipleEvents( + HsaEvent* Events[], //IN + HSAuint32 NumEvents, //IN + bool WaitOnAll, //IN + HSAuint32 Milliseconds //IN + ); + +/** + new TEMPORARY function definition - to be used only on "Triniti + Southern Islands" platform + If used on other platforms the function will return HSAKMT_STATUS_ERROR +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtReportQueue( + HSA_QUEUEID QueueId, //IN + HsaQueueReport* QueueReport //OUT + ); + +/** + Creates a GPU queue with user-mode access rights +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtCreateQueue( + HSAuint32 NodeId, //IN + HSA_QUEUE_TYPE Type, //IN + HSAuint32 QueuePercentage, //IN + HSA_QUEUE_PRIORITY Priority, //IN + void* QueueAddress, //IN + HSAuint64 QueueSizeInBytes, //IN + HsaEvent* Event, //IN + HsaQueueResource* QueueResource //OUT + ); + +/** + Updates a queue +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtUpdateQueue( + HSA_QUEUEID QueueId, //IN + HSAuint32 QueuePercentage,//IN + HSA_QUEUE_PRIORITY Priority, //IN + void* QueueAddress, //IN + HSAuint64 QueueSize, //IN + HsaEvent* Event //IN + ); + +/** + Destroys a queue +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDestroyQueue( + HSA_QUEUEID QueueId //IN + ); + +/** + Allows an HSA process to set/change the default and alternate memory coherency, before starting to dispatch. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetMemoryPolicy( + HSAuint32 Node, //IN + HSAuint32 DefaultPolicy, //IN + HSAuint32 AlternatePolicy, //IN + void* MemoryAddressAlternate, //IN (page-aligned) + HSAuint64 MemorySizeInBytes //IN (page-aligned) + ); +/** + Allocates a memory buffer that may be accessed by the GPU +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtAllocMemory( + HSAuint32 PreferredNode, //IN + HSAuint64 SizeInBytes, //IN (multiple of page size) + HsaMemFlags MemFlags, //IN + void** MemoryAddress //OUT (page-aligned) + ); + +/** + Frees a memory buffer +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtFreeMemory( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 SizeInBytes //IN + ); + +/** + Registers with KFD a memory buffer that may be accessed by the GPU + This function will never be required for Linux +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemory( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 MemorySizeInBytes //IN (page-aligned) + ); + + +/** + Unregisters with KFD a memory buffer + This function will never be required for Linux +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDeregisterMemory( + void* MemoryAddress //IN + ); + + +/** + Ensures that the memory is resident and can be accessed by GPU + Not implemented yet +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtMapMemoryToGPU( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 MemorySizeInBytes, //IN (page-aligned) + HSAuint64* AlternateVAGPU //OUT (page-aligned) + ); + +/** + Releases the residency of the memory + Not implemented yet +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtUnmapMemoryToGPU( + void* MemoryAddress //IN (page-aligned) + ); + + +/** + Notifies the kernel driver that a process wants to use GPU debugging facilities +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgRegister( + HSAuint32 NodeId //IN + ); + +/** + Detaches the debugger process from the HW debug established by hsaKmtDbgRegister() API +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgUnregister( + HSAuint32 NodeId //IN + ); + +/** + Controls a wavefront +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgWavefrontControl( + HSAuint32 NodeId, //IN + HSA_DBG_WAVEOP Operand, //IN + HSA_DBG_WAVEMODE Mode, //IN + HSAuint32 TrapId, //IN + HsaDbgWaveMessage* DbgWaveMsgRing //IN + ); + +/** + Sets watch points on memory address ranges to generate exception events when the + watched addresses are accessed +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgAddressWatch( + HSAuint32 NodeId, //IN + HSAuint32 NumWatchPoints, //IN + HSA_DBG_WATCH_MODE WatchMode[], //IN + void* WatchAddress[], //IN + HSAuint64 WatchMask[], //IN, optional + HsaEvent* WatchEvent[] //IN, optional + ); + +/** + Gets GPU and CPU clock counters for particular Node +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetClockCounters( + HSAuint32 NodeId, //IN + HsaClockCounters* Counters //OUT + ); + +/** + Retrieves information on the available HSA counters +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcGetCounterProperties( + HSAuint32 NodeId, //IN + HsaCounterProperties** CounterProperties //OUT + ); + +/** + Registers a set of (HW) counters to be used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcRegisterTrace( + HSAuint32 NodeId, //IN + HSAuint32 NumberOfCounters, //IN + HsaCounter* Counters, //IN + HsaPmcTraceRoot* TraceRoot //OUT + ); + +/** + Unregisters a set of (HW) counters used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcUnregisterTrace( + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ); + +/** + Allows a user mode process to get exclusive access to the defined set of (HW) counters + used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcAcquireTraceAccess( + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ); + +/** + Allows a user mode process to release exclusive access to the defined set of (HW) counters + used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcReleaseTraceAccess( + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ); + +/** + Starts tracing operation on a previously established set of performance counters +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcStartTrace( + HSATraceId TraceId, //IN + void* TraceBuffer, //IN (page aligned) + HSAuint64 TraceBufferSizeBytes //IN (page aligned) + ); + +/** + Forces an update of all the counters that a previously started trace operation has registered +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcQueryTrace( + HSATraceId TraceId //IN + ); + +/** + Stops tracing operation on a previously established set of performance counters +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcStopTrace( + HSATraceId TraceId //IN + ); + +#ifdef __cplusplus +} //extern "C" +#endif + +#endif //_HSAKMT_H_ + diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h new file mode 100644 index 0000000000..178a86d178 --- /dev/null +++ b/include/hsakmttypes.h @@ -0,0 +1,851 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _HSAKMTTYPES_H_ +#define _HSAKMTTYPES_H_ + +//the definitions and THUNK API are version specific - define the version numbers here +#define HSAKMT_VERSION_MAJOR 0 +#define HSAKMT_VERSION_MINOR 99 + + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_WIN64) || defined(_WINDOWS) || defined(_WIN32) + + #if defined(_WIN32) + #define HSAKMTAPI __stdcall + #else + #define HSAKMTAPI + #endif + + typedef unsigned char HSAuint8; + typedef char HSAint8; + typedef unsigned short HSAuint16; + typedef signed short HSAint16; + typedef unsigned __int32 HSAuint32; + typedef signed __int64 HSAint64; + typedef unsigned __int64 HSAuint64; + +#elif defined(__linux__) + +#include +#include + + #define HSAKMTAPI + + typedef uint8_t HSAuint8; + typedef int8_t HSAint8; + typedef uint16_t HSAuint16; + typedef int16_t HSAint16; + typedef uint32_t HSAuint32; + typedef int64_t HSAint64; + typedef uint64_t HSAuint64; + +#endif + +typedef void* HSA_HANDLE; +typedef HSAuint64 HSA_QUEUEID; + +// This is included in order to force the alignments to be 4 bytes so that +// it avoids extra padding added by the compiler when a 64-bit binary is generated. +#pragma pack(push, hsakmttypes_h, 4) + +// +// HSA STATUS codes returned by the KFD Interfaces +// + +typedef enum _HSAKMT_STATUS +{ + HSAKMT_STATUS_SUCCESS = 0, // Operation successful + HSAKMT_STATUS_ERROR = 1, // General error return if not otherwise specified + HSAKMT_STATUS_DRIVER_MISMATCH = 2, // User mode component is not compatible with kernel HSA driver + + HSAKMT_STATUS_INVALID_PARAMETER = 3, // KFD identifies input parameters invalid + HSAKMT_STATUS_INVALID_HANDLE = 4, // KFD identifies handle parameter invalid + HSAKMT_STATUS_INVALID_NODE_UNIT = 5, // KFD identifies node or unit parameter invalid + + HSAKMT_STATUS_NO_MEMORY = 6, // No memory available (when allocating queues or memory) + HSAKMT_STATUS_BUFFER_TOO_SMALL = 7, // A buffer needed to handle a request is too small + + HSAKMT_STATUS_NOT_IMPLEMENTED = 10, // KFD function is not implemented for this set of paramters + HSAKMT_STATUS_NOT_SUPPORTED = 11, // KFD function is not supported on this node + HSAKMT_STATUS_UNAVAILABLE = 12, // KFD function is not available currently on this node (but + // may be at a later time) + + HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED = 20, // KFD driver path not opened + HSAKMT_STATUS_KERNEL_COMMUNICATION_ERROR = 21, // user-kernel mode communication failure + HSAKMT_STATUS_KERNEL_ALREADY_OPENED = 22, // KFD driver path already opened + HSAKMT_STATUS_HSAMMU_UNAVAILABLE = 23, // ATS/PRI 1.1 (Address Translation Services) not available + // (IOMMU driver not installed or not-available) + + HSAKMT_STATUS_WAIT_FAILURE = 30, // The wait operation failed + HSAKMT_STATUS_WAIT_TIMEOUT = 31, // The wait operation timed out + + HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED = 35, // Memory buffer already registered + HSAKMT_STATUS_MEMORY_NOT_REGISTERED = 36, // Memory buffer not registered + HSAKMT_STATUS_MEMORY_ALIGNMENT = 37, // Memory parameter not aligned + +} HSAKMT_STATUS; + +// +// HSA KFD interface version information. Calling software has to validate that it meets +// the minimum interface version as described in the API specification. +// All future structures will be extended in a backward compatible fashion. +// + +typedef struct _HsaVersionInfo +{ + HSAuint32 KernelInterfaceMajorVersion; // supported kernel interface major version + HSAuint32 KernelInterfaceMinorVersion; // supported kernel interface minor version +} HsaVersionInfo; + +// +// HSA Topology Discovery Infrastructure structure definitions. +// The infrastructure implementation is based on design specified in the Kernel HSA Driver ADD +// The discoverable data is retrieved from ACPI structures in the platform infrastructure, as defined +// in the "Heterogeneous System Architecture Detail Topology" specification. +// +// The following structure is returned on a call to hsaKmtAcquireSystemProperties() as output. +// When the call is made within a process context, a "snapshot" of the topology information +// is taken within the KFD to avoid any changes during the enumeration process. +// The Snapshot is released when hsaKmtReleaseSystemProperties() is called +// or when the process exits or is terminated. +// + +typedef struct _HsaSystemProperties +{ + HSAuint32 NumNodes; // the number of "H-NUMA" memory nodes. + // each node represents a discoverable node of the system + // All other enumeration is done on a per-node basis + + HSAuint32 PlatformOem; // identifies HSA platform, reflects the OEMID in the CRAT + HSAuint32 PlatformId; // HSA platform ID, reflects OEM TableID in the CRAT + HSAuint32 PlatformRev; // HSA platform revision, reflects Platform Table Revision ID +} HsaSystemProperties; + + +typedef union +{ + HSAuint32 Value; + struct + { + unsigned int HotPluggable : 1; // the node may be removed by some system action + // (event will be sent) + unsigned int HSAMMUPresent : 1; // This node has an ATS/PRI 1.1 compatible + // translation agent in the system (e.g. IOMMUv2) + unsigned int SharedWithGraphics : 1; // this HSA nodes' GPU function is also used for OS primary + // graphics render (= UI) + unsigned int QueueSizePowerOfTwo : 1; // This node GPU requires the queue size to be a power of 2 value + unsigned int QueueSize32bit : 1; // This node GPU requires the queue size to be less than 4GB + unsigned int QueueIdleEvent : 1; // This node GPU supports notification on Queue Idle + unsigned int VALimit : 1; // This node GPU has limited VA range for platform + // (typical 40bit). Affects shared VM use for 64bit apps + unsigned int WatchPointsSupported: 1; // Indicates if Watchpoints are available on the node. + unsigned int WatchPointsTotalBits: 4; // ld(Watchpoints) available. To determine the number use 2^value + + unsigned int Reserved : 20; + } ui32; +} HSA_CAPABILITY; + + +// +// HSA node properties. This structure is an output parameter of hsaKmtGetNodeProperties() +// The application or runtime can use the information herein to size the topology management structures +// Unless there is some very weird setup, there is at most one "GPU" device (with a certain number +// of throughput compute units (= SIMDs) associated with a H-NUMA node. +// + +#define HSA_PUBLIC_NAME_SIZE 128 + +typedef struct _HsaNodeProperties +{ + HSAuint32 NumCPUCores; // # of latency (= CPU) cores present on this HSA node. + // This value is 0 for a HSA node with no such cores, + // e.g a "discrete HSA GPU" + HSAuint32 NumFComputeCores; // # of HSA throughtput (= GPU) FCompute cores ("SIMD") present in a node. + // This value is 0 if no FCompute cores are present (e.g. pure "CPU node"). + HSAuint32 NumMemoryBanks; // # of discoverable memory bank affinity properties on this "H-NUMA" node. + HSAuint32 NumCaches; // # of discoverable cache affinity properties on this "H-NUMA" node. + + HSAuint32 NumIOLinks; // # of discoverable IO link affinity properties of this node + // connecting to other nodes. + + HSAuint32 CComputeIdLo; // low value of the logical processor ID of the latency (= CPU) + // cores available on this node + HSAuint32 FComputeIdLo; // low value of the logical processor ID of the throughput (= GPU) + // units available on this node + + HSA_CAPABILITY Capability; // see above + + HSAuint32 MaxWavesPerSIMD; // This identifies the max. number of launched waves per SIMD. + // If NumFComputeCores is 0, this value is ignored. + HSAuint32 LDSSizeInKB; // Size of Local Data Store in Kilobytes per SIMD Wavefront + HSAuint32 GDSSizeInKB; // Size of Global Data Store in Kilobytes shared across SIMD Wavefronts + + HSAuint32 WaveFrontSize; // Number of SIMD cores per wavefront executed, typically 64, + // may be 32 or a different value for some HSA based architectures + + HSAuint32 NumShaderBanks; // Number of Shader Banks or Shader Engines, typical values are 1 or 2 + + + HSAuint32 NumArrays; // Number of SIMD arrays per engine + HSAuint32 NumCUPerArray; // Number of Compute Units (CU) per SIMD array + HSAuint32 NumSIMDPerCU; // Number of SIMD representing a Compute Unit (CU) + + HSAuint32 MaxSlotsScratchCU; // Number of temp. memory ("scratch") wave slots available to access, + // may be 0 if HW has no restrictions + + HSAuint32 EngineId; // Identifier (rev) of teh GPU uEngine or Firmware, may be 0 + + HSAuint16 VendorId; // GPU vendor id; 0 on latency (= CPU)-only nodes + HSAuint16 DeviceId; // GPU device id; 0 on latency (= CPU)-only nodes + + HSAuint32 LocationId; // GPU BDF (Bus/Device/function number) - identifies the device + // location in the overall system + HSAuint64 LocalMemSize; // Local memory size + HSAuint32 MaxEngineClockMhzFCompute; // maximum engine clocks for CPU and + HSAuint32 MaxEngineClockMhzCCompute; // GPU function, including any boost caopabilities, + + HSAuint16 MarketingName[HSA_PUBLIC_NAME_SIZE]; // Public name of the "device" on the node (board or APU name). + // Unicode string +} HsaNodeProperties; + + +typedef enum _HSA_HEAPTYPE +{ + HSA_HEAPTYPE_SYSTEM = 0, + HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC = 1, // CPU "visible" part of GPU device local memory (for discrete GPU) + HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE = 2, // CPU "invisible" part of GPU device local memory (for discrete GPU) + // All HSA accessible memory is per definition "CPU visible" + // "Private memory" is relevant for graphics interop only. + HSA_HEAPTYPE_GPU_GDS = 3, // GPU internal memory (GDS) + HSA_HEAPTYPE_GPU_LDS = 4, // GPU internal memory (LDS) + HSA_HEAPTYPE_GPU_SCRATCH = 5, // GPU special memory (scratch) + + HSA_HEAPTYPE_NUMHEAPTYPES, + HSA_HEAPTYPE_SIZE = 0xFFFFFFFF +} HSA_HEAPTYPE; + +typedef union +{ + HSAuint32 MemoryProperty; + struct + { + unsigned int HotPluggable : 1; // the memory may be removed by some system action, + // memory should be used for temporary data + unsigned int NonVolatile : 1; // memory content is preserved across a power-off cycle. + unsigned int Reserved :30; + } ui32; +} HSA_MEMORYPROPERTY; + + +// +// Discoverable HSA Memory properties. +// The structure is the output parameter of the hsaKmtGetNodeMemoryProperties() function +// + +typedef struct _HsaMemoryProperties +{ + HSA_HEAPTYPE HeapType; // system or frame buffer, + union + { + HSAuint64 SizeInBytes; // physical memory size of the memory range in bytes + struct + { + HSAuint32 SizeInBytesLow; // physical memory size of the memory range in bytes (lower 32bit) + HSAuint32 SizeInBytesHigh; // physical memory size of the memory range in bytes (higher 32bit) + } ui32; + }; + HSA_MEMORYPROPERTY Flags; // See definitions above + + HSAuint32 Width; // memory width - the number of parallel bits of the memory interface + HSAuint32 MemoryClockMax; // memory clock for the memory, this allows computing the available bandwidth + // to the memory when needed + HSAuint64 VirtualBaseAddress; // if set to value != 0, indicates the virtual base address of the memory + // in process virtual space +} HsaMemoryProperties; + +// +// Discoverable Cache Properties. (optional). +// The structure is the output parameter of the hsaKmtGetNodeMemoryProperties() function +// Any of the parameters may be 0 (= not defined) +// + +#define HSA_CPU_SIBLINGS 256 +#define HSA_PROCESSORID_ALL 0xFFFFFFFF + +typedef union +{ + HSAuint32 Value; + struct + { + unsigned int Data : 1; + unsigned int Instruction : 1; + unsigned int CPU : 1; + unsigned int HSACU : 1; + unsigned int Reserved :28; + } ui32; +} HsaCacheType; + +typedef struct _HaCacheProperties +{ + HSAuint32 ProcessorIdLow; // Identifies the processor number + + HSAuint32 CacheLevel; // Integer representing level: 1, 2, 3, 4, etc + HSAuint32 CacheSize; // Size of the cache + HSAuint32 CacheLineSize; // Cache line size in bytes + HSAuint32 CacheLinesPerTag; // Cache lines per Cache Tag + HSAuint32 CacheAssociativity; // Cache Associativity + HSAuint32 CacheLatency; // Cache latency in ns + HsaCacheType CacheType; + HSAuint32 SiblingMap[HSA_CPU_SIBLINGS]; +} HsaCacheProperties; + + +// +// Discoverable CPU Compute Properties. (optional). +// The structure is the output parameter of the hsaKmtGetCComputeProperties() function +// Any of the parameters may be 0 (= not defined) +// + +typedef struct _HsaCComputeProperties +{ + HSAuint32 SiblingMap[HSA_CPU_SIBLINGS]; +} HsaCComputeProperties; + +// +// Discoverable IoLink Properties (optional). +// The structure is the output parameter of the hsaKmtGetIoLinkProperties() function. +// Any of the parameters may be 0 (= not defined) +// + +typedef enum _HSA_IOLINKTYPE { + HSA_IOLINKTYPE_UNDEFINED = 0, + HSA_IOLINKTYPE_HYPERTRANSPORT = 1, + HSA_IOLINKTYPE_PCIEXPRESS = 2, + HSA_IOLINKTYPE_AMBA = 3, + HSA_IOLINKTYPE_MIPI = 4, + HSA_IOLINKTYPE_OTHER = 5, + HSA_IOLINKTYPE_NUMIOLINKTYPES, + HSA_IOLINKTYPE_SIZE = 0xFFFFFFFF +} HSA_IOLINKTYPE; + +typedef union +{ + HSAuint32 LinkProperty; + struct + { + unsigned int Override : 1; // bus link properties are determined by this structure + // not by the HSA_IOLINKTYPE. The other flags are valid + // only if this bit is set to one + unsigned int NonCoherent : 1; // The link doesn't support coherent transactions + // memory accesses across must not be set to "host cacheable"! + unsigned int NoAtomics32bit : 1; // The link doesn't support 32bit-wide atomic transactions + unsigned int NoAtomics64bit : 1; // The link doesn't support 64bit-wide atomic transactions + unsigned int Reserved :28; + } ui32; +} HSA_LINKPROPERTY; + + +typedef struct _HsaIoLinkProperties +{ + HSA_IOLINKTYPE IoLinkType; // see above + HSAuint32 VersionMajor; // Bus interface version (optional) + HSAuint32 VersionMinor; // Bus interface version (optional) + + HSAuint32 NodeFrom; // + HSAuint32 NodeTo; // + + HSAuint32 Weight; // weight factor (derived from CDIT) + + HSAuint32 MinimumLatency; // minimum cost of time to transfer (rounded to ns) + HSAuint32 MaximumLatency; // maximum cost of time to transfer (rounded to ns) + HSAuint32 MinimumBandwidth; // minimum interface Bandwidth in MB/s + HSAuint32 MaximumBandwidth; // maximum interface Bandwidth in MB/s + HSAuint32 RecTransferSize; // recommended transfer size to reach maximum bandwidth in Bytes + HSA_LINKPROPERTY Flags; // override flags (may be active for specific platforms) +} HsaIoLinkProperties; + +// +// Memory allocation definitions for the KFD HSA interface +// + +typedef struct _HsaMemFlags +{ + union + { + struct + { + unsigned int NonPaged : 1; // default = 0: pageable memory + unsigned int CachePolicy : 2; // see HSA_CACHING_TYPE + unsigned int ReadOnly : 1; // default = 0: Read/Write memory + unsigned int PageSize : 2; // see HSA_PAGE_SIZE + unsigned int HostAccess : 1; // default = 0: GPU access only + unsigned int NoSubstitute: 1; // default = 0: if specific memory is not available on node (e.g. on + // discrete GPU local), allocation may fall back to system memory node 0 + // memory (= always available). Otherwise no allocation is possible. + unsigned int GDSMemory : 1; // default = 0: If set, the allocation will occur in GDS heap. + // HostAccess must be 0, all other flags (except NoSubstitute) should + // be 0 when setting this entry to 1. GDS allocation may fail due to + // limited resources. Application code is required to work without + // any allocated GDS memory using regular memory. + // Allocation fails on any node without GPU function. + unsigned int Scratch : 1; // default = 0: If set, the allocation will occur in GPU "scratch area". + // HostAccess must be 0, all other flags (except NoSubstitute) should be 0 + // when setting this entry to 1. Scratch allocation may fail due to limited + // resources. Application code is required to work without any allocation. + // Allocation fails on any node without GPU function. + unsigned int Reserved : 22; + } ui32; + HSAuint32 Value; + }; +} HsaMemFlags; + +typedef enum _HSA_CACHING_TYPE +{ + HSA_CACHING_CACHED = 0, + HSA_CACHING_NONCACHED = 1, + HSA_CACHING_WRITECOMBINED = 2, + HSA_CACHING_RESERVED = 3, + HSA_CACHING_NUM_CACHING, + HSA_CACHING_SIZE = 0xFFFFFFFF +} HSA_CACHING_TYPE; + +typedef enum _HSA_PAGE_SIZE +{ + HSA_PAGE_SIZE_4KB = 0, + HSA_PAGE_SIZE_64KB = 1, //64KB pages, not generally available in systems + HSA_PAGE_SIZE_2MB = 2, + HSA_PAGE_SIZE_1GB = 3, //1GB pages, not generally available in systems +} HSA_PAGE_SIZE; + + +typedef enum _HSA_DEVICE +{ + HSA_DEVICE_CPU = 0, + HSA_DEVICE_GPU = 1, + MAX_HSA_DEVICE = 2 +} HSA_DEVICE; + + +typedef enum _HSA_QUEUE_PRIORITY +{ + HSA_QUEUE_PRIORITY_MINIMUM = -3, + HSA_QUEUE_PRIORITY_LOW = -2, + HSA_QUEUE_PRIORITY_BELOW_NORMAL = -1, + HSA_QUEUE_PRIORITY_NORMAL = 0, + HSA_QUEUE_PRIORITY_ABOVE_NORMAL = 1, + HSA_QUEUE_PRIORITY_HIGH = 2, + HSA_QUEUE_PRIORITY_MAXIMUM = 3, + HSA_QUEUE_PRIORITY_NUM_PRIORITY, + HSA_QUEUE_PRIORITY_SIZE = 0xFFFFFFFF +} HSA_QUEUE_PRIORITY; + +typedef enum _HSA_QUEUE_TYPE +{ + HSA_QUEUE_COMPUTE = 1, // AMD PM4 compatible Compute Queue + HSA_QUEUE_SDMA = 2, // SDMA Queue, used for data transport and format conversion (e.g. (de-)tiling, etc). + HSA_QUEUE_MULTIMEDIA_DECODE = 3, // reserved, for HSA multimedia decode queue + HSA_QUEUE_MULTIMEDIA_ENCODE = 4, // reserved, for HSA multimedia encode queue + + // the following values indicate a queue type permitted to reference OS graphics + // resources through the interoperation API. See [5] "HSA Graphics Interoperation + // specification" for more details on use of such resources. + + HSA_QUEUE_COMPUTE_OS = 11, // AMD PM4 compatible Compute Queue + HSA_QUEUE_SDMA_OS = 12, // SDMA Queue, used for data transport and format conversion (e.g. (de-)tiling, etc). + HSA_QUEUE_MULTIMEDIA_DECODE_OS = 13, // reserved, for HSA multimedia decode queue + HSA_QUEUE_MULTIMEDIA_ENCODE_OS = 14, // reserved, for HSA multimedia encode queue + + HSA_QUEUE_COMPUTE_AQL = 21, // HSA AQL packet compatible Compute Queue + HSA_QUEUE_DMA_AQL = 22, // HSA AQL packet compatible DMA Queue + + // more types in the future + + HSA_QUEUE_TYPE_SIZE = 0xFFFFFFFF //aligns to 32bit enum +} HSA_QUEUE_TYPE; + +typedef struct _HsaQueueResource +{ + HSA_QUEUEID QueueId; /** queue ID */ + /** Doorbell address to notify HW of a new dispatch */ + union + { + HSAuint32* Queue_DoorBell; + HSAuint64* Queue_DoorBell_aql; + HSAuint64 QueueDoorBell; + }; + + /** virtual address to notify HW of queue write ptr value */ + union + { + HSAuint32* Queue_write_ptr; + HSAuint64* Queue_write_ptr_aql; + HSAuint64 QueueWptrValue; + }; + + /** virtual address updated by HW to indicate current read location */ + union + { + HSAuint32* Queue_read_ptr; + HSAuint64* Queue_read_ptr_aql; + HSAuint64 QueueRptrValue; + }; + +} HsaQueueResource; + + +//TEMPORARY structure definition - to be used only on "Triniti + Southern Islands" platform +typedef struct _HsaQueueReport +{ + HSAuint32 VMID; //Required on SI to dispatch IB in primary ring + void* QueueAddress; //virtual address of UM mapped compute ring + HSAuint64 QueueSize; //size of the UM mapped compute ring +} HsaQueueReport; + + + +typedef enum _HSA_DBG_WAVEOP +{ + HSA_DBG_WAVEOP_HALT = 1, //Halts a wavefront + HSA_DBG_WAVEOP_RESUME = 2, //Resumes a wavefront + HSA_DBG_WAVEOP_KILL = 3, //Kills a wavefront + HSA_DBG_WAVEOP_DEBUG = 4, //Causes wavefront to enter debug mode + HSA_DBG_WAVEOP_TRAP = 5, //Causes wavefront to take a trap + HSA_DBG_NUM_WAVEOP = 5, + HSA_DBG_MAX_WAVEOP = 0xFFFFFFFF +} HSA_DBG_WAVEOP; + +typedef enum _HSA_DBG_WAVEMODE +{ + HSA_DBG_WAVEMODE_SINGLE = 0, //send command to a single wave + //Broadcast to all wavefronts of all processes is not supported for HSA user mode + HSA_DBG_WAVEMODE_BROADCAST_PROCESS = 2, //send to waves within current process + HSA_DBG_WAVEMODE_BROADCAST_PROCESS_CU = 3, //send to waves within current process on CU + HSA_DBG_NUM_WAVEMODE = 3, + HSA_DBG_MAX_WAVEMODE = 0xFFFFFFFF +} HSA_DBG_WAVEMODE; + + +typedef enum _HSA_DBG_WAVEMSG_TYPE +{ + HSA_DBG_WAVEMSG_AUTO = 0, + HSA_DBG_WAVEMSG_USER = 1, + HSA_DBG_WAVEMSG_ERROR = 2, + HSA_DBG_NUM_WAVEMSG, + HSA_DBG_MAX_WAVEMSG = 0xFFFFFFFF +} HSA_DBG_WAVEMSG_TYPE; + +typedef enum _HSA_DBG_WATCH_MODE +{ + HSA_DBG_WATCH_READ = 0, //Read operations only + HSA_DBG_WATCH_NONREAD = 1, //Write or Atomic operations only + HSA_DBG_WATCH_ATOMIC = 2, //Atomic Operations only + HSA_DBG_WATCH_ALL = 3, //Read, Write or Atomic operations + HSA_DBG_WATCH_NUM, + HSA_DBG_WATCH_SIZE = 0xFFFFFFFF +} HSA_DBG_WATCH_MODE; + + +//This structure is hardware specific and may change in the future +typedef struct _HsaDbgWaveMsgAMDGen2 +{ + HSAuint32 Value; + HSAuint32 Reserved2; + +} HsaDbgWaveMsgAMDGen2; + +typedef union _HsaDbgWaveMessageAMD +{ + HsaDbgWaveMsgAMDGen2 WaveMsgInfoGen2; + //for future HsaDbgWaveMsgAMDGen3; +} HsaDbgWaveMessageAMD; + +typedef struct _HsaDbgWaveMessage +{ + void* MemoryVA; // ptr to associated host-accessible data + HsaDbgWaveMessageAMD DbgWaveMsg; +} HsaDbgWaveMessage; + + +// +// HSA sync primitive, Event and HW Exception notification API definitions +// The API functions allow the runtime to define a so-called sync-primitive, a SW object +// combining a user-mode provided "syncvar" and a scheduler event that can be signaled +// through a defined GPU interrupt. A syncvar is a process virtual memory location of +// a certain size that can be accessed by CPU and GPU shader code within the process to set +// and query the content within that memory. The definition of the content is determined by +// the HSA runtime and potentially GPU shader code interfacing with the HSA runtime. +// The syncvar values may be commonly written through an PM4 WRITE_DATA packet in the +// user mode instruction stream. +// The OS scheduler event is typically associated and signaled by an interrupt issued by +// the GPU, but other HSA system interrupt conditions from other HW (e.g. IOMMUv2) may be +// surfaced by the KFD by this mechanism, too. +// + +// these are the new definitions for events +typedef enum _HSA_EVENTTYPE +{ + HSA_EVENTTYPE_SIGNAL = 0, //user-mode generated GPU signal + HSA_EVENTTYPE_NODECHANGE = 1, //HSA node change (attach/detach) + HSA_EVENTTYPE_DEVICESTATECHANGE = 2, //HSA device state change( start/stop ) + HSA_EVENTTYPE_HW_EXCEPTION = 3, //GPU shader exception event + HSA_EVENTTYPE_SYSTEM_EVENT = 4, //GPU SYSCALL with parameter info + HSA_EVENTTYPE_DEBUG_EVENT = 5, //GPU signal for debugging + HSA_EVENTTYPE_PROFILE_EVENT = 6, //GPU signal for profiling + HSA_EVENTTYPE_QUEUE_EVENT = 7, //GPU signal queue idle state (EOP pm4) + //... + HSA_EVENTTYPE_MAXID, + HSA_EVENTTYPE_TYPE_SIZE = 0xFFFFFFFF +} HSA_EVENTTYPE; + +typedef HSAuint32 HSA_EVENTID; + +// +// Subdefinitions for various event types: Syncvar +// + +typedef struct _HsaSyncVar +{ + union + { + void* UserData; //pointer to user mode data + HSAuint64 UserDataPtrValue; //64bit compatibility of value + } SyncVar; + HSAuint64 SyncVarSize; +} HsaSyncVar; + +// +// Subdefinitions for various event types: NodeChange +// + +typedef enum _HSA_EVENTTYPE_NODECHANGE_FLAGS +{ + HSA_EVENTTYPE_NODECHANGE_ADD = 0, + HSA_EVENTTYPE_NODECHANGE_REMOVE = 1, + HSA_EVENTTYPE_NODECHANGE_SIZE = 0xFFFFFFFF +} HSA_EVENTTYPE_NODECHANGE_FLAGS; + +typedef struct _HsaNodeChange +{ + HSA_EVENTTYPE_NODECHANGE_FLAGS Flags; // HSA node added/removed on the platform +} HsaNodeChange; + +// +// Sub-definitions for various event types: DeviceStateChange +// + +typedef enum _HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS +{ + HSA_EVENTTYPE_DEVICESTATUSCHANGE_START = 0, //device started (and available) + HSA_EVENTTYPE_DEVICESTATUSCHANGE_STOP = 1, //device stopped (i.e. unavailable) + HSA_EVENTTYPE_DEVICESTATUSCHANGE_SIZE = 0xFFFFFFFF +} HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS; + +typedef struct _HsaDeviceStateChange +{ + HSAuint32 NodeId; // F-NUMA node that contains the device + HSA_DEVICE Device; // device type: GPU or CPU + HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS Flags; // event flags +} HsaDeviceStateChange; + + +typedef struct _HsaEventData +{ + HSA_EVENTTYPE EventType; //event type + union + { + // return data associated with HSA_EVENTTYPE_SIGNAL and other events + HsaSyncVar SyncVar; + + // data associated with HSA_EVENTTYPE_NODE_CHANGE + HsaNodeChange NodeChangeState; + + // data associated with HSA_EVENTTYPE_DEVICE_STATE_CHANGE + HsaDeviceStateChange DeviceState; + } EventData; + + // the following data entries are internal to the KFD & thunk itself. + + HSAuint64 HWData1; // internal thunk store for Event data (OsEventHandle) + HSAuint64 HWData2; // internal thunk store for Event data (HWAddress) + HSAuint32 HWData3; // internal thunk store for Event data (HWData) +} HsaEventData; + + +typedef struct _HsaEventDescriptor +{ + HSA_EVENTTYPE EventType; // event type to allocate + HSAuint32 NodeId; // H-NUMA node containing GPU device that is event source + HsaSyncVar SyncVar; // pointer to user mode syncvar data, syncvar->UserDataPtrValue may be NULL +} HsaEventDescriptor; + + +typedef struct _HsaEvent +{ + HSA_EVENTID EventId; + HsaEventData EventData; +} HsaEvent; + +typedef enum _HsaEventTimeout +{ + HSA_EVENTTIMEOUT_IMMEDIATE = 0, + HSA_EVENTTIMEOUT_INFINITE = 0xFFFFFFFF +} HsaEventTimeOut; + +typedef struct _HsaClockCounters +{ + HSAuint64 GPUClockCounter; + HSAuint64 CPUClockCounter; + HSAuint64 SystemClockCounter; + HSAuint64 SystemClockFrequencyHz; +} HsaClockCounters; + +#ifndef DEFINE_GUID +typedef struct _HSA_UUID +{ + HSAuint32 Data1; + HSAuint16 Data2; + HSAuint16 Data3; + HSAuint8 Data4[8]; +} HSA_UUID; + +#define HSA_DEFINE_UUID(name, dw, w1, w2, b1, b2, b3, b4, b5, b6, b7, b8) \ + static const HSA_UUID name = {dw, w1, w2, {b1, b2, b3, b4, b5, b6, b7, b8}} +#else +#define HSA_UUID GUID +#define HSA_DEFINE_UUID DEFINE_GUID +#endif + + +// GUID that identifies the GPU Shader Sequencer (SQ) block +// {B5C396B6-D310-47E4-86FC-5CC3043AF508} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_SQ, +0xb5c396b6, 0xd310, 0x47e4, 0x86, 0xfc, 0x5c, 0xc3, 0x4, 0x3a, 0xf5, 0x8); + +// GUID that identifies the GPU Memory Controller (MC) block +// {13900B57-4956-4D98-81D0-68521937F59C} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_MC, +0x13900b57, 0x4956, 0x4d98, 0x81, 0xd0, 0x68, 0x52, 0x19, 0x37, 0xf5, 0x9c); + +// GUID that identifies the IMOMMUv2 HW device +// {80969879-B0F6-4BE6-97F6-6A6300F5101D} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_IOMMUV2, +0x80969879, 0xb0f6, 0x4be6, 0x97, 0xf6, 0x6a, 0x63, 0x0, 0xf5, 0x10, 0x1d); + +// GUID that identifies the KFD +// {EA9B5AE1-6C3F-44B3-8954-DAF07565A90A} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_KERNEL_DRIVER, +0xea9b5ae1, 0x6c3f, 0x44b3, 0x89, 0x54, 0xda, 0xf0, 0x75, 0x65, 0xa9, 0xa); + +typedef enum _HSA_PROFILE_TYPE +{ + HSA_PROFILE_TYPE_PRIVILEGED_IMMEDIATE = 0, //immediate access counter (KFD access only) + HSA_PROFILE_TYPE_PRIVILEGED_STREAMING = 1, //streaming counter, HW continuously + //writes to memory on updates (KFD access only) + HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE = 2, //user-queue accessible counter + HSA_PROFILE_TYPE_NONPRIV_STREAMING = 3, //user-queue accessible counter + //... + HSA_PROFILE_TYPE_NUM, + + HSA_PROFILE_TYPE_SIZE = 0xFFFFFFFF // In order to align to 32-bit value +} HSA_PROFILE_TYPE; + + +typedef struct _HsaCounterFlags +{ + union + { + struct + { + unsigned int Global : 1; // counter is global + // (not tied to VMID/WAVE/CU, ...) + unsigned int Resettable : 1; // counter can be reset by SW + // (always to 0?) + unsigned int ReadOnly : 1; // counter is read-only + // (but may be reset, if indicated) + unsigned int Stream : 1; // counter has streaming capability + // (after trigger, updates buffer) + unsigned int Reserved : 28; + } ui32; + HSAuint32 Value; + }; +} HsaCounterFlags; + + +typedef struct _HsaCounter +{ + HSA_PROFILE_TYPE Type; // specifies the counter type + HSAuint64 CounterId; // indicates counter register offset + HSAuint32 CounterSizeInBits; // indicates relevant counter bits + HSAuint64 CounterMask; // bitmask for counter value (if applicable) + HsaCounterFlags Flags; // Property flags (see above) + HSAuint32 BlockIndex; // identifies block the counter belongs to, + // value may be 0 to NumBlocks +} HsaCounter; + + +typedef struct _HsaCounterBlockProperties +{ + HSA_UUID BlockId; // specifies the block location + HSAuint32 NumCounters; // How many counters are available? + // (sizes Counters[] array below) + HSAuint32 NumConcurrent; // How many counter slots are available + // in block? + HsaCounter Counters[1]; // Start of counter array + // (NumCounters elements total) +} HsaCounterBlockProperties; + + +typedef struct _HsaCounterProperties +{ + HSAuint32 NumBlocks; // How many profilable block are available? + // (sizes Blocks[] array below) + HSAuint32 NumConcurrent; // How many blocks slots can be queried + // concurrently by HW? + HsaCounterBlockProperties Blocks[1]; // Start of block array + // (NumBlocks elements total) +} HsaCounterProperties; + +typedef HSAuint64 HSATraceId; + +typedef struct _HsaPmcTraceRoot +{ + HSAuint64 TraceBufferMinSizeBytes;// (page aligned) + HSAuint32 NumberOfPasses; + HSATraceId TraceId; +} HsaPmcTraceRoot; + +#pragma pack(pop, hsakmttypes_h) + + +#ifdef __cplusplus +} //extern "C" +#endif + +#endif //_HSAKMTTYPES_H_ From 85ba9efa183f5cee639ef656a5bb861c0e8ce247 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 29 Jul 2014 11:16:00 +0300 Subject: [PATCH 0005/1247] Add skeleton driver which supports open and close Signed-off-by: Oded Gabbay --- Makefile | 12 ++++ src/Makefile | 52 +++++++++++++++ src/debug.c | 79 +++++++++++++++++++++++ src/events.c | 153 ++++++++++++++++++++++++++++++++++++++++++++ src/globals.c | 33 ++++++++++ src/libhsakmt.h | 75 ++++++++++++++++++++++ src/libhsakmt.ver | 46 +++++++++++++ src/memory.c | 118 ++++++++++++++++++++++++++++++++++ src/openclose.c | 100 +++++++++++++++++++++++++++++ src/perfctr.c | 160 ++++++++++++++++++++++++++++++++++++++++++++++ src/queues.c | 75 ++++++++++++++++++++++ src/time.c | 39 +++++++++++ src/topology.c | 104 ++++++++++++++++++++++++++++++ src/version.c | 40 ++++++++++++ 14 files changed, 1086 insertions(+) create mode 100644 Makefile create mode 100644 src/Makefile create mode 100644 src/debug.c create mode 100644 src/events.c create mode 100644 src/globals.c create mode 100644 src/libhsakmt.h create mode 100644 src/libhsakmt.ver create mode 100644 src/memory.c create mode 100644 src/openclose.c create mode 100644 src/perfctr.c create mode 100644 src/queues.c create mode 100644 src/time.c create mode 100644 src/topology.c create mode 100644 src/version.c diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..e356d53936 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +.PHONY: all clean lnx64a lnx + +all: lnx lnx64a + +lnx64a: + $(MAKE) -C src lnx64a + +lnx: + $(MAKE) -C src lnx + +clean: + $(MAKE) -C src clean diff --git a/src/Makefile b/src/Makefile new file mode 100644 index 0000000000..406bda0d9a --- /dev/null +++ b/src/Makefile @@ -0,0 +1,52 @@ +# Include directories +INCLUDES += ../include +CFLAGS += $(foreach DIR,$(INCLUDES),-I$(DIR)) + +LIB_NAME = libhsakmt.so +LIB_MAJOR_VER = 1 + +# Compiler options +CFLAGS += -fPIC # Position-independent code required to build shared library +CFLAGS += -W -Wall -Wextra -Werror -Wno-unused-parameter +CFLAGS += -Wformat-security -Wswitch-default -Wundef \ + -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual \ + -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes \ + -Wmissing-declarations -Wredundant-decls \ + -Wunreachable-code +CFLAGS += -std=gnu99 -ggdb -pthread -fvisibility=hidden -O2 + +LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) + +OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ + events.o openclose.o queues.o topology.o + +.PHONY: all lnx lnx64a clean + +# Default target +all: lnx lnx64a + +BUILD_ROOT = ../build +BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) + +TARGET = $(addprefix $(BUILDDIR)/,$(OBJS)) + +$(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER): $(TARGET) + gcc -shared $(LDFLAGS) -o $@ $^ + +$(BUILDDIR)/$(LIB_NAME): $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) + @ln -sf $(LIB_NAME).$(LIB_MAJOR_VER) $(BUILDDIR)/$(LIB_NAME) + +lnx: CFLAGS += -m32 +lnx: LDFLAGS += -m32 +lnx: $(BUILDDIR)/$(LIB_NAME) + +lnx64a: $(BUILDDIR)/$(LIB_NAME) + +clean: + rm -rf $(BUILD_ROOT) + +#Rule +$(BUILDDIR)/%.o: %.c + @echo Compiling $^ + @mkdir -p $(dir $@) + gcc $(CFLAGS) -c $< -o $@ diff --git a/src/debug.c b/src/debug.c new file mode 100644 index 0000000000..dab3a51973 --- /dev/null +++ b/src/debug.c @@ -0,0 +1,79 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgRegister( + HSAuint32 NodeId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgUnregister( + HSAuint32 NodeId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgWavefrontControl( + HSAuint32 NodeId, //IN + HSA_DBG_WAVEOP Operand, //IN + HSA_DBG_WAVEMODE Mode, //IN + HSAuint32 TrapId, //IN + HsaDbgWaveMessage* DbgWaveMsgRing //IN (? - see thunk API doc!) + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgAddressWatch( + HSAuint32 NodeId, //IN + HSAuint32 NumWatchPoints, //IN + HSA_DBG_WATCH_MODE WatchMode[], //IN + void* WatchAddress[], //IN + HSAuint64 WatchMask[], //IN, optional + HsaEvent* WatchEvent[] //IN, optional + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/src/events.c b/src/events.c new file mode 100644 index 0000000000..cb07116aa8 --- /dev/null +++ b/src/events.c @@ -0,0 +1,153 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include +#include +#include +#include + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtCreateEvent( + HsaEventDescriptor* EventDesc, //IN + bool ManualReset, //IN + bool IsSignaled, //IN + HsaEvent** Event //OUT + ) +{ + CHECK_KFD_OPEN(); + + if (EventDesc->EventType >= HSA_EVENTTYPE_MAXID) + { + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + HsaEvent* e = malloc(sizeof(HsaEvent)); + if (e == NULL) + { + return HSAKMT_STATUS_ERROR; + } + + *Event = e; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDestroyEvent( + HsaEvent* Event //IN + ) +{ + CHECK_KFD_OPEN(); + + free(Event); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetEvent( + HsaEvent* Event //IN + ) +{ + CHECK_KFD_OPEN(); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtResetEvent( + HsaEvent* Event //IN + ) +{ + CHECK_KFD_OPEN(); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueryEventState( + HsaEvent* Event //IN + ) +{ + CHECK_KFD_OPEN(); + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtWaitOnEvent( + HsaEvent* Event, //IN + HSAuint32 Milliseconds //IN + ) +{ + CHECK_KFD_OPEN(); + + if (Milliseconds == HSA_EVENTTIMEOUT_INFINITE) + { + while (1) { pause(); } + } + else if (Milliseconds != HSA_EVENTTIMEOUT_IMMEDIATE) + { + struct timespec req; + + req.tv_sec = Milliseconds / 1000; + req.tv_nsec = (long)(Milliseconds % 1000) * 1000000; + + while (1) + { + struct timespec rem; + + int err = nanosleep(&req, &rem); + if (err == -1 && errno == EINTR) + { + req = rem; + } + else + { + break; // success or other error + } + } + } + + return HSAKMT_STATUS_WAIT_TIMEOUT; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtWaitOnMultipleEvents( + HsaEvent* Events[], //IN + HSAuint32 NumEvents, //IN + bool WaitOnAll, //IN + HSAuint32 Milliseconds //IN + ) +{ + CHECK_KFD_OPEN(); + + return hsaKmtWaitOnEvent(NULL, Milliseconds); +} diff --git a/src/globals.c b/src/globals.c new file mode 100644 index 0000000000..cad6b1f989 --- /dev/null +++ b/src/globals.c @@ -0,0 +1,33 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" + +// HSAKMT global data + +int kfd_fd; +unsigned long kfd_open_count; +unsigned long system_properties_count; +pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; diff --git a/src/libhsakmt.h b/src/libhsakmt.h new file mode 100644 index 0000000000..8972ba9341 --- /dev/null +++ b/src/libhsakmt.h @@ -0,0 +1,75 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef LIBHSAKMT_H_INCLUDED +#define LIBHSAKMT_H_INCLUDED + +#include "hsakmt.h" +#include +#include +#include + +extern int kfd_fd; +extern unsigned long kfd_open_count; +extern pthread_mutex_t hsakmt_mutex; + +#undef HSAKMTAPI +#define HSAKMTAPI __attribute__((visibility ("default"))) + +/*Avoid pointer-to-int-cast warning*/ +#define PORT_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr)) + +/*Avoid int-to-pointer-cast warning*/ +#define PORT_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v)) + +#define CHECK_KFD_OPEN() \ + do { if (kfd_open_count == 0) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) + +#define PAGE_SIZE 4096 + +#define CHECK_PAGE_MULTIPLE(x) \ + do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) + +#define PAGE_ALIGN_UP(x) (((uint64_t)(x) + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE-1)) +#define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) + +/* + * Even though the toplogy code doesn't limit us to maximum number of nodes, + * the current HSA spec says the maximum is 8 nodes + */ +#define MAX_NODES 8 + +HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); +uint16_t get_device_id_by_node(HSAuint32 node_id); + +extern int kfd_ioctl(int cmdcode, void* data); + +/* Void pointer arithmetic (or remove -Wpointer-arith to allow void pointers arithmetic) */ +#define VOID_PTR_ADD(ptr,n) (void*)((uint8_t*)(ptr) + n)/*ptr + offset*/ +#define VOID_PTR_SUB(ptr,n) (void*)((uint8_t*)(ptr) - n)/*ptr - offset*/ +#define VOID_PTRS_SUB(ptr1,ptr2) (uint64_t)((uint8_t*)(ptr1) - (uint8_t*)(ptr2)) /*ptr1 - ptr2*/ + + +#endif diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver new file mode 100644 index 0000000000..9c6e6cb1fd --- /dev/null +++ b/src/libhsakmt.ver @@ -0,0 +1,46 @@ +HSAKMT_1 +{ +global: +hsaKmtOpenKFD; +hsaKmtCloseKFD; +hsaKmtGetVersion; +hsaKmtAcquireSystemProperties; +hsaKmtReleaseSystemProperties; +hsaKmtGetNodeProperties; +hsaKmtGetNodeMemoryProperties; +hsaKmtGetNodeCacheProperties; +hsaKmtGetNodeIoLinkProperties; +hsaKmtCreateEvent; +hsaKmtDestroyEvent; +hsaKmtSetEvent; +hsaKmtResetEvent; +hsaKmtQueryEventState; +hsaKmtWaitOnEvent; +hsaKmtWaitOnMultipleEvents; +hsaKmtCreateQueue; +hsaKmtUpdateQueue; +hsaKmtDestroyQueue; +hsaKmtSetMemoryPolicy; +hsaKmtAllocMemory; +hsaKmtFreeMemory; +hsaKmtRegisterMemory; +hsaKmtDeregisterMemory; +hsaKmtMapMemoryToGPU; +hsaKmtUnmapMemoryToGPU; +hsaKmtDbgRegister; +hsaKmtDbgUnregister; +hsaKmtDbgWavefrontControl; +hsaKmtDbgAddressWatch; +hsaKmtGetClockCounters; +hsaKmtPmcGetCounterProperties; +hsaKmtPmcRegisterTrace; +hsaKmtPmcUnregisterTrace; +hsaKmtPmcAcquireTraceAccess; +hsaKmtPmcReleaseTraceAccess; +hsaKmtPmcStartTrace; +hsaKmtPmcQueryTrace; +hsaKmtPmcStopTrace; + +local: *; +}; + diff --git a/src/memory.c b/src/memory.c new file mode 100644 index 0000000000..acc45dc89c --- /dev/null +++ b/src/memory.c @@ -0,0 +1,118 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include "linux/kfd_ioctl.h" +#include +#include +#include + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetMemoryPolicy( + HSAuint32 Node, + HSAuint32 DefaultPolicy, + HSAuint32 AlternatePolicy, + void* MemoryAddressAlternate, + HSAuint64 MemorySizeInBytes + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtAllocMemory( + HSAuint32 PreferredNode, //IN + HSAuint64 SizeInBytes, //IN (multiple of page size) + HsaMemFlags MemFlags, //IN + void** MemoryAddress //OUT (page-aligned) + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtFreeMemory( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 SizeInBytes //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemory( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 MemorySizeInBytes //IN (page-aligned) + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDeregisterMemory( + void* MemoryAddress //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtMapMemoryToGPU( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 MemorySizeInBytes, //IN (page-aligned) + HSAuint64* AlternateVAGPU //OUT (page-aligned) + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtUnmapMemoryToGPU( + void* MemoryAddress //IN (page-aligned) + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/src/openclose.c b/src/openclose.c new file mode 100644 index 0000000000..f79b9607c9 --- /dev/null +++ b/src/openclose.c @@ -0,0 +1,100 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" + +#include +#include +#include +#include +#include + +static const char kfd_device_name[] = "/dev/kfd"; + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtOpenKFD(void) +{ + HSAKMT_STATUS result; + + pthread_mutex_lock(&hsakmt_mutex); + + if (kfd_open_count == 0) + { + int fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); + + if (fd != -1) + { + kfd_fd = fd; + kfd_open_count = 1; + result = HSAKMT_STATUS_SUCCESS; + } + else + { + result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; + } + } + else + { + kfd_open_count++; + result = HSAKMT_STATUS_SUCCESS; + } + + pthread_mutex_unlock(&hsakmt_mutex); + + return result; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtCloseKFD(void) +{ + HSAKMT_STATUS result; + + pthread_mutex_lock(&hsakmt_mutex); + + if (kfd_open_count > 0) + { + if (--kfd_open_count == 0) + { + close(kfd_fd); + } + + result = HSAKMT_STATUS_SUCCESS; + } + else + { + result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; + } + + pthread_mutex_unlock(&hsakmt_mutex); + + return result; +} + +extern int kfd_ioctl(int cmdcode, void* data) +{ + return ioctl(kfd_fd, cmdcode, data); +} diff --git a/src/perfctr.c b/src/perfctr.c new file mode 100644 index 0000000000..0f10f6ac71 --- /dev/null +++ b/src/perfctr.c @@ -0,0 +1,160 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include "libhsakmt.h" +#include "linux/kfd_ioctl.h" + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcGetCounterProperties( + HSAuint32 NodeId, //IN + HsaCounterProperties** CounterProperties //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +/** + Registers a set of (HW) counters to be used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcRegisterTrace( + HSAuint32 NodeId, //IN + HSAuint32 NumberOfCounters, //IN + HsaCounter* Counters, //IN + HsaPmcTraceRoot* TraceRoot //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +/** + Unregisters a set of (HW) counters used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcUnregisterTrace( + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + + +/** + Allows a user mode process to get exclusive access to the defined set of (HW) counters + used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcAcquireTraceAccess( + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + + +/** + Allows a user mode process to release exclusive access to the defined set of (HW) counters + used for tracing/profiling +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcReleaseTraceAccess( + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + + +/** + Starts tracing operation on a previously established set of performance counters +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcStartTrace( + HSATraceId TraceId, //IN + void* TraceBuffer, //IN (page aligned) + HSAuint64 TraceBufferSizeBytes //IN (page aligned) + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + + +/** + Forces an update of all the counters that a previously started trace operation has registered +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcQueryTrace( + HSATraceId TraceId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + + +/** + Stops tracing operation on a previously established set of performance counters +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPmcStopTrace( + HSATraceId TraceId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/src/queues.c b/src/queues.c new file mode 100644 index 0000000000..3c1c4ff3df --- /dev/null +++ b/src/queues.c @@ -0,0 +1,75 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include "linux/kfd_ioctl.h" +#include +#include + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtCreateQueue( + HSAuint32 NodeId, //IN + HSA_QUEUE_TYPE Type, //IN + HSAuint32 QueuePercentage, //IN + HSA_QUEUE_PRIORITY Priority, //IN + void* QueueAddress, //IN + HSAuint64 QueueSizeInBytes, //IN + HsaEvent* Event, //IN + HsaQueueResource* QueueResource //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtUpdateQueue( + HSA_QUEUEID QueueId, //IN + HSAuint32 QueuePercentage,//IN + HSA_QUEUE_PRIORITY Priority, //IN + void* QueueAddress, //IN + HSAuint64 QueueSize, //IN + HsaEvent* Event //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDestroyQueue( + HSA_QUEUEID QueueId //IN + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/src/time.c b/src/time.c new file mode 100644 index 0000000000..df6ad0b2bb --- /dev/null +++ b/src/time.c @@ -0,0 +1,39 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include "linux/kfd_ioctl.h" + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetClockCounters( + HSAuint32 NodeId, //IN + HsaClockCounters* Counters //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/src/topology.c b/src/topology.c new file mode 100644 index 0000000000..e52f17b28e --- /dev/null +++ b/src/topology.c @@ -0,0 +1,104 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "libhsakmt.h" + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtAcquireSystemProperties( + HsaSystemProperties* SystemProperties //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtReleaseSystemProperties(void) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeProperties( + HSAuint32 NodeId, //IN + HsaNodeProperties* NodeProperties //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeMemoryProperties( + HSAuint32 NodeId, //IN + HSAuint32 NumBanks, //IN + HsaMemoryProperties* MemoryProperties //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeCacheProperties( + HSAuint32 NodeId, //IN + HSAuint32 ProcessorId, //IN + HSAuint32 NumCaches, //IN + HsaCacheProperties* CacheProperties //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetNodeIoLinkProperties( + HSAuint32 NodeId, //IN + HSAuint32 NumIoLinks, //IN + HsaIoLinkProperties* IoLinkProperties //OUT + ) +{ + CHECK_KFD_OPEN(); + + return HSAKMT_STATUS_NOT_SUPPORTED; +} diff --git a/src/version.c b/src/version.c new file mode 100644 index 0000000000..434b625717 --- /dev/null +++ b/src/version.c @@ -0,0 +1,40 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetVersion( + HsaVersionInfo* VersionInfo //OUT + ) +{ + CHECK_KFD_OPEN(); + + VersionInfo->KernelInterfaceMajorVersion = HSAKMT_VERSION_MAJOR; + VersionInfo->KernelInterfaceMinorVersion = HSAKMT_VERSION_MINOR; + + return HSAKMT_STATUS_SUCCESS; +} From 1905152a3387f65da8080f8d46e484dc0437ebda Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Tue, 29 Jul 2014 11:21:10 +0300 Subject: [PATCH 0006/1247] Add flat memory manager module Signed-off-by: Alexey Skidanov Signed-off-by: Oded Gabbay --- src/Makefile | 2 +- src/fmm.c | 486 ++++++++++++++++++++++++++++++++++++++++++++++++ src/fmm.h | 60 ++++++ src/openclose.c | 6 +- 4 files changed, 552 insertions(+), 2 deletions(-) create mode 100644 src/fmm.c create mode 100644 src/fmm.h diff --git a/src/Makefile b/src/Makefile index 406bda0d9a..748fa2671c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -18,7 +18,7 @@ CFLAGS += -std=gnu99 -ggdb -pthread -fvisibility=hidden -O2 LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ - events.o openclose.o queues.o topology.o + events.o openclose.o queues.o topology.o fmm.o .PHONY: all lnx lnx64a clean diff --git a/src/fmm.c b/src/fmm.c new file mode 100644 index 0000000000..4edbb3096b --- /dev/null +++ b/src/fmm.c @@ -0,0 +1,486 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "fmm.h" +#include "linux/kfd_ioctl.h" +#include "libhsakmt.h" +#include +#include +#include +#include + +#define NON_VALID_GPU_ID 0 +#define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) +#define INIT_APERTURE(base_value, limit_value) {.base = (void*)base_value, .limit = (void*)limit_value } +#define INIT_MANAGEBLE_APERTURE(base_value, limit_value) {.base = (void*)base_value,.limit = (void*)limit_value, .vm_ranges = NULL, .vm_objects = NULL, .fmm_mutex = PTHREAD_MUTEX_INITIALIZER} +#define INIT_GPU_MEM \ +{ .gpu_id = NON_VALID_GPU_ID,\ + .lds_aperture = INIT_APERTURE(0, 0), \ + .scratch_aperture = INIT_MANAGEBLE_APERTURE(0, 0),\ + .gpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0)\ +} + +#define INIT_GPUs_MEM {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = INIT_GPU_MEM} +struct vm_object{ + void* start; + HSAuint64 size; + HSAuint64 handle; // opaque + struct vm_object* next; + struct vm_object* prev; +}; +typedef struct vm_object vm_object_t; + +struct vm_area{ + void* start; + void* end; + struct vm_area* next; + struct vm_area* prev; +}; +typedef struct vm_area vm_area_t; + +typedef struct { + void* base; + void* limit; + vm_area_t* vm_ranges; + vm_object_t* vm_objects; + pthread_mutex_t fmm_mutex; +} manageble_aperture_t; + +typedef struct { + void* base; + void* limit; +} aperture_t; + +typedef struct{ + HSAuint32 gpu_id; + aperture_t lds_aperture; + manageble_aperture_t scratch_aperture; + manageble_aperture_t gpuvm_aperture; +}gpu_mem_t; + +static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; + +static vm_area_t* vm_create_and_init_area(void* start, void* end){ + vm_area_t* area = (vm_area_t*)malloc(sizeof(vm_area_t));// TODO: Memory pool ??? + if (area){ + area->start = start; + area->end = end; + area->next = area->prev = NULL; + } + + return area; +} + +static vm_object_t* vm_create_and_init_object(void* start, uint64_t size, uint64_t handle){ + vm_object_t* object = (vm_object_t*)malloc(sizeof(vm_object_t)); // TODO: Memory pool ??? + if (object){ + object->start = start; + object->size = size; + object->handle = handle; + object->next = object->prev = NULL; + } + + return object; +} + + +static void vm_remove_area(manageble_aperture_t* app, vm_area_t* area){ + vm_area_t* next; + vm_area_t* prev; + + next = area->next; + prev = area->prev; + + if (prev == NULL )// The first element + app->vm_ranges = next; + else + prev->next = next; + + if(next) // If not the last element + next->prev = prev; + + free(area); + +} + +static void vm_remove_object(manageble_aperture_t* app, vm_object_t* object){ + vm_object_t* next; + vm_object_t* prev; + + next = object->next; + prev = object->prev; + + if (prev == NULL )// The first element + app->vm_objects = next; + else + prev->next = next; + + if(next) // If not the last element + next->prev = prev; + + free(object); + +} + + + +static void vm_add_area_after(vm_area_t* after_this, vm_area_t* new_area){ + vm_area_t* next = after_this->next; + after_this->next = new_area; + new_area->next = next; + + new_area->prev = after_this; + if (next) + next->prev = new_area; +} + +static void vm_add_object_before(vm_object_t* before_this, vm_object_t* new_object){ + vm_object_t* prev = before_this->prev; + before_this->prev = new_object; + new_object->next = before_this; + + new_object->prev = prev; + if (prev) + prev->next = new_object; +} + +static void vm_split_area(manageble_aperture_t* app, vm_area_t* area, void* address, uint64_t MemorySizeInBytes){ + + // The existing area is split to: [area->start, address - 1] and [address + MemorySizeInBytes, area->end] + vm_area_t* new_area = vm_create_and_init_area(VOID_PTR_ADD(address,MemorySizeInBytes), area->end); + + // Shrink the existing area + area->end = VOID_PTR_SUB(address,1); + + vm_add_area_after(area, new_area); + +} + +static vm_object_t* vm_find_object_by_address(manageble_aperture_t* app, void* address, uint64_t size){ + vm_object_t* cur = app->vm_objects; + + // Look up the appropriate address range containing the given address + while(cur){ + if(cur->start == address && cur->size == size) + break; + cur = cur->next; + }; + + return cur; // NULL if not found +} + +static vm_area_t* vm_find(manageble_aperture_t* app, void* address){ + vm_area_t* cur = app->vm_ranges; + + // Look up the appropriate address range containing the given address + while(cur){ + if(cur->start <= address && cur->end >= address) + break; + cur = cur->next; + }; + + return cur; // NULL if not found +} + +static bool aperture_is_valid(void* app_base, void* app_limit){ + if (app_base && app_limit && app_base < app_limit) + return true; + return false; +} + +/* + * Assumes that fmm_mutex is locked on entry. + */ +static int aperture_release(manageble_aperture_t* app, void* address, uint64_t MemorySizeInBytes){ + int rc = -1; + vm_area_t* area; + + area = vm_find(app, address); + vm_object_t* object = vm_find_object_by_address(app, address, MemorySizeInBytes); + if (object && area){ + vm_remove_object(app, object); + if (VOID_PTRS_SUB(area->end, area->start) + 1 > MemorySizeInBytes){ // the size of the released block is less than the size of area + if (area->start == address){ // shrink from the start + area->start = VOID_PTR_ADD(area->start,MemorySizeInBytes); + } else if (VOID_PTRS_SUB(area->end, address) + 1 == MemorySizeInBytes){ // shrink from the end + area->end = VOID_PTR_SUB(area->end, MemorySizeInBytes); + } else { // split the area + vm_split_area(app, area, address, MemorySizeInBytes); + } + rc = 0; + } else if (VOID_PTRS_SUB(area->end, area->start) + 1 == MemorySizeInBytes){ // the size of the released block is exactly the same as the size of area + vm_remove_area(app, area); + rc = 0; + } else { + //Inconsistent data. Fail it? + rc = -1; + } + } + + return rc; +} + +/* + * returns allocated address or NULL. Assumes, that fmm_mutex is locked on entry. + */ +static void* aperture_allocate(manageble_aperture_t* app, uint64_t MemorySizeInBytes){ + vm_area_t* cur, *next, *new_area, *start; + vm_object_t* new_object; + void* new_address = NULL; + next = NULL; + new_area = NULL; + + cur = app->vm_ranges; + if (cur){ // not empty + + // Look up the appropriate address space "hole" or end of the list + while(cur){ + next = cur->next; + + // End of the list reached + if (!next) + break; + + // address space "hole" + if ((VOID_PTRS_SUB(next->start,cur->end) >= MemorySizeInBytes)) + break; + + cur = next; + }; + + // If the new range is inside the reserved aperture + if (VOID_PTRS_SUB(app->limit, cur->end) + 1 >= MemorySizeInBytes){ + // cur points to the last inspected element: the tail of the list or the found "hole" + // Just extend the existing region + new_address = VOID_PTR_ADD(cur->end, 1); + cur->end = VOID_PTR_ADD(cur->end, MemorySizeInBytes); + } else + new_address = NULL; + + } else { // empty - create the first area + start = (void*)app->base; + new_area = vm_create_and_init_area(start, VOID_PTR_ADD(start, (MemorySizeInBytes - 1))); + if (new_area){ + app->vm_ranges = new_area; + new_address = new_area->start; + } + } + + // Allocate new object + if (new_address){ + new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, 0); + if (new_object){ + if (app->vm_objects == NULL){ // empty list + // Update head + app->vm_objects = new_object; + } else { + // Add it before the first element + vm_add_object_before(app->vm_objects, new_object); + // Update head + app->vm_objects = new_object; + } + } else{ + // Failed to allocate object: remove just allocated range and return NULL + aperture_release(app, new_address, MemorySizeInBytes); + new_address = NULL; + } + } + + return new_address; + +} + + + +static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id){ + int32_t i; + + for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ + if(gpu_mem[i].gpu_id == gpu_id) + return i; + } + + return -1; +} + +bool fmm_is_inside_some_aperture(void* address){ + + int32_t i; + + for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ + if(gpu_mem[i].gpu_id != NON_VALID_GPU_ID){ + if ((address>= gpu_mem[i].lds_aperture.base) && (address<= gpu_mem[i].lds_aperture.limit)) + return true; + if ((address>= gpu_mem[i].gpuvm_aperture.base) && (address<= gpu_mem[i].gpuvm_aperture.limit)) + return true; + if ((address>= gpu_mem[i].scratch_aperture.base) && (address<= gpu_mem[i].scratch_aperture.limit)) + return true; + } + } + + return false; +} + +#ifdef DEBUG_PRINT_APERTURE +static void aperture_print(aperture_t* app){ + printf("\t Base: %p\n", app->base); + printf("\t Limit: %p\n", app->limit); +} + +static void manageble_aperture_print(manageble_aperture_t* app){ + vm_area_t* cur = app->vm_ranges; + vm_object_t *object = app->vm_objects; + + printf("\t Base: %p\n", app->base); + printf("\t Limit: %p\n", app->limit); + printf("\t Ranges: \n"); + while(cur){ + printf("\t\t Range [%p - %p] \n", cur->start, cur->end); + cur = cur->next; + }; + printf("\t Objects: \n"); + while(object){ + printf("\t\t Object [%p - %" PRIu64 "] \n", object->start, object->size); + object = object->next; + }; +} + +void fmm_print(uint32_t gpu_id){ + int32_t i = gpu_mem_find_by_gpu_id(gpu_id); + if(i >= 0){ // Found + printf("LDS aperture: \n"); + aperture_print(&gpu_mem[i].lds_aperture); + printf("GPUVM aperture: \n"); + manageble_aperture_print(&gpu_mem[i].gpuvm_aperture); + printf("Scratch aperture: \n"); + manageble_aperture_print(&gpu_mem[i].scratch_aperture); + + } +} +#else +void fmm_print(uint32_t gpu_id){ + +} +#endif + + +void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes){ + + void* mem = NULL; + int32_t i = gpu_mem_find_by_gpu_id(gpu_id); + + // If not found or aperture isn't properly initialized/supported + if(i < 0 || !aperture_is_valid(gpu_mem[i].scratch_aperture.base, gpu_mem[i].scratch_aperture.limit)) + return NULL; + + pthread_mutex_lock(&gpu_mem[i].scratch_aperture.fmm_mutex); + mem = aperture_allocate(&gpu_mem[i].scratch_aperture, MemorySizeInBytes); + pthread_mutex_unlock(&gpu_mem[i].scratch_aperture.fmm_mutex); + + return mem; +} + +void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes){ + + void* mem = NULL; + int32_t i = gpu_mem_find_by_gpu_id(gpu_id); + + // If not found or aperture isn't properly initialized/supported + if(i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, gpu_mem[i].gpuvm_aperture.limit)) + return NULL; + + pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + mem = aperture_allocate(&gpu_mem[i].gpuvm_aperture, MemorySizeInBytes); + pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + + return mem; +} + + +int fmm_release(void* address, uint64_t MemorySizeInBytes){ + + uint32_t i; + int32_t rc = -1; + + for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ + if(gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + continue; + + if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit){ + pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + rc = aperture_release(&gpu_mem[i].gpuvm_aperture, address, MemorySizeInBytes); + pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + fmm_print(gpu_mem[i].gpu_id); + } else if (address >= gpu_mem[i].scratch_aperture.base && address <= gpu_mem[i].scratch_aperture.limit) + pthread_mutex_lock(&gpu_mem[i].scratch_aperture.fmm_mutex); + rc = aperture_release(&gpu_mem[i].scratch_aperture, address, MemorySizeInBytes); + pthread_mutex_unlock(&gpu_mem[i].scratch_aperture.fmm_mutex); + } + + return rc; +} + +HSAKMT_STATUS fmm_init_process_apertures(){ + struct kfd_ioctl_get_process_apertures_args args; + uint8_t node_id; + + if (0 == kfd_ioctl(KFD_IOC_GET_PROCESS_APERTURES, (void*)&args)){ + for(node_id = 0; node_id < args.num_of_nodes; node_id++){ + gpu_mem[node_id].gpu_id = args.process_apertures[node_id].gpu_id; + gpu_mem[node_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); + gpu_mem[node_id].lds_aperture.limit = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_limit); + gpu_mem[node_id].gpuvm_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_base); + gpu_mem[node_id].gpuvm_aperture.limit = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_limit); + gpu_mem[node_id].scratch_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_base); + gpu_mem[node_id].scratch_aperture.limit = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_limit); + } + + return HSAKMT_STATUS_SUCCESS; + } + + return HSAKMT_STATUS_ERROR; + +} + +HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id){ + int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); + if (slot<0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + switch(aperture_type){ + case FMM_GPUVM: + return aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, gpu_mem[slot].gpuvm_aperture.limit) ? PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.base) : 0; + break; + case FMM_SCRATCH: + return aperture_is_valid(gpu_mem[slot].scratch_aperture.base, gpu_mem[slot].scratch_aperture.limit) ? PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.base) : 0; + break; + case FMM_LDS: + return aperture_is_valid(gpu_mem[slot].lds_aperture.base, gpu_mem[slot].lds_aperture.limit) ? PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.base) : 0; + break; + default: + return 0; + } + +} diff --git a/src/fmm.h b/src/fmm.h new file mode 100644 index 0000000000..5924247645 --- /dev/null +++ b/src/fmm.h @@ -0,0 +1,60 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef FMM_H_ +#define FMM_H_ + +#include "hsakmttypes.h" +#include + +typedef enum { + FMM_FIRST_APERTURE_TYPE = 0, + FMM_GPUVM = FMM_FIRST_APERTURE_TYPE, + FMM_LDS, + FMM_SCRATCH, + FMM_LAST_APERTURE_TYPE +} aperture_type_e; + +typedef struct { + aperture_type_e app_type; + uint64_t size; + void* start_address; +} aperture_properties_t; + +HSAKMT_STATUS fmm_init_process_apertures(void); +/* + * Memory interface + */ +void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); +void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes); +void fmm_print(uint32_t node); +bool fmm_is_inside_some_aperture(void* address); +int fmm_release(void* address, HSAuint64 MemorySizeInBytes); + +/* Topology interface*/ +HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); +HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); +HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id); +#endif /* FMM_H_ */ diff --git a/src/openclose.c b/src/openclose.c index f79b9607c9..dad0cb9084 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -30,6 +30,7 @@ #include #include #include +#include "fmm.h" static const char kfd_device_name[] = "/dev/kfd"; @@ -49,7 +50,10 @@ hsaKmtOpenKFD(void) { kfd_fd = fd; kfd_open_count = 1; - result = HSAKMT_STATUS_SUCCESS; + + result = fmm_init_process_apertures(); + if (result != HSAKMT_STATUS_SUCCESS) + close(fd); } else { From 8f560d5b8333d2ffd09e05384759fa641f95e0f4 Mon Sep 17 00:00:00 2001 From: Evgeny Pinchuk Date: Tue, 29 Jul 2014 13:52:02 +0300 Subject: [PATCH 0007/1247] Add topology module Signed-off-by: Evgeny Pinchuk Signed-off-by: Oded Gabbay --- src/topology.c | 846 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 837 insertions(+), 9 deletions(-) diff --git a/src/topology.c b/src/topology.c index e52f17b28e..492b5a3412 100644 --- a/src/topology.c +++ b/src/topology.c @@ -30,6 +30,665 @@ #include #include "libhsakmt.h" +#include "fmm.h" +#define PAGE_SIZE 4096 +#define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) +#define NUM_OF_HEAPS 2 +/* SYSFS related */ +#define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" +#define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" +#define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" + +typedef struct { + uint32_t gpu_id; + HsaNodeProperties node; + HsaMemoryProperties *mem; /* node->NumBanks elements */ + HsaCacheProperties *cache; + HsaIoLinkProperties *link; +} node_t; + +static HsaSystemProperties *system = NULL; +static node_t *node = NULL; + +static HSAKMT_STATUS topology_take_snapshot(void); +static HSAKMT_STATUS topology_drop_snapshot(void); + +static void +free_node(node_t *n) +{ + assert(n); + + if (n == NULL) + return; + + if ((n)->mem) + free((n)->mem); + if ((n)->cache) + free((n)->cache); + if ((n)->link) + free((n)->link); +} + +static HSAKMT_STATUS +topology_sysfs_get_generation(uint32_t *gen) { + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(gen); + fd = fopen(KFD_SYSFS_PATH_GENERATION_ID, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + if (fscanf(fd, "%ul", gen) != 1) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + +err: + fclose(fd); + return ret; +} + +static HSAKMT_STATUS +topology_sysfs_get_system_props(HsaSystemProperties *props) { + FILE *fd; + DIR *dirp; + char *read_buf, *p; + char prop_name[256]; + long long unsigned int prop_val; + uint32_t node_count, prog; + struct dirent *dir; + int read_size; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + + assert(props); + fd = fopen(KFD_SYSFS_PATH_SYSTEM_PROPERTIES, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + + read_buf = malloc(PAGE_SIZE); + if (!read_buf) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err1; + } + + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto err2; + } + + /* Since we're using the buffer as a string, we make sure the string terminates */ + if(read_size >= PAGE_SIZE) + read_size = PAGE_SIZE-1; + read_buf[read_size] = 0; + + /* + * Read the system properties + */ + prog = 0; + p = read_buf; + while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name,"platform_oem") == 0) + props->PlatformOem = (uint32_t)prop_val; + else if (strcmp(prop_name,"platform_id") == 0) + props->PlatformId = (uint32_t)prop_val; + else if (strcmp(prop_name,"platform_rev") == 0) + props->PlatformRev = (uint32_t)prop_val; + } + + /* + * Discover the number of nodes + */ + node_count = 0; + dirp = opendir(KFD_SYSFS_PATH_NODES); + if(dirp) { + /* + * Assuming that inside nodes folder there are only folders + * which represent the node numbers + */ + while ((dir = readdir(dirp)) != 0) { + if ((strcmp(dir->d_name, ".") == 0) || + (strcmp(dir->d_name, "..") == 0)) + continue; + node_count++; + } + closedir(dirp); + } + props->NumNodes = node_count; + + +err2: + free(read_buf); +err1: + fclose(fd); + return ret; +} + +static HSAKMT_STATUS +topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) { + FILE *fd; + char path[256]; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(gpu_id); + snprintf(path, 256, "%s/%d/gpu_id", KFD_SYSFS_PATH_NODES, node_id); + fd = fopen(path, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + if (fscanf(fd, "%ul", gpu_id) != 1) { + ret = HSAKMT_STATUS_ERROR; + } + fclose(fd); + + return ret; +} + +static HSAKMT_STATUS +topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id) { + FILE *fd; + char *read_buf, *p; + char prop_name[256]; + char path[256]; + long long unsigned int prop_val; + uint32_t i, prog; + int read_size; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(props); + assert(gpu_id); + /* Retrieve the GPU ID */ + ret = topology_sysfs_get_gpu_id(node_id, gpu_id); + + /* Retrieve the marketing name of the node */ + snprintf(path, 256, "%s/%d/name", KFD_SYSFS_PATH_NODES, node_id); + fd = fopen(path, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + + read_buf = malloc(PAGE_SIZE); + if (!read_buf) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err1; + } + + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto err2; + } + p = memchr(read_buf, '\n', read_size); + if ((!p) || ((p-read_buf) > HSA_PUBLIC_NAME_SIZE)) { + ret = HSAKMT_STATUS_ERROR; + goto err2; + } + /* + * Convert UTF8 to UTF16 + */ + for (i = 0; (i < HSA_PUBLIC_NAME_SIZE) && (read_buf[i] != '\n'); i++) + props->MarketingName[i] = read_buf[i]; + props->MarketingName[i] = 0; + fclose(fd); + + /* Retrieve the node properties */ + snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, node_id); + fd = fopen(path, "r"); + if (!fd) { + free(read_buf); + return HSAKMT_STATUS_ERROR; + } + + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto err2; + } + + /* Since we're using the buffer as a string, we make sure the string terminates */ + if(read_size >= PAGE_SIZE) + read_size = PAGE_SIZE-1; + read_buf[read_size] = 0; + + /* + * Read the node properties + */ + prog = 0; + p = read_buf; + while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name,"cpu_cores_count") == 0) + props->NumCPUCores = (uint32_t)prop_val; + else if (strcmp(prop_name,"simd_count") == 0) + props->NumFComputeCores = (uint32_t)prop_val; + else if (strcmp(prop_name,"mem_banks_count") == 0) + props->NumMemoryBanks = (uint32_t)prop_val; + else if (strcmp(prop_name,"caches_count") == 0) + props->NumCaches = (uint32_t)prop_val; + else if (strcmp(prop_name,"io_links_count") == 0) + props->NumIOLinks = (uint32_t)prop_val; + else if (strcmp(prop_name,"cpu_core_id_base") == 0) + props->CComputeIdLo = (uint32_t)prop_val; + else if (strcmp(prop_name,"simd_id_base") == 0) + props->FComputeIdLo = (uint32_t)prop_val; + else if (strcmp(prop_name,"capability") == 0) + props->Capability.Value = (uint32_t)prop_val; + else if (strcmp(prop_name,"max_waves_per_simd") == 0) + props->MaxWavesPerSIMD = (uint32_t)prop_val; + else if (strcmp(prop_name,"lds_size_in_kb") == 0) + props->LDSSizeInKB = (uint32_t)prop_val; + else if (strcmp(prop_name,"gds_size_in_kb") == 0) + props->GDSSizeInKB = (uint32_t)prop_val; + else if (strcmp(prop_name,"wave_front_size") == 0) + props->WaveFrontSize = (uint32_t)prop_val; + else if (strcmp(prop_name,"array_count") == 0) + props->NumShaderBanks = (uint32_t)prop_val; + else if (strcmp(prop_name,"simd_arrays_per_engine") == 0) + props->NumArrays = (uint32_t)prop_val; + else if (strcmp(prop_name,"cu_per_simd_array") == 0) + props->NumCUPerArray = (uint32_t)prop_val; + else if (strcmp(prop_name,"simd_per_cu") == 0) + props->NumSIMDPerCU = (uint32_t)prop_val; + else if (strcmp(prop_name,"max_slots_scratch_cu") == 0) + props->MaxSlotsScratchCU = (uint32_t)prop_val; + else if (strcmp(prop_name,"engine_id") == 0) + props->EngineId = (uint32_t)prop_val; + else if (strcmp(prop_name,"vendor_id") == 0) + props->VendorId = (uint32_t)prop_val; + else if (strcmp(prop_name,"device_id") == 0) + props->DeviceId = (uint32_t)prop_val; + else if (strcmp(prop_name,"location_id") == 0) + props->LocationId = (uint32_t)prop_val; + else if (strcmp(prop_name,"max_engine_clk_fcompute") == 0) + props->MaxEngineClockMhzFCompute = (uint32_t)prop_val; + else if (strcmp(prop_name,"max_engine_clk_ccompute") == 0) + props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; + else if (strcmp(prop_name,"local_mem_size") == 0) + props->LocalMemSize = (uint32_t)prop_val; + } + +err2: + free(read_buf); +err1: + fclose(fd); + return ret; +} + +static HSAKMT_STATUS +topology_sysfs_get_mem_props(uint32_t node_id, uint32_t mem_id, HsaMemoryProperties *props) { + FILE *fd; + char *read_buf, *p; + char prop_name[256]; + char path[256]; + long long unsigned int prop_val; + uint32_t prog; + int read_size; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(props); + snprintf(path, 256, "%s/%d/mem_banks/%d/properties", KFD_SYSFS_PATH_NODES, node_id, mem_id); + fd = fopen(path, "r"); + if (!fd) { + return HSAKMT_STATUS_ERROR; + } + read_buf = malloc(PAGE_SIZE); + if (!read_buf) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err1; + } + + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto err2; + } + + /* Since we're using the buffer as a string, we make sure the string terminates */ + if(read_size >= PAGE_SIZE) + read_size = PAGE_SIZE-1; + read_buf[read_size] = 0; + + prog = 0; + p = read_buf; + while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name,"heap_type") == 0) + props->HeapType = (uint32_t)prop_val; + else if (strcmp(prop_name,"size_in_bytes") == 0) + props->SizeInBytes = prop_val; + else if (strcmp(prop_name,"flags") == 0) + props->Flags.MemoryProperty = (uint32_t)prop_val; + else if (strcmp(prop_name,"width") == 0) + props->Width = (uint32_t)prop_val; + else if (strcmp(prop_name,"mem_clk_max") == 0) + props->MemoryClockMax = (uint32_t)prop_val; + } + +err2: + free(read_buf); +err1: + fclose(fd); + return ret; +} + +static HSAKMT_STATUS +topology_sysfs_get_cache_props(uint32_t node_id, uint32_t cache_id, HsaCacheProperties *props) { + FILE *fd; + char *read_buf, *p; + char prop_name[256]; + char path[256]; + long long unsigned int prop_val; + uint32_t i, prog; + int read_size; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(props); + snprintf(path, 256, "%s/%d/caches/%d/properties", KFD_SYSFS_PATH_NODES, node_id, cache_id); + fd = fopen(path, "r"); + if (!fd) { + return HSAKMT_STATUS_ERROR; + } + read_buf = malloc(PAGE_SIZE); + if (!read_buf) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err1; + } + + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto err2; + } + + /* Since we're using the buffer as a string, we make sure the string terminates */ + if(read_size >= PAGE_SIZE) + read_size = PAGE_SIZE-1; + read_buf[read_size] = 0; + + prog = 0; + p = read_buf; + while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name,"processor_id_low") == 0) + props->ProcessorIdLow = (uint32_t)prop_val; + else if (strcmp(prop_name,"level") == 0) + props->CacheLevel = (uint32_t)prop_val; + else if (strcmp(prop_name,"size") == 0) + props->CacheSize = (uint32_t)prop_val; + else if (strcmp(prop_name,"cache_line_size") == 0) + props->CacheLineSize = (uint32_t)prop_val; + else if (strcmp(prop_name,"cache_lines_per_tag") == 0) + props->CacheLinesPerTag = (uint32_t)prop_val; + else if (strcmp(prop_name,"association") == 0) + props->CacheAssociativity = (uint32_t)prop_val; + else if (strcmp(prop_name,"latency") == 0) + props->CacheLatency = (uint32_t)prop_val; + else if (strcmp(prop_name,"type") == 0) + props->CacheType.Value = (uint32_t)prop_val; + else if (strcmp(prop_name, "sibling_map") == 0) + break; + } + + prog = 0; + if ((sscanf(p, "sibling_map %n", &prog)) == 0 && prog) { + i = 0; + while ((i < HSA_CPU_SIBLINGS) && + (sscanf(p+=prog, "%u%*[,\n]%n", &props->SiblingMap[i++], + &prog) == 1)); + } + +err2: + free(read_buf); +err1: + fclose(fd); + return ret; +} + +static HSAKMT_STATUS +topology_sysfs_get_iolink_props(uint32_t node_id, uint32_t iolink_id, HsaIoLinkProperties *props) { + FILE *fd; + char *read_buf, *p; + char prop_name[256]; + char path[256]; + long long unsigned int prop_val; + uint32_t prog; + int read_size; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(props); + snprintf(path, 256, "%s/%d/io_link/%d/properties", KFD_SYSFS_PATH_NODES, node_id, iolink_id); + fd = fopen(path, "r"); + if (!fd) { + return HSAKMT_STATUS_ERROR; + } + read_buf = malloc(PAGE_SIZE); + if (!read_buf) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err1; + } + + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto err2; + } + + /* Since we're using the buffer as a string, we make sure the string terminates */ + if(read_size >= PAGE_SIZE) + read_size = PAGE_SIZE-1; + read_buf[read_size] = 0; + + prog = 0; + p = read_buf; + while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name,"type") == 0) + props->IoLinkType = (uint32_t)prop_val; + else if (strcmp(prop_name,"version_major") == 0) + props->VersionMajor = (uint32_t)prop_val; + else if (strcmp(prop_name,"version_minor") == 0) + props->VersionMinor = (uint32_t)prop_val; + else if (strcmp(prop_name,"node_from") == 0) + props->NodeFrom = (uint32_t)prop_val; + else if (strcmp(prop_name,"node_to") == 0) + props->NodeTo = (uint32_t)prop_val; + else if (strcmp(prop_name,"weight") == 0) + props->Weight = (uint32_t)prop_val; + else if (strcmp(prop_name,"min_latency") == 0) + props->MinimumLatency = (uint32_t)prop_val; + else if (strcmp(prop_name,"max_latency") == 0) + props->MaximumLatency = (uint32_t)prop_val; + else if (strcmp(prop_name,"min_bandwidth") == 0) + props->MinimumBandwidth = (uint32_t)prop_val; + else if (strcmp(prop_name,"max_bandwidth") == 0) + props->MaximumBandwidth = (uint32_t)prop_val; + else if (strcmp(prop_name,"recommended_transfer_size") == 0) + props->RecTransferSize = (uint32_t)prop_val; + else if (strcmp(prop_name,"flags") == 0) + props->Flags.LinkProperty = (uint32_t)prop_val; + } + + +err2: + free(read_buf); +err1: + fclose(fd); + return ret; +} + +HSAKMT_STATUS +topology_take_snapshot(void) +{ + uint32_t gen_start, gen_end, i, j, mem_id, cache_id, link_id; + HsaSystemProperties sys_props; + node_t *temp_nodes = 0; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + +retry: + ret = topology_sysfs_get_generation(&gen_start); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = topology_sysfs_get_system_props(&sys_props); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + if(sys_props.NumNodes > 0) { + temp_nodes = calloc(sys_props.NumNodes * sizeof(node_t),1); + if (!temp_nodes) + return HSAKMT_STATUS_NO_MEMORY; + for (i = 0; i < sys_props.NumNodes; i++) { + ret = topology_sysfs_get_node_props(i, + &temp_nodes[i].node, + &temp_nodes[i].gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + for (j=0; j < i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + if (temp_nodes[i].node.NumMemoryBanks) { + temp_nodes[i].mem = calloc(temp_nodes[i].node.NumMemoryBanks * sizeof(HsaMemoryProperties), 1); + if (!temp_nodes[i].mem) { + ret = HSAKMT_STATUS_NO_MEMORY; + for (j=0; j <= i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + for (mem_id = 0; mem_id < temp_nodes[i].node.NumMemoryBanks; mem_id++) { + ret = topology_sysfs_get_mem_props(i, mem_id, &temp_nodes[i].mem[mem_id]); + if (ret != HSAKMT_STATUS_SUCCESS) { + for (j=0; j <= i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + } + } + + if (temp_nodes[i].node.NumCaches) { + temp_nodes[i].cache = calloc(temp_nodes[i].node.NumCaches * sizeof(HsaCacheProperties), 1); + if (!temp_nodes[i].cache) { + ret = HSAKMT_STATUS_NO_MEMORY; + for (j=0; j <= i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + for (cache_id = 0; cache_id < temp_nodes[i].node.NumCaches; cache_id++) { + ret = topology_sysfs_get_cache_props(i, cache_id, &temp_nodes[i].cache[cache_id]); + if (ret != HSAKMT_STATUS_SUCCESS) { + for (j=0; j <= i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + } + } + + if (temp_nodes[i].node.NumIOLinks) { + temp_nodes[i].link = calloc(temp_nodes[i].node.NumIOLinks * sizeof(HsaIoLinkProperties), 1); + if (!temp_nodes[i].link) { + ret = HSAKMT_STATUS_NO_MEMORY; + for (j=0; j <= i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + for (link_id = 0; link_id < temp_nodes[i].node.NumIOLinks; link_id++) { + ret = topology_sysfs_get_iolink_props(i, link_id, &temp_nodes[i].link[link_id]); + if (ret != HSAKMT_STATUS_SUCCESS) { + for (j=0; j <= i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + } + } + + } + } + + ret = topology_sysfs_get_generation(&gen_end); + if (ret != HSAKMT_STATUS_SUCCESS) { + if (temp_nodes) { + for (j=0; j < sys_props.NumNodes; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + } + goto err; + } + + if (gen_start != gen_end) { + if (temp_nodes) { + for (j=0; j < sys_props.NumNodes; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + temp_nodes = 0; + } + goto retry; + } + + if (!system) { + system = malloc(sizeof(HsaSystemProperties)); + if (!system) { + if (temp_nodes) { + for (j=0; j < sys_props.NumNodes; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + } + return HSAKMT_STATUS_NO_MEMORY; + } + } + + *system = sys_props; + if (node) + free(node); + node = temp_nodes; +err: + + return ret; +} + +/* + * Drop the Snashot of the HSA topology information. + * Assume lock is held. + */ +HSAKMT_STATUS +topology_drop_snapshot(void) +{ + HSAKMT_STATUS err; + + if (!!system != !!node) { + printf("Probable inconsistency?\n"); + err = HSAKMT_STATUS_SUCCESS; + goto out; + } + + if (node) { + uint64_t nodeid; + + /* Remove state */ + for (nodeid = 0; nodeid < system->NumNodes; nodeid++) { + free_node(&node[nodeid]); + } + + free(node); + node = NULL; + } + + free(system); + system = NULL; + err = HSAKMT_STATUS_SUCCESS; + +out: + return err; +} + +HSAKMT_STATUS +validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) +{ + if (nodeid >= MAX_NODES || !node || !system || system->NumNodes <= nodeid) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (gpu_id) + *gpu_id = node[nodeid].gpu_id; + + return HSAKMT_STATUS_SUCCESS; +} HSAKMT_STATUS HSAKMTAPI @@ -37,9 +696,26 @@ hsaKmtAcquireSystemProperties( HsaSystemProperties* SystemProperties //OUT ) { + HSAKMT_STATUS err; CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + if (!SystemProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pthread_mutex_lock(&hsakmt_mutex); + + err = topology_take_snapshot(); + if (err != HSAKMT_STATUS_SUCCESS) + goto out; + + assert(system); + + *SystemProperties = *system; + err = HSAKMT_STATUS_SUCCESS; + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; } HSAKMT_STATUS @@ -48,7 +724,15 @@ hsaKmtReleaseSystemProperties(void) { CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + HSAKMT_STATUS err; + + pthread_mutex_lock(&hsakmt_mutex); + + err = topology_drop_snapshot(); + + pthread_mutex_unlock(&hsakmt_mutex); + + return err; } HSAKMT_STATUS @@ -58,9 +742,39 @@ hsaKmtGetNodeProperties( HsaNodeProperties* NodeProperties //OUT ) { - CHECK_KFD_OPEN(); + HSAKMT_STATUS err; + uint32_t gpu_id; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (!NodeProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_KFD_OPEN(); + pthread_mutex_lock(&hsakmt_mutex); + + /* KFD ADD page 18, snapshot protocol violation */ + if (system == NULL) { + err = HSAKMT_STATUS_INVALID_NODE_UNIT; + assert(system); + goto out; + } + + if (NodeId >= system->NumNodes) { + err = HSAKMT_STATUS_INVALID_PARAMETER; + goto out; + } + + err = validate_nodeid(NodeId, &gpu_id); + if (err != HSAKMT_STATUS_SUCCESS) + return err; + + *NodeProperties = node[NodeId].node; + NodeProperties->NumMemoryBanks += NUM_OF_HEAPS; + + err = HSAKMT_STATUS_SUCCESS; + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; } HSAKMT_STATUS @@ -71,9 +785,58 @@ hsaKmtGetNodeMemoryProperties( HsaMemoryProperties* MemoryProperties //OUT ) { - CHECK_KFD_OPEN(); + HSAKMT_STATUS err; + uint32_t i, gpu_id; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (!MemoryProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_KFD_OPEN(); + pthread_mutex_lock(&hsakmt_mutex); + + /* KFD ADD page 18, snapshot protocol violation */ + if (system == NULL) { + err = HSAKMT_STATUS_INVALID_NODE_UNIT; + assert(system); + goto out; + } + + /* Check still necessary */ + if (NodeId >= system->NumNodes ) { + err = HSAKMT_STATUS_INVALID_PARAMETER; + goto out; + } + + err = validate_nodeid(NodeId, &gpu_id); + if (err != HSAKMT_STATUS_SUCCESS) + return err; + + for (i = 0; i < MIN(node[NodeId].node.NumMemoryBanks, NumBanks); i++) { + assert(node[NodeId].mem); + MemoryProperties[i] = node[NodeId].mem[i]; + } + + /*Add LDS*/ + if (i < NumBanks){ + MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_LDS; + MemoryProperties[i].SizeInBytes = node[NodeId].node.LDSSizeInKB * 1024; + MemoryProperties[i].VirtualBaseAddress = fmm_get_aperture_base(FMM_LDS, gpu_id); + i++; + } + + /*Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE*/ + if (i < NumBanks){ + MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; + MemoryProperties[i].SizeInBytes = node[NodeId].node.LocalMemSize; + MemoryProperties[i].VirtualBaseAddress = fmm_get_aperture_base(FMM_GPUVM, gpu_id); + i++; + } + + err = HSAKMT_STATUS_SUCCESS; + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; } HSAKMT_STATUS @@ -85,9 +848,37 @@ hsaKmtGetNodeCacheProperties( HsaCacheProperties* CacheProperties //OUT ) { - CHECK_KFD_OPEN(); + HSAKMT_STATUS err; + uint32_t i; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (!CacheProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_KFD_OPEN(); + pthread_mutex_lock(&hsakmt_mutex); + + /* KFD ADD page 18, snapshot protocol violation */ + if (system == NULL) { + err = HSAKMT_STATUS_INVALID_NODE_UNIT; + assert(system); + goto out; + } + + if (NodeId >= system->NumNodes || NumCaches > node[NodeId].node.NumCaches) { + err = HSAKMT_STATUS_INVALID_PARAMETER; + goto out; + } + + for (i = 0; i < MIN(node[NodeId].node.NumCaches, NumCaches); i++) { + assert(node[NodeId].cache); + CacheProperties[i] = node[NodeId].cache[i]; + } + + err = HSAKMT_STATUS_SUCCESS; + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; } HSAKMT_STATUS @@ -98,7 +889,44 @@ hsaKmtGetNodeIoLinkProperties( HsaIoLinkProperties* IoLinkProperties //OUT ) { + HSAKMT_STATUS err; + uint32_t i; + + if (!IoLinkProperties) + return HSAKMT_STATUS_INVALID_PARAMETER; + CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + pthread_mutex_lock(&hsakmt_mutex); + + /* KFD ADD page 18, snapshot protocol violation */ + if (system == NULL) { + err = HSAKMT_STATUS_INVALID_NODE_UNIT; + assert(system); + goto out; + } + + if (NodeId >= system->NumNodes || NumIoLinks > node[NodeId].node.NumIOLinks) { + err = HSAKMT_STATUS_INVALID_PARAMETER; + goto out; + } + + for (i = 0; i < MIN(node[NodeId].node.NumIOLinks, NumIoLinks); i++) { + assert(node[NodeId].link); + IoLinkProperties[i] = node[NodeId].link[i]; + } + + err = HSAKMT_STATUS_SUCCESS; + +out: + pthread_mutex_unlock(&hsakmt_mutex); + return err; +} + +uint16_t get_device_id_by_node(HSAuint32 node_id) +{ + if (!node || !system || system->NumNodes <= node_id) + return 0; + + return node[node_id].node.DeviceId; } From 6a8a380ba38dfbf2ec578fd6d7360eae6e729916 Mon Sep 17 00:00:00 2001 From: Andrew Lewycky Date: Tue, 29 Jul 2014 13:56:31 +0300 Subject: [PATCH 0008/1247] Add memory module Signed-off-by: Andrew Lewycky Signed-off-by: Oded Gabbay --- src/memory.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/src/memory.c b/src/memory.c index acc45dc89c..37cb88f1ef 100644 --- a/src/memory.c +++ b/src/memory.c @@ -28,6 +28,7 @@ #include #include #include +#include "fmm.h" HSAKMT_STATUS HSAKMTAPI @@ -39,9 +40,49 @@ hsaKmtSetMemoryPolicy( HSAuint64 MemorySizeInBytes ) { + HSAKMT_STATUS result; + uint32_t gpu_id; + CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + result = validate_nodeid(Node, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + // We accept any legal policy and alternate address location. You get CC everywhere anyway. + if ((DefaultPolicy != HSA_CACHING_CACHED && DefaultPolicy != HSA_CACHING_NONCACHED) + || (AlternatePolicy != HSA_CACHING_CACHED && AlternatePolicy != HSA_CACHING_NONCACHED)) + { + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + CHECK_PAGE_MULTIPLE(MemoryAddressAlternate); + CHECK_PAGE_MULTIPLE(MemorySizeInBytes); + + struct kfd_ioctl_set_memory_policy_args args; + memset(&args, 0, sizeof(args)); + + args.gpu_id = gpu_id; + args.default_policy = (DefaultPolicy == HSA_CACHING_CACHED) ? KFD_IOC_CACHE_POLICY_COHERENT : KFD_IOC_CACHE_POLICY_NONCOHERENT; + args.alternate_policy = (AlternatePolicy == HSA_CACHING_CACHED) ? KFD_IOC_CACHE_POLICY_COHERENT : KFD_IOC_CACHE_POLICY_NONCOHERENT; + args.alternate_aperture_base = (uintptr_t)MemoryAddressAlternate; + args.alternate_aperture_size = MemorySizeInBytes; + + int err = kfd_ioctl(KFD_IOC_SET_MEMORY_POLICY, &args); + + return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; +} + +static HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags) +{ + switch (pageSizeFlags) + { + case HSA_PAGE_SIZE_4KB: return 4*1024; + case HSA_PAGE_SIZE_64KB: return 64*1024; + case HSA_PAGE_SIZE_2MB: return 2*1024*1024; + case HSA_PAGE_SIZE_1GB: return 1024*1024*1024; + default: assert(false); return 4*1024; + } } HSAKMT_STATUS @@ -54,8 +95,29 @@ hsaKmtAllocMemory( ) { CHECK_KFD_OPEN(); + HSAKMT_STATUS result; + uint32_t gpu_id; + + result = validate_nodeid(PreferredNode, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + // The required size should be page aligned (GDS?) + HSAuint64 page_size = PageSizeFromFlags(MemFlags.ui32.PageSize); + if ((SizeInBytes & (page_size-1)) && !MemFlags.ui32.GDSMemory){ + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged){ + int err = posix_memalign(MemoryAddress, page_size, SizeInBytes); + if (err == 0) + return HSAKMT_STATUS_SUCCESS; + else + return HSAKMT_STATUS_NO_MEMORY; + } + else + return HSAKMT_STATUS_INVALID_PARAMETER; - return HSAKMT_STATUS_NOT_SUPPORTED; } HSAKMT_STATUS @@ -65,9 +127,17 @@ hsaKmtFreeMemory( HSAuint64 SizeInBytes //IN ) { + HSAKMT_STATUS hsa_status = HSAKMT_STATUS_SUCCESS; CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + if (fmm_is_inside_some_aperture(MemoryAddress)){ + if (fmm_release( MemoryAddress, SizeInBytes)) + hsa_status = HSAKMT_STATUS_INVALID_PARAMETER; + } + else + free(MemoryAddress); + + return hsa_status; } HSAKMT_STATUS @@ -79,7 +149,7 @@ hsaKmtRegisterMemory( { CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS @@ -90,7 +160,7 @@ hsaKmtDeregisterMemory( { CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS @@ -103,7 +173,13 @@ hsaKmtMapMemoryToGPU( { CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + // We don't support GPUVM in the stub, there should never be a request for a GPUVA. + if (AlternateVAGPU) + { + *AlternateVAGPU = 0; + } + + return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS @@ -114,5 +190,5 @@ hsaKmtUnmapMemoryToGPU( { CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + return HSAKMT_STATUS_SUCCESS; } From 678287d98b2948d488dba604a3e87cfff94a0dbc Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Tue, 29 Jul 2014 13:56:50 +0300 Subject: [PATCH 0009/1247] Add queues module Signed-off-by: Ben Goz Signed-off-by: Oded Gabbay --- src/libhsakmt.h | 1 + src/queues.c | 138 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 136 insertions(+), 3 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 8972ba9341..4d337e612b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -67,6 +67,7 @@ uint16_t get_device_id_by_node(HSAuint32 node_id); extern int kfd_ioctl(int cmdcode, void* data); /* Void pointer arithmetic (or remove -Wpointer-arith to allow void pointers arithmetic) */ +#define VOID_PTR_ADD32(ptr,n) (void*)((uint32_t*)(ptr) + n)/*ptr + offset*/ #define VOID_PTR_ADD(ptr,n) (void*)((uint8_t*)(ptr) + n)/*ptr + offset*/ #define VOID_PTR_SUB(ptr,n) (void*)((uint8_t*)(ptr) - n)/*ptr - offset*/ #define VOID_PTRS_SUB(ptr1,ptr2) (uint64_t)((uint8_t*)(ptr1) - (uint8_t*)(ptr2)) /*ptr1 - ptr2*/ diff --git a/src/queues.c b/src/queues.c index 3c1c4ff3df..1292b2034a 100644 --- a/src/queues.c +++ b/src/queues.c @@ -27,6 +27,28 @@ #include "linux/kfd_ioctl.h" #include #include +#include +#include +#include + +/* 1024 doorbells, 4 bytes each doorbell */ +#define DOORBELLS_PAGE_SIZE 1024 * 4 + +struct queue +{ + uint32_t queue_id; + uint32_t wptr; + uint32_t rptr; +}; + +struct process_doorbells +{ + bool need_mmap; + void* doorbells; + pthread_mutex_t doorbells_mutex; +}; + +struct process_doorbells doorbells[] = {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = {.need_mmap = true, .doorbells = NULL, .doorbells_mutex = PTHREAD_MUTEX_INITIALIZER}}; HSAKMT_STATUS HSAKMTAPI @@ -41,9 +63,82 @@ hsaKmtCreateQueue( HsaQueueResource* QueueResource //OUT ) { + HSAKMT_STATUS result; + uint32_t gpu_id; + int err; + void* ptr; CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + struct queue *q = malloc(sizeof(struct queue)); + if (q == NULL) + { + return HSAKMT_STATUS_NO_MEMORY; + } + + memset(q, 0, sizeof(*q)); + + struct kfd_ioctl_create_queue_args args; + memset(&args, 0, sizeof(args)); + + args.gpu_id = gpu_id; + + switch (Type) + { + case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; + case HSA_QUEUE_SDMA: free(q); return HSAKMT_STATUS_NOT_IMPLEMENTED; + case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; + default: free(q); return HSAKMT_STATUS_INVALID_PARAMETER; + } + + if (Type != HSA_QUEUE_COMPUTE_AQL) + { + QueueResource->QueueRptrValue = (uintptr_t)&q->rptr; + QueueResource->QueueWptrValue = (uintptr_t)&q->wptr; + } + + args.read_pointer_address = QueueResource->QueueRptrValue; + args.write_pointer_address = QueueResource->QueueWptrValue; + args.ring_base_address = (uintptr_t)QueueAddress; + args.ring_size = QueueSizeInBytes; + args.queue_percentage = QueuePercentage; + args.queue_priority = Priority; + + err = kfd_ioctl(KFD_IOC_CREATE_QUEUE, &args); + + if (err == -1) + { + free(q); + return HSAKMT_STATUS_ERROR; + } + + q->queue_id = args.queue_id; + + pthread_mutex_lock(&doorbells[NodeId].doorbells_mutex); + + if (doorbells[NodeId].need_mmap) { + ptr = mmap(0, DOORBELLS_PAGE_SIZE, PROT_READ|PROT_WRITE, + MAP_SHARED, kfd_fd, args.doorbell_offset); + + if (ptr == MAP_FAILED) { + pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); + hsaKmtDestroyQueue(q->queue_id); + return HSAKMT_STATUS_ERROR; + } + + doorbells[NodeId].need_mmap = false; + doorbells[NodeId].doorbells = ptr; + } + + pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); + + QueueResource->QueueId = PORT_VPTR_TO_UINT64(q); + QueueResource->Queue_DoorBell = VOID_PTR_ADD32(doorbells[NodeId].doorbells, q->queue_id); + + return HSAKMT_STATUS_SUCCESS; } @@ -58,9 +153,26 @@ hsaKmtUpdateQueue( HsaEvent* Event //IN ) { + struct kfd_ioctl_update_queue_args arg; + struct queue *q = PORT_UINT64_TO_VPTR(QueueId); + CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + if (q == NULL) + return (HSAKMT_STATUS_INVALID_PARAMETER); + arg.queue_id = (HSAuint32)q->queue_id; + arg.ring_base_address = (uintptr_t)QueueAddress; + arg.ring_size = QueueSize; + arg.queue_percentage = QueuePercentage; + arg.queue_priority = Priority; + + int err = kfd_ioctl(KFD_IOC_UPDATE_QUEUE, &arg); + if (err == -1) + { + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS @@ -71,5 +183,25 @@ hsaKmtDestroyQueue( { CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + struct queue *q = PORT_UINT64_TO_VPTR(QueueId); + struct kfd_ioctl_destroy_queue_args args; + + if (q == NULL) + return (HSAKMT_STATUS_INVALID_PARAMETER); + + memset(&args, 0, sizeof(args)); + + args.queue_id = q->queue_id; + + int err = kfd_ioctl(KFD_IOC_DESTROY_QUEUE, &args); + + if (err == -1) + { + return HSAKMT_STATUS_ERROR; + } + else + { + free(q); + return HSAKMT_STATUS_SUCCESS; + } } From 5f106cf3e4be1de36ea1380cb1d43845c30ccec3 Mon Sep 17 00:00:00 2001 From: Evgeny Pinchuk Date: Tue, 29 Jul 2014 13:57:10 +0300 Subject: [PATCH 0010/1247] Add clock counters module Signed-off-by: Evgeny Pinchuk Signed-off-by: Oded Gabbay --- src/time.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/time.c b/src/time.c index df6ad0b2bb..91c1a1209b 100644 --- a/src/time.c +++ b/src/time.c @@ -33,7 +33,29 @@ hsaKmtGetClockCounters( HsaClockCounters* Counters //OUT ) { + HSAKMT_STATUS result; + uint32_t gpu_id; + struct kfd_ioctl_get_clock_counters_args args; + int err; + CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + args.gpu_id = gpu_id; + + err = kfd_ioctl(KFD_IOC_GET_CLOCK_COUNTERS, &args); + if (err < 0) { + result = HSAKMT_STATUS_ERROR; + } else { + /* At this point the result is already HSAKMT_STATUS_SUCCESS */ + Counters->GPUClockCounter = args.gpu_clock_counter; + Counters->CPUClockCounter = args.cpu_clock_counter; + Counters->SystemClockCounter = args.system_clock_counter; + Counters->SystemClockFrequencyHz = args.system_clock_freq; + } + + return result; } From a4f446e8cd6de16d67cead169b2ea2ada67833c9 Mon Sep 17 00:00:00 2001 From: Evgeny Pinchuk Date: Tue, 29 Jul 2014 13:58:01 +0300 Subject: [PATCH 0011/1247] Add pmc table module Signed-off-by: Evgeny Pinchuk Signed-off-by: Oded Gabbay --- src/Makefile | 2 +- src/pmc_table.c | 98 +++++++++++++++++++++++++++++++++++++++++++++++++ src/pmc_table.h | 50 +++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 src/pmc_table.c create mode 100644 src/pmc_table.h diff --git a/src/Makefile b/src/Makefile index 748fa2671c..c842a5e59f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -18,7 +18,7 @@ CFLAGS += -std=gnu99 -ggdb -pthread -fvisibility=hidden -O2 LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ - events.o openclose.o queues.o topology.o fmm.o + events.o openclose.o queues.o topology.o fmm.o pmc_table.o .PHONY: all lnx lnx64a clean diff --git a/src/pmc_table.c b/src/pmc_table.c new file mode 100644 index 0000000000..3fa90e95e8 --- /dev/null +++ b/src/pmc_table.c @@ -0,0 +1,98 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include "pmc_table.h" + + +static uint32_t kaveri_sq_counter_ids[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 168, 169, 170, + 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, + 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, + 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, + 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 +}; + +static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 16, + .num_of_counters = sizeof(kaveri_sq_counter_ids)/sizeof(*kaveri_sq_counter_ids), + .counter_ids = kaveri_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + +HSAKMT_STATUS +get_block_properties(uint16_t dev_id, + enum perf_block_id block_id, + struct perf_counter_block *block) +{ + HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; + if (block_id > PERFCOUNTER_BLOCKID__MAX || block_id < PERFCOUNTER_BLOCKID__FIRST) + return HSAKMT_STATUS_INVALID_PARAMETER; + + switch(dev_id) { + case 0x1304: + case 0x1305: + case 0x1306: + case 0x1307: + case 0x1309: + case 0x130A: + case 0x130B: + case 0x130C: + case 0x130D: + case 0x130E: + case 0x130F: + case 0x1310: + case 0x1311: + case 0x1312: + case 0x1313: + case 0x1315: + case 0x1316: + case 0x1317: + case 0x1318: + case 0x131B: + case 0x131C: + case 0x131D: + *block = kaveri_blocks[block_id]; + break; + default: + rc = HSAKMT_STATUS_INVALID_PARAMETER; + } + + return rc; +} + + diff --git a/src/pmc_table.h b/src/pmc_table.h new file mode 100644 index 0000000000..35ed07eb4c --- /dev/null +++ b/src/pmc_table.h @@ -0,0 +1,50 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef PMC_TABLE_H +#define PMC_TABLE_H + +#include "libhsakmt.h" + +enum perf_block_id { + PERFCOUNTER_BLOCKID__FIRST = 0, + PERFCOUNTER_BLOCKID__SQ = PERFCOUNTER_BLOCKID__FIRST, + PERFCOUNTER_BLOCKID__MAX +}; + +struct perf_counter_block { + uint32_t num_of_slots; + uint32_t num_of_counters; + uint32_t *counter_ids; + uint32_t counter_size_in_bits; + uint64_t counter_mask; +}; + +HSAKMT_STATUS +get_block_properties(uint16_t dev_id, + enum perf_block_id block_id, + struct perf_counter_block *block); + +#endif // PMC_TABLE_H From 2c865d510ff7c66e5d0ce3c67379dd4b0ede4859 Mon Sep 17 00:00:00 2001 From: Evgeny Pinchuk Date: Tue, 29 Jul 2014 13:59:07 +0300 Subject: [PATCH 0012/1247] Add pmc module Signed-off-by: Evgeny Pinchuk Signed-off-by: Oded Gabbay --- src/perfctr.c | 222 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 206 insertions(+), 16 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 0f10f6ac71..e0d6ad0b88 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -25,8 +25,41 @@ #include #include "libhsakmt.h" +#include "pmc_table.h" #include "linux/kfd_ioctl.h" +#define BITS_PER_BYTE CHAR_BIT + +#define HSA_PERF_MAGIC4CC 0x54415348 + +enum perf_trace_state { + PERF_TRACE_STATE__STOPPED = 0, + PERF_TRACE_STATE__STARTED +}; + +struct perf_trace { + uint32_t magic4cc; + uint32_t gpu_id; + enum perf_trace_state state; +}; + +static HsaCounterProperties *counter_props[MAX_NODES] = {NULL}; + +static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) +{ + int rc = 0; + switch (block_id) { + case PERFCOUNTER_BLOCKID__SQ: + *uuid = HSA_PROFILEBLOCK_AMD_SQ; + break; + default: + /* If we reach this point, it's a bug */ + rc = -1; + } + + return rc; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties( @@ -34,9 +67,71 @@ hsaKmtPmcGetCounterProperties( HsaCounterProperties** CounterProperties //OUT ) { - CHECK_KFD_OPEN(); + HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; + uint32_t gpu_id, i, block_id; + uint16_t dev_id; + uint32_t counter_props_size = 0; + uint32_t total_counters = 0; + uint32_t total_concurrent = 0; + struct perf_counter_block block = {0}; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (CounterProperties == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (validate_nodeid(NodeId, &gpu_id) != 0) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + + + if (counter_props[NodeId] == NULL) { + dev_id = get_device_id_by_node(NodeId); + for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { + rc = get_block_properties(dev_id, i, &block); + if (rc != HSAKMT_STATUS_SUCCESS) + return rc; + total_concurrent += block.num_of_slots; + total_counters += block.num_of_counters; + } + + counter_props_size = sizeof(HsaCounterProperties) + + sizeof(HsaCounterBlockProperties)*(PERFCOUNTER_BLOCKID__MAX-1) + + sizeof(HsaCounter)*(total_counters-1); + + counter_props[NodeId] = malloc(counter_props_size); + + if (counter_props[NodeId] == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + counter_props[NodeId]->NumBlocks = PERFCOUNTER_BLOCKID__MAX; + counter_props[NodeId]->NumConcurrent = total_concurrent; + + for (block_id = 0; block_id < PERFCOUNTER_BLOCKID__MAX; block_id++) + { + rc = get_block_properties(dev_id, block_id, &block); + if (rc != HSAKMT_STATUS_SUCCESS) { + free(counter_props[NodeId]); + return rc; + } + + /* Filling the SQ block */ + blockid2uuid(block_id, &counter_props[NodeId]->Blocks[block_id].BlockId); + counter_props[NodeId]->Blocks[block_id].NumCounters = block.num_of_counters; + counter_props[NodeId]->Blocks[block_id].NumConcurrent = block.num_of_slots; + + for (i = 0; i < block.num_of_counters; i++) { + counter_props[NodeId]->Blocks[block_id].Counters[i].BlockIndex = block_id; + counter_props[NodeId]->Blocks[block_id].Counters[i].CounterId = block.counter_ids[i]; + counter_props[NodeId]->Blocks[block_id].Counters[i].CounterSizeInBits = block.counter_size_in_bits; + counter_props[NodeId]->Blocks[block_id].Counters[i].CounterMask = block.counter_mask; + counter_props[NodeId]->Blocks[block_id].Counters[i].Flags.ui32.Global = 1; + counter_props[NodeId]->Blocks[block_id].Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; + } + } + } + + *CounterProperties = counter_props[NodeId]; + + return HSAKMT_STATUS_SUCCESS; } /** @@ -52,9 +147,42 @@ hsaKmtPmcRegisterTrace( HsaPmcTraceRoot* TraceRoot //OUT ) { - CHECK_KFD_OPEN(); + uint32_t gpu_id, i; + uint64_t min_buf_size = 0; + uint32_t concurrent_counters[PERFCOUNTER_BLOCKID__MAX] = {0}; + struct perf_trace *trace = NULL; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (Counters == NULL || TraceRoot == NULL || NumberOfCounters == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (validate_nodeid(NodeId, &gpu_id) != 0) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + /* Calculating the minimum buffer size */ + for (i = 0; i < NumberOfCounters; i++) { + if (Counters[i].BlockIndex >= PERFCOUNTER_BLOCKID__MAX) + return HSAKMT_STATUS_INVALID_PARAMETER; + min_buf_size += Counters[i].CounterSizeInBits/BITS_PER_BYTE; + concurrent_counters[Counters[i].BlockIndex]++; + } + + /* Verifying that the number of counters per block is not larger than the amount of slots */ + if (concurrent_counters[PERFCOUNTER_BLOCKID__SQ] > counter_props[NodeId]->Blocks[PERFCOUNTER_BLOCKID__SQ].NumConcurrent) + return HSAKMT_STATUS_INVALID_PARAMETER; + + trace = malloc(sizeof(trace)); + if (trace == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + trace->magic4cc = HSA_PERF_MAGIC4CC; + trace->gpu_id = gpu_id; + trace->state = PERF_TRACE_STATE__STOPPED; + + TraceRoot->NumberOfPasses = 1; + TraceRoot->TraceBufferMinSizeBytes = PAGE_ALIGN_UP(min_buf_size); + TraceRoot->TraceId = PORT_VPTR_TO_UINT64(trace); + + return HSAKMT_STATUS_SUCCESS; } /** @@ -68,9 +196,33 @@ hsaKmtPmcUnregisterTrace( HSATraceId TraceId //IN ) { - CHECK_KFD_OPEN(); + uint32_t gpu_id; + struct perf_trace *trace; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (validate_nodeid(NodeId, &gpu_id) != 0) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; + + if (trace->gpu_id != gpu_id) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + /* If the trace is in the running state, stop it */ + if (trace->state == PERF_TRACE_STATE__STARTED) { + HSAKMT_STATUS status = hsaKmtPmcStopTrace(TraceId); + if (status != HSAKMT_STATUS_SUCCESS) + return status; + } + + free(trace); + + return HSAKMT_STATUS_SUCCESS; } @@ -86,9 +238,17 @@ hsaKmtPmcAcquireTraceAccess( HSATraceId TraceId //IN ) { - CHECK_KFD_OPEN(); + struct perf_trace *trace; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; + + return HSAKMT_STATUS_SUCCESS; } @@ -104,9 +264,17 @@ hsaKmtPmcReleaseTraceAccess( HSATraceId TraceId //IN ) { - CHECK_KFD_OPEN(); + struct perf_trace *trace; - return HSAKMT_STATUS_NOT_SUPPORTED; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; + + return HSAKMT_STATUS_SUCCESS; } @@ -122,9 +290,17 @@ hsaKmtPmcStartTrace( HSAuint64 TraceBufferSizeBytes //IN (page aligned) ) { - CHECK_KFD_OPEN(); + struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - return HSAKMT_STATUS_NOT_SUPPORTED; + if (TraceId == 0 || TraceBuffer == NULL || TraceBufferSizeBytes == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; + + trace->state = PERF_TRACE_STATE__STARTED; + + return HSAKMT_STATUS_SUCCESS; } @@ -138,9 +314,15 @@ hsaKmtPmcQueryTrace( HSATraceId TraceId //IN ) { - CHECK_KFD_OPEN(); + struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - return HSAKMT_STATUS_NOT_SUPPORTED; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; + + return HSAKMT_STATUS_SUCCESS; } @@ -154,7 +336,15 @@ hsaKmtPmcStopTrace( HSATraceId TraceId //IN ) { - CHECK_KFD_OPEN(); + struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - return HSAKMT_STATUS_NOT_SUPPORTED; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; + + trace->state = PERF_TRACE_STATE__STOPPED; + + return HSAKMT_STATUS_SUCCESS; } From f8757c0e067a52cfcfafae71bf312e44644f845a Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Thu, 4 Dec 2014 11:39:41 +0200 Subject: [PATCH 0013/1247] Fix makefile to include dependency on header files Signed-off-by: Oded Gabbay --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index c842a5e59f..b74312ab43 100644 --- a/src/Makefile +++ b/src/Makefile @@ -46,7 +46,7 @@ clean: rm -rf $(BUILD_ROOT) #Rule -$(BUILDDIR)/%.o: %.c +$(BUILDDIR)/%.o: %.c ../include/hsakmt.h ../include/hsakmttypes.h ../include/linux/kfd_ioctl.h @echo Compiling $^ @mkdir -p $(dir $@) gcc $(CFLAGS) -c $< -o $@ From f081249d172bb4eb7d1d73e60c33b051b2a44ab0 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 28 Oct 2014 16:49:56 +0200 Subject: [PATCH 0014/1247] Implement hsaKmtGetVersion Reviewed-by: Alexey Skidanov Signed-off-by: Oded Gabbay --- src/version.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/version.c b/src/version.c index 434b625717..7d927c2dc3 100644 --- a/src/version.c +++ b/src/version.c @@ -24,6 +24,9 @@ */ #include "libhsakmt.h" +#include +#include +#include "linux/kfd_ioctl.h" HSAKMT_STATUS HSAKMTAPI @@ -33,8 +36,14 @@ hsaKmtGetVersion( { CHECK_KFD_OPEN(); - VersionInfo->KernelInterfaceMajorVersion = HSAKMT_VERSION_MAJOR; - VersionInfo->KernelInterfaceMinorVersion = HSAKMT_VERSION_MINOR; + struct kfd_ioctl_get_version_args args; + memset(&args, 0, sizeof(args)); + + if (kfd_ioctl(KFD_IOC_GET_VERSION, &args) == -1) + return HSAKMT_STATUS_ERROR; + + VersionInfo->KernelInterfaceMajorVersion = args.major_version; + VersionInfo->KernelInterfaceMinorVersion = args.minor_version; return HSAKMT_STATUS_SUCCESS; } From c12b26c9d55136e3a7e8fd286b34e481ea791ee4 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 8 Dec 2014 15:52:51 +0200 Subject: [PATCH 0015/1247] Don't build 32-bit library by default Signed-off-by: Oded Gabbay --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e356d53936..c899cca992 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: all clean lnx64a lnx -all: lnx lnx64a +all: lnx64a lnx64a: $(MAKE) -C src lnx64a From 4a16f41259f83b7b16e1fa3a41538588ccef18d1 Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Tue, 11 Nov 2014 13:23:09 +0200 Subject: [PATCH 0016/1247] Add chip revision and fw version to topology Reviewed-by: Oded Gabbay Signed-off-by: Alexey Skidanov --- src/topology.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/topology.c b/src/topology.c index 492b5a3412..e50b7458d8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -52,6 +52,7 @@ static node_t *node = NULL; static HSAKMT_STATUS topology_take_snapshot(void); static HSAKMT_STATUS topology_drop_snapshot(void); +static int get_cpu_stepping(uint16_t* stepping); static void free_node(node_t *n) @@ -192,7 +193,9 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 char path[256]; long long unsigned int prop_val; uint32_t i, prog; + uint16_t stepping = 0, fw_version = 0; int read_size; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); @@ -289,8 +292,8 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 props->NumSIMDPerCU = (uint32_t)prop_val; else if (strcmp(prop_name,"max_slots_scratch_cu") == 0) props->MaxSlotsScratchCU = (uint32_t)prop_val; - else if (strcmp(prop_name,"engine_id") == 0) - props->EngineId = (uint32_t)prop_val; + else if (strcmp(prop_name,"fw_version") == 0) + fw_version = (uint16_t)prop_val; else if (strcmp(prop_name,"vendor_id") == 0) props->VendorId = (uint32_t)prop_val; else if (strcmp(prop_name,"device_id") == 0) @@ -303,8 +306,12 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; else if (strcmp(prop_name,"local_mem_size") == 0) props->LocalMemSize = (uint32_t)prop_val; + } + get_cpu_stepping(&stepping); + props->EngineId = ((stepping << 16) | fw_version); + err2: free(read_buf); err1: @@ -930,3 +937,41 @@ uint16_t get_device_id_by_node(HSAuint32 node_id) return node[node_id].node.DeviceId; } + +static int get_cpu_stepping(uint16_t* stepping) +{ + int ret; + FILE* fd = fopen("/proc/cpuinfo", "r"); + if (!fd) + return -1; + + char* read_buf = malloc(PAGE_SIZE); + if (!read_buf) { + ret = -1; + goto err1; + } + + int read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = -2; + goto err2; + } + + /* Since we're using the buffer as a string, we make sure the string terminates */ + if(read_size >= PAGE_SIZE) + read_size = PAGE_SIZE-1; + read_buf[read_size] = 0; + + *stepping = 0; + + char* p = strstr(read_buf, "stepping"); + if (p) + sscanf(p , "stepping\t: %hu\n", stepping); + +err2: + free(read_buf); +err1: + fclose(fd); + + return ret; +} From 0ecaa965238131e0d91a356d81d757fb002e7924 Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Mon, 8 Dec 2014 17:05:56 +0200 Subject: [PATCH 0017/1247] Add exclusive trace access Signed-off-by: Alexey Skidanov --- src/openclose.c | 13 +++++++++++++ src/perfctr.c | 24 ++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/openclose.c b/src/openclose.c index dad0cb9084..8290d00502 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -33,6 +33,8 @@ #include "fmm.h" static const char kfd_device_name[] = "/dev/kfd"; +static const char tmp_file[] = "/var/lock/.amd_hsa_thunk_lock"; +int amd_hsa_thunk_lock_fd = 0; HSAKMT_STATUS HSAKMTAPI @@ -59,6 +61,11 @@ hsaKmtOpenKFD(void) { result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } + + amd_hsa_thunk_lock_fd = open(tmp_file, + O_CREAT | //create the file if it's not present. + O_RDWR, //only need write access for the internal locking semantics. + S_IRUSR | S_IWUSR); //permissions on the file, 600 here. } else { @@ -84,6 +91,12 @@ hsaKmtCloseKFD(void) if (--kfd_open_count == 0) { close(kfd_fd); + + if (amd_hsa_thunk_lock_fd > 0) { + close(amd_hsa_thunk_lock_fd); + unlink(tmp_file); + } + } result = HSAKMT_STATUS_SUCCESS; diff --git a/src/perfctr.c b/src/perfctr.c index e0d6ad0b88..64ab168fda 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -27,6 +27,7 @@ #include "libhsakmt.h" #include "pmc_table.h" #include "linux/kfd_ioctl.h" +#include #define BITS_PER_BYTE CHAR_BIT @@ -43,6 +44,8 @@ struct perf_trace { enum perf_trace_state state; }; +extern int amd_hsa_thunk_lock_fd; + static HsaCounterProperties *counter_props[MAX_NODES] = {NULL}; static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) @@ -248,7 +251,15 @@ hsaKmtPmcAcquireTraceAccess( if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; - return HSAKMT_STATUS_SUCCESS; + if (amd_hsa_thunk_lock_fd > 0) { + if (lockf( amd_hsa_thunk_lock_fd, F_TLOCK, 0 ) != 0) + return HSAKMT_STATUS_ERROR; + else + return HSAKMT_STATUS_SUCCESS; + } + else { + return HSAKMT_STATUS_ERROR; + } } @@ -274,7 +285,16 @@ hsaKmtPmcReleaseTraceAccess( if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; - return HSAKMT_STATUS_SUCCESS; + if (amd_hsa_thunk_lock_fd > 0) { + if (lockf( amd_hsa_thunk_lock_fd, F_ULOCK, 0 ) != 0) + return HSAKMT_STATUS_ERROR; + else + return HSAKMT_STATUS_SUCCESS; + } + else { + return HSAKMT_STATUS_ERROR; + } + } From 9f647b07ff9c355e7e9ff6fd82b82fb87187cfc6 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 29 Dec 2014 13:34:32 +0200 Subject: [PATCH 0018/1247] Change names of IOCTLS Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 37 ++++++++++++++++++++++--------------- src/fmm.c | 2 +- src/memory.c | 2 +- src/queues.c | 6 +++--- src/time.c | 2 +- src/version.c | 2 +- 6 files changed, 29 insertions(+), 22 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 7acef41fc2..af94f31e33 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -128,27 +128,34 @@ struct kfd_ioctl_get_process_apertures_args { uint32_t pad; }; -#define KFD_IOC_MAGIC 'K' +#define AMDKFD_IOCTL_BASE 'K' +#define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) +#define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) +#define AMDKFD_IOW(nr, type) _IOW(AMDKFD_IOCTL_BASE, nr, type) +#define AMDKFD_IOWR(nr, type) _IOWR(AMDKFD_IOCTL_BASE, nr, type) -#define KFD_IOC_GET_VERSION \ - _IOR(KFD_IOC_MAGIC, 1, struct kfd_ioctl_get_version_args) +#define AMDKFD_IOC_GET_VERSION \ + AMDKFD_IOR(0x01, struct kfd_ioctl_get_version_args) -#define KFD_IOC_CREATE_QUEUE \ - _IOWR(KFD_IOC_MAGIC, 2, struct kfd_ioctl_create_queue_args) +#define AMDKFD_IOC_CREATE_QUEUE \ + AMDKFD_IOWR(0x02, struct kfd_ioctl_create_queue_args) -#define KFD_IOC_DESTROY_QUEUE \ - _IOWR(KFD_IOC_MAGIC, 3, struct kfd_ioctl_destroy_queue_args) +#define AMDKFD_IOC_DESTROY_QUEUE \ + AMDKFD_IOWR(0x03, struct kfd_ioctl_destroy_queue_args) -#define KFD_IOC_SET_MEMORY_POLICY \ - _IOW(KFD_IOC_MAGIC, 4, struct kfd_ioctl_set_memory_policy_args) +#define AMDKFD_IOC_SET_MEMORY_POLICY \ + AMDKFD_IOW(0x04, struct kfd_ioctl_set_memory_policy_args) -#define KFD_IOC_GET_CLOCK_COUNTERS \ - _IOWR(KFD_IOC_MAGIC, 5, struct kfd_ioctl_get_clock_counters_args) +#define AMDKFD_IOC_GET_CLOCK_COUNTERS \ + AMDKFD_IOWR(0x05, struct kfd_ioctl_get_clock_counters_args) -#define KFD_IOC_GET_PROCESS_APERTURES \ - _IOR(KFD_IOC_MAGIC, 6, struct kfd_ioctl_get_process_apertures_args) +#define AMDKFD_IOC_GET_PROCESS_APERTURES \ + AMDKFD_IOR(0x06, struct kfd_ioctl_get_process_apertures_args) -#define KFD_IOC_UPDATE_QUEUE \ - _IOW(KFD_IOC_MAGIC, 7, struct kfd_ioctl_update_queue_args) +#define AMDKFD_IOC_UPDATE_QUEUE \ + AMDKFD_IOW(0x07, struct kfd_ioctl_update_queue_args) + +#define AMDKFD_COMMAND_START 0x01 +#define AMDKFD_COMMAND_END 0x08 #endif diff --git a/src/fmm.c b/src/fmm.c index 4edbb3096b..a274c32090 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -446,7 +446,7 @@ HSAKMT_STATUS fmm_init_process_apertures(){ struct kfd_ioctl_get_process_apertures_args args; uint8_t node_id; - if (0 == kfd_ioctl(KFD_IOC_GET_PROCESS_APERTURES, (void*)&args)){ + if (0 == kfd_ioctl(AMDKFD_IOC_GET_PROCESS_APERTURES, (void*)&args)){ for(node_id = 0; node_id < args.num_of_nodes; node_id++){ gpu_mem[node_id].gpu_id = args.process_apertures[node_id].gpu_id; gpu_mem[node_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); diff --git a/src/memory.c b/src/memory.c index 37cb88f1ef..0b73b5260d 100644 --- a/src/memory.c +++ b/src/memory.c @@ -68,7 +68,7 @@ hsaKmtSetMemoryPolicy( args.alternate_aperture_base = (uintptr_t)MemoryAddressAlternate; args.alternate_aperture_size = MemorySizeInBytes; - int err = kfd_ioctl(KFD_IOC_SET_MEMORY_POLICY, &args); + int err = kfd_ioctl(AMDKFD_IOC_SET_MEMORY_POLICY, &args); return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } diff --git a/src/queues.c b/src/queues.c index 1292b2034a..be250a2502 100644 --- a/src/queues.c +++ b/src/queues.c @@ -107,7 +107,7 @@ hsaKmtCreateQueue( args.queue_percentage = QueuePercentage; args.queue_priority = Priority; - err = kfd_ioctl(KFD_IOC_CREATE_QUEUE, &args); + err = kfd_ioctl(AMDKFD_IOC_CREATE_QUEUE, &args); if (err == -1) { @@ -166,7 +166,7 @@ hsaKmtUpdateQueue( arg.queue_percentage = QueuePercentage; arg.queue_priority = Priority; - int err = kfd_ioctl(KFD_IOC_UPDATE_QUEUE, &arg); + int err = kfd_ioctl(AMDKFD_IOC_UPDATE_QUEUE, &arg); if (err == -1) { return HSAKMT_STATUS_ERROR; @@ -193,7 +193,7 @@ hsaKmtDestroyQueue( args.queue_id = q->queue_id; - int err = kfd_ioctl(KFD_IOC_DESTROY_QUEUE, &args); + int err = kfd_ioctl(AMDKFD_IOC_DESTROY_QUEUE, &args); if (err == -1) { diff --git a/src/time.c b/src/time.c index 91c1a1209b..37e070d102 100644 --- a/src/time.c +++ b/src/time.c @@ -46,7 +46,7 @@ hsaKmtGetClockCounters( args.gpu_id = gpu_id; - err = kfd_ioctl(KFD_IOC_GET_CLOCK_COUNTERS, &args); + err = kfd_ioctl(AMDKFD_IOC_GET_CLOCK_COUNTERS, &args); if (err < 0) { result = HSAKMT_STATUS_ERROR; } else { diff --git a/src/version.c b/src/version.c index 7d927c2dc3..8908b1793e 100644 --- a/src/version.c +++ b/src/version.c @@ -39,7 +39,7 @@ hsaKmtGetVersion( struct kfd_ioctl_get_version_args args; memset(&args, 0, sizeof(args)); - if (kfd_ioctl(KFD_IOC_GET_VERSION, &args) == -1) + if (kfd_ioctl(AMDKFD_IOC_GET_VERSION, &args) == -1) return HSAKMT_STATUS_ERROR; VersionInfo->KernelInterfaceMajorVersion = args.major_version; From 94c0329fc4f37197a139fbd70cee621299f5036f Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 29 Dec 2014 15:37:42 +0200 Subject: [PATCH 0019/1247] Move from direct call to kfd_ioctl to wrapping kmtIoctl Signed-off-by: Oded Gabbay --- src/Makefile | 3 ++- src/fmm.c | 2 +- src/libhsakmt.c | 18 ++++++++++++++++++ src/libhsakmt.h | 3 +-- src/memory.c | 2 +- src/openclose.c | 5 ----- src/queues.c | 6 +++--- src/time.c | 2 +- src/version.c | 2 +- 9 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 src/libhsakmt.c diff --git a/src/Makefile b/src/Makefile index b74312ab43..5608ab7927 100644 --- a/src/Makefile +++ b/src/Makefile @@ -18,7 +18,8 @@ CFLAGS += -std=gnu99 -ggdb -pthread -fvisibility=hidden -O2 LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ - events.o openclose.o queues.o topology.o fmm.o pmc_table.o + events.o openclose.o queues.o topology.o fmm.o pmc_table.o \ + libhsakmt.o .PHONY: all lnx lnx64a clean diff --git a/src/fmm.c b/src/fmm.c index a274c32090..a90fb957e9 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -446,7 +446,7 @@ HSAKMT_STATUS fmm_init_process_apertures(){ struct kfd_ioctl_get_process_apertures_args args; uint8_t node_id; - if (0 == kfd_ioctl(AMDKFD_IOC_GET_PROCESS_APERTURES, (void*)&args)){ + if (0 == kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void*)&args)){ for(node_id = 0; node_id < args.num_of_nodes; node_id++){ gpu_mem[node_id].gpu_id = args.process_apertures[node_id].gpu_id; gpu_mem[node_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); diff --git a/src/libhsakmt.c b/src/libhsakmt.c new file mode 100644 index 0000000000..d7f79d3562 --- /dev/null +++ b/src/libhsakmt.c @@ -0,0 +1,18 @@ +#include +#include + +#include "libhsakmt.h" + +/** + * Call ioctl, restarting if it is interupted + */ +int +kmtIoctl(int fd, unsigned long request, void *arg) +{ + int ret; + + do { + ret = ioctl(fd, request, arg); + } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); + return ret; +} diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 4d337e612b..fa8dc0e24a 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -64,7 +64,7 @@ extern pthread_mutex_t hsakmt_mutex; HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); uint16_t get_device_id_by_node(HSAuint32 node_id); -extern int kfd_ioctl(int cmdcode, void* data); +extern int kmtIoctl(int fd, unsigned long request, void *arg); /* Void pointer arithmetic (or remove -Wpointer-arith to allow void pointers arithmetic) */ #define VOID_PTR_ADD32(ptr,n) (void*)((uint32_t*)(ptr) + n)/*ptr + offset*/ @@ -72,5 +72,4 @@ extern int kfd_ioctl(int cmdcode, void* data); #define VOID_PTR_SUB(ptr,n) (void*)((uint8_t*)(ptr) - n)/*ptr - offset*/ #define VOID_PTRS_SUB(ptr1,ptr2) (uint64_t)((uint8_t*)(ptr1) - (uint8_t*)(ptr2)) /*ptr1 - ptr2*/ - #endif diff --git a/src/memory.c b/src/memory.c index 0b73b5260d..d65730a336 100644 --- a/src/memory.c +++ b/src/memory.c @@ -68,7 +68,7 @@ hsaKmtSetMemoryPolicy( args.alternate_aperture_base = (uintptr_t)MemoryAddressAlternate; args.alternate_aperture_size = MemorySizeInBytes; - int err = kfd_ioctl(AMDKFD_IOC_SET_MEMORY_POLICY, &args); + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args); return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } diff --git a/src/openclose.c b/src/openclose.c index 8290d00502..d5b91e23dd 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -110,8 +110,3 @@ hsaKmtCloseKFD(void) return result; } - -extern int kfd_ioctl(int cmdcode, void* data) -{ - return ioctl(kfd_fd, cmdcode, data); -} diff --git a/src/queues.c b/src/queues.c index be250a2502..1feb9bade9 100644 --- a/src/queues.c +++ b/src/queues.c @@ -107,7 +107,7 @@ hsaKmtCreateQueue( args.queue_percentage = QueuePercentage; args.queue_priority = Priority; - err = kfd_ioctl(AMDKFD_IOC_CREATE_QUEUE, &args); + err = kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_QUEUE, &args); if (err == -1) { @@ -166,7 +166,7 @@ hsaKmtUpdateQueue( arg.queue_percentage = QueuePercentage; arg.queue_priority = Priority; - int err = kfd_ioctl(AMDKFD_IOC_UPDATE_QUEUE, &arg); + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_UPDATE_QUEUE, &arg); if (err == -1) { return HSAKMT_STATUS_ERROR; @@ -193,7 +193,7 @@ hsaKmtDestroyQueue( args.queue_id = q->queue_id; - int err = kfd_ioctl(AMDKFD_IOC_DESTROY_QUEUE, &args); + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_QUEUE, &args); if (err == -1) { diff --git a/src/time.c b/src/time.c index 37e070d102..45709f9bd6 100644 --- a/src/time.c +++ b/src/time.c @@ -46,7 +46,7 @@ hsaKmtGetClockCounters( args.gpu_id = gpu_id; - err = kfd_ioctl(AMDKFD_IOC_GET_CLOCK_COUNTERS, &args); + err = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_CLOCK_COUNTERS, &args); if (err < 0) { result = HSAKMT_STATUS_ERROR; } else { diff --git a/src/version.c b/src/version.c index 8908b1793e..95bfec6523 100644 --- a/src/version.c +++ b/src/version.c @@ -39,7 +39,7 @@ hsaKmtGetVersion( struct kfd_ioctl_get_version_args args; memset(&args, 0, sizeof(args)); - if (kfd_ioctl(AMDKFD_IOC_GET_VERSION, &args) == -1) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1) return HSAKMT_STATUS_ERROR; VersionInfo->KernelInterfaceMajorVersion = args.major_version; From 0a8bc868b58a4d6ef0e1420bd6ca0be65749eef7 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 8 Dec 2014 10:04:31 +0200 Subject: [PATCH 0020/1247] set thunk-kfd version to match stg Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index af94f31e33..35f28b2010 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -27,7 +27,7 @@ #include #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 0 +#define KFD_IOCTL_MINOR_VERSION 2 struct kfd_ioctl_get_version_args { uint32_t major_version; /* from KFD */ From 8afe1ef4e812687ffae21c479ab5ae018a062483 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Tue, 1 Jul 2014 16:51:57 +0300 Subject: [PATCH 0021/1247] Add SDMA queues support Signed-off-by: Ben Goz Signed-off-by: Oded Gabbay --- src/queues.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/queues.c b/src/queues.c index 1feb9bade9..650a1f27be 100644 --- a/src/queues.c +++ b/src/queues.c @@ -73,7 +73,7 @@ hsaKmtCreateQueue( if (result != HSAKMT_STATUS_SUCCESS) return result; - struct queue *q = malloc(sizeof(struct queue)); + struct queue *q = (struct queue *)malloc(sizeof(*q)); if (q == NULL) { return HSAKMT_STATUS_NO_MEMORY; @@ -89,7 +89,7 @@ hsaKmtCreateQueue( switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: free(q); return HSAKMT_STATUS_NOT_IMPLEMENTED; + case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; default: free(q); return HSAKMT_STATUS_INVALID_PARAMETER; } From 03604633cac20f0e1cb453087a3d2256c7d03317 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 8 Dec 2014 10:01:26 +0200 Subject: [PATCH 0022/1247] kfd_ioctl: events Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 65 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 35f28b2010..e0a1330a66 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -128,6 +128,54 @@ struct kfd_ioctl_get_process_apertures_args { uint32_t pad; }; +/* Matching HSA_EVENTTYPE */ +#define KFD_IOC_EVENT_SIGNAL 0 +#define KFD_IOC_EVENT_NODECHANGE 1 +#define KFD_IOC_EVENT_DEVICESTATECHANGE 2 +#define KFD_IOC_EVENT_HW_EXCEPTION 3 +#define KFD_IOC_EVENT_SYSTEM_EVENT 4 +#define KFD_IOC_EVENT_DEBUG_EVENT 5 +#define KFD_IOC_EVENT_PROFILE_EVENT 6 +#define KFD_IOC_EVENT_QUEUE_EVENT 7 +#define KFD_IOC_EVENT_MEMORY 8 + +#define KFD_IOC_WAIT_RESULT_COMPLETE 0 +#define KFD_IOC_WAIT_RESULT_TIMEOUT 1 +#define KFD_IOC_WAIT_RESULT_FAIL 2 + +struct kfd_ioctl_create_event_args { + uint64_t event_trigger_address; /* from KFD - signal events only */ + uint32_t event_trigger_data; /* from KFD - signal events only */ + uint32_t event_type; /* to KFD */ + uint32_t auto_reset; /* to KFD */ + uint32_t node_id; /* to KFD - only valid for certain event types */ + uint32_t event_id; /* from KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_destroy_event_args { + uint32_t event_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_set_event_args { + uint32_t event_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_reset_event_args { + uint32_t event_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_wait_events_args { + uint64_t events_ptr; /* to KFD */ + uint32_t num_events; /* to KFD */ + uint32_t wait_for_all; /* to KFD */ + uint32_t timeout; /* to KFD */ + uint32_t wait_result; /* from KFD */ +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -155,7 +203,22 @@ struct kfd_ioctl_get_process_apertures_args { #define AMDKFD_IOC_UPDATE_QUEUE \ AMDKFD_IOW(0x07, struct kfd_ioctl_update_queue_args) +#define AMDKFD_IOC_CREATE_EVENT \ + AMDKFD_IOWR(0x08, struct kfd_ioctl_create_event_args) + +#define AMDKFD_IOC_DESTROY_EVENT \ + AMDKFD_IOW(0x09, struct kfd_ioctl_destroy_event_args) + +#define AMDKFD_IOC_SET_EVENT \ + AMDKFD_IOW(0x0A, struct kfd_ioctl_set_event_args) + +#define AMDKFD_IOC_RESET_EVENT \ + AMDKFD_IOW(0x0B, struct kfd_ioctl_reset_event_args) + +#define AMDKFD_IOC_WAIT_EVENTS \ + AMDKFD_IOWR(0x0C, struct kfd_ioctl_wait_events_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x08 +#define AMDKFD_COMMAND_END 0x0D #endif From f66e7d71b18b196242f01a91608b4f431f3dfe81 Mon Sep 17 00:00:00 2001 From: Andrew Lewycky Date: Mon, 13 Oct 2014 11:30:18 +0300 Subject: [PATCH 0023/1247] Add HSA Signals module Signed-off-by: Andrew Lewycky Signed-off-by: Oded Gabbay Adjust to new ioctl format Signed-off-by: Oded Gabbay --- src/events.c | 127 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 96 insertions(+), 31 deletions(-) diff --git a/src/events.c b/src/events.c index cb07116aa8..c62f4a4a20 100644 --- a/src/events.c +++ b/src/events.c @@ -25,9 +25,17 @@ #include "libhsakmt.h" #include +#include #include #include #include +#include "linux/kfd_ioctl.h" + +static bool IsSystemEventType(HSA_EVENTTYPE type) +{ + // Debug events behave as signal events. + return (type != HSA_EVENTTYPE_SIGNAL && type != HSA_EVENTTYPE_DEBUG_EVENT); +} HSAKMT_STATUS HSAKMTAPI @@ -51,6 +59,30 @@ hsaKmtCreateEvent( return HSAKMT_STATUS_ERROR; } + memset(e, 0, sizeof(*e)); + + struct kfd_ioctl_create_event_args args; + memset(&args, 0, sizeof(args)); + + args.event_type = EventDesc->EventType; + args.auto_reset = !ManualReset; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) == -1) + return HSAKMT_STATUS_ERROR; + + e->EventId = args.event_id; + e->EventData.HWData1 = args.event_id; + e->EventData.HWData2 = args.event_trigger_address; + e->EventData.HWData3 = args.event_trigger_data; + + if (IsSignaled && !IsSystemEventType(e->EventData.EventType)) { + struct kfd_ioctl_set_event_args set_args; + memset(&set_args, 0, sizeof(set_args)); + set_args.event_id = args.event_id; + + kmtIoctl(kfd_fd, AMDKFD_IOC_SET_EVENT, &set_args); + } + *Event = e; return HSAKMT_STATUS_SUCCESS; @@ -64,6 +96,14 @@ hsaKmtDestroyEvent( { CHECK_KFD_OPEN(); + struct kfd_ioctl_destroy_event_args args; + memset(&args, 0, sizeof(args)); + + args.event_id = Event->EventId; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_EVENT, &args) == -1) + return HSAKMT_STATUS_ERROR; + free(Event); return HSAKMT_STATUS_SUCCESS; @@ -76,6 +116,19 @@ hsaKmtSetEvent( ) { CHECK_KFD_OPEN(); + + /* Although the spec is doesn't say, don't allow system-defined events to be signaled. */ + if (IsSystemEventType(Event->EventData.EventType)) + return HSAKMT_STATUS_ERROR; + + struct kfd_ioctl_set_event_args args; + memset(&args, 0, sizeof(args)); + + args.event_id = Event->EventId; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_EVENT, &args) == -1) + return HSAKMT_STATUS_ERROR; + return HSAKMT_STATUS_SUCCESS; } @@ -86,6 +139,19 @@ hsaKmtResetEvent( ) { CHECK_KFD_OPEN(); + + /* Although the spec is doesn't say, don't allow system-defined events to be signaled. */ + if (IsSystemEventType(Event->EventData.EventType)) + return HSAKMT_STATUS_ERROR; + + struct kfd_ioctl_reset_event_args args; + memset(&args, 0, sizeof(args)); + + args.event_id = Event->EventId; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_RESET_EVENT, &args) == -1) + return HSAKMT_STATUS_ERROR; + return HSAKMT_STATUS_SUCCESS; } @@ -96,6 +162,7 @@ hsaKmtQueryEventState( ) { CHECK_KFD_OPEN(); + return HSAKMT_STATUS_SUCCESS; } @@ -106,36 +173,7 @@ hsaKmtWaitOnEvent( HSAuint32 Milliseconds //IN ) { - CHECK_KFD_OPEN(); - - if (Milliseconds == HSA_EVENTTIMEOUT_INFINITE) - { - while (1) { pause(); } - } - else if (Milliseconds != HSA_EVENTTIMEOUT_IMMEDIATE) - { - struct timespec req; - - req.tv_sec = Milliseconds / 1000; - req.tv_nsec = (long)(Milliseconds % 1000) * 1000000; - - while (1) - { - struct timespec rem; - - int err = nanosleep(&req, &rem); - if (err == -1 && errno == EINTR) - { - req = rem; - } - else - { - break; // success or other error - } - } - } - - return HSAKMT_STATUS_WAIT_TIMEOUT; + return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); } HSAKMT_STATUS @@ -149,5 +187,32 @@ hsaKmtWaitOnMultipleEvents( { CHECK_KFD_OPEN(); - return hsaKmtWaitOnEvent(NULL, Milliseconds); + uint32_t *event_ids = malloc(NumEvents * sizeof(uint32_t)); + for (HSAuint32 i = 0; i < NumEvents; i++) { + event_ids[i] = Events[i]->EventId; + } + + struct kfd_ioctl_wait_events_args args; + memset(&args, 0, sizeof(args)); + + args.wait_for_all = WaitOnAll; + args.timeout = Milliseconds; + args.num_events = NumEvents; + args.events_ptr = (uint64_t)(uintptr_t)event_ids; + + HSAKMT_STATUS result; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_WAIT_EVENTS, &args) == -1) { + result = HSAKMT_STATUS_ERROR; + } + else if (args.wait_result == KFD_IOC_WAIT_RESULT_TIMEOUT) { + result = HSAKMT_STATUS_WAIT_TIMEOUT; + } + else { + result = HSAKMT_STATUS_SUCCESS; + } + + free(event_ids); + + return result; } From f7c3c3bd1b29852041805b1415da07efc9783101 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 8 Dec 2014 10:02:12 +0200 Subject: [PATCH 0024/1247] kfd_ioctl: debug Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 40 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index e0a1330a66..3545a8af1e 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -128,6 +128,32 @@ struct kfd_ioctl_get_process_apertures_args { uint32_t pad; }; +#define MAX_ALLOWED_NUM_POINTS 100 +#define MAX_ALLOWED_AW_BUFF_SIZE 4096 +#define MAX_ALLOWED_WAC_BUFF_SIZE 128 + +struct kfd_ioctl_dbg_register_args { + uint32_t gpu_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_dbg_unregister_args { + uint32_t gpu_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_dbg_address_watch_args { + uint32_t gpu_id; /* to KFD */ + uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ + unsigned char content[0]; +}; + +struct kfd_ioctl_dbg_wave_control_args { + uint32_t gpu_id; /* to KFD */ + uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ + unsigned char content[0]; +}; + /* Matching HSA_EVENTTYPE */ #define KFD_IOC_EVENT_SIGNAL 0 #define KFD_IOC_EVENT_NODECHANGE 1 @@ -218,7 +244,19 @@ struct kfd_ioctl_wait_events_args { #define AMDKFD_IOC_WAIT_EVENTS \ AMDKFD_IOWR(0x0C, struct kfd_ioctl_wait_events_args) +#define AMDKFD_IOC_DBG_REGISTER \ + AMDKFD_IOW(0x0D, struct kfd_ioctl_dbg_register_args) + +#define AMDKFD_IOC_DBG_UNREGISTER \ + AMDKFD_IOW(0x0E, struct kfd_ioctl_dbg_unregister_args) + +#define AMDKFD_IOC_DBG_ADDRESS_WATCH \ + AMDKFD_IOW(0x0F, struct kfd_ioctl_dbg_address_watch_args) + +#define AMDKFD_IOC_DBG_WAVE_CONTROL \ + AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x0D +#define AMDKFD_COMMAND_END 0x11 #endif From 39f693aec09d2cedc1bba187ff8e252f14aaac09 Mon Sep 17 00:00:00 2001 From: Yair Shachar Date: Mon, 13 Oct 2014 11:29:03 +0300 Subject: [PATCH 0025/1247] Add H/W debug module Signed-off-by: Yair Shachar Signed-off-by: Oded Gabbay Adjust to new ioctl format Signed-off-by: Oded Gabbay --- src/debug.c | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 198 insertions(+), 4 deletions(-) diff --git a/src/debug.c b/src/debug.c index dab3a51973..155222570e 100644 --- a/src/debug.c +++ b/src/debug.c @@ -24,6 +24,9 @@ */ #include "libhsakmt.h" +#include "linux/kfd_ioctl.h" +#include +#include HSAKMT_STATUS HSAKMTAPI @@ -31,22 +34,57 @@ hsaKmtDbgRegister( HSAuint32 NodeId //IN ) { + HSAKMT_STATUS result; + uint32_t gpu_id; CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + struct kfd_ioctl_dbg_register_args args; + memset(&args, 0, sizeof(args)); + args.gpu_id = gpu_id; + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER, &args); + + if (err == 0) + result = HSAKMT_STATUS_SUCCESS; + else + result = HSAKMT_STATUS_ERROR; + + return (result); } +/* =============================================================================== */ + HSAKMT_STATUS HSAKMTAPI hsaKmtDbgUnregister( HSAuint32 NodeId //IN ) { + HSAKMT_STATUS result; + uint32_t gpu_id; CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + struct kfd_ioctl_dbg_unregister_args args; + memset(&args, 0, sizeof(args)); + args.gpu_id = gpu_id; + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER, &args); + if (err == 0) + result = HSAKMT_STATUS_SUCCESS; + else + result = HSAKMT_STATUS_ERROR; + + return (result); } +/* =============================================================================== */ + HSAKMT_STATUS HSAKMTAPI hsaKmtDbgWavefrontControl( @@ -57,11 +95,74 @@ hsaKmtDbgWavefrontControl( HsaDbgWaveMessage* DbgWaveMsgRing //IN (? - see thunk API doc!) ) { + HSAKMT_STATUS result; + uint32_t gpu_id; + + struct kfd_ioctl_dbg_wave_control_args *args; + CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + +/* Determine Size of the ioctl buffer */ + + uint32_t buff_size = sizeof(Operand)+ + sizeof(Mode) + sizeof(TrapId) + + sizeof(DbgWaveMsgRing->DbgWaveMsg)+ sizeof(DbgWaveMsgRing->MemoryVA) + sizeof(*args); + + + args = (struct kfd_ioctl_dbg_wave_control_args*) malloc(buff_size); + if (args == NULL) + { + return HSAKMT_STATUS_ERROR; + } + + memset(args, 0, buff_size); + + args->gpu_id = gpu_id; + args->buf_size_in_bytes = buff_size; + + /* increment pointer to the start of the non fixed part */ + + unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); + + /* insert items, and increment pointer accordingly */ + + *((HSA_DBG_WAVEOP*)run_ptr) = Operand; + run_ptr += sizeof(Operand); + + + *((HSA_DBG_WAVEMODE*)run_ptr) = Mode; + run_ptr += sizeof(Mode); + + + *((HSAuint32*)run_ptr) = TrapId; + run_ptr += sizeof(TrapId); + + *((HsaDbgWaveMessageAMD*)run_ptr) = DbgWaveMsgRing->DbgWaveMsg; + run_ptr += sizeof(DbgWaveMsgRing->DbgWaveMsg); + + *((void**)run_ptr) = DbgWaveMsgRing->MemoryVA; + run_ptr += sizeof(DbgWaveMsgRing->MemoryVA); + + /* send to kernel */ + + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_WAVE_CONTROL, args); + + free (args); + + if (err == 0) + return HSAKMT_STATUS_SUCCESS; + else + return HSAKMT_STATUS_ERROR; } + +/* =============================================================================== */ + HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch( @@ -73,7 +174,100 @@ hsaKmtDbgAddressWatch( HsaEvent* WatchEvent[] //IN, optional ) { + HSAKMT_STATUS result; + uint32_t gpu_id; + + /* determine the size of the watch mask and event buffers + * the value is NULL if and only if no vector data should be attached + * + */ + + uint32_t watch_mask_items = WatchMask[0] > 0 ? NumWatchPoints:1; + uint32_t watch_event_items = WatchEvent != NULL ? NumWatchPoints:0; + + struct kfd_ioctl_dbg_address_watch_args *args; + HSAuint32 i = 0; + CHECK_KFD_OPEN(); - return HSAKMT_STATUS_NOT_SUPPORTED; + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + if (NumWatchPoints > MAX_ALLOWED_NUM_POINTS) + return HSAKMT_STATUS_INVALID_PARAMETER; + +/* Size and structure of the ioctl buffer is dynamic in this case + * Here we calculate the buff size. + */ + + uint32_t buff_size =sizeof(NumWatchPoints)+ + ( sizeof(WatchMode[0]) + + sizeof(WatchAddress[0]))*NumWatchPoints + + watch_mask_items*sizeof(HSAuint64) + + watch_event_items*sizeof(HsaEvent*)+ + sizeof(*args); + + + args = (struct kfd_ioctl_dbg_address_watch_args*) malloc(buff_size); + if (args == NULL) + { + return HSAKMT_STATUS_ERROR; + } + + memset(args, 0, buff_size); + + args->gpu_id = gpu_id; + args->buf_size_in_bytes = buff_size; + + /* increment pointer to the start of the non fixed part */ + + unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); + + /* insert items, and increment pointer accordingly */ + + *((HSAuint32*)run_ptr) = NumWatchPoints; + run_ptr += sizeof(NumWatchPoints); + + for (i=0; i < NumWatchPoints; i++) + { + *((HSA_DBG_WATCH_MODE*)run_ptr) = WatchMode[i]; + run_ptr += sizeof(WatchMode[i]); + } + + for (i=0; i < NumWatchPoints; i++) + { + *((void**)run_ptr) = WatchAddress[i]; + run_ptr += sizeof(WatchAddress[i]); + } + + for (i=0; i < watch_mask_items; i++) + { + *((HSAuint64*)run_ptr) = WatchMask[i]; + run_ptr += sizeof(WatchMask[i]); + } + + for (i=0; i < watch_event_items; i++) + { + *((HsaEvent**)run_ptr) = WatchEvent[i]; + run_ptr += sizeof(WatchEvent[i]); + } + + /* send to kernel */ + + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_ADDRESS_WATCH, args); + + free (args); + + if (err == 0) + { + return HSAKMT_STATUS_SUCCESS; + } + else + { + return HSAKMT_STATUS_ERROR; + } } + +/* =============================================================================== */ + From d14eb7aa00ebff2ce6d4885a3a3e7cd08e1ebc21 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 8 Dec 2014 10:02:27 +0200 Subject: [PATCH 0026/1247] kfd_ioctl: local memory Add two new IOCTLs Remove old IOCTLs Signed-off-by: Alexey Skidanov Reviewed-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 3545a8af1e..a0e9563cdd 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -202,6 +202,18 @@ struct kfd_ioctl_wait_events_args { uint32_t wait_result; /* from KFD */ }; +struct kfd_ioctl_map_memory_to_gpu_args { + uint64_t va_addr; /* to KFD */ + uint64_t size; /* to KFD */ + uint64_t handle; /* from KFD */ + uint32_t gpu_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_unmap_memory_from_gpu_args { + uint64_t handle; /* to KFD */ +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -256,7 +268,13 @@ struct kfd_ioctl_wait_events_args { #define AMDKFD_IOC_DBG_WAVE_CONTROL \ AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args) +#define AMDKFD_IOC_MAP_MEMORY_TO_GPU \ + AMDKFD_IOWR(0x11, struct kfd_ioctl_map_memory_to_gpu_args) + +#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ + AMDKFD_IOWR(0x12, struct kfd_ioctl_unmap_memory_from_gpu_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x11 +#define AMDKFD_COMMAND_END 0x13 #endif From d995344ae73342d3a31290066ce1f5908ed9ad3f Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 20 Oct 2014 08:48:28 +0300 Subject: [PATCH 0027/1247] Add new definitions of HSA functions and types Signed-off-by: Oded Gabbay --- include/hsakmt.h | 29 +++++++++++++++++++++++++++++ src/libhsakmt.ver | 2 ++ 2 files changed, 31 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index 3983aa0412..a57946e9da 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -405,6 +405,35 @@ hsaKmtUnmapMemoryToGPU( ); +/** + Notifies the kernel driver that a process wants to use GPU debugging facilities +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtMapGraphicHandle( + HSAuint32 NodeId, //IN + HSAuint64 GraphicDeviceHandle, //IN + HSAuint64 GraphicResourceHandle, //IN + HSAuint64 GraphicResourceOffset, //IN + HSAuint64 GraphicResourceSize, //IN + HSAuint64* FlatMemoryAddress //OUT + ); + + +/** + Stub for Unmap Graphic Handle +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtUnmapGraphicHandle( + HSAuint32 NodeId, //IN + HSAuint64 FlatMemoryAddress, //IN + HSAuint64 SizeInBytes //IN + ); + + /** Notifies the kernel driver that a process wants to use GPU debugging facilities */ diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 9c6e6cb1fd..72917e16af 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -40,6 +40,8 @@ hsaKmtPmcReleaseTraceAccess; hsaKmtPmcStartTrace; hsaKmtPmcQueryTrace; hsaKmtPmcStopTrace; +hsaKmtMapGraphicHandle; +hsaKmtUnmapGraphicHandle; local: *; }; From 7972bb00c5af76cafe28ba0d5b4219c8fe532d03 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 8 Dec 2014 10:02:34 +0200 Subject: [PATCH 0028/1247] kfd_ioctl: interop Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index a0e9563cdd..9906419ec0 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -214,6 +214,15 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { uint64_t handle; /* to KFD */ }; +struct kfd_ioctl_open_graphic_handle_args { + uint64_t va_addr; /* to KFD */ + uint64_t handle; /* from KFD */ + uint32_t gpu_id; /* to KFD */ + int graphic_device_fd; /* to KFD */ + uint32_t graphic_handle; /* to KFD */ + uint32_t pad; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -274,7 +283,10 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { #define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ AMDKFD_IOWR(0x12, struct kfd_ioctl_unmap_memory_from_gpu_args) +#define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ + AMDKFD_IOWR(0x13, struct kfd_ioctl_open_graphic_handle_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x13 +#define AMDKFD_COMMAND_END 0x14 #endif From 5d82c65602303066076d2e11732aa3d8f72d0110 Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Mon, 13 Oct 2014 11:29:39 +0300 Subject: [PATCH 0029/1247] Add GPU local memory module Add graphic interop support Local memory rework Signed-off-by: Alexey Skidanov Reviewed-by: Oded Gabbay Adjust to new ioctl format Signed-off-by: Oded Gabbay --- src/fmm.c | 294 +++++++++++++++++++++++++++++++++++++-------------- src/fmm.h | 8 +- src/memory.c | 79 +++++++++++--- 3 files changed, 288 insertions(+), 93 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index a90fb957e9..6e3d8a1ece 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -45,8 +45,8 @@ #define INIT_GPUs_MEM {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = INIT_GPU_MEM} struct vm_object{ void* start; - HSAuint64 size; - HSAuint64 handle; // opaque + uint64_t size; + uint64_t handle; // opaque struct vm_object* next; struct vm_object* prev; }; @@ -74,7 +74,7 @@ typedef struct { } aperture_t; typedef struct{ - HSAuint32 gpu_id; + uint32_t gpu_id; aperture_t lds_aperture; manageble_aperture_t scratch_aperture; manageble_aperture_t gpuvm_aperture; @@ -183,7 +183,7 @@ static vm_object_t* vm_find_object_by_address(manageble_aperture_t* app, void* a // Look up the appropriate address range containing the given address while(cur){ - if(cur->start == address && cur->size == size) + if(cur->start == address && (cur->size == size || size == 0)) break; cur = cur->next; }; @@ -213,105 +213,96 @@ static bool aperture_is_valid(void* app_base, void* app_limit){ /* * Assumes that fmm_mutex is locked on entry. */ -static int aperture_release(manageble_aperture_t* app, void* address, uint64_t MemorySizeInBytes){ - int rc = -1; +static void aperture_release_area(manageble_aperture_t* app, void* address, uint64_t MemorySizeInBytes){ vm_area_t* area; area = vm_find(app, address); - vm_object_t* object = vm_find_object_by_address(app, address, MemorySizeInBytes); - if (object && area){ - vm_remove_object(app, object); - if (VOID_PTRS_SUB(area->end, area->start) + 1 > MemorySizeInBytes){ // the size of the released block is less than the size of area - if (area->start == address){ // shrink from the start + if(area) { + if(VOID_PTRS_SUB(area->end, area->start) + 1 > MemorySizeInBytes) { // the size of the released block is less than the size of area + if(area->start == address) { // shrink from the start area->start = VOID_PTR_ADD(area->start,MemorySizeInBytes); - } else if (VOID_PTRS_SUB(area->end, address) + 1 == MemorySizeInBytes){ // shrink from the end + } else if(VOID_PTRS_SUB(area->end, address) + 1 == MemorySizeInBytes) { // shrink from the end area->end = VOID_PTR_SUB(area->end, MemorySizeInBytes); } else { // split the area vm_split_area(app, area, address, MemorySizeInBytes); } - rc = 0; - } else if (VOID_PTRS_SUB(area->end, area->start) + 1 == MemorySizeInBytes){ // the size of the released block is exactly the same as the size of area + } else if(VOID_PTRS_SUB(area->end, area->start) + 1 == MemorySizeInBytes) { // the size of the released block is exactly the same as the size of area vm_remove_area(app, area); - rc = 0; - } else { - //Inconsistent data. Fail it? - rc = -1; } } - - return rc; } /* * returns allocated address or NULL. Assumes, that fmm_mutex is locked on entry. */ -static void* aperture_allocate(manageble_aperture_t* app, uint64_t MemorySizeInBytes){ +static void* aperture_allocate_area(manageble_aperture_t* app, uint64_t MemorySizeInBytes, uint64_t offset){ vm_area_t* cur, *next, *new_area, *start; - vm_object_t* new_object; void* new_address = NULL; next = NULL; new_area = NULL; cur = app->vm_ranges; - if (cur){ // not empty + if(cur) { // not empty // Look up the appropriate address space "hole" or end of the list - while(cur){ + while (cur) { next = cur->next; // End of the list reached - if (!next) + if(!next) break; // address space "hole" - if ((VOID_PTRS_SUB(next->start,cur->end) >= MemorySizeInBytes)) + if((VOID_PTRS_SUB(next->start,cur->end) >= MemorySizeInBytes)) break; cur = next; }; // If the new range is inside the reserved aperture - if (VOID_PTRS_SUB(app->limit, cur->end) + 1 >= MemorySizeInBytes){ + if(VOID_PTRS_SUB(app->limit, cur->end) + 1 >= MemorySizeInBytes) { // cur points to the last inspected element: the tail of the list or the found "hole" // Just extend the existing region new_address = VOID_PTR_ADD(cur->end, 1); cur->end = VOID_PTR_ADD(cur->end, MemorySizeInBytes); - } else - new_address = NULL; + } else new_address = NULL; } else { // empty - create the first area - start = (void*)app->base; + start = VOID_PTR_ADD(app->base, offset); // Some offset from the base new_area = vm_create_and_init_area(start, VOID_PTR_ADD(start, (MemorySizeInBytes - 1))); - if (new_area){ + if(new_area) { app->vm_ranges = new_area; new_address = new_area->start; } } - // Allocate new object - if (new_address){ - new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, 0); - if (new_object){ - if (app->vm_objects == NULL){ // empty list - // Update head - app->vm_objects = new_object; - } else { - // Add it before the first element - vm_add_object_before(app->vm_objects, new_object); - // Update head - app->vm_objects = new_object; - } - } else{ - // Failed to allocate object: remove just allocated range and return NULL - aperture_release(app, new_address, MemorySizeInBytes); - new_address = NULL; - } - } - return new_address; } +/* + * returns 0 on success. Assumes, that fmm_mutex is locked on entry. + */ +static int aperture_allocate_object(manageble_aperture_t* app, void* new_address, uint64_t handle, uint64_t MemorySizeInBytes){ + vm_object_t* new_object; + + // Allocate new object + new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, handle); + if(!new_object) + return -1; + + if(app->vm_objects == NULL ) { // empty list + // Update head + app->vm_objects = new_object; + } else { + // Add it before the first element + vm_add_object_before(app->vm_objects, new_object); + // Update head + app->vm_objects = new_object; + } + + return 0; +} static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id){ @@ -387,21 +378,12 @@ void fmm_print(uint32_t gpu_id){ void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes){ - - void* mem = NULL; - int32_t i = gpu_mem_find_by_gpu_id(gpu_id); - - // If not found or aperture isn't properly initialized/supported - if(i < 0 || !aperture_is_valid(gpu_mem[i].scratch_aperture.base, gpu_mem[i].scratch_aperture.limit)) - return NULL; - - pthread_mutex_lock(&gpu_mem[i].scratch_aperture.fmm_mutex); - mem = aperture_allocate(&gpu_mem[i].scratch_aperture, MemorySizeInBytes); - pthread_mutex_unlock(&gpu_mem[i].scratch_aperture.fmm_mutex); - - return mem; + // Not supported yet + return NULL; } +// The offset from GPUVM aperture base address to ensure that address 0 (after base subtraction) won't be used +#define GPUVM_APP_OFFSET 0x10000 void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes){ void* mem = NULL; @@ -409,37 +391,88 @@ void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes){ // If not found or aperture isn't properly initialized/supported if(i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, gpu_mem[i].gpuvm_aperture.limit)) - return NULL; + return NULL ; + // Allocate address space pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - mem = aperture_allocate(&gpu_mem[i].gpuvm_aperture, MemorySizeInBytes); - pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, MemorySizeInBytes, GPUVM_APP_OFFSET); + pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); return mem; } +void* fmm_open_graphic_handle(uint32_t gpu_id, + int32_t graphic_device_handle, + uint32_t graphic_handle, + uint64_t MemorySizeInBytes){ -int fmm_release(void* address, uint64_t MemorySizeInBytes){ + void* mem = NULL; + int32_t i = gpu_mem_find_by_gpu_id(gpu_id); + struct kfd_ioctl_open_graphic_handle_args open_graphic_handle_args; + struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; + + // If not found or aperture isn't properly initialized/supported + if (i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, gpu_mem[i].gpuvm_aperture.limit)) + return NULL; + + pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + // Allocate address space + mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, MemorySizeInBytes, GPUVM_APP_OFFSET); + if (!mem) + goto out; + + // Allocate local memory + open_graphic_handle_args.gpu_id = gpu_id; + open_graphic_handle_args.graphic_device_fd = graphic_device_handle; + open_graphic_handle_args.graphic_handle = graphic_handle; + open_graphic_handle_args.va_addr = VOID_PTRS_SUB(mem, gpu_mem[i].gpuvm_aperture.base); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_OPEN_GRAPHIC_HANDLE, &open_graphic_handle_args)) + goto release_area; + + // Allocate object + if (aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, open_graphic_handle_args.handle, MemorySizeInBytes)) + goto release_mem; + + pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + + // That's all. Just return the new address + return mem; + +release_mem: + unmap_args.handle = open_graphic_handle_args.handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); +release_area: + aperture_release_area(&gpu_mem[i].gpuvm_aperture, mem, MemorySizeInBytes); +out: + pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + + return NULL ; +} + + +void fmm_release(void* address, uint64_t MemorySizeInBytes){ uint32_t i; - int32_t rc = -1; + bool found = false; - for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ + for (i = 0; i < NUM_OF_SUPPORTED_GPUS && !found; i++) { if(gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; - if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit){ - pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - rc = aperture_release(&gpu_mem[i].gpuvm_aperture, address, MemorySizeInBytes); - pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + if(address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { + found = true; + pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + aperture_release_area(&gpu_mem[i].gpuvm_aperture, address, MemorySizeInBytes); + pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); fmm_print(gpu_mem[i].gpu_id); - } else if (address >= gpu_mem[i].scratch_aperture.base && address <= gpu_mem[i].scratch_aperture.limit) - pthread_mutex_lock(&gpu_mem[i].scratch_aperture.fmm_mutex); - rc = aperture_release(&gpu_mem[i].scratch_aperture, address, MemorySizeInBytes); - pthread_mutex_unlock(&gpu_mem[i].scratch_aperture.fmm_mutex); + + } } - return rc; + // If memory address isn't inside of any defined aperture - it refers to the system memory + if (!found) { + free(address); + } } HSAKMT_STATUS fmm_init_process_apertures(){ @@ -484,3 +517,106 @@ HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id) } } + + +static bool _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t* aperture, void* address, uint64_t size, uint64_t* gpuvm_address) { + + struct kfd_ioctl_map_memory_to_gpu_args args; + struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; + + // Check that address space was previously reserved + if (vm_find(aperture, address) == NULL) + return false; + + // Allocate local memory + args.gpu_id = gpu_id; + args.size = size; + args.va_addr = VOID_PTRS_SUB(address, aperture->base); //va_addr is 40 bit GPUVM address + if(kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) + return false; + + // Allocate object + pthread_mutex_lock(&aperture->fmm_mutex); + if (aperture_allocate_object(aperture, address, args.handle, size)) + goto err_object_allocation_failed; + pthread_mutex_unlock(&aperture->fmm_mutex); + + *gpuvm_address = args.va_addr; + + return true; + +err_object_allocation_failed: + pthread_mutex_unlock(&aperture->fmm_mutex); + unmap_args.handle = args.handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); + + *gpuvm_address = 0; + return false; +} + +bool fmm_map_to_gpu(void* address, uint64_t size, uint64_t* gpuvm_address) { + + int32_t i; + uint64_t pi; + + // Find an aperture the requested address belongs to + for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ + if(gpu_mem[i].gpu_id != NON_VALID_GPU_ID){ + if ((address>= gpu_mem[i].gpuvm_aperture.base) && (address<= gpu_mem[i].gpuvm_aperture.limit)) { + // map it + return _fmm_map_to_gpu(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); + } + } + } + + // If address isn't Local memory address, we assume that this is + // system memory address accessed through IOMMU. + // Thus we "prefetch" it + for(pi = 0; pi < size / PAGE_SIZE; pi++) { + ((char*)address)[pi*PAGE_SIZE] = 0; + } + return true; +} + +static bool _fmm_unmap_from_gpu(manageble_aperture_t* aperture, void* address) { + + vm_object_t* object; + struct kfd_ioctl_unmap_memory_from_gpu_args args; + + pthread_mutex_lock(&aperture->fmm_mutex); + + // Find the object to retrieve the handle + object = vm_find_object_by_address(aperture, address, 0); + if (!object) + goto err; + + args.handle = object->handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + + vm_remove_object(aperture, object); + + pthread_mutex_unlock(&aperture->fmm_mutex); + return true; + +err: + pthread_mutex_unlock(&aperture->fmm_mutex); + return false; + +} + +bool fmm_unmap_from_gpu(void* address) { + + int32_t i; + + // Find the aperture the requested address belongs to + for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ + if(gpu_mem[i].gpu_id != NON_VALID_GPU_ID){ + if ((address>= gpu_mem[i].gpuvm_aperture.base) && (address<= gpu_mem[i].gpuvm_aperture.limit)) { + // unmap it + return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, address); + } + } + } + + return true; +} diff --git a/src/fmm.h b/src/fmm.h index 5924247645..ca0a692ead 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -49,9 +49,15 @@ HSAKMT_STATUS fmm_init_process_apertures(void); */ void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes); +void* fmm_open_graphic_handle(uint32_t gpu_id, + int32_t graphic_device_handle, + uint32_t graphic_handle, + uint64_t MemorySizeInBytes); void fmm_print(uint32_t node); bool fmm_is_inside_some_aperture(void* address); -int fmm_release(void* address, HSAuint64 MemorySizeInBytes); +void fmm_release(void* address, HSAuint64 MemorySizeInBytes); +bool fmm_map_to_gpu(void* address, uint64_t size, uint64_t* gpuvm_address); +bool fmm_unmap_from_gpu(void* address); /* Topology interface*/ HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); diff --git a/src/memory.c b/src/memory.c index d65730a336..9c8e63f55b 100644 --- a/src/memory.c +++ b/src/memory.c @@ -115,6 +115,13 @@ hsaKmtAllocMemory( else return HSAKMT_STATUS_NO_MEMORY; } + else if(!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged){ + *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); + if (*MemoryAddress) + return HSAKMT_STATUS_SUCCESS; + else + return HSAKMT_STATUS_NO_MEMORY; + } else return HSAKMT_STATUS_INVALID_PARAMETER; @@ -127,17 +134,10 @@ hsaKmtFreeMemory( HSAuint64 SizeInBytes //IN ) { - HSAKMT_STATUS hsa_status = HSAKMT_STATUS_SUCCESS; CHECK_KFD_OPEN(); - if (fmm_is_inside_some_aperture(MemoryAddress)){ - if (fmm_release( MemoryAddress, SizeInBytes)) - hsa_status = HSAKMT_STATUS_INVALID_PARAMETER; - } - else - free(MemoryAddress); - - return hsa_status; + fmm_release( MemoryAddress, SizeInBytes); + return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS @@ -173,13 +173,16 @@ hsaKmtMapMemoryToGPU( { CHECK_KFD_OPEN(); - // We don't support GPUVM in the stub, there should never be a request for a GPUVA. if (AlternateVAGPU) - { *AlternateVAGPU = 0; + + if (fmm_map_to_gpu(MemoryAddress, MemorySizeInBytes, AlternateVAGPU)){ + return HSAKMT_STATUS_SUCCESS; + } + else { + return HSAKMT_STATUS_NOT_IMPLEMENTED; } - return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS @@ -189,6 +192,56 @@ hsaKmtUnmapMemoryToGPU( ) { CHECK_KFD_OPEN(); + if (fmm_unmap_from_gpu(MemoryAddress)) + return HSAKMT_STATUS_SUCCESS; + else + return HSAKMT_STATUS_NOT_IMPLEMENTED; - return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtMapGraphicHandle( + HSAuint32 NodeId, //IN + HSAuint64 GraphicDeviceHandle, //IN + HSAuint64 GraphicResourceHandle, //IN + HSAuint64 GraphicResourceOffset, //IN + HSAuint64 GraphicResourceSize, //IN + HSAuint64* FlatMemoryAddress //OUT + ) +{ + CHECK_KFD_OPEN(); + HSAKMT_STATUS result; + uint32_t gpu_id; + void *graphic_handle; + + if (GraphicResourceOffset != 0) + return HSAKMT_STATUS_NOT_IMPLEMENTED; + + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + graphic_handle = fmm_open_graphic_handle(gpu_id, + GraphicDeviceHandle, + GraphicResourceHandle, + GraphicResourceSize); + + *FlatMemoryAddress = PORT_VPTR_TO_UINT64(graphic_handle); + + if (*FlatMemoryAddress) + return HSAKMT_STATUS_SUCCESS; + else + return HSAKMT_STATUS_NO_MEMORY; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtUnmapGraphicHandle( + HSAuint32 NodeId, //IN + HSAuint64 FlatMemoryAddress, //IN + HSAuint64 SizeInBytes //IN + ) +{ + return hsaKmtUnmapMemoryToGPU(PORT_UINT64_TO_VPTR(FlatMemoryAddress)); } From bd7e10b0ec67ebd755ab38e1a353dbe1ede9f121 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Sun, 7 Dec 2014 14:36:21 +0200 Subject: [PATCH 0030/1247] Adding support in allocating executable memory Signed-off-by: Ben Goz --- include/hsakmttypes.h | 3 ++- src/memory.c | 38 ++++++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 178a86d178..3eb1dff41f 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -420,7 +420,8 @@ typedef struct _HsaMemFlags // when setting this entry to 1. Scratch allocation may fail due to limited // resources. Application code is required to work without any allocation. // Allocation fails on any node without GPU function. - unsigned int Reserved : 22; + unsigned int ExecAccess : 1; + unsigned int Reserved : 21; } ui32; HSAuint32 Value; }; diff --git a/src/memory.c b/src/memory.c index 9c8e63f55b..d37d1b16ed 100644 --- a/src/memory.c +++ b/src/memory.c @@ -26,8 +26,12 @@ #include "libhsakmt.h" #include "linux/kfd_ioctl.h" #include +#include #include #include +#include +#include +#include #include "fmm.h" HSAKMT_STATUS @@ -97,6 +101,7 @@ hsaKmtAllocMemory( CHECK_KFD_OPEN(); HSAKMT_STATUS result; uint32_t gpu_id; + int err; result = validate_nodeid(PreferredNode, &gpu_id); if (result != HSAKMT_STATUS_SUCCESS) @@ -108,23 +113,28 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_INVALID_PARAMETER; } - if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged){ - int err = posix_memalign(MemoryAddress, page_size, SizeInBytes); - if (err == 0) - return HSAKMT_STATUS_SUCCESS; - else + if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged) { + err = posix_memalign(MemoryAddress, page_size, SizeInBytes); + if (err != 0) return HSAKMT_STATUS_NO_MEMORY; + if (MemFlags.ui32.ExecAccess) { + err = mprotect(*MemoryAddress, SizeInBytes, PROT_READ | PROT_WRITE | PROT_EXEC); + if (err != 0) { + free(*MemoryAddress); + return err; + } + } + return HSAKMT_STATUS_SUCCESS; } - else if(!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged){ - *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); - if (*MemoryAddress) - return HSAKMT_STATUS_SUCCESS; - else - return HSAKMT_STATUS_NO_MEMORY; - } - else - return HSAKMT_STATUS_INVALID_PARAMETER; + if(!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged){ + *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); + if (*MemoryAddress == NULL) + return HSAKMT_STATUS_NO_MEMORY; + return HSAKMT_STATUS_SUCCESS; + } + + return HSAKMT_STATUS_INVALID_PARAMETER; } HSAKMT_STATUS From c5585b0f267761bc29dcc8876a3f29baefe242c6 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Sun, 7 Dec 2014 14:50:28 +0200 Subject: [PATCH 0031/1247] Supporting QCM for gfx801 Signed-off-by: Ben Goz --- src/queues.c | 148 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 141 insertions(+), 7 deletions(-) diff --git a/src/queues.c b/src/queues.c index 650a1f27be..b25a7d503c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -30,15 +30,73 @@ #include #include #include +#include +#include +#include /* 1024 doorbells, 4 bytes each doorbell */ #define DOORBELLS_PAGE_SIZE 1024 * 4 +struct device_info +{ + uint32_t ctx_save_restore_size; + uint32_t eop_buffer_size; +}; + +struct device_info kaveri_device_info = { + .ctx_save_restore_size = 0, + .eop_buffer_size = 0, +}; + +struct device_info carrizo_device_info = { + .ctx_save_restore_size = 2756608, + .eop_buffer_size = 4096, +}; + +struct device_id +{ + uint16_t dev_id; + struct device_info *dev_info; +}; + +struct device_id supported_devices[] = { + { 0x1304, &kaveri_device_info }, /* Kaveri */ + { 0x1305, &kaveri_device_info }, /* Kaveri */ + { 0x1306, &kaveri_device_info }, /* Kaveri */ + { 0x1307, &kaveri_device_info }, /* Kaveri */ + { 0x1309, &kaveri_device_info }, /* Kaveri */ + { 0x130A, &kaveri_device_info }, /* Kaveri */ + { 0x130B, &kaveri_device_info }, /* Kaveri */ + { 0x130C, &kaveri_device_info }, /* Kaveri */ + { 0x130D, &kaveri_device_info }, /* Kaveri */ + { 0x130E, &kaveri_device_info }, /* Kaveri */ + { 0x130F, &kaveri_device_info }, /* Kaveri */ + { 0x1310, &kaveri_device_info }, /* Kaveri */ + { 0x1311, &kaveri_device_info }, /* Kaveri */ + { 0x1312, &kaveri_device_info }, /* Kaveri */ + { 0x1313, &kaveri_device_info }, /* Kaveri */ + { 0x1315, &kaveri_device_info }, /* Kaveri */ + { 0x1316, &kaveri_device_info }, /* Kaveri */ + { 0x1317, &kaveri_device_info }, /* Kaveri */ + { 0x1318, &kaveri_device_info }, /* Kaveri */ + { 0x131B, &kaveri_device_info }, /* Kaveri */ + { 0x131C, &kaveri_device_info }, /* Kaveri */ + { 0x131D, &kaveri_device_info }, /* Kaveri */ + { 0x9870, &carrizo_device_info }, /* Carrizo */ + { 0x9874, &carrizo_device_info }, /* Carrizo */ + { 0x9875, &carrizo_device_info }, /* Carrizo */ + { 0x9876, &carrizo_device_info }, /* Carrizo */ + { 0x9877, &carrizo_device_info }, /* Carrizo */ + { 0, NULL } +}; + struct queue { uint32_t queue_id; uint32_t wptr; uint32_t rptr; + void *eop_buffer; + void *ctx_save_restore; }; struct process_doorbells @@ -50,6 +108,74 @@ struct process_doorbells struct process_doorbells doorbells[] = {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = {.need_mmap = true, .doorbells = NULL, .doorbells_mutex = PTHREAD_MUTEX_INITIALIZER}}; +static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) +{ + int i = 0; + while (supported_devices[i].dev_id != 0) { + if (supported_devices[i].dev_id == dev_id) { + return supported_devices[i].dev_info; + } + i++; + } + + return NULL; +} + +static void free_queue(struct queue *q) +{ + if (q->eop_buffer) + free(q->eop_buffer); + if (q->ctx_save_restore) + free(q->ctx_save_restore); + free(q); +} + +static void* allocate_exec_aligned_memory(uint32_t size, uint32_t align) +{ + void *ptr; + int retval; + + retval = posix_memalign(&ptr, align, size); + if (retval != 0) + return NULL; + + retval = mprotect(ptr, size, PROT_READ | PROT_WRITE | PROT_EXEC); + if (retval != 0) { + free(ptr); + return NULL; + } + + memset(ptr, 0, size); + return ptr; +} + +static int handle_concrete_asic(struct device_info *dev_info, struct queue *q, + struct kfd_ioctl_create_queue_args *args) +{ + if (dev_info) { + if (dev_info->eop_buffer_size > 0) { + q->eop_buffer = + allocate_exec_aligned_memory(dev_info->eop_buffer_size, PAGE_SIZE); + if (q->eop_buffer == NULL) { + return HSAKMT_STATUS_NO_MEMORY; + } + args->eop_buffer_address = (uintptr_t)q->eop_buffer; + args->eop_buffer_size = dev_info->eop_buffer_size; + } + if (dev_info->ctx_save_restore_size > 0) { + args->ctx_save_restore_size = dev_info->ctx_save_restore_size; + q->ctx_save_restore = + allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, PAGE_SIZE); + if (q->ctx_save_restore == NULL) {; + return HSAKMT_STATUS_NO_MEMORY; + } + args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; + } + } + + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue( @@ -65,6 +191,8 @@ hsaKmtCreateQueue( { HSAKMT_STATUS result; uint32_t gpu_id; + uint16_t dev_id; + struct device_info *dev_info; int err; void* ptr; CHECK_KFD_OPEN(); @@ -73,25 +201,30 @@ hsaKmtCreateQueue( if (result != HSAKMT_STATUS_SUCCESS) return result; - struct queue *q = (struct queue *)malloc(sizeof(*q)); + struct queue *q = malloc(sizeof(*q)); if (q == NULL) - { return HSAKMT_STATUS_NO_MEMORY; - } - memset(q, 0, sizeof(*q)); struct kfd_ioctl_create_queue_args args; memset(&args, 0, sizeof(args)); + dev_id = get_device_id_by_node(NodeId); + dev_info = get_device_info_by_dev_id(dev_id); args.gpu_id = gpu_id; + err = handle_concrete_asic(dev_info, q, &args); + if (err != HSAKMT_STATUS_SUCCESS) { + free_queue(q); + return err; + } + switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; - default: free(q); return HSAKMT_STATUS_INVALID_PARAMETER; + default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } if (Type != HSA_QUEUE_COMPUTE_AQL) @@ -111,7 +244,7 @@ hsaKmtCreateQueue( if (err == -1) { - free(q); + free_queue(q); return HSAKMT_STATUS_ERROR; } @@ -126,6 +259,7 @@ hsaKmtCreateQueue( if (ptr == MAP_FAILED) { pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); hsaKmtDestroyQueue(q->queue_id); + free_queue(q); return HSAKMT_STATUS_ERROR; } @@ -201,7 +335,7 @@ hsaKmtDestroyQueue( } else { - free(q); + free_queue(q); return HSAKMT_STATUS_SUCCESS; } } From 402ff3f881cadd8e19baad1ded5f40f10044300c Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Sun, 7 Dec 2014 14:51:53 +0200 Subject: [PATCH 0032/1247] Workaround for non cached memory issue on gfx801 A0 Signed-off-by: Ben Goz Adjust to new ioctl format Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 8 ++++++- src/queues.c | 46 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 9906419ec0..da4668300d 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -286,7 +286,13 @@ struct kfd_ioctl_open_graphic_handle_args { #define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ AMDKFD_IOWR(0x13, struct kfd_ioctl_open_graphic_handle_args) +#define AMDKFD_IOC_NC_PAGE_ALLOC \ + AMDKFD_IOWR(0x14, unsigned long) + +#define AMDKFD_IOC_NC_PAGE_FREE \ + AMDKFD_IOWR(0x15, unsigned long) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x14 +#define AMDKFD_COMMAND_END 0x16 #endif diff --git a/src/queues.c b/src/queues.c index b25a7d503c..4e3ab889d2 100644 --- a/src/queues.c +++ b/src/queues.c @@ -94,7 +94,7 @@ struct queue { uint32_t queue_id; uint32_t wptr; - uint32_t rptr; + uint32_t *rptr; void *eop_buffer; void *ctx_save_restore; }; @@ -108,6 +108,29 @@ struct process_doorbells struct process_doorbells doorbells[] = {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = {.need_mmap = true, .doorbells = NULL, .doorbells_mutex = PTHREAD_MUTEX_INITIALIZER}}; +static uint32_t *nc_rptr_page = NULL; + +static void* alloc_nc_page(void) +{ + void *ptr; + if (nc_rptr_page) + return nc_rptr_page; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_NC_PAGE_ALLOC, &ptr) == 0) { + nc_rptr_page = ptr; + return ptr; + } + return NULL; + +} + +static void free_nc_page(void *page_addr) +{ + if (page_addr) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_NC_PAGE_FREE, page_addr) == 0) + page_addr = NULL; +} + static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) { int i = 0; @@ -227,10 +250,18 @@ hsaKmtCreateQueue( default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } - if (Type != HSA_QUEUE_COMPUTE_AQL) - { + if (Type != HSA_QUEUE_COMPUTE_AQL) { QueueResource->QueueRptrValue = (uintptr_t)&q->rptr; QueueResource->QueueWptrValue = (uintptr_t)&q->wptr; + + /* CZ A0 workaround */ + if (dev_info->eop_buffer_size > 0) { + if (alloc_nc_page() == NULL) { + free_queue(q); + return HSAKMT_STATUS_NO_MEMORY; + } + QueueResource->Queue_read_ptr = (HSAuint32 *)nc_rptr_page; + } } args.read_pointer_address = QueueResource->QueueRptrValue; @@ -249,6 +280,13 @@ hsaKmtCreateQueue( } q->queue_id = args.queue_id; + if (Type != HSA_QUEUE_COMPUTE_AQL && + dev_info->eop_buffer_size > 0) + { + q->rptr = &nc_rptr_page[args.queue_id]; + QueueResource->Queue_read_ptr = (HSAuint32 *)&nc_rptr_page[args.queue_id]; + *q->rptr = 0; + } pthread_mutex_lock(&doorbells[NodeId].doorbells_mutex); @@ -335,6 +373,8 @@ hsaKmtDestroyQueue( } else { + if (nc_rptr_page != NULL) + free_nc_page((void *)nc_rptr_page); free_queue(q); return HSAKMT_STATUS_SUCCESS; } From 36ccd7f867f78679ed4fb7c11cec2e14cc676839 Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Thu, 27 Nov 2014 15:54:21 +0200 Subject: [PATCH 0033/1247] Adding memory event type and definition of memory exception data Signed-off-by: Alexey Skidanov Reviewed-by: Oded Gabbay --- include/hsakmttypes.h | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 3eb1dff41f..88c145455a 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -620,6 +620,7 @@ typedef enum _HSA_EVENTTYPE HSA_EVENTTYPE_DEBUG_EVENT = 5, //GPU signal for debugging HSA_EVENTTYPE_PROFILE_EVENT = 6, //GPU signal for profiling HSA_EVENTTYPE_QUEUE_EVENT = 7, //GPU signal queue idle state (EOP pm4) + HSA_EVENTTYPE_MEMORY = 8, //GPU signal for signaling memory access faults and memory subsystem issues //... HSA_EVENTTYPE_MAXID, HSA_EVENTTYPE_TYPE_SIZE = 0xFFFFFFFF @@ -675,20 +676,54 @@ typedef struct _HsaDeviceStateChange HSA_EVENTTYPE_DEVICESTATECHANGE_FLAGS Flags; // event flags } HsaDeviceStateChange; +// +// Sub-definitions for various event types: Memory exception +// + +typedef enum _HSA_EVENTID_MEMORYFLAGS +{ + HSA_EVENTID_MEMORY_RECOVERABLE = 0, //access fault, recoverable after page adjustment + HSA_EVENTID_MEMORY_FATAL_PROCESS = 1, //memory access requires process context destruction, unrecoverable + HSA_EVENTID_MEMORY_FATAL_VM = 2, //memory access requires all GPU VA context destruction, unrecoverable +} HSA_EVENTID_MEMORYFLAGS; + +typedef struct _HsaAccessAttributeFailure +{ + unsigned int NotPresent : 1; // Page not present or supervisor privilege + unsigned int ReadOnly : 1; // Write access to a read-only page + unsigned int NoExecute : 1; // Execute access to a page marked NX + unsigned int GpuAccess : 1; // Host access only + unsigned int ECC : 1; // ECC failure (if supported by HW) + unsigned int Reserved : 27; // must be 0 +} HsaAccessAttributeFailure; + +// data associated with HSA_EVENTID_MEMORY +typedef struct _HsaMemoryAccessFault +{ + HSAuint32 NodeId; // H-NUMA node that contains the device where the memory access occurred + HSAuint64 VirtualAddress; // virtual address this occurred on + HsaAccessAttributeFailure Failure; // failure attribute + HSA_EVENTID_MEMORYFLAGS Flags; // event flags +} HsaMemoryAccessFault; typedef struct _HsaEventData { HSA_EVENTTYPE EventType; //event type + union { // return data associated with HSA_EVENTTYPE_SIGNAL and other events - HsaSyncVar SyncVar; + HsaSyncVar SyncVar; // data associated with HSA_EVENTTYPE_NODE_CHANGE - HsaNodeChange NodeChangeState; + HsaNodeChange NodeChangeState; // data associated with HSA_EVENTTYPE_DEVICE_STATE_CHANGE - HsaDeviceStateChange DeviceState; + HsaDeviceStateChange DeviceState; + + // data associated with HSA_EVENTTYPE_MEMORY + HsaMemoryAccessFault MemoryAccessFault; + } EventData; // the following data entries are internal to the KFD & thunk itself. From d7196d41222a2b314d8634834f76fda4a0273fd6 Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Thu, 27 Nov 2014 16:21:53 +0200 Subject: [PATCH 0034/1247] IOCTL: Adding memory exception data Signed-off-by: Alexey Skidanov Reviewed-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index da4668300d..693b3f64f9 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -194,8 +194,35 @@ struct kfd_ioctl_reset_event_args { uint32_t pad; }; + +struct kfd_memory_exception_failure { + bool NotPresent; /* Page not present or supervisor privilege */ + bool ReadOnly; /* Write access to a read-only page */ + bool NoExecute; /* Execute access to a page marked NX */ + bool pad; +}; + +/* memory exception data*/ +struct kfd_hsa_memory_exception_data { + uint64_t va; + uint32_t gpu_id; + struct kfd_memory_exception_failure failure; +}; + +/* Event data*/ +struct kfd_event_data { + union { + struct kfd_hsa_memory_exception_data memory_exception_data; + }; /* From KFD */ + uint64_t kfd_event_data_ext; /* pointer to an extension structure + for future exception types */ + uint32_t event_id; /* to KFD */ + uint32_t pad; +}; + struct kfd_ioctl_wait_events_args { - uint64_t events_ptr; /* to KFD */ + uint64_t events_ptr; /* pointed to struct + kfd_event_data array, to KFD */ uint32_t num_events; /* to KFD */ uint32_t wait_for_all; /* to KFD */ uint32_t timeout; /* to KFD */ From c3b51fd8270f0536cc8f0899aea95ebaa52003d3 Mon Sep 17 00:00:00 2001 From: Alexey Skidanov Date: Thu, 27 Nov 2014 16:24:06 +0200 Subject: [PATCH 0035/1247] Adding memory exception notification through hsaKmtWaitOnMultipleEvents Signed-off-by: Alexey Skidanov Reviewed-by: Oded Gabbay --- src/events.c | 24 +++++++++++++++++++----- src/libhsakmt.h | 1 + src/topology.c | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/events.c b/src/events.c index c62f4a4a20..5d674bcd5f 100644 --- a/src/events.c +++ b/src/events.c @@ -71,6 +71,7 @@ hsaKmtCreateEvent( return HSAKMT_STATUS_ERROR; e->EventId = args.event_id; + e->EventData.EventType = EventDesc->EventType; e->EventData.HWData1 = args.event_id; e->EventData.HWData2 = args.event_trigger_address; e->EventData.HWData3 = args.event_trigger_data; @@ -187,9 +188,10 @@ hsaKmtWaitOnMultipleEvents( { CHECK_KFD_OPEN(); - uint32_t *event_ids = malloc(NumEvents * sizeof(uint32_t)); + struct kfd_event_data *event_data = malloc(NumEvents * sizeof(struct kfd_event_data)); for (HSAuint32 i = 0; i < NumEvents; i++) { - event_ids[i] = Events[i]->EventId; + event_data[i].event_id = Events[i]->EventId; + event_data[i].kfd_event_data_ext = (uint64_t)(uintptr_t)NULL; } struct kfd_ioctl_wait_events_args args; @@ -198,7 +200,7 @@ hsaKmtWaitOnMultipleEvents( args.wait_for_all = WaitOnAll; args.timeout = Milliseconds; args.num_events = NumEvents; - args.events_ptr = (uint64_t)(uintptr_t)event_ids; + args.events_ptr = (uint64_t)(uintptr_t)event_data; HSAKMT_STATUS result; @@ -210,9 +212,21 @@ hsaKmtWaitOnMultipleEvents( } else { result = HSAKMT_STATUS_SUCCESS; + for (HSAuint32 i = 0; i < NumEvents; i++) { + if (Events[i]->EventData.EventType == HSA_EVENTTYPE_MEMORY) { + Events[i]->EventData.EventData.MemoryAccessFault.VirtualAddress = event_data[i].memory_exception_data.va; + result = gpuid_to_nodeid(event_data[i].memory_exception_data.gpu_id, &Events[i]->EventData.EventData.MemoryAccessFault.NodeId); + if (result != HSAKMT_STATUS_SUCCESS) + goto out; + Events[i]->EventData.EventData.MemoryAccessFault.Failure.NotPresent = event_data[i].memory_exception_data.failure.NotPresent; + Events[i]->EventData.EventData.MemoryAccessFault.Failure.ReadOnly = event_data[i].memory_exception_data.failure.ReadOnly; + Events[i]->EventData.EventData.MemoryAccessFault.Failure.NoExecute = event_data[i].memory_exception_data.failure.NoExecute; + Events[i]->EventData.EventData.MemoryAccessFault.Flags = HSA_EVENTID_MEMORY_FATAL_PROCESS; + } + } } - - free(event_ids); +out: + free(event_data); return result; } diff --git a/src/libhsakmt.h b/src/libhsakmt.h index fa8dc0e24a..0d73c8fa8b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -62,6 +62,7 @@ extern pthread_mutex_t hsakmt_mutex; #define MAX_NODES 8 HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); +HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint16_t get_device_id_by_node(HSAuint32 node_id); extern int kmtIoctl(int fd, unsigned long request, void *arg); diff --git a/src/topology.c b/src/topology.c index e50b7458d8..3275aba9c2 100644 --- a/src/topology.c +++ b/src/topology.c @@ -697,6 +697,20 @@ validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) return HSAKMT_STATUS_SUCCESS; } +HSAKMT_STATUS +gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id){ + uint64_t node_idx; + for(node_idx = 0; node_idx < system->NumNodes; node_idx++){ + if (node[node_idx].gpu_id == gpu_id){ + *node_id = node_idx; + return HSAKMT_STATUS_SUCCESS; + } + } + + return HSAKMT_STATUS_INVALID_NODE_UNIT; + +} + HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties( From e7a5cda5a51cf10f55cc30b1032a9224129bef1c Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Sun, 14 Dec 2014 13:49:41 +0200 Subject: [PATCH 0036/1247] fix a bug in rptr page release Adjust to new ioctl format Signed-off-by: Ben Goz Reviewed-by: Oded Gabbay --- src/queues.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/queues.c b/src/queues.c index 4e3ab889d2..aadd0dd307 100644 --- a/src/queues.c +++ b/src/queues.c @@ -124,11 +124,11 @@ static void* alloc_nc_page(void) } -static void free_nc_page(void *page_addr) +static void free_nc_page(void **page_addr) { - if (page_addr) - if (kmtIoctl(kfd_fd, AMDKFD_IOC_NC_PAGE_FREE, page_addr) == 0) - page_addr = NULL; + if (*page_addr) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_NC_PAGE_FREE, *page_addr) == 0) + *page_addr = NULL; } static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) @@ -374,7 +374,8 @@ hsaKmtDestroyQueue( else { if (nc_rptr_page != NULL) - free_nc_page((void *)nc_rptr_page); + free_nc_page((void **)&nc_rptr_page); + free_queue(q); return HSAKMT_STATUS_SUCCESS; } From 8cc0b1629353070a5b166b4acda519cbe021f087 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 19 Jan 2015 17:18:47 +0200 Subject: [PATCH 0037/1247] match kfd_ioctl.h to kernel repo Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 693b3f64f9..a8b750f30a 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -319,7 +319,7 @@ struct kfd_ioctl_open_graphic_handle_args { #define AMDKFD_IOC_NC_PAGE_FREE \ AMDKFD_IOWR(0x15, unsigned long) -#define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x16 +#define AMDKFD_COMMAND_START 0x01 +#define AMDKFD_COMMAND_END 0x16 #endif From 930ec8f04dd6dc947314b57aaa73ee130b242738 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Thu, 29 Jan 2015 16:45:36 +0200 Subject: [PATCH 0038/1247] Check if event is not null before handling events Signed-off-by: Ben Goz Reviewed-by: Oded Gabbay --- src/events.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/events.c b/src/events.c index 5d674bcd5f..ab2df5b333 100644 --- a/src/events.c +++ b/src/events.c @@ -67,8 +67,11 @@ hsaKmtCreateEvent( args.event_type = EventDesc->EventType; args.auto_reset = !ManualReset; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) == -1) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) != 0) { + free(e); + *Event = NULL; return HSAKMT_STATUS_ERROR; + } e->EventId = args.event_id; e->EventData.EventType = EventDesc->EventType; @@ -97,13 +100,17 @@ hsaKmtDestroyEvent( { CHECK_KFD_OPEN(); + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + struct kfd_ioctl_destroy_event_args args; memset(&args, 0, sizeof(args)); args.event_id = Event->EventId; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_EVENT, &args) == -1) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_EVENT, &args) != 0) { return HSAKMT_STATUS_ERROR; + } free(Event); @@ -118,6 +125,9 @@ hsaKmtSetEvent( { CHECK_KFD_OPEN(); + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + /* Although the spec is doesn't say, don't allow system-defined events to be signaled. */ if (IsSystemEventType(Event->EventData.EventType)) return HSAKMT_STATUS_ERROR; @@ -141,6 +151,9 @@ hsaKmtResetEvent( { CHECK_KFD_OPEN(); + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + /* Although the spec is doesn't say, don't allow system-defined events to be signaled. */ if (IsSystemEventType(Event->EventData.EventType)) return HSAKMT_STATUS_ERROR; @@ -164,6 +177,9 @@ hsaKmtQueryEventState( { CHECK_KFD_OPEN(); + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + return HSAKMT_STATUS_SUCCESS; } @@ -174,6 +190,9 @@ hsaKmtWaitOnEvent( HSAuint32 Milliseconds //IN ) { + if (!Event) + return HSAKMT_STATUS_INVALID_HANDLE; + return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); } @@ -188,6 +207,9 @@ hsaKmtWaitOnMultipleEvents( { CHECK_KFD_OPEN(); + if (!Events) + return HSAKMT_STATUS_INVALID_HANDLE; + struct kfd_event_data *event_data = malloc(NumEvents * sizeof(struct kfd_event_data)); for (HSAuint32 i = 0; i < NumEvents; i++) { event_data[i].event_id = Events[i]->EventId; From f24aa620edb01dc1221b505b8a5f05ed9795614a Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Thu, 29 Jan 2015 16:55:14 +0200 Subject: [PATCH 0039/1247] fix return value in Map/Unmap functions Signed-off-by: Oded Gabbay Reviewed-by: Yair Shachar --- src/memory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/memory.c b/src/memory.c index d37d1b16ed..0d42180d29 100644 --- a/src/memory.c +++ b/src/memory.c @@ -190,7 +190,7 @@ hsaKmtMapMemoryToGPU( return HSAKMT_STATUS_SUCCESS; } else { - return HSAKMT_STATUS_NOT_IMPLEMENTED; + return HSAKMT_STATUS_ERROR; } } @@ -205,7 +205,7 @@ hsaKmtUnmapMemoryToGPU( if (fmm_unmap_from_gpu(MemoryAddress)) return HSAKMT_STATUS_SUCCESS; else - return HSAKMT_STATUS_NOT_IMPLEMENTED; + return HSAKMT_STATUS_ERROR; } From 25441796bdb0938b7d61f7a3d596790a32c5ac4d Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Sun, 1 Feb 2015 14:33:33 +0200 Subject: [PATCH 0040/1247] Supporting new thunk spec - adding relevant memflags Signed-off-by: Ben Goz Reviewed-by: Oded Gabbay --- include/hsakmttypes.h | 22 ++++++++++++++++++++-- src/memory.c | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 88c145455a..d3cae8d689 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -420,8 +420,26 @@ typedef struct _HsaMemFlags // when setting this entry to 1. Scratch allocation may fail due to limited // resources. Application code is required to work without any allocation. // Allocation fails on any node without GPU function. - unsigned int ExecAccess : 1; - unsigned int Reserved : 21; + unsigned int AtomicAccessFull: 1; // default = 0: If set, the memory will be allocated and mapped to allow + // atomic ops processing. On AMD APU, this will use the ATC path on system + // memory, irrespective of the NonPaged flag setting (= if NonPaged is set, + // the memory is pagelocked but mapped through IOMMUv2 instead of GPUVM). + // All atomic ops must be supported on this memory. + unsigned int AtomicAccessPartial: 1; // default = 0: See above for AtomicAccessFull description, however + // focused on AMD discrete GPU that support PCIe atomics; the memory + // allocation is mapped to allow for PCIe atomics to operate on system + // memory, irrespective of NonPaged set or the presence of an ATC path + // in the system. The atomic operations supported are limited to SWAP, + // CompareAndSwap (CAS) and FetchAdd (this PCIe op allows both atomic + // increment and decrement via 2-complement arithmetic), which are the + // only atomic ops directly supported in PCI Express. + // On AMD APU, setting this flag will allocate the same type of memory + // as AtomicAccessFull, but it will be considered compatible with + // discrete GPU atomic operations access. + unsigned int ExecuteAccess: 1; // default = 0: Identifies if memory is primarily used for data or accessed + // for executable code (e.g. queue memory) by the host CPU or the device. + // Influences the page attribute setting within the allocation + unsigned int Reserved : 19; } ui32; HSAuint32 Value; }; diff --git a/src/memory.c b/src/memory.c index 0d42180d29..a2c1eb49b6 100644 --- a/src/memory.c +++ b/src/memory.c @@ -117,7 +117,7 @@ hsaKmtAllocMemory( err = posix_memalign(MemoryAddress, page_size, SizeInBytes); if (err != 0) return HSAKMT_STATUS_NO_MEMORY; - if (MemFlags.ui32.ExecAccess) { + if (MemFlags.ui32.ExecuteAccess) { err = mprotect(*MemoryAddress, SizeInBytes, PROT_READ | PROT_WRITE | PROT_EXEC); if (err != 0) { free(*MemoryAddress); From f0a7fcccc91d0f830c05d46bf5ea7b3c32fd0385 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 3 Feb 2015 11:13:13 +0200 Subject: [PATCH 0041/1247] Match as much as possible hsakmt.h and hsakmttypes.h to p4 Signed-off-by: Oded Gabbay --- include/hsakmt.h | 16 ++++++++++++++-- include/hsakmttypes.h | 5 +++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index a57946e9da..fd2788803d 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -382,7 +382,6 @@ hsaKmtDeregisterMemory( /** Ensures that the memory is resident and can be accessed by GPU - Not implemented yet */ HSAKMT_STATUS @@ -395,7 +394,6 @@ hsaKmtMapMemoryToGPU( /** Releases the residency of the memory - Not implemented yet */ HSAKMT_STATUS @@ -586,6 +584,20 @@ hsaKmtPmcStopTrace( HSATraceId TraceId //IN ); +/** + Sets trap handler and trap buffer to be used for all queues associated with the specified NodeId within this process context +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetTrapHandler( + HSAuint32 NodeId, //IN + void* TrapHandlerBaseAddress, //IN + HSAuint64 TrapHandlerSizeInBytes, //IN + void* TrapBufferBaseAddress, //IN + HSAuint64 TrapBufferSizeInBytes //IN + ); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index d3cae8d689..ec10255b50 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -420,7 +420,7 @@ typedef struct _HsaMemFlags // when setting this entry to 1. Scratch allocation may fail due to limited // resources. Application code is required to work without any allocation. // Allocation fails on any node without GPU function. - unsigned int AtomicAccessFull: 1; // default = 0: If set, the memory will be allocated and mapped to allow + unsigned int AtomicAccessFull: 1; // default = 0: If set, the memory will be allocated and mapped to allow // atomic ops processing. On AMD APU, this will use the ATC path on system // memory, irrespective of the NonPaged flag setting (= if NonPaged is set, // the memory is pagelocked but mapped through IOMMUv2 instead of GPUVM). @@ -440,6 +440,7 @@ typedef struct _HsaMemFlags // for executable code (e.g. queue memory) by the host CPU or the device. // Influences the page attribute setting within the allocation unsigned int Reserved : 19; + } ui32; HSAuint32 Value; }; @@ -720,7 +721,7 @@ typedef struct _HsaMemoryAccessFault { HSAuint32 NodeId; // H-NUMA node that contains the device where the memory access occurred HSAuint64 VirtualAddress; // virtual address this occurred on - HsaAccessAttributeFailure Failure; // failure attribute + HsaAccessAttributeFailure Failure; // failure attribute HSA_EVENTID_MEMORYFLAGS Flags; // event flags } HsaMemoryAccessFault; From 89188a6d6c8ea92f44113438513570b2a1b9318e Mon Sep 17 00:00:00 2001 From: Yair Shachar Date: Sun, 8 Feb 2015 21:44:47 +0200 Subject: [PATCH 0042/1247] Modify HW dbg thunk,to adapt to new KFD-thunk interface Signed-off-by: Yair Shachar --- include/linux/kfd_ioctl.h | 4 ++-- src/debug.c | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index a8b750f30a..08558f6678 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -145,13 +145,13 @@ struct kfd_ioctl_dbg_unregister_args { struct kfd_ioctl_dbg_address_watch_args { uint32_t gpu_id; /* to KFD */ uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ - unsigned char content[0]; + void *content_ptr; /* a pointer to the actual content */ }; struct kfd_ioctl_dbg_wave_control_args { uint32_t gpu_id; /* to KFD */ uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ - unsigned char content[0]; + void *content_ptr; /* a pointer to the actual content */ }; /* Matching HSA_EVENTTYPE */ diff --git a/src/debug.c b/src/debug.c index 155222570e..eff9023d90 100644 --- a/src/debug.c +++ b/src/debug.c @@ -129,6 +129,9 @@ hsaKmtDbgWavefrontControl( unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); + /* save variable content pointer for kfd */ + args->content_ptr = (void *)run_ptr; + /* insert items, and increment pointer accordingly */ *((HSA_DBG_WAVEOP*)run_ptr) = Operand; @@ -220,10 +223,13 @@ hsaKmtDbgAddressWatch( args->gpu_id = gpu_id; args->buf_size_in_bytes = buff_size; + /* increment pointer to the start of the non fixed part */ unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); + /* save variable content pointer for kfd */ + args->content_ptr = (void *)run_ptr; /* insert items, and increment pointer accordingly */ *((HSAuint32*)run_ptr) = NumWatchPoints; From 5c0f8660b6da9fb327982cebb1533294279bbe40 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sun, 15 Feb 2015 16:06:35 +0200 Subject: [PATCH 0043/1247] Revert "Workaround for non cached memory issue on gfx801 A0" This reverts commit 402ff3f881cadd8e19baad1ded5f40f10044300c. Signed-off-by: Oded Gabbay Acked-by: Ben Goz Conflicts: include/linux/kfd_ioctl.h src/queues.c --- include/linux/kfd_ioctl.h | 8 +------ src/queues.c | 47 +++------------------------------------ 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 08558f6678..75f5517d9c 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -313,13 +313,7 @@ struct kfd_ioctl_open_graphic_handle_args { #define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ AMDKFD_IOWR(0x13, struct kfd_ioctl_open_graphic_handle_args) -#define AMDKFD_IOC_NC_PAGE_ALLOC \ - AMDKFD_IOWR(0x14, unsigned long) - -#define AMDKFD_IOC_NC_PAGE_FREE \ - AMDKFD_IOWR(0x15, unsigned long) - #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x16 +#define AMDKFD_COMMAND_END 0x14 #endif diff --git a/src/queues.c b/src/queues.c index aadd0dd307..b25a7d503c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -94,7 +94,7 @@ struct queue { uint32_t queue_id; uint32_t wptr; - uint32_t *rptr; + uint32_t rptr; void *eop_buffer; void *ctx_save_restore; }; @@ -108,29 +108,6 @@ struct process_doorbells struct process_doorbells doorbells[] = {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = {.need_mmap = true, .doorbells = NULL, .doorbells_mutex = PTHREAD_MUTEX_INITIALIZER}}; -static uint32_t *nc_rptr_page = NULL; - -static void* alloc_nc_page(void) -{ - void *ptr; - if (nc_rptr_page) - return nc_rptr_page; - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_NC_PAGE_ALLOC, &ptr) == 0) { - nc_rptr_page = ptr; - return ptr; - } - return NULL; - -} - -static void free_nc_page(void **page_addr) -{ - if (*page_addr) - if (kmtIoctl(kfd_fd, AMDKFD_IOC_NC_PAGE_FREE, *page_addr) == 0) - *page_addr = NULL; -} - static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) { int i = 0; @@ -250,18 +227,10 @@ hsaKmtCreateQueue( default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } - if (Type != HSA_QUEUE_COMPUTE_AQL) { + if (Type != HSA_QUEUE_COMPUTE_AQL) + { QueueResource->QueueRptrValue = (uintptr_t)&q->rptr; QueueResource->QueueWptrValue = (uintptr_t)&q->wptr; - - /* CZ A0 workaround */ - if (dev_info->eop_buffer_size > 0) { - if (alloc_nc_page() == NULL) { - free_queue(q); - return HSAKMT_STATUS_NO_MEMORY; - } - QueueResource->Queue_read_ptr = (HSAuint32 *)nc_rptr_page; - } } args.read_pointer_address = QueueResource->QueueRptrValue; @@ -280,13 +249,6 @@ hsaKmtCreateQueue( } q->queue_id = args.queue_id; - if (Type != HSA_QUEUE_COMPUTE_AQL && - dev_info->eop_buffer_size > 0) - { - q->rptr = &nc_rptr_page[args.queue_id]; - QueueResource->Queue_read_ptr = (HSAuint32 *)&nc_rptr_page[args.queue_id]; - *q->rptr = 0; - } pthread_mutex_lock(&doorbells[NodeId].doorbells_mutex); @@ -373,9 +335,6 @@ hsaKmtDestroyQueue( } else { - if (nc_rptr_page != NULL) - free_nc_page((void **)&nc_rptr_page); - free_queue(q); return HSAKMT_STATUS_SUCCESS; } From 027ca02f2e193c6779c629b5e35dc1bd531534ce Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sun, 22 Feb 2015 13:06:50 +0200 Subject: [PATCH 0044/1247] reformat memory and fmm functions according to kernel coding style Signed-off-by: Oded Gabbay --- src/fmm.c | 629 ++++++++++++++++++++++++++++++--------------------- src/fmm.h | 4 +- src/memory.c | 142 ++++++------ 3 files changed, 451 insertions(+), 324 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 6e3d8a1ece..be67d5d41a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -33,58 +33,73 @@ #define NON_VALID_GPU_ID 0 #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) -#define INIT_APERTURE(base_value, limit_value) {.base = (void*)base_value, .limit = (void*)limit_value } -#define INIT_MANAGEBLE_APERTURE(base_value, limit_value) {.base = (void*)base_value,.limit = (void*)limit_value, .vm_ranges = NULL, .vm_objects = NULL, .fmm_mutex = PTHREAD_MUTEX_INITIALIZER} -#define INIT_GPU_MEM \ -{ .gpu_id = NON_VALID_GPU_ID,\ - .lds_aperture = INIT_APERTURE(0, 0), \ - .scratch_aperture = INIT_MANAGEBLE_APERTURE(0, 0),\ - .gpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0)\ + +#define INIT_APERTURE(base_value, limit_value) { \ + .base = (void *) base_value, \ + .limit = (void *) limit_value \ + } + +#define INIT_MANAGEBLE_APERTURE(base_value, limit_value) { \ + .base = (void *) base_value, \ + .limit = (void *) limit_value, \ + .vm_ranges = NULL, \ + .vm_objects = NULL, \ + .fmm_mutex = PTHREAD_MUTEX_INITIALIZER \ + } + +#define INIT_GPU_MEM { \ + .gpu_id = NON_VALID_GPU_ID, \ + .lds_aperture = INIT_APERTURE(0, 0), \ + .scratch_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ + .gpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0) \ } #define INIT_GPUs_MEM {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = INIT_GPU_MEM} -struct vm_object{ - void* start; + +struct vm_object { + void *start; uint64_t size; - uint64_t handle; // opaque - struct vm_object* next; - struct vm_object* prev; + uint64_t handle; /* opaque */ + struct vm_object *next; + struct vm_object *prev; }; typedef struct vm_object vm_object_t; -struct vm_area{ - void* start; - void* end; - struct vm_area* next; - struct vm_area* prev; +struct vm_area { + void *start; + void *end; + struct vm_area *next; + struct vm_area *prev; }; typedef struct vm_area vm_area_t; typedef struct { - void* base; - void* limit; - vm_area_t* vm_ranges; - vm_object_t* vm_objects; + void *base; + void *limit; + vm_area_t *vm_ranges; + vm_object_t *vm_objects; pthread_mutex_t fmm_mutex; } manageble_aperture_t; typedef struct { - void* base; - void* limit; + void *base; + void *limit; } aperture_t; -typedef struct{ +typedef struct { uint32_t gpu_id; aperture_t lds_aperture; manageble_aperture_t scratch_aperture; manageble_aperture_t gpuvm_aperture; -}gpu_mem_t; +} gpu_mem_t; static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; -static vm_area_t* vm_create_and_init_area(void* start, void* end){ - vm_area_t* area = (vm_area_t*)malloc(sizeof(vm_area_t));// TODO: Memory pool ??? - if (area){ +static vm_area_t *vm_create_and_init_area(void *start, void *end) +{ + vm_area_t *area = (vm_area_t *) malloc(sizeof(vm_area_t)); + + if (area) { area->start = start; area->end = end; area->next = area->prev = NULL; @@ -93,9 +108,12 @@ static vm_area_t* vm_create_and_init_area(void* start, void* end){ return area; } -static vm_object_t* vm_create_and_init_object(void* start, uint64_t size, uint64_t handle){ - vm_object_t* object = (vm_object_t*)malloc(sizeof(vm_object_t)); // TODO: Memory pool ??? - if (object){ +static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, + uint64_t handle) +{ + vm_object_t *object = (vm_object_t *) malloc(sizeof(vm_object_t)); + + if (object) { object->start = start; object->size = size; object->handle = handle; @@ -106,48 +124,49 @@ static vm_object_t* vm_create_and_init_object(void* start, uint64_t size, uint64 } -static void vm_remove_area(manageble_aperture_t* app, vm_area_t* area){ - vm_area_t* next; - vm_area_t* prev; +static void vm_remove_area(manageble_aperture_t *app, vm_area_t *area) +{ + vm_area_t *next; + vm_area_t *prev; next = area->next; prev = area->prev; - if (prev == NULL )// The first element + if (prev == NULL) /* The first element */ app->vm_ranges = next; else prev->next = next; - if(next) // If not the last element + if (next) /* If not the last element */ next->prev = prev; free(area); - } -static void vm_remove_object(manageble_aperture_t* app, vm_object_t* object){ - vm_object_t* next; - vm_object_t* prev; +static void vm_remove_object(manageble_aperture_t *app, vm_object_t *object) +{ + vm_object_t *next; + vm_object_t *prev; next = object->next; prev = object->prev; - if (prev == NULL )// The first element + if (prev == NULL) /* The first element */ app->vm_objects = next; else prev->next = next; - if(next) // If not the last element + if (next) /* If not the last element */ next->prev = prev; free(object); } +static void vm_add_area_after(vm_area_t *after_this, vm_area_t *new_area) +{ + vm_area_t *next = after_this->next; - -static void vm_add_area_after(vm_area_t* after_this, vm_area_t* new_area){ - vm_area_t* next = after_this->next; after_this->next = new_area; new_area->next = next; @@ -156,8 +175,11 @@ static void vm_add_area_after(vm_area_t* after_this, vm_area_t* new_area){ next->prev = new_area; } -static void vm_add_object_before(vm_object_t* before_this, vm_object_t* new_object){ - vm_object_t* prev = before_this->prev; +static void vm_add_object_before(vm_object_t *before_this, + vm_object_t *new_object) +{ + vm_object_t *prev = before_this->prev; + before_this->prev = new_object; new_object->next = before_this; @@ -166,45 +188,54 @@ static void vm_add_object_before(vm_object_t* before_this, vm_object_t* new_obje prev->next = new_object; } -static void vm_split_area(manageble_aperture_t* app, vm_area_t* area, void* address, uint64_t MemorySizeInBytes){ +static void vm_split_area(manageble_aperture_t *app, vm_area_t *area, + void *address, uint64_t MemorySizeInBytes) +{ + /* + * The existing area is split to: [area->start, address - 1] + * and [address + MemorySizeInBytes, area->end] + */ + vm_area_t *new_area = vm_create_and_init_area( + VOID_PTR_ADD(address, MemorySizeInBytes), + area->end); - // The existing area is split to: [area->start, address - 1] and [address + MemorySizeInBytes, area->end] - vm_area_t* new_area = vm_create_and_init_area(VOID_PTR_ADD(address,MemorySizeInBytes), area->end); - - // Shrink the existing area - area->end = VOID_PTR_SUB(address,1); + /* Shrink the existing area */ + area->end = VOID_PTR_SUB(address, 1); vm_add_area_after(area, new_area); - } -static vm_object_t* vm_find_object_by_address(manageble_aperture_t* app, void* address, uint64_t size){ - vm_object_t* cur = app->vm_objects; +static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, + void *address, uint64_t size) +{ + vm_object_t *cur = app->vm_objects; - // Look up the appropriate address range containing the given address - while(cur){ - if(cur->start == address && (cur->size == size || size == 0)) + /* Look up the appropriate address range containing the given address */ + while (cur) { + if (cur->start == address && (cur->size == size || size == 0)) break; cur = cur->next; }; - return cur; // NULL if not found + return cur; /* NULL if not found */ } -static vm_area_t* vm_find(manageble_aperture_t* app, void* address){ - vm_area_t* cur = app->vm_ranges; +static vm_area_t *vm_find(manageble_aperture_t *app, void *address) +{ + vm_area_t *cur = app->vm_ranges; - // Look up the appropriate address range containing the given address - while(cur){ - if(cur->start <= address && cur->end >= address) + /* Look up the appropriate address range containing the given address */ + while (cur) { + if (cur->start <= address && cur->end >= address) break; cur = cur->next; }; - return cur; // NULL if not found + return cur; /* NULL if not found */ } -static bool aperture_is_valid(void* app_base, void* app_limit){ +static bool aperture_is_valid(void *app_base, void *app_limit) +{ if (app_base && app_limit && app_base < app_limit) return true; return false; @@ -213,329 +244,405 @@ static bool aperture_is_valid(void* app_base, void* app_limit){ /* * Assumes that fmm_mutex is locked on entry. */ -static void aperture_release_area(manageble_aperture_t* app, void* address, uint64_t MemorySizeInBytes){ - vm_area_t* area; +static void aperture_release_area(manageble_aperture_t *app, void *address, + uint64_t MemorySizeInBytes) +{ + vm_area_t *area; + uint64_t SizeOfRegion; area = vm_find(app, address); - if(area) { - if(VOID_PTRS_SUB(area->end, area->start) + 1 > MemorySizeInBytes) { // the size of the released block is less than the size of area - if(area->start == address) { // shrink from the start - area->start = VOID_PTR_ADD(area->start,MemorySizeInBytes); - } else if(VOID_PTRS_SUB(area->end, address) + 1 == MemorySizeInBytes) { // shrink from the end - area->end = VOID_PTR_SUB(area->end, MemorySizeInBytes); - } else { // split the area - vm_split_area(app, area, address, MemorySizeInBytes); - } - } else if(VOID_PTRS_SUB(area->end, area->start) + 1 == MemorySizeInBytes) { // the size of the released block is exactly the same as the size of area - vm_remove_area(app, area); - } + if (!area) + return; + + SizeOfRegion = VOID_PTRS_SUB(area->end, area->start) + 1; + + /* check if block is whole region or part of it */ + if (SizeOfRegion == MemorySizeInBytes) { + vm_remove_area(app, area); + } else if (SizeOfRegion > MemorySizeInBytes) { + /* shrink from the start */ + if (area->start == address) + area->start = + VOID_PTR_ADD(area->start, MemorySizeInBytes); + /* shrink from the end */ + else if (VOID_PTRS_SUB(area->end, address) + 1 == + MemorySizeInBytes) + area->end = VOID_PTR_SUB(area->end, MemorySizeInBytes); + /* split the area */ + else + vm_split_area(app, area, address, MemorySizeInBytes); } } /* - * returns allocated address or NULL. Assumes, that fmm_mutex is locked on entry. + * returns allocated address or NULL. Assumes, that fmm_mutex is locked + * on entry. */ -static void* aperture_allocate_area(manageble_aperture_t* app, uint64_t MemorySizeInBytes, uint64_t offset){ - vm_area_t* cur, *next, *new_area, *start; - void* new_address = NULL; +static void *aperture_allocate_area(manageble_aperture_t *app, + uint64_t MemorySizeInBytes, + uint64_t offset) +{ + vm_area_t *cur, *next, *new_area, *start; + void *new_address = NULL; + next = NULL; new_area = NULL; cur = app->vm_ranges; - if(cur) { // not empty - - // Look up the appropriate address space "hole" or end of the list + if (cur) { /* not empty */ + /* + * Look up the appropriate address space "hole" or end of + * the list + */ while (cur) { next = cur->next; - // End of the list reached - if(!next) + /* End of the list reached */ + if (!next) break; - // address space "hole" - if((VOID_PTRS_SUB(next->start,cur->end) >= MemorySizeInBytes)) + /* address space "hole" */ + if ((VOID_PTRS_SUB(next->start, cur->end) >= + MemorySizeInBytes)) break; cur = next; }; - // If the new range is inside the reserved aperture - if(VOID_PTRS_SUB(app->limit, cur->end) + 1 >= MemorySizeInBytes) { - // cur points to the last inspected element: the tail of the list or the found "hole" - // Just extend the existing region + /* If the new range is inside the reserved aperture */ + if (VOID_PTRS_SUB(app->limit, cur->end) + 1 >= + MemorySizeInBytes) { + /* + * cur points to the last inspected element: the tail + * of the list or the found "hole". + * Just extend the existing region + */ new_address = VOID_PTR_ADD(cur->end, 1); cur->end = VOID_PTR_ADD(cur->end, MemorySizeInBytes); - } else new_address = NULL; - - } else { // empty - create the first area - start = VOID_PTR_ADD(app->base, offset); // Some offset from the base - new_area = vm_create_and_init_area(start, VOID_PTR_ADD(start, (MemorySizeInBytes - 1))); - if(new_area) { + } else { + new_address = NULL; + } + } else { /* empty - create the first area */ + /* Some offset from the base */ + start = VOID_PTR_ADD(app->base, offset); + new_area = vm_create_and_init_area(start, + VOID_PTR_ADD(start, (MemorySizeInBytes - 1))); + if (new_area) { app->vm_ranges = new_area; new_address = new_area->start; } } return new_address; - } -/* - * returns 0 on success. Assumes, that fmm_mutex is locked on entry. - */ -static int aperture_allocate_object(manageble_aperture_t* app, void* new_address, uint64_t handle, uint64_t MemorySizeInBytes){ - vm_object_t* new_object; +/* returns 0 on success. Assumes, that fmm_mutex is locked on entry */ +static int aperture_allocate_object(manageble_aperture_t *app, + void *new_address, + uint64_t handle, + uint64_t MemorySizeInBytes) +{ + vm_object_t *new_object; - // Allocate new object - new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, handle); - if(!new_object) + /* Allocate new object */ + new_object = vm_create_and_init_object(new_address, + MemorySizeInBytes, + handle); + if (!new_object) return -1; - if(app->vm_objects == NULL ) { // empty list - // Update head - app->vm_objects = new_object; - } else { - // Add it before the first element + /* check for non-empty list */ + if (app->vm_objects != NULL) + /* Add it before the first element */ vm_add_object_before(app->vm_objects, new_object); - // Update head - app->vm_objects = new_object; - } + + app->vm_objects = new_object; /* Update head */ return 0; } - -static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id){ +static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) +{ int32_t i; - for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ - if(gpu_mem[i].gpu_id == gpu_id) + for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS ; i++) + if (gpu_mem[i].gpu_id == gpu_id) return i; - } return -1; } -bool fmm_is_inside_some_aperture(void* address){ - +bool fmm_is_inside_some_aperture(void *address) +{ int32_t i; - for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ - if(gpu_mem[i].gpu_id != NON_VALID_GPU_ID){ - if ((address>= gpu_mem[i].lds_aperture.base) && (address<= gpu_mem[i].lds_aperture.limit)) - return true; - if ((address>= gpu_mem[i].gpuvm_aperture.base) && (address<= gpu_mem[i].gpuvm_aperture.limit)) - return true; - if ((address>= gpu_mem[i].scratch_aperture.base) && (address<= gpu_mem[i].scratch_aperture.limit)) - return true; - } + for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS ; i++) { + if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + continue; + if ((address >= gpu_mem[i].lds_aperture.base) && + (address <= gpu_mem[i].lds_aperture.limit)) + return true; + if ((address >= gpu_mem[i].gpuvm_aperture.base) && + (address <= gpu_mem[i].gpuvm_aperture.limit)) + return true; + if ((address >= gpu_mem[i].scratch_aperture.base) && + (address <= gpu_mem[i].scratch_aperture.limit)) + return true; } return false; } #ifdef DEBUG_PRINT_APERTURE -static void aperture_print(aperture_t* app){ +static void aperture_print(aperture_t *app) +{ printf("\t Base: %p\n", app->base); printf("\t Limit: %p\n", app->limit); } -static void manageble_aperture_print(manageble_aperture_t* app){ - vm_area_t* cur = app->vm_ranges; +static void manageble_aperture_print(manageble_aperture_t *app) +{ + vm_area_t *cur = app->vm_ranges; vm_object_t *object = app->vm_objects; printf("\t Base: %p\n", app->base); printf("\t Limit: %p\n", app->limit); - printf("\t Ranges: \n"); - while(cur){ - printf("\t\t Range [%p - %p] \n", cur->start, cur->end); + printf("\t Ranges:\n"); + while (cur) { + printf("\t\t Range [%p - %p]\n", cur->start, cur->end); cur = cur->next; }; - printf("\t Objects: \n"); - while(object){ - printf("\t\t Object [%p - %" PRIu64 "] \n", object->start, object->size); + printf("\t Objects:\n"); + while (object) { + printf("\t\t Object [%p - %" PRIu64 "]\n", + object->start, object->size); object = object->next; }; } -void fmm_print(uint32_t gpu_id){ +void fmm_print(uint32_t gpu_id) +{ int32_t i = gpu_mem_find_by_gpu_id(gpu_id); - if(i >= 0){ // Found - printf("LDS aperture: \n"); - aperture_print(&gpu_mem[i].lds_aperture); - printf("GPUVM aperture: \n"); - manageble_aperture_print(&gpu_mem[i].gpuvm_aperture); - printf("Scratch aperture: \n"); - manageble_aperture_print(&gpu_mem[i].scratch_aperture); + if (i >= 0) { /* Found */ + printf("LDS aperture:\n"); + aperture_print(&gpu_mem[i].lds_aperture); + printf("GPUVM aperture:\n"); + manageble_aperture_print(&gpu_mem[i].gpuvm_aperture); + printf("Scratch aperture:\n"); + manageble_aperture_print(&gpu_mem[i].scratch_aperture); } } #else -void fmm_print(uint32_t gpu_id){ - +void fmm_print(uint32_t gpu_id) +{ } #endif -void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes){ - // Not supported yet +void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) +{ + /* Not supported yet */ return NULL; } -// The offset from GPUVM aperture base address to ensure that address 0 (after base subtraction) won't be used +/* + * The offset from GPUVM aperture base address to ensure that address 0 + * (after base subtraction) won't be used + */ #define GPUVM_APP_OFFSET 0x10000 -void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes){ - - void* mem = NULL; +void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) +{ + void *mem = NULL; int32_t i = gpu_mem_find_by_gpu_id(gpu_id); - // If not found or aperture isn't properly initialized/supported - if(i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, gpu_mem[i].gpuvm_aperture.limit)) - return NULL ; + /* If not found or aperture isn't properly initialized/supported */ + if (i < 0 || + !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, + gpu_mem[i].gpuvm_aperture.limit)) + return NULL; - // Allocate address space + /* Allocate address space */ pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, MemorySizeInBytes, GPUVM_APP_OFFSET); + mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, + MemorySizeInBytes, GPUVM_APP_OFFSET); pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); return mem; } -void* fmm_open_graphic_handle(uint32_t gpu_id, +void *fmm_open_graphic_handle(uint32_t gpu_id, int32_t graphic_device_handle, uint32_t graphic_handle, - uint64_t MemorySizeInBytes){ + uint64_t MemorySizeInBytes) +{ - void* mem = NULL; + void *mem = NULL; int32_t i = gpu_mem_find_by_gpu_id(gpu_id); struct kfd_ioctl_open_graphic_handle_args open_graphic_handle_args; struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; - // If not found or aperture isn't properly initialized/supported - if (i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, gpu_mem[i].gpuvm_aperture.limit)) + /* If not found or aperture isn't properly initialized/supported */ + if (i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, + gpu_mem[i].gpuvm_aperture.limit)) return NULL; pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - // Allocate address space - mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, MemorySizeInBytes, GPUVM_APP_OFFSET); + /* Allocate address space */ + mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, + MemorySizeInBytes, GPUVM_APP_OFFSET); if (!mem) goto out; - // Allocate local memory + /* Allocate local memory */ open_graphic_handle_args.gpu_id = gpu_id; open_graphic_handle_args.graphic_device_fd = graphic_device_handle; open_graphic_handle_args.graphic_handle = graphic_handle; - open_graphic_handle_args.va_addr = VOID_PTRS_SUB(mem, gpu_mem[i].gpuvm_aperture.base); - if (kmtIoctl(kfd_fd, AMDKFD_IOC_OPEN_GRAPHIC_HANDLE, &open_graphic_handle_args)) + open_graphic_handle_args.va_addr = + VOID_PTRS_SUB(mem, gpu_mem[i].gpuvm_aperture.base); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_OPEN_GRAPHIC_HANDLE, + &open_graphic_handle_args)) goto release_area; - // Allocate object - if (aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, open_graphic_handle_args.handle, MemorySizeInBytes)) + /* Allocate object */ + if (aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, + open_graphic_handle_args.handle, + MemorySizeInBytes)) goto release_mem; pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - // That's all. Just return the new address + /* That's all. Just return the new address */ return mem; release_mem: unmap_args.handle = open_graphic_handle_args.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); release_area: - aperture_release_area(&gpu_mem[i].gpuvm_aperture, mem, MemorySizeInBytes); + aperture_release_area(&gpu_mem[i].gpuvm_aperture, mem, + MemorySizeInBytes); out: pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - return NULL ; + return NULL; } -void fmm_release(void* address, uint64_t MemorySizeInBytes){ - +void fmm_release(void *address, uint64_t MemorySizeInBytes) +{ uint32_t i; bool found = false; - for (i = 0; i < NUM_OF_SUPPORTED_GPUS && !found; i++) { - if(gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS && !found ; i++) { + if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; - if(address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { + if (address >= gpu_mem[i].gpuvm_aperture.base && + address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - aperture_release_area(&gpu_mem[i].gpuvm_aperture, address, MemorySizeInBytes); + aperture_release_area(&gpu_mem[i].gpuvm_aperture, address, + MemorySizeInBytes); pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); fmm_print(gpu_mem[i].gpu_id); - } } - // If memory address isn't inside of any defined aperture - it refers to the system memory - if (!found) { + /* + * If memory address isn't inside of any defined aperture - it refers + * to the system memory + */ + if (!found) free(address); - } } -HSAKMT_STATUS fmm_init_process_apertures(){ +HSAKMT_STATUS fmm_init_process_apertures(void) +{ struct kfd_ioctl_get_process_apertures_args args; uint8_t node_id; - if (0 == kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void*)&args)){ - for(node_id = 0; node_id < args.num_of_nodes; node_id++){ - gpu_mem[node_id].gpu_id = args.process_apertures[node_id].gpu_id; - gpu_mem[node_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); - gpu_mem[node_id].lds_aperture.limit = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_limit); - gpu_mem[node_id].gpuvm_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_base); - gpu_mem[node_id].gpuvm_aperture.limit = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_limit); - gpu_mem[node_id].scratch_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_base); - gpu_mem[node_id].scratch_aperture.limit = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_limit); - } + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) + return HSAKMT_STATUS_ERROR; - return HSAKMT_STATUS_SUCCESS; + for (node_id = 0 ; node_id < args.num_of_nodes ; node_id++) { + gpu_mem[node_id].gpu_id = + args.process_apertures[node_id].gpu_id; + + gpu_mem[node_id].lds_aperture.base = + PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); + + gpu_mem[node_id].lds_aperture.limit = + PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_limit); + + gpu_mem[node_id].gpuvm_aperture.base = + PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_base); + + gpu_mem[node_id].gpuvm_aperture.limit = + PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_limit); + + gpu_mem[node_id].scratch_aperture.base = + PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_base); + + gpu_mem[node_id].scratch_aperture.limit = + PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_limit); } - return HSAKMT_STATUS_ERROR; - + return HSAKMT_STATUS_SUCCESS; } -HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id){ +HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id) +{ int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); - if (slot<0) + + if (slot < 0) return HSAKMT_STATUS_INVALID_PARAMETER; - switch(aperture_type){ + switch (aperture_type) { case FMM_GPUVM: - return aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, gpu_mem[slot].gpuvm_aperture.limit) ? PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.base) : 0; + return aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, + gpu_mem[slot].gpuvm_aperture.limit) ? + PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.base) : 0; break; + case FMM_SCRATCH: - return aperture_is_valid(gpu_mem[slot].scratch_aperture.base, gpu_mem[slot].scratch_aperture.limit) ? PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.base) : 0; + return aperture_is_valid(gpu_mem[slot].scratch_aperture.base, + gpu_mem[slot].scratch_aperture.limit) ? + PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.base) : 0; break; + case FMM_LDS: - return aperture_is_valid(gpu_mem[slot].lds_aperture.base, gpu_mem[slot].lds_aperture.limit) ? PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.base) : 0; + return aperture_is_valid(gpu_mem[slot].lds_aperture.base, + gpu_mem[slot].lds_aperture.limit) ? + PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.base) : 0; break; + default: return 0; } - } - -static bool _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t* aperture, void* address, uint64_t size, uint64_t* gpuvm_address) { - +static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, + void *address, uint64_t size, + uint64_t *gpuvm_address) +{ struct kfd_ioctl_map_memory_to_gpu_args args; struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; - // Check that address space was previously reserved + /* Check that address space was previously reserved */ if (vm_find(aperture, address) == NULL) - return false; + return -1; - // Allocate local memory + /* Allocate local memory */ args.gpu_id = gpu_id; args.size = size; - args.va_addr = VOID_PTRS_SUB(address, aperture->base); //va_addr is 40 bit GPUVM address - if(kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) - return false; - // Allocate object + /* va_addr is 40 bit GPUVM address */ + args.va_addr = VOID_PTRS_SUB(address, aperture->base); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) + goto err_map_ioctl_failed; + + /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); if (aperture_allocate_object(aperture, address, args.handle, size)) goto err_object_allocation_failed; @@ -543,49 +650,54 @@ static bool _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t* aperture, voi *gpuvm_address = args.va_addr; - return true; + return 0; err_object_allocation_failed: pthread_mutex_unlock(&aperture->fmm_mutex); unmap_args.handle = args.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); +err_map_ioctl_failed: *gpuvm_address = 0; - return false; + return -1; } -bool fmm_map_to_gpu(void* address, uint64_t size, uint64_t* gpuvm_address) { - +int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) +{ int32_t i; uint64_t pi; - // Find an aperture the requested address belongs to - for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ - if(gpu_mem[i].gpu_id != NON_VALID_GPU_ID){ - if ((address>= gpu_mem[i].gpuvm_aperture.base) && (address<= gpu_mem[i].gpuvm_aperture.limit)) { - // map it - return _fmm_map_to_gpu(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); - } - } + /* Find an aperture the requested address belongs to */ + for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + continue; + + if ((address >= gpu_mem[i].gpuvm_aperture.base) && + (address <= gpu_mem[i].gpuvm_aperture.limit)) + /* map it */ + return _fmm_map_to_gpu(gpu_mem[i].gpu_id, + &gpu_mem[i].gpuvm_aperture, + address, size, gpuvm_address); } - // If address isn't Local memory address, we assume that this is - // system memory address accessed through IOMMU. - // Thus we "prefetch" it - for(pi = 0; pi < size / PAGE_SIZE; pi++) { - ((char*)address)[pi*PAGE_SIZE] = 0; - } - return true; + /* + * If address isn't Local memory address, we assume that this is + * system memory address accessed through IOMMU. Thus we "prefetch" it + */ + for (pi = 0; pi < size / PAGE_SIZE; pi++) + ((char *) address)[pi * PAGE_SIZE] = 0; + + return 0; } -static bool _fmm_unmap_from_gpu(manageble_aperture_t* aperture, void* address) { - - vm_object_t* object; +static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) +{ + vm_object_t *object; struct kfd_ioctl_unmap_memory_from_gpu_args args; pthread_mutex_lock(&aperture->fmm_mutex); - // Find the object to retrieve the handle + /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); if (!object) goto err; @@ -596,27 +708,28 @@ static bool _fmm_unmap_from_gpu(manageble_aperture_t* aperture, void* address) { vm_remove_object(aperture, object); pthread_mutex_unlock(&aperture->fmm_mutex); - return true; + return 0; err: pthread_mutex_unlock(&aperture->fmm_mutex); - return false; - + return -1; } -bool fmm_unmap_from_gpu(void* address) { - +int fmm_unmap_from_gpu(void *address) +{ int32_t i; - // Find the aperture the requested address belongs to - for(i = 0; i < NUM_OF_SUPPORTED_GPUS; i++){ - if(gpu_mem[i].gpu_id != NON_VALID_GPU_ID){ - if ((address>= gpu_mem[i].gpuvm_aperture.base) && (address<= gpu_mem[i].gpuvm_aperture.limit)) { - // unmap it - return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, address); - } - } + /* Find the aperture the requested address belongs to */ + for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + continue; + + if ((address >= gpu_mem[i].gpuvm_aperture.base) && + (address <= gpu_mem[i].gpuvm_aperture.limit)) + /* unmap it */ + return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, + address); } - return true; + return 0; } diff --git a/src/fmm.h b/src/fmm.h index ca0a692ead..c3b99d0b9e 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -56,8 +56,8 @@ void* fmm_open_graphic_handle(uint32_t gpu_id, void fmm_print(uint32_t node); bool fmm_is_inside_some_aperture(void* address); void fmm_release(void* address, HSAuint64 MemorySizeInBytes); -bool fmm_map_to_gpu(void* address, uint64_t size, uint64_t* gpuvm_address); -bool fmm_unmap_from_gpu(void* address); +int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); +int fmm_unmap_from_gpu(void *address); /* Topology interface*/ HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); diff --git a/src/memory.c b/src/memory.c index a2c1eb49b6..68d86d5e5e 100644 --- a/src/memory.c +++ b/src/memory.c @@ -37,13 +37,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy( - HSAuint32 Node, - HSAuint32 DefaultPolicy, - HSAuint32 AlternatePolicy, - void* MemoryAddressAlternate, - HSAuint64 MemorySizeInBytes - ) + HSAuint32 Node, + HSAuint32 DefaultPolicy, + HSAuint32 AlternatePolicy, + void *MemoryAddressAlternate, + HSAuint64 MemorySizeInBytes +) { + struct kfd_ioctl_set_memory_policy_args args; HSAKMT_STATUS result; uint32_t gpu_id; @@ -53,23 +54,31 @@ hsaKmtSetMemoryPolicy( if (result != HSAKMT_STATUS_SUCCESS) return result; - // We accept any legal policy and alternate address location. You get CC everywhere anyway. - if ((DefaultPolicy != HSA_CACHING_CACHED && DefaultPolicy != HSA_CACHING_NONCACHED) - || (AlternatePolicy != HSA_CACHING_CACHED && AlternatePolicy != HSA_CACHING_NONCACHED)) - { + /* + * We accept any legal policy and alternate address location. + * You get CC everywhere anyway. + */ + if ((DefaultPolicy != HSA_CACHING_CACHED && + DefaultPolicy != HSA_CACHING_NONCACHED) || + (AlternatePolicy != HSA_CACHING_CACHED && + AlternatePolicy != HSA_CACHING_NONCACHED)) return HSAKMT_STATUS_INVALID_PARAMETER; - } CHECK_PAGE_MULTIPLE(MemoryAddressAlternate); CHECK_PAGE_MULTIPLE(MemorySizeInBytes); - struct kfd_ioctl_set_memory_policy_args args; memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; - args.default_policy = (DefaultPolicy == HSA_CACHING_CACHED) ? KFD_IOC_CACHE_POLICY_COHERENT : KFD_IOC_CACHE_POLICY_NONCOHERENT; - args.alternate_policy = (AlternatePolicy == HSA_CACHING_CACHED) ? KFD_IOC_CACHE_POLICY_COHERENT : KFD_IOC_CACHE_POLICY_NONCOHERENT; - args.alternate_aperture_base = (uintptr_t)MemoryAddressAlternate; + args.default_policy = (DefaultPolicy == HSA_CACHING_CACHED) ? + KFD_IOC_CACHE_POLICY_COHERENT : + KFD_IOC_CACHE_POLICY_NONCOHERENT; + + args.alternate_policy = (AlternatePolicy == HSA_CACHING_CACHED) ? + KFD_IOC_CACHE_POLICY_COHERENT : + KFD_IOC_CACHE_POLICY_NONCOHERENT; + + args.alternate_aperture_base = (uintptr_t) MemoryAddressAlternate; args.alternate_aperture_size = MemorySizeInBytes; int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args); @@ -79,46 +88,51 @@ hsaKmtSetMemoryPolicy( static HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags) { - switch (pageSizeFlags) - { + switch (pageSizeFlags) { case HSA_PAGE_SIZE_4KB: return 4*1024; case HSA_PAGE_SIZE_64KB: return 64*1024; case HSA_PAGE_SIZE_2MB: return 2*1024*1024; case HSA_PAGE_SIZE_1GB: return 1024*1024*1024; - default: assert(false); return 4*1024; + default: + assert(false); + return 4*1024; } } HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory( - HSAuint32 PreferredNode, //IN - HSAuint64 SizeInBytes, //IN (multiple of page size) - HsaMemFlags MemFlags, //IN - void** MemoryAddress //OUT (page-aligned) - ) + HSAuint32 PreferredNode, /* IN */ + HSAuint64 SizeInBytes, /* IN (multiple of page size) */ + HsaMemFlags MemFlags, /* IN */ + void **MemoryAddress /* OUT (page-aligned) */ +) { - CHECK_KFD_OPEN(); HSAKMT_STATUS result; + HSAuint64 page_size; uint32_t gpu_id; int err; + CHECK_KFD_OPEN(); + result = validate_nodeid(PreferredNode, &gpu_id); if (result != HSAKMT_STATUS_SUCCESS) return result; - // The required size should be page aligned (GDS?) - HSAuint64 page_size = PageSizeFromFlags(MemFlags.ui32.PageSize); - if ((SizeInBytes & (page_size-1)) && !MemFlags.ui32.GDSMemory){ + /* The required size should be page aligned (GDS?) */ + page_size = PageSizeFromFlags(MemFlags.ui32.PageSize); + if ((SizeInBytes & (page_size-1)) && !MemFlags.ui32.GDSMemory) return HSAKMT_STATUS_INVALID_PARAMETER; - } if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged) { err = posix_memalign(MemoryAddress, page_size, SizeInBytes); if (err != 0) return HSAKMT_STATUS_NO_MEMORY; + if (MemFlags.ui32.ExecuteAccess) { - err = mprotect(*MemoryAddress, SizeInBytes, PROT_READ | PROT_WRITE | PROT_EXEC); + err = mprotect(*MemoryAddress, SizeInBytes, + PROT_READ | PROT_WRITE | PROT_EXEC); + if (err != 0) { free(*MemoryAddress); return err; @@ -127,10 +141,12 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_SUCCESS; } - if(!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged){ - *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); + if (!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged) { + *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); + if (*MemoryAddress == NULL) return HSAKMT_STATUS_NO_MEMORY; + return HSAKMT_STATUS_SUCCESS; } @@ -140,22 +156,22 @@ hsaKmtAllocMemory( HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory( - void* MemoryAddress, //IN (page-aligned) - HSAuint64 SizeInBytes //IN - ) + void *MemoryAddress, /* IN (page-aligned) */ + HSAuint64 SizeInBytes /* IN */ +) { CHECK_KFD_OPEN(); - fmm_release( MemoryAddress, SizeInBytes); + fmm_release(MemoryAddress, SizeInBytes); return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory( - void* MemoryAddress, //IN (page-aligned) - HSAuint64 MemorySizeInBytes //IN (page-aligned) - ) + void *MemoryAddress, /* IN (page-aligned) */ + HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ +) { CHECK_KFD_OPEN(); @@ -165,8 +181,8 @@ hsaKmtRegisterMemory( HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory( - void* MemoryAddress //IN - ) + void *MemoryAddress /* IN */ +) { CHECK_KFD_OPEN(); @@ -176,50 +192,47 @@ hsaKmtDeregisterMemory( HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU( - void* MemoryAddress, //IN (page-aligned) - HSAuint64 MemorySizeInBytes, //IN (page-aligned) - HSAuint64* AlternateVAGPU //OUT (page-aligned) - ) + void *MemoryAddress, /* IN (page-aligned) */ + HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ + HSAuint64 *AlternateVAGPU /* OUT (page-aligned) */ +) { CHECK_KFD_OPEN(); if (AlternateVAGPU) *AlternateVAGPU = 0; - if (fmm_map_to_gpu(MemoryAddress, MemorySizeInBytes, AlternateVAGPU)){ + if (!fmm_map_to_gpu(MemoryAddress, MemorySizeInBytes, AlternateVAGPU)) return HSAKMT_STATUS_SUCCESS; - } - else { + else return HSAKMT_STATUS_ERROR; - } - } HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU( - void* MemoryAddress //IN (page-aligned) - ) + void *MemoryAddress /* IN (page-aligned) */ +) { CHECK_KFD_OPEN(); - if (fmm_unmap_from_gpu(MemoryAddress)) + if (!fmm_unmap_from_gpu(MemoryAddress)) return HSAKMT_STATUS_SUCCESS; else return HSAKMT_STATUS_ERROR; - } HSAKMT_STATUS HSAKMTAPI hsaKmtMapGraphicHandle( - HSAuint32 NodeId, //IN - HSAuint64 GraphicDeviceHandle, //IN - HSAuint64 GraphicResourceHandle, //IN - HSAuint64 GraphicResourceOffset, //IN - HSAuint64 GraphicResourceSize, //IN - HSAuint64* FlatMemoryAddress //OUT - ) + HSAuint32 NodeId, /* IN */ + HSAuint64 GraphicDeviceHandle, /* IN */ + HSAuint64 GraphicResourceHandle, /* IN */ + HSAuint64 GraphicResourceOffset, /* IN */ + HSAuint64 GraphicResourceSize, /* IN */ + HSAuint64 *FlatMemoryAddress /* OUT */ +) { + CHECK_KFD_OPEN(); HSAKMT_STATUS result; uint32_t gpu_id; @@ -248,10 +261,11 @@ hsaKmtMapGraphicHandle( HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapGraphicHandle( - HSAuint32 NodeId, //IN - HSAuint64 FlatMemoryAddress, //IN - HSAuint64 SizeInBytes //IN - ) + HSAuint32 NodeId, /* IN */ + HSAuint64 FlatMemoryAddress, /* IN */ + HSAuint64 SizeInBytes /* IN */ +) { + return hsaKmtUnmapMemoryToGPU(PORT_UINT64_TO_VPTR(FlatMemoryAddress)); } From e62d46ba1c40e1a5bbd27911b50cf35f19fc134c Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 23 Feb 2015 15:54:01 +0200 Subject: [PATCH 0045/1247] Don't alloc/free VRAM in map/unmap paths Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 26 ++++-- src/fmm.c | 165 +++++++++++++++++++++++++++++++------- 2 files changed, 154 insertions(+), 37 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 75f5517d9c..442f2e4622 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -229,7 +229,7 @@ struct kfd_ioctl_wait_events_args { uint32_t wait_result; /* from KFD */ }; -struct kfd_ioctl_map_memory_to_gpu_args { +struct kfd_ioctl_alloc_memory_of_gpu_args { uint64_t va_addr; /* to KFD */ uint64_t size; /* to KFD */ uint64_t handle; /* from KFD */ @@ -237,6 +237,14 @@ struct kfd_ioctl_map_memory_to_gpu_args { uint32_t pad; }; +struct kfd_ioctl_free_memory_of_gpu_args { + uint64_t handle; /* to KFD */ +}; + +struct kfd_ioctl_map_memory_to_gpu_args { + uint64_t handle; /* to KFD */ +}; + struct kfd_ioctl_unmap_memory_from_gpu_args { uint64_t handle; /* to KFD */ }; @@ -304,16 +312,22 @@ struct kfd_ioctl_open_graphic_handle_args { #define AMDKFD_IOC_DBG_WAVE_CONTROL \ AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args) +#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU \ + AMDKFD_IOWR(0x11, struct kfd_ioctl_alloc_memory_of_gpu_args) + +#define AMDKFD_IOC_FREE_MEMORY_OF_GPU \ + AMDKFD_IOWR(0x12, struct kfd_ioctl_free_memory_of_gpu_args) + #define AMDKFD_IOC_MAP_MEMORY_TO_GPU \ - AMDKFD_IOWR(0x11, struct kfd_ioctl_map_memory_to_gpu_args) + AMDKFD_IOWR(0x13, struct kfd_ioctl_map_memory_to_gpu_args) #define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ - AMDKFD_IOWR(0x12, struct kfd_ioctl_unmap_memory_from_gpu_args) + AMDKFD_IOWR(0x14, struct kfd_ioctl_unmap_memory_from_gpu_args) #define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ - AMDKFD_IOWR(0x13, struct kfd_ioctl_open_graphic_handle_args) + AMDKFD_IOWR(0x15, struct kfd_ioctl_open_graphic_handle_args) -#define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x14 +#define AMDKFD_COMMAND_START 0x01 +#define AMDKFD_COMMAND_END 0x16 #endif diff --git a/src/fmm.c b/src/fmm.c index be67d5d41a..bda55cf3a9 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -372,6 +372,67 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) return -1; } +static manageble_aperture_t *find_valid_gpuvm_apperture_of_gpu(uint32_t gpu_id) +{ + manageble_aperture_t *aperture; + int32_t gpu_mem_id; + + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return NULL; + + aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; + + /* Check that aperture is properly initialized/supported */ + if (!aperture_is_valid(aperture->base, aperture->limit)) + return NULL; + + return aperture; +} + +static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, + uint64_t MemorySizeInBytes) +{ + struct kfd_ioctl_alloc_memory_of_gpu_args args; + struct kfd_ioctl_free_memory_of_gpu_args free_args; + manageble_aperture_t *aperture; + + if (!mem) + return -1; + + /* Retrieve gpuvm aperture according to gpu_id */ + aperture = find_valid_gpuvm_apperture_of_gpu(gpu_id); + if (!aperture) + return -1; + + /* Allocate memory from amdkfd */ + args.gpu_id = gpu_id; + args.size = MemorySizeInBytes; + + /* va_addr is 40 bit GPUVM address */ + args.va_addr = VOID_PTRS_SUB(mem, aperture->base); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) + return -1; + + /* Allocate object */ + pthread_mutex_lock(&aperture->fmm_mutex); + if (aperture_allocate_object(aperture, mem, args.handle, + MemorySizeInBytes)) + goto err_object_allocation_failed; + pthread_mutex_unlock(&aperture->fmm_mutex); + + return 0; + +err_object_allocation_failed: + pthread_mutex_unlock(&aperture->fmm_mutex); + free_args.handle = args.handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args); + + return -1; +} + bool fmm_is_inside_some_aperture(void *address) { int32_t i; @@ -453,20 +514,40 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) #define GPUVM_APP_OFFSET 0x10000 void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) { + manageble_aperture_t *aperture; + int32_t gpu_mem_id; void *mem = NULL; - int32_t i = gpu_mem_find_by_gpu_id(gpu_id); - /* If not found or aperture isn't properly initialized/supported */ - if (i < 0 || - !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, - gpu_mem[i].gpuvm_aperture.limit)) + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return NULL; + + aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; + + /* Check that aperture is properly initialized/supported */ + if (!aperture_is_valid(aperture->base, aperture->limit)) return NULL; /* Allocate address space */ - pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, + pthread_mutex_lock(&aperture->fmm_mutex); + mem = aperture_allocate_area(aperture, MemorySizeInBytes, GPUVM_APP_OFFSET); - pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* + * Now that we have the area reserved, allocate memory in the device + * itself + */ + if (fmm_allocate_memory_in_device(gpu_id, mem, MemorySizeInBytes)) { + /* + * allocation of memory in device failed. + * Release region in aperture + */ + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, MemorySizeInBytes); + pthread_mutex_unlock(&aperture->fmm_mutex); + } return mem; } @@ -528,6 +609,38 @@ out: return NULL; } +static void __fmm_release(uint32_t gpu_id, void *address, + uint64_t MemorySizeInBytes) +{ + struct kfd_ioctl_free_memory_of_gpu_args args; + manageble_aperture_t *aperture; + vm_object_t *object; + + if (!address) + return; + + /* Retrieve gpuvm aperture according to gpu_id */ + aperture = find_valid_gpuvm_apperture_of_gpu(gpu_id); + if (!aperture) + return; + + pthread_mutex_lock(&aperture->fmm_mutex); + + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (!object) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return; + } + + args.handle = object->handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); + + vm_remove_object(aperture, object); + aperture_release_area(aperture, address, MemorySizeInBytes); + + pthread_mutex_unlock(&aperture->fmm_mutex); +} void fmm_release(void *address, uint64_t MemorySizeInBytes) { @@ -541,10 +654,7 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; - pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - aperture_release_area(&gpu_mem[i].gpuvm_aperture, address, - MemorySizeInBytes); - pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); + __fmm_release(i, address, MemorySizeInBytes); fmm_print(gpu_mem[i].gpu_id); } } @@ -627,37 +737,32 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, uint64_t *gpuvm_address) { struct kfd_ioctl_map_memory_to_gpu_args args; - struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; + vm_object_t *object; /* Check that address space was previously reserved */ if (vm_find(aperture, address) == NULL) return -1; - /* Allocate local memory */ - args.gpu_id = gpu_id; - args.size = size; + pthread_mutex_lock(&aperture->fmm_mutex); - /* va_addr is 40 bit GPUVM address */ - args.va_addr = VOID_PTRS_SUB(address, aperture->base); + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (!object) + goto err_object_not_found; + + args.handle = object->handle; if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - /* Allocate object */ - pthread_mutex_lock(&aperture->fmm_mutex); - if (aperture_allocate_object(aperture, address, args.handle, size)) - goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); - *gpuvm_address = args.va_addr; + *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); return 0; -err_object_allocation_failed: - pthread_mutex_unlock(&aperture->fmm_mutex); - unmap_args.handle = args.handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); - err_map_ioctl_failed: +err_object_not_found: + pthread_mutex_unlock(&aperture->fmm_mutex); *gpuvm_address = 0; return -1; } @@ -705,11 +810,9 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); - vm_remove_object(aperture, object); - pthread_mutex_unlock(&aperture->fmm_mutex); - return 0; + return 0; err: pthread_mutex_unlock(&aperture->fmm_mutex); return -1; From f3ac33230e44244f8c0e2df6a87b9d4a866f6d82 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sun, 1 Mar 2015 11:08:46 +0200 Subject: [PATCH 0046/1247] Disable SDMA queue creation Signed-off-by: Oded Gabbay --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index b25a7d503c..bf7d681043 100644 --- a/src/queues.c +++ b/src/queues.c @@ -222,7 +222,7 @@ hsaKmtCreateQueue( switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; + case HSA_QUEUE_SDMA: free_queue(q); return HSAKMT_STATUS_UNAVAILABLE; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } From 417e6f75013738202eb9967f142da22fe7f08fd4 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sun, 1 Mar 2015 14:36:45 +0200 Subject: [PATCH 0047/1247] Revert "Disable SDMA queue creation" This reverts commit f3ac33230e44244f8c0e2df6a87b9d4a866f6d82. --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index bf7d681043..b25a7d503c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -222,7 +222,7 @@ hsaKmtCreateQueue( switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: free_queue(q); return HSAKMT_STATUS_UNAVAILABLE; + case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } From 13e0d87733d0ef18715fbfce1d817d2784ff5881 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sun, 1 Mar 2015 11:08:46 +0200 Subject: [PATCH 0048/1247] Disable SDMA queue creation Signed-off-by: Oded Gabbay --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index b25a7d503c..bf7d681043 100644 --- a/src/queues.c +++ b/src/queues.c @@ -222,7 +222,7 @@ hsaKmtCreateQueue( switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; + case HSA_QUEUE_SDMA: free_queue(q); return HSAKMT_STATUS_UNAVAILABLE; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } From aba396bc527568605f894817eafc85fb776f4406 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 3 Mar 2015 15:22:49 +0200 Subject: [PATCH 0049/1247] Call __fmm_release with gpu id and not index __fmm_release expects gpu_id as parameter. fmm_release passed it instead the index of the GPU in the gpu_mem array. Signed-off-by: Oded Gabbay --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index bda55cf3a9..6f5e651054 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -627,7 +627,7 @@ static void __fmm_release(uint32_t gpu_id, void *address, pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, MemorySizeInBytes); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return; @@ -654,7 +654,7 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; - __fmm_release(i, address, MemorySizeInBytes); + __fmm_release(gpu_mem[i].gpu_id, address, MemorySizeInBytes); fmm_print(gpu_mem[i].gpu_id); } } From ac56c6baff140c05be0bd7ca4806f192af639ca2 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Thu, 19 Mar 2015 10:09:37 +0200 Subject: [PATCH 0050/1247] Fix return value when local memory alloc fails Signed-off-by: Oded Gabbay Reviewed-by: Ben Goz --- src/fmm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index 6f5e651054..eacbac070a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -547,6 +547,9 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, MemorySizeInBytes); pthread_mutex_unlock(&aperture->fmm_mutex); + + /* Assign NULL to mem to indicate failure to calling function */ + mem = NULL; } return mem; From b27de08442537dd9f853a6e401f21799316b82a8 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 18 Mar 2015 15:33:43 +0200 Subject: [PATCH 0051/1247] Adding support for mmap events page from Thunk Signed-off-by: Ben Goz --- include/linux/kfd_ioctl.h | 4 ++-- src/events.c | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 442f2e4622..09a22cc681 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -170,13 +170,13 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_IOC_WAIT_RESULT_FAIL 2 struct kfd_ioctl_create_event_args { - uint64_t event_trigger_address; /* from KFD - signal events only */ + uint64_t event_page_offset; /* from KFD */ uint32_t event_trigger_data; /* from KFD - signal events only */ uint32_t event_type; /* to KFD */ uint32_t auto_reset; /* to KFD */ uint32_t node_id; /* to KFD - only valid for certain event types */ uint32_t event_id; /* from KFD */ - uint32_t pad; + uint32_t event_slot_index; /* from KFD */ }; struct kfd_ioctl_destroy_event_args { diff --git a/src/events.c b/src/events.c index ab2df5b333..8b057b0fc7 100644 --- a/src/events.c +++ b/src/events.c @@ -29,8 +29,11 @@ #include #include #include +#include #include "linux/kfd_ioctl.h" +static HSAuint64 *events_page = NULL; + static bool IsSystemEventType(HSA_EVENTTYPE type) { // Debug events behave as signal events. @@ -73,10 +76,22 @@ hsaKmtCreateEvent( return HSAKMT_STATUS_ERROR; } + if (events_page == NULL && args.event_page_offset > 0) { + events_page = mmap(NULL, 4096, PROT_WRITE | PROT_READ, + MAP_SHARED, kfd_fd, args.event_page_offset); + if (events_page == NULL) { + hsaKmtDestroyEvent(e); + return HSAKMT_STATUS_ERROR; + } + } + + if (args.event_page_offset > 0 && args.event_slot_index < 256) + e->EventData.HWData2 = (HSAuint64)&events_page[args.event_slot_index]; + e->EventId = args.event_id; e->EventData.EventType = EventDesc->EventType; e->EventData.HWData1 = args.event_id; - e->EventData.HWData2 = args.event_trigger_address; + e->EventData.HWData3 = args.event_trigger_data; if (IsSignaled && !IsSystemEventType(e->EventData.EventType)) { From 30eb386026401c210f996193757535245b5f87c4 Mon Sep 17 00:00:00 2001 From: Xihan Zhang Date: Wed, 8 Apr 2015 17:53:11 +0800 Subject: [PATCH 0052/1247] Add Performance Counter Support on gfx801. Signed-off-by: Xihan Zhang Acked-by: Oded Gabbay --- src/pmc_table.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 3fa90e95e8..0390639928 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -44,6 +44,23 @@ static uint32_t kaveri_sq_counter_ids[] = { 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 }; +static uint32_t carrizo_sq_counter_ids[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 168, 169, 170, + 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, + 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, + 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, + 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 16, @@ -54,6 +71,16 @@ static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 16, + .num_of_counters = sizeof(carrizo_sq_counter_ids)/sizeof(*carrizo_sq_counter_ids), + .counter_ids = carrizo_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + HSAKMT_STATUS get_block_properties(uint16_t dev_id, enum perf_block_id block_id, @@ -88,6 +115,15 @@ get_block_properties(uint16_t dev_id, case 0x131D: *block = kaveri_blocks[block_id]; break; + + case 0x9870: + case 0x9874: + case 0x9875: + case 0x9876: + case 0x9877: + *block = carrizo_blocks[block_id]; + break; + default: rc = HSAKMT_STATUS_INVALID_PARAMETER; } From 5ed05c99b3cbddbca389127b3656f38493f8339a Mon Sep 17 00:00:00 2001 From: Xihan Zhang Date: Fri, 10 Apr 2015 22:26:44 +0800 Subject: [PATCH 0053/1247] Add execution property in register memory for gfx801. Signed-off-by: Xihan Zhang --- include/hsakmt.h | 3 ++- src/memory.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index fd2788803d..321df6e6c9 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -364,7 +364,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory( void* MemoryAddress, //IN (page-aligned) - HSAuint64 MemorySizeInBytes //IN (page-aligned) + HSAuint64 MemorySizeInBytes, //IN (page-aligned) + int ExecuteAccess ); diff --git a/src/memory.c b/src/memory.c index 68d86d5e5e..0be2a0f738 100644 --- a/src/memory.c +++ b/src/memory.c @@ -170,11 +170,23 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory( void *MemoryAddress, /* IN (page-aligned) */ - HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ + HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ + int ExecuteAccess ) { CHECK_KFD_OPEN(); + int err; + /* for CZ ib execution */ + if (ExecuteAccess) { + err = mprotect(MemoryAddress, MemorySizeInBytes, + PROT_READ | PROT_WRITE | PROT_EXEC); + + if (err != 0) { + free(MemoryAddress); + return err; + } + } return HSAKMT_STATUS_SUCCESS; } From 112f7e751a8bc6c827a25f9360ef3c0e87360391 Mon Sep 17 00:00:00 2001 From: Xihan Zhang Date: Tue, 28 Apr 2015 23:42:49 +0800 Subject: [PATCH 0054/1247] Enable creating SDMA queue. Signed-off-by: Xihan Zhang Reviewed-by: Ben Goz --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index bf7d681043..b25a7d503c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -222,7 +222,7 @@ hsaKmtCreateQueue( switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: free_queue(q); return HSAKMT_STATUS_UNAVAILABLE; + case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } From 4c4df3803559eccea008c0e0ac17546ed535b8ce Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 28 Apr 2015 17:50:00 +0300 Subject: [PATCH 0055/1247] Revert "Add execution property in register memory for gfx801." This reverts commit 5ed05c99b3cbddbca389127b3656f38493f8339a. --- include/hsakmt.h | 3 +-- src/memory.c | 14 +------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 321df6e6c9..fd2788803d 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -364,8 +364,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory( void* MemoryAddress, //IN (page-aligned) - HSAuint64 MemorySizeInBytes, //IN (page-aligned) - int ExecuteAccess + HSAuint64 MemorySizeInBytes //IN (page-aligned) ); diff --git a/src/memory.c b/src/memory.c index 0be2a0f738..68d86d5e5e 100644 --- a/src/memory.c +++ b/src/memory.c @@ -170,23 +170,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory( void *MemoryAddress, /* IN (page-aligned) */ - HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ - int ExecuteAccess + HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ ) { CHECK_KFD_OPEN(); - int err; - /* for CZ ib execution */ - if (ExecuteAccess) { - err = mprotect(MemoryAddress, MemorySizeInBytes, - PROT_READ | PROT_WRITE | PROT_EXEC); - - if (err != 0) { - free(MemoryAddress); - return err; - } - } return HSAKMT_STATUS_SUCCESS; } From eb2d3cfcdfbfb805d51a40751e56c29c7ed6e1a6 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Sat, 2 May 2015 12:10:04 +0300 Subject: [PATCH 0056/1247] Add missing DoorbellType field to HSA_CAPABILITY Signed-off-by: Oded Gabbay --- include/hsakmttypes.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index ec10255b50..a7e0a81ab5 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -168,7 +168,10 @@ typedef union unsigned int WatchPointsSupported: 1; // Indicates if Watchpoints are available on the node. unsigned int WatchPointsTotalBits: 4; // ld(Watchpoints) available. To determine the number use 2^value - unsigned int Reserved : 20; + unsigned int DoorbellType : 2; // 0: This node has pre-1.0 doorbell characteristic + // 1: This node has 1.0 doorbell characteristic + // 2,3: reserved for future use + unsigned int Reserved : 18; } ui32; } HSA_CAPABILITY; From a70a98b30b69ffc7ecb78fcb7e9d3f6ea7fd8d76 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Thu, 30 Apr 2015 18:08:36 +0300 Subject: [PATCH 0057/1247] Increase limit of signal events to 4096 Signed-off-by: Oded Gabbay Reviewed-by: Ben Goz --- include/linux/kfd_ioctl.h | 2 ++ src/events.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 09a22cc681..3316967170 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -169,6 +169,8 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_IOC_WAIT_RESULT_TIMEOUT 1 #define KFD_IOC_WAIT_RESULT_FAIL 2 +#define KFD_SIGNAL_EVENT_LIMIT 4096 + struct kfd_ioctl_create_event_args { uint64_t event_page_offset; /* from KFD */ uint32_t event_trigger_data; /* from KFD - signal events only */ diff --git a/src/events.c b/src/events.c index 8b057b0fc7..d64f601ebd 100644 --- a/src/events.c +++ b/src/events.c @@ -77,7 +77,7 @@ hsaKmtCreateEvent( } if (events_page == NULL && args.event_page_offset > 0) { - events_page = mmap(NULL, 4096, PROT_WRITE | PROT_READ, + events_page = mmap(NULL, KFD_SIGNAL_EVENT_LIMIT * 8, PROT_WRITE | PROT_READ, MAP_SHARED, kfd_fd, args.event_page_offset); if (events_page == NULL) { hsaKmtDestroyEvent(e); @@ -85,7 +85,7 @@ hsaKmtCreateEvent( } } - if (args.event_page_offset > 0 && args.event_slot_index < 256) + if (args.event_page_offset > 0 && args.event_slot_index < KFD_SIGNAL_EVENT_LIMIT) e->EventData.HWData2 = (HSAuint64)&events_page[args.event_slot_index]; e->EventId = args.event_id; From 8aa0791ddb3b4d0bbf602d6004cca3669c278368 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 5 May 2015 10:51:50 +0300 Subject: [PATCH 0058/1247] Don't report local mem aperture if local mem size is 0 Signed-off-by: Oded Gabbay --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 3275aba9c2..903b6f78a8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -846,7 +846,7 @@ hsaKmtGetNodeMemoryProperties( } /*Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE*/ - if (i < NumBanks){ + if ((i < NumBanks) && (node[NodeId].node.LocalMemSize > 0)) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; MemoryProperties[i].SizeInBytes = node[NodeId].node.LocalMemSize; MemoryProperties[i].VirtualBaseAddress = fmm_get_aperture_base(FMM_GPUVM, gpu_id); From 2e76017278ff364d432b37213c6a200332826897 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Mon, 18 May 2015 11:01:42 +0300 Subject: [PATCH 0059/1247] increase event limit to provide 4K events Signed-off-by: Oded Gabbay --- include/linux/kfd_ioctl.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 3316967170..de1cc83c9b 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -169,7 +169,14 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_IOC_WAIT_RESULT_TIMEOUT 1 #define KFD_IOC_WAIT_RESULT_FAIL 2 -#define KFD_SIGNAL_EVENT_LIMIT 4096 +/* + * The added 512 is because, currently, 8*(4096/256) signal events are + * reserved for debugger events, and we want to provide at least 4K signal + * events for EOP usage. + * We add 512 to make the allocated size (KFD_SIGNAL_EVENT_LIMIT * 8) be + * page aligned. + */ +#define KFD_SIGNAL_EVENT_LIMIT (4096 + 512) struct kfd_ioctl_create_event_args { uint64_t event_page_offset; /* from KFD */ From 29c083f6950f36a87ade5142b6fc24ab3cb42e4f Mon Sep 17 00:00:00 2001 From: Moses Reuben Date: Tue, 9 Jun 2015 05:37:05 +0300 Subject: [PATCH 0060/1247] adding support for scratch memory Signed-off-by: Moses Reuben --- include/linux/kfd_ioctl.h | 6 ++- src/fmm.c | 79 ++++++++++++++++++++++++++++++++++++--- src/fmm.h | 1 + src/memory.c | 10 ++++- src/topology.c | 10 ++++- 5 files changed, 98 insertions(+), 8 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index de1cc83c9b..3f9807eb3c 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -336,7 +336,11 @@ struct kfd_ioctl_open_graphic_handle_args { #define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ AMDKFD_IOWR(0x15, struct kfd_ioctl_open_graphic_handle_args) +#define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ + AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) + + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x16 +#define AMDKFD_COMMAND_END 0x17 #endif diff --git a/src/fmm.c b/src/fmm.c index eacbac070a..67e46162da 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -90,6 +90,7 @@ typedef struct { uint32_t gpu_id; aperture_t lds_aperture; manageble_aperture_t scratch_aperture; + manageble_aperture_t scratch_physical; manageble_aperture_t gpuvm_aperture; } gpu_mem_t; @@ -410,7 +411,6 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, args.gpu_id = gpu_id; args.size = MemorySizeInBytes; - /* va_addr is 40 bit GPUVM address */ args.va_addr = VOID_PTRS_SUB(mem, aperture->base); if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) @@ -500,11 +500,45 @@ void fmm_print(uint32_t gpu_id) } #endif - void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { - /* Not supported yet */ - return NULL; + manageble_aperture_t *aperture; + manageble_aperture_t *aperture_phy; + struct kfd_ioctl_alloc_memory_of_gpu_args args; + int32_t gpu_mem_id; + void *mem = NULL; + + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return NULL; + + aperture = &gpu_mem[gpu_mem_id].scratch_aperture; + aperture_phy = &gpu_mem[gpu_mem_id].scratch_physical; + + /* Check that aperture is properly initialized/supported */ + if (!aperture_is_valid(aperture->base, aperture->limit)) + return NULL; + + /* Allocate address space */ + mem = mmap(0, MemorySizeInBytes + 16 * PAGE_SIZE, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem == NULL) + return NULL; + + /* Allocate memory from amdkfd */ + args.gpu_id = gpu_id; + args.size = MemorySizeInBytes; + + /* va_addr is 40 bit GPUVM address */ + args.va_addr = (((uint64_t)mem) >> 16) + 1; + + aperture_phy->base = mem; + aperture_phy->limit = (void*)(((uint64_t)mem) + MemorySizeInBytes + 16 * PAGE_SIZE); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH, &args)) + return NULL; + + return (void*)(((((uint64_t)mem) >> 16) + 1) << 16); } /* @@ -653,8 +687,13 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS && !found ; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; + if (address >= gpu_mem[i].scratch_physical.base && + address <= gpu_mem[i].scratch_physical.limit){ + munmap(gpu_mem[i].scratch_physical.base,(uint64_t)gpu_mem[i].scratch_physical.limit - (uint64_t)gpu_mem[i].scratch_physical.base); + return; + } - if (address >= gpu_mem[i].gpuvm_aperture.base && + if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; __fmm_release(gpu_mem[i].gpu_id, address, MemorySizeInBytes); @@ -704,6 +743,36 @@ HSAKMT_STATUS fmm_init_process_apertures(void) return HSAKMT_STATUS_SUCCESS; } +HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id) +{ + int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); + + if (slot < 0) + return HSAKMT_STATUS_INVALID_PARAMETER; + + switch (aperture_type) { + case FMM_GPUVM: + return aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, + gpu_mem[slot].gpuvm_aperture.limit) ? + PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.limit) : 0; + break; + + case FMM_SCRATCH: + return aperture_is_valid(gpu_mem[slot].scratch_aperture.base, + gpu_mem[slot].scratch_aperture.limit) ? + PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.limit) : 0; + break; + + case FMM_LDS: + return aperture_is_valid(gpu_mem[slot].lds_aperture.base, + gpu_mem[slot].lds_aperture.limit) ? + PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.limit) : 0; + break; + + default: + return 0; + } +} HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id) { int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); diff --git a/src/fmm.h b/src/fmm.h index c3b99d0b9e..d326c83f06 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -63,4 +63,5 @@ int fmm_unmap_from_gpu(void *address); HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id); +HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id); #endif /* FMM_H_ */ diff --git a/src/memory.c b/src/memory.c index 68d86d5e5e..202e76de3f 100644 --- a/src/memory.c +++ b/src/memory.c @@ -124,7 +124,7 @@ hsaKmtAllocMemory( if ((SizeInBytes & (page_size-1)) && !MemFlags.ui32.GDSMemory) return HSAKMT_STATUS_INVALID_PARAMETER; - if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged) { + if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { err = posix_memalign(MemoryAddress, page_size, SizeInBytes); if (err != 0) return HSAKMT_STATUS_NO_MEMORY; @@ -149,6 +149,14 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_SUCCESS; } + if (MemFlags.ui32.HostAccess && MemFlags.ui32.Scratch ) { + *MemoryAddress = fmm_allocate_scratch(gpu_id, SizeInBytes); + + if (*MemoryAddress == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + return HSAKMT_STATUS_SUCCESS; + } return HSAKMT_STATUS_INVALID_PARAMETER; } diff --git a/src/topology.c b/src/topology.c index 903b6f78a8..bd2a5c2a1f 100644 --- a/src/topology.c +++ b/src/topology.c @@ -33,7 +33,7 @@ #include "fmm.h" #define PAGE_SIZE 4096 #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) -#define NUM_OF_HEAPS 2 +#define NUM_OF_HEAPS 3 /* SYSFS related */ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" @@ -853,6 +853,14 @@ hsaKmtGetNodeMemoryProperties( i++; } + /*Add SCRATCH*/ + if (i < NumBanks){ + MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_SCRATCH; + MemoryProperties[i].VirtualBaseAddress = fmm_get_aperture_base(FMM_SCRATCH, gpu_id); + MemoryProperties[i].SizeInBytes = fmm_get_aperture_limit(FMM_SCRATCH, gpu_id) - MemoryProperties[i].VirtualBaseAddress; + i++; + } + err = HSAKMT_STATUS_SUCCESS; out: From fc4e07daa32ccfef1135e15d56f94edaa0b6d14b Mon Sep 17 00:00:00 2001 From: Flora Cui Date: Tue, 23 Jun 2015 15:25:38 +0800 Subject: [PATCH 0061/1247] Add interface to set CU mask Signed-off-by: Flora Cui Acked-by: Ben Goz --- include/hsakmt.h | 12 ++++++++++++ include/linux/kfd_ioctl.h | 10 +++++++++- src/libhsakmt.ver | 1 + src/queues.c | 30 ++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index fd2788803d..bdb8f2bdd3 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -318,6 +318,18 @@ hsaKmtDestroyQueue( HSA_QUEUEID QueueId //IN ); +/** + Set cu mask for a queue +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetQueueCUMask( + HSA_QUEUEID QueueId, //IN + HSAuint32 CUMaskCount, //IN + HSAuint32* QueueCUMask //IN + ); + /** Allows an HSA process to set/change the default and alternate memory coherency, before starting to dispatch. */ diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 3f9807eb3c..91aa69a345 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -75,6 +75,12 @@ struct kfd_ioctl_update_queue_args { uint32_t queue_priority; /* to KFD */ }; +struct kfd_ioctl_set_cu_mask_args { + uint32_t queue_id; /* to KFD */ + uint32_t num_cu_mask; /* to KFD */ + uint64_t cu_mask_ptr; /* to KFD */ +}; + /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ #define KFD_IOC_CACHE_POLICY_COHERENT 0 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 @@ -339,8 +345,10 @@ struct kfd_ioctl_open_graphic_handle_args { #define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) +#define AMDKFD_IOC_SET_CU_MASK \ + AMDKFD_IOW(0x17, struct kfd_ioctl_set_cu_mask_args) #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x17 +#define AMDKFD_COMMAND_END 0x18 #endif diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 72917e16af..3c8064d43c 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -20,6 +20,7 @@ hsaKmtWaitOnMultipleEvents; hsaKmtCreateQueue; hsaKmtUpdateQueue; hsaKmtDestroyQueue; +hsaKmtSetQueueCUMask; hsaKmtSetMemoryPolicy; hsaKmtAllocMemory; hsaKmtFreeMemory; diff --git a/src/queues.c b/src/queues.c index b25a7d503c..e720220b95 100644 --- a/src/queues.c +++ b/src/queues.c @@ -339,3 +339,33 @@ hsaKmtDestroyQueue( return HSAKMT_STATUS_SUCCESS; } } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetQueueCUMask( + HSA_QUEUEID QueueId, //IN + HSAuint32 CUMaskCount, //IN + HSAuint32* QueueCUMask //IN + ) +{ + struct queue *q = PORT_UINT64_TO_VPTR(QueueId); + struct kfd_ioctl_set_cu_mask_args args; + + CHECK_KFD_OPEN(); + + if (CUMaskCount == 0 || QueueCUMask == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + memset(&args, 0, sizeof(args)); + args.queue_id = q->queue_id; + args.num_cu_mask = CUMaskCount; + args.cu_mask_ptr = (uintptr_t)QueueCUMask; + + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_CU_MASK, &args); + if (err == -1) + { + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} From a3925a3a19d6509f5d57428024387e2dd5bdbf8a Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 28 Jul 2015 12:22:57 -0400 Subject: [PATCH 0062/1247] Enable version info via thunk interface - Replace HSAuint32 with HSA_ENGINE_ID for EngineId type so it explicitely presents version information for ucode and GfxIP - Created a GfxIP lookup table to pass the version information. This lookup searches for matching device ID. Signed-off-by: Amber Lin Acked-by: John Bridgman Reviewed-by: Felix Kuehling --- include/hsakmttypes.h | 13 ++++++++- src/topology.c | 63 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index a7e0a81ab5..0133fdfd9b 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -148,6 +148,17 @@ typedef struct _HsaSystemProperties HSAuint32 PlatformRev; // HSA platform revision, reflects Platform Table Revision ID } HsaSystemProperties; +typedef union +{ + HSAuint32 Value; + struct + { + unsigned int uCode : 10; // ucode packet processor version + unsigned int Major : 6; // GFXIP Major engine version + unsigned int Minor : 8; // GFXIP Minor engine version + unsigned int Stepping : 8; // GFXIP Stepping info + }; +} HSA_ENGINE_ID; typedef union { @@ -223,7 +234,7 @@ typedef struct _HsaNodeProperties HSAuint32 MaxSlotsScratchCU; // Number of temp. memory ("scratch") wave slots available to access, // may be 0 if HW has no restrictions - HSAuint32 EngineId; // Identifier (rev) of teh GPU uEngine or Firmware, may be 0 + HSA_ENGINE_ID EngineId; // Identifier (rev) of the GPU uEngine or Firmware, may be 0 HSAuint16 VendorId; // GPU vendor id; 0 on latency (= CPU)-only nodes HSAuint16 DeviceId; // GPU device id; 0 on latency (= CPU)-only nodes diff --git a/src/topology.c b/src/topology.c index bd2a5c2a1f..523df0d8da 100644 --- a/src/topology.c +++ b/src/topology.c @@ -52,7 +52,44 @@ static node_t *node = NULL; static HSAKMT_STATUS topology_take_snapshot(void); static HSAKMT_STATUS topology_drop_snapshot(void); -static int get_cpu_stepping(uint16_t* stepping); +//static int get_cpu_stepping(uint16_t* stepping); + +static struct hsa_gfxip_table { + unsigned short device_id; // Device ID + unsigned char major; // GFXIP Major engine version + unsigned char minor; // GFXIP Minor engine version + unsigned char stepping; // GFXIP Stepping info +} gfxip_lookup_table[] = { + /* Kaveri Family */ + { 0x1304, 7, 0, 0 }, + { 0x1305, 7, 0, 0 }, + { 0x1306, 7, 0, 0 }, + { 0x1307, 7, 0, 0 }, + { 0x1309, 7, 0, 0 }, + { 0x130A, 7, 0, 0 }, + { 0x130B, 7, 0, 0 }, + { 0x130C, 7, 0, 0 }, + { 0x130D, 7, 0, 0 }, + { 0x130E, 7, 0, 0 }, + { 0x130F, 7, 0, 0 }, + { 0x1310, 7, 0, 0 }, + { 0x1311, 7, 0, 0 }, + { 0x1312, 7, 0, 0 }, + { 0x1313, 7, 0, 0 }, + { 0x1315, 7, 0, 0 }, + { 0x1316, 7, 0, 0 }, + { 0x1317, 7, 0, 0 }, + { 0x1318, 7, 0, 0 }, + { 0x131B, 7, 0, 0 }, + { 0x131C, 7, 0, 0 }, + { 0x131D, 7, 0, 0 }, + /* Carrizo Family */ + { 0x9870, 8, 0, 1 }, + { 0x9874, 8, 0, 1 }, + { 0x9875, 8, 0, 1 }, + { 0x9876, 8, 0, 1 }, + { 0x9877, 8, 0, 1 } +}; static void free_node(node_t *n) @@ -192,8 +229,8 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 char prop_name[256]; char path[256]; long long unsigned int prop_val; - uint32_t i, prog; - uint16_t stepping = 0, fw_version = 0; + uint32_t i, prog, table_size; + uint16_t fw_version = 0; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -309,8 +346,22 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 } - get_cpu_stepping(&stepping); - props->EngineId = ((stepping << 16) | fw_version); +// get_cpu_stepping(&stepping); + props->EngineId.uCode = fw_version & 0x3ff; + props->EngineId.Major = 0; + props->EngineId.Minor = 0; + props->EngineId.Stepping = 0; + table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); + for (i=0; iDeviceId) { + props->EngineId.Major = gfxip_lookup_table[i].major & 0x3f; + props->EngineId.Minor = gfxip_lookup_table[i].minor; + props->EngineId.Stepping = gfxip_lookup_table[i].stepping; + break; + } + } + assert(props->EngineId.Major); + //TODO: error handler when Device ID lookup fails err2: free(read_buf); @@ -960,6 +1011,7 @@ uint16_t get_device_id_by_node(HSAuint32 node_id) return node[node_id].node.DeviceId; } +#if 0 static int get_cpu_stepping(uint16_t* stepping) { int ret; @@ -997,3 +1049,4 @@ err1: return ret; } +#endif From bb4a5cddd96833e63e70b1238dac6e0719ab4fa4 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 5 Aug 2015 13:33:42 +0300 Subject: [PATCH 0063/1247] Revert "Enable creating SDMA queue." This reverts commit 112f7e751a8bc6c827a25f9360ef3c0e87360391. --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index e720220b95..5bcbe63087 100644 --- a/src/queues.c +++ b/src/queues.c @@ -222,7 +222,7 @@ hsaKmtCreateQueue( switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; + case HSA_QUEUE_SDMA: free_queue(q); return HSAKMT_STATUS_UNAVAILABLE; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } From 2dff5cabfa0f9e141ca91c6910e105859b8f7963 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Fri, 24 Jul 2015 13:35:22 -0400 Subject: [PATCH 0064/1247] Minor fix in libhsathunk for KFDMemory test Signed-off-by: shaoyun liu(shaoyun.liu@amd.com) Reviewed-by: Ben Goz(Ben.Goz@amd.com) --- src/events.c | 3 ++- src/memory.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/events.c b/src/events.c index d64f601ebd..f4217c2c58 100644 --- a/src/events.c +++ b/src/events.c @@ -79,8 +79,9 @@ hsaKmtCreateEvent( if (events_page == NULL && args.event_page_offset > 0) { events_page = mmap(NULL, KFD_SIGNAL_EVENT_LIMIT * 8, PROT_WRITE | PROT_READ, MAP_SHARED, kfd_fd, args.event_page_offset); - if (events_page == NULL) { + if (events_page == MAP_FAILED) { hsaKmtDestroyEvent(e); + events_page = NULL; return HSAKMT_STATUS_ERROR; } } diff --git a/src/memory.c b/src/memory.c index 202e76de3f..7adc4fb914 100644 --- a/src/memory.c +++ b/src/memory.c @@ -121,7 +121,8 @@ hsaKmtAllocMemory( /* The required size should be page aligned (GDS?) */ page_size = PageSizeFromFlags(MemFlags.ui32.PageSize); - if ((SizeInBytes & (page_size-1)) && !MemFlags.ui32.GDSMemory) + if ((!MemoryAddress) || (!SizeInBytes) || + (SizeInBytes & (page_size-1))) return HSAKMT_STATUS_INVALID_PARAMETER; if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { From fb8378a18b5aa5c7874d0e65758991cffa781703 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Sun, 23 Aug 2015 17:42:27 +0300 Subject: [PATCH 0065/1247] Support gfx802 dGPU Signed-off-by: Ben Goz --- include/linux/kfd_ioctl.h | 18 +- src/events.c | 11 ++ src/fmm.c | 351 ++++++++++++++++++++++++++++++++------ src/fmm.h | 3 + src/globals.c | 1 + src/libhsakmt.h | 10 ++ src/memory.c | 28 ++- src/queues.c | 129 ++++++++++++-- src/topology.c | 80 +++++---- 9 files changed, 513 insertions(+), 118 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 91aa69a345..39fd4d5be4 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -249,7 +249,7 @@ struct kfd_ioctl_alloc_memory_of_gpu_args { uint64_t size; /* to KFD */ uint64_t handle; /* from KFD */ uint32_t gpu_id; /* to KFD */ - uint32_t pad; + uint64_t mmap_offset; /* from KFD */ }; struct kfd_ioctl_free_memory_of_gpu_args { @@ -273,6 +273,12 @@ struct kfd_ioctl_open_graphic_handle_args { uint32_t pad; }; +struct kfd_ioctl_set_process_dgpu_aperture_args { + uint32_t node_id; + uint64_t dgpu_base; + uint64_t dgpu_limit; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -342,13 +348,17 @@ struct kfd_ioctl_open_graphic_handle_args { #define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ AMDKFD_IOWR(0x15, struct kfd_ioctl_open_graphic_handle_args) -#define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ - AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) +#define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ + AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) #define AMDKFD_IOC_SET_CU_MASK \ AMDKFD_IOW(0x17, struct kfd_ioctl_set_cu_mask_args) +#define AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE \ + AMDKFD_IOW(0x18, struct kfd_ioctl_set_process_dgpu_aperture_args) + + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x18 +#define AMDKFD_COMMAND_END 0x19 #endif diff --git a/src/events.c b/src/events.c index f4217c2c58..7cedacb459 100644 --- a/src/events.c +++ b/src/events.c @@ -30,7 +30,9 @@ #include #include #include +#include #include "linux/kfd_ioctl.h" +#include "fmm.h" static HSAuint64 *events_page = NULL; @@ -70,6 +72,15 @@ hsaKmtCreateEvent( args.event_type = EventDesc->EventType; args.auto_reset = !ManualReset; + /* dGPU code */ + if (is_dgpu && events_page == NULL) { + events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, 0x9000); + if (!events_page) { + return HSAKMT_STATUS_ERROR; + } + fmm_get_handle(events_page, &args.event_page_offset); + } + if (kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) != 0) { free(e); *Event = NULL; diff --git a/src/fmm.c b/src/fmm.c index 67e46162da..df864779ea 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -92,10 +92,16 @@ typedef struct { manageble_aperture_t scratch_aperture; manageble_aperture_t scratch_physical; manageble_aperture_t gpuvm_aperture; + manageble_aperture_t dgpu_aperture; } gpu_mem_t; static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; +static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit); +static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); +static void __fmm_release(uint32_t gpu_id, void *address, + uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); + static vm_area_t *vm_create_and_init_area(void *start, void *end) { vm_area_t *area = (vm_area_t *) malloc(sizeof(vm_area_t)); @@ -373,45 +379,24 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) return -1; } -static manageble_aperture_t *find_valid_gpuvm_apperture_of_gpu(uint32_t gpu_id) -{ - manageble_aperture_t *aperture; - int32_t gpu_mem_id; - - /* Retrieve gpu_mem id according to gpu_id */ - gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); - if (gpu_mem_id < 0) - return NULL; - - aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; - - /* Check that aperture is properly initialized/supported */ - if (!aperture_is_valid(aperture->base, aperture->limit)) - return NULL; - - return aperture; -} - static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, - uint64_t MemorySizeInBytes) + uint64_t MemorySizeInBytes, + manageble_aperture_t *aperture, + uint64_t *mmap_offset) { struct kfd_ioctl_alloc_memory_of_gpu_args args; struct kfd_ioctl_free_memory_of_gpu_args free_args; - manageble_aperture_t *aperture; if (!mem) return -1; - /* Retrieve gpuvm aperture according to gpu_id */ - aperture = find_valid_gpuvm_apperture_of_gpu(gpu_id); - if (!aperture) - return -1; - /* Allocate memory from amdkfd */ args.gpu_id = gpu_id; args.size = MemorySizeInBytes; - args.va_addr = VOID_PTRS_SUB(mem, aperture->base); + args.va_addr = (uint64_t)mem; + if (!mmap_offset) + args.va_addr = VOID_PTRS_SUB(mem, aperture->base); if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) return -1; @@ -423,6 +408,9 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); + if (mmap_offset) + *mmap_offset = args.mmap_offset; + return 0; err_object_allocation_failed: @@ -541,24 +529,10 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) return (void*)(((((uint64_t)mem) >> 16) + 1) << 16); } -/* - * The offset from GPUVM aperture base address to ensure that address 0 - * (after base subtraction) won't be used - */ -#define GPUVM_APP_OFFSET 0x10000 -void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) +static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, + manageble_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset) { - manageble_aperture_t *aperture; - int32_t gpu_mem_id; void *mem = NULL; - - /* Retrieve gpu_mem id according to gpu_id */ - gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); - if (gpu_mem_id < 0) - return NULL; - - aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; - /* Check that aperture is properly initialized/supported */ if (!aperture_is_valid(aperture->base, aperture->limit)) return NULL; @@ -566,14 +540,15 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, - MemorySizeInBytes, GPUVM_APP_OFFSET); + MemorySizeInBytes, offset); pthread_mutex_unlock(&aperture->fmm_mutex); /* * Now that we have the area reserved, allocate memory in the device * itself */ - if (fmm_allocate_memory_in_device(gpu_id, mem, MemorySizeInBytes)) { + if (fmm_allocate_memory_in_device(gpu_id, mem, + MemorySizeInBytes, aperture, mmap_offset)) { /* * allocation of memory in device failed. * Release region in aperture @@ -589,6 +564,89 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) return mem; } +/* + * The offset from GPUVM aperture base address to ensure that address 0 + * (after base subtraction) won't be used + */ +#define GPUVM_APP_OFFSET 0x10000 +void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) +{ + manageble_aperture_t *aperture; + int32_t gpu_mem_id; + + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return NULL; + + aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; + + return __fmm_allocate_device(gpu_id, MemorySizeInBytes, + aperture, GPUVM_APP_OFFSET, NULL); +} + +static void* fmm_allocate_host_cpu(uint32_t gpu_id, + uint64_t MemorySizeInBytes, HsaMemFlags flags) +{ + int err; + HSAuint64 page_size; + void *mem = NULL; + + page_size = PageSizeFromFlags(flags.ui32.PageSize); + err = posix_memalign(&mem, page_size, MemorySizeInBytes); + if (err != 0) + return NULL; + + if (flags.ui32.ExecuteAccess) { + err = mprotect(mem, MemorySizeInBytes, + PROT_READ | PROT_WRITE | PROT_EXEC); + + if (err != 0) { + free(mem); + return NULL; + } + } + return mem; +} + +static void* fmm_allocate_host_gpu(uint32_t gpu_id, + uint64_t MemorySizeInBytes, HsaMemFlags flags) +{ + void *mem; + manageble_aperture_t *aperture; + int32_t gpu_mem_id; + uint64_t mmap_offset; + + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return NULL; + + aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; + + MemorySizeInBytes += 0x8000 - (MemorySizeInBytes % 0x8000); + + mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, + aperture, 0, &mmap_offset); + + void *ret = mmap(mem, MemorySizeInBytes, + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + if (ret == MAP_FAILED) { + __fmm_release(gpu_id, mem, MemorySizeInBytes, aperture); + return NULL; + } + + return ret; +} + +void* fmm_allocate_host(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags, uint16_t dev_id) +{ + if (topology_is_dgpu(dev_id)) + return fmm_allocate_host_gpu(gpu_id, MemorySizeInBytes, flags); + return fmm_allocate_host_cpu(gpu_id, MemorySizeInBytes, flags); +} + void *fmm_open_graphic_handle(uint32_t gpu_id, int32_t graphic_device_handle, uint32_t graphic_handle, @@ -647,20 +705,14 @@ out: } static void __fmm_release(uint32_t gpu_id, void *address, - uint64_t MemorySizeInBytes) + uint64_t MemorySizeInBytes, manageble_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args; - manageble_aperture_t *aperture; vm_object_t *object; if (!address) return; - /* Retrieve gpuvm aperture according to gpu_id */ - aperture = find_valid_gpuvm_apperture_of_gpu(gpu_id); - if (!aperture) - return; - pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ @@ -696,7 +748,16 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; - __fmm_release(gpu_mem[i].gpu_id, address, MemorySizeInBytes); + __fmm_release(gpu_mem[i].gpu_id, address, + MemorySizeInBytes, &gpu_mem[i].gpuvm_aperture); + fmm_print(gpu_mem[i].gpu_id); + } + + if (address >= gpu_mem[i].dgpu_aperture.base && + address <= gpu_mem[i].dgpu_aperture.limit) { + found = true; + __fmm_release(gpu_mem[i].gpu_id, address, + MemorySizeInBytes, &gpu_mem[i].dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); } } @@ -713,6 +774,8 @@ HSAKMT_STATUS fmm_init_process_apertures(void) { struct kfd_ioctl_get_process_apertures_args args; uint8_t node_id; + uint32_t gpu_id; + HsaNodeProperties props; if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) return HSAKMT_STATUS_ERROR; @@ -721,6 +784,17 @@ HSAKMT_STATUS fmm_init_process_apertures(void) gpu_mem[node_id].gpu_id = args.process_apertures[node_id].gpu_id; + + if (topology_sysfs_get_node_props(node_id, &props, &gpu_id) == + HSAKMT_STATUS_SUCCESS) { + if (topology_is_dgpu(props.DeviceId)) { + dgpu_mem_init(node_id, &gpu_mem[node_id].dgpu_aperture.base, + &gpu_mem[node_id].dgpu_aperture.limit); + set_dgpu_aperture(node_id, (uint64_t)gpu_mem[node_id].dgpu_aperture.base, + (uint64_t)gpu_mem[node_id].dgpu_aperture.limit); + } + } + gpu_mem[node_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); @@ -804,6 +878,34 @@ HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id) } } +static int _fmm_map_to_gpu_gtt(uint32_t gpu_id, manageble_aperture_t *aperture, + void *address, uint64_t size) +{ + struct kfd_ioctl_map_memory_to_gpu_args args; + vm_object_t *object; + + pthread_mutex_lock(&aperture->fmm_mutex); + + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (!object) { + goto err_object_not_found; + } + + args.handle = object->handle; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) + goto err_map_ioctl_failed; + + pthread_mutex_unlock(&aperture->fmm_mutex); + + return 0; + +err_map_ioctl_failed: +err_object_not_found: + pthread_mutex_unlock(&aperture->fmm_mutex); + return -1; +} + static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) @@ -855,6 +957,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) return _fmm_map_to_gpu(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); + if ((address >= gpu_mem[i].dgpu_aperture.base) && + (address <= gpu_mem[i].dgpu_aperture.limit)) + /* map it */ + return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, + &gpu_mem[i].dgpu_aperture, + address, size); } /* @@ -904,7 +1012,144 @@ int fmm_unmap_from_gpu(void *address) /* unmap it */ return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, address); + else if ((address >= gpu_mem[i].dgpu_aperture.base) && + (address <= gpu_mem[i].dgpu_aperture.limit)) + /* unmap it */ + return _fmm_unmap_from_gpu(&gpu_mem[i].dgpu_aperture, + address); } return 0; } + +/* Tonga dGPU specific functions */ +static bool is_dgpu_mem_init = false; +static void *dgpu_shared_aperture_base = NULL; +static void *dgpu_shared_aperture_limit = NULL; + +static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit) +{ + struct kfd_ioctl_set_process_dgpu_aperture_args args; + + args.node_id = node_id; + args.dgpu_base = base; + args.dgpu_limit = limit; + + return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE, &args); +} + +static void *reserve_address(void *addr, long long unsigned int len) +{ + void *ret_addr; + + if (len <= 0) + return NULL; + + ret_addr = mmap(addr, len, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); + if (addr == MAP_FAILED) + return NULL; + + return ret_addr; +} + +#define ADDRESS_RANGE_LIMIT_MASK 0xFFFFFFFFFF + +static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) +{ + bool found; + HSAKMT_STATUS ret; + void *addr, *ret_addr; + uint32_t max_len; + long long unsigned int temp; + uint32_t gpu_id; + HsaNodeProperties props; + + if (is_dgpu_mem_init) { + if (base) + base = dgpu_shared_aperture_base; + if (limit) + limit = dgpu_shared_aperture_limit; + return HSAKMT_STATUS_SUCCESS; + } + + ret = topology_sysfs_get_node_props(node_id, &props, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + max_len = (uint32_t)props.LocalMemSize; + found = false; + + for (addr = (void *)PAGE_SIZE, ret_addr = NULL; + ret_addr != addr; + addr = (void *)((unsigned long)addr + 0x8000)) + { + ret_addr = reserve_address(addr, max_len); + if (!ret_addr) + continue; + temp = (long long unsigned int)ret_addr + max_len; + if (temp < ADDRESS_RANGE_LIMIT_MASK) { + found = true; + break; + } + else + munmap(ret_addr, max_len); + } + + if (found) { + if (base) + *base = ret_addr; + dgpu_shared_aperture_base = ret_addr; + if (limit) + *limit = (void *)((long long unsigned int)ret_addr + max_len); + dgpu_shared_aperture_limit = (void *)((long long unsigned int)ret_addr + max_len); + is_dgpu_mem_init = true; + return HSAKMT_STATUS_SUCCESS; + } + + return HSAKMT_STATUS_ERROR; +} + +bool fmm_get_handle(void *address, uint64_t *handle) +{ + int32_t i; + manageble_aperture_t *aperture; + vm_object_t *object; + bool found; + + found = false; + aperture = NULL; + + /* Find the aperture the requested address belongs to */ + for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + continue; + + if ((address >= gpu_mem[i].gpuvm_aperture.base) && + (address <= gpu_mem[i].gpuvm_aperture.limit)) { + aperture = &gpu_mem[i].gpuvm_aperture; + break; + } + + else if ((address >= gpu_mem[i].dgpu_aperture.base) && + (address <= gpu_mem[i].dgpu_aperture.limit)) { + aperture = &gpu_mem[i].dgpu_aperture; + break; + } + } + + if (!aperture) + return false; + + pthread_mutex_lock(&aperture->fmm_mutex); + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (object && handle) { + *handle = object->handle; + found = true; + } + pthread_mutex_unlock(&aperture->fmm_mutex); + + + return found; +} diff --git a/src/fmm.h b/src/fmm.h index d326c83f06..c2ff77e074 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -49,6 +49,8 @@ HSAKMT_STATUS fmm_init_process_apertures(void); */ void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes); +void* fmm_allocate_host(uint32_t gpu_id, uint64_t MemorySizeInBytes, + HsaMemFlags flags, uint16_t dev_id); void* fmm_open_graphic_handle(uint32_t gpu_id, int32_t graphic_device_handle, uint32_t graphic_handle, @@ -58,6 +60,7 @@ bool fmm_is_inside_some_aperture(void* address); void fmm_release(void* address, HSAuint64 MemorySizeInBytes); int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); int fmm_unmap_from_gpu(void *address); +bool fmm_get_handle(void *address, uint64_t *handle); /* Topology interface*/ HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); diff --git a/src/globals.c b/src/globals.c index cad6b1f989..d055cecbde 100644 --- a/src/globals.c +++ b/src/globals.c @@ -31,3 +31,4 @@ int kfd_fd; unsigned long kfd_open_count; unsigned long system_properties_count; pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; +bool is_dgpu = false; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 0d73c8fa8b..37584ac66d 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -34,6 +34,7 @@ extern int kfd_fd; extern unsigned long kfd_open_count; extern pthread_mutex_t hsakmt_mutex; +extern bool is_dgpu; #undef HSAKMTAPI #define HSAKMTAPI __attribute__((visibility ("default"))) @@ -65,6 +66,15 @@ HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint16_t get_device_id_by_node(HSAuint32 node_id); +HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id); +HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id); +bool topology_is_dgpu(uint16_t gpu_id); + +HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); + +void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align); +void free_exec_aligned_memory_gpu(void *addr, uint32_t size); + extern int kmtIoctl(int fd, unsigned long request, void *arg); /* Void pointer arithmetic (or remove -Wpointer-arith to allow void pointers arithmetic) */ diff --git a/src/memory.c b/src/memory.c index 7adc4fb914..40020f702f 100644 --- a/src/memory.c +++ b/src/memory.c @@ -86,7 +86,7 @@ hsaKmtSetMemoryPolicy( return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } -static HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags) +HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags) { switch (pageSizeFlags) { case HSA_PAGE_SIZE_4KB: return 4*1024; @@ -109,9 +109,8 @@ hsaKmtAllocMemory( ) { HSAKMT_STATUS result; - HSAuint64 page_size; uint32_t gpu_id; - int err; + HSAuint64 page_size; CHECK_KFD_OPEN(); @@ -119,26 +118,18 @@ hsaKmtAllocMemory( if (result != HSAKMT_STATUS_SUCCESS) return result; - /* The required size should be page aligned (GDS?) */ page_size = PageSizeFromFlags(MemFlags.ui32.PageSize); + if ((!MemoryAddress) || (!SizeInBytes) || - (SizeInBytes & (page_size-1))) + (SizeInBytes & (page_size-1))) { return HSAKMT_STATUS_INVALID_PARAMETER; + } if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { - err = posix_memalign(MemoryAddress, page_size, SizeInBytes); - if (err != 0) - return HSAKMT_STATUS_NO_MEMORY; - - if (MemFlags.ui32.ExecuteAccess) { - err = mprotect(*MemoryAddress, SizeInBytes, - PROT_READ | PROT_WRITE | PROT_EXEC); - - if (err != 0) { - free(*MemoryAddress); - return err; - } - } + *MemoryAddress = fmm_allocate_host(gpu_id, SizeInBytes, MemFlags, + get_device_id_by_node(PreferredNode)); + if (*MemoryAddress == NULL) + return HSAKMT_STATUS_ERROR; return HSAKMT_STATUS_SUCCESS; } @@ -224,6 +215,7 @@ hsaKmtUnmapMemoryToGPU( ) { CHECK_KFD_OPEN(); + if (!fmm_unmap_from_gpu(MemoryAddress)) return HSAKMT_STATUS_SUCCESS; else diff --git a/src/queues.c b/src/queues.c index 5bcbe63087..24b737071d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -34,25 +34,42 @@ #include #include +#define TONGA_PAGE_SIZE 0x9000 + /* 1024 doorbells, 4 bytes each doorbell */ #define DOORBELLS_PAGE_SIZE 1024 * 4 +enum asic_family_type { + CHIP_KAVERI = 0, + CHIP_CARRIZO, + CHIP_TONGA +}; + struct device_info { + enum asic_family_type asic_family; uint32_t ctx_save_restore_size; uint32_t eop_buffer_size; }; struct device_info kaveri_device_info = { + .asic_family = CHIP_KAVERI, .ctx_save_restore_size = 0, .eop_buffer_size = 0, }; struct device_info carrizo_device_info = { + .asic_family = CHIP_CARRIZO, .ctx_save_restore_size = 2756608, .eop_buffer_size = 4096, }; +struct device_info tonga_device_info = { + .asic_family = CHIP_TONGA, + .ctx_save_restore_size = TONGA_PAGE_SIZE, + .eop_buffer_size = TONGA_PAGE_SIZE, +}; + struct device_id { uint16_t dev_id; @@ -87,6 +104,8 @@ struct device_id supported_devices[] = { { 0x9875, &carrizo_device_info }, /* Carrizo */ { 0x9876, &carrizo_device_info }, /* Carrizo */ { 0x9877, &carrizo_device_info }, /* Carrizo */ + { 0x6939, &tonga_device_info }, + { 0x692b, &tonga_device_info }, { 0, NULL } }; @@ -97,6 +116,7 @@ struct queue uint32_t rptr; void *eop_buffer; void *ctx_save_restore; + enum asic_family_type type; }; struct process_doorbells @@ -121,7 +141,7 @@ static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) return NULL; } -static void free_queue(struct queue *q) +static void free_queue_cpu(struct queue *q) { if (q->eop_buffer) free(q->eop_buffer); @@ -130,7 +150,7 @@ static void free_queue(struct queue *q) free(q); } -static void* allocate_exec_aligned_memory(uint32_t size, uint32_t align) +static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) { void *ptr; int retval; @@ -149,13 +169,89 @@ static void* allocate_exec_aligned_memory(uint32_t size, uint32_t align) return ptr; } +void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align) +{ + void *mem; + HSAuint64 gpu_va; + HsaMemFlags flags; + HSAKMT_STATUS ret; + + flags.Value = 0; + flags.ui32.HostAccess = 1; + flags.ui32.ExecuteAccess = 1; + flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + + size += align - (size % align); + + ret = hsaKmtAllocMemory(0, size, flags, &mem); + if (ret != HSAKMT_STATUS_SUCCESS) { + return NULL; + } + if (hsaKmtMapMemoryToGPU(mem, size, &gpu_va) != HSAKMT_STATUS_SUCCESS) { + hsaKmtFreeMemory(mem, size); + return NULL; + } + + return mem; +} + +void free_exec_aligned_memory_gpu(void *addr, uint32_t size) +{ + size += TONGA_PAGE_SIZE - (size % TONGA_PAGE_SIZE); + + if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) { + hsaKmtFreeMemory(addr, size); + } +} + +static void* allocate_exec_aligned_memory(uint32_t size, uint32_t align, enum asic_family_type type) +{ + if (type == CHIP_TONGA) + return allocate_exec_aligned_memory_gpu(size, TONGA_PAGE_SIZE); + return allocate_exec_aligned_memory_cpu(size, align); +} + +static void release_exec_aligned_memory_gpu(void *addr, uint32_t size) +{ + if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) + hsaKmtFreeMemory(addr, (HSAuint64)size); +} + +static void release_exec_aligned_memory(void *addr, uint32_t size, enum asic_family_type type) +{ + if (type == CHIP_TONGA) + release_exec_aligned_memory_gpu(addr, TONGA_PAGE_SIZE); + else + free(addr); +} + +static void free_queue_gpu(struct queue *q) +{ + if (q->eop_buffer) { + hsaKmtUnmapMemoryToGPU(q->eop_buffer); + hsaKmtFreeMemory(q->eop_buffer, TONGA_PAGE_SIZE); + } + if (q->ctx_save_restore) { + hsaKmtUnmapMemoryToGPU(q->ctx_save_restore); + hsaKmtFreeMemory(q->ctx_save_restore, TONGA_PAGE_SIZE); + } + release_exec_aligned_memory((void *)q, sizeof(*q), q->type); +} + +static void free_queue(struct queue *q, enum asic_family_type type) +{ + if (type == CHIP_TONGA) + return free_queue_gpu(q); + return free_queue_cpu(q); +} + static int handle_concrete_asic(struct device_info *dev_info, struct queue *q, struct kfd_ioctl_create_queue_args *args) { if (dev_info) { if (dev_info->eop_buffer_size > 0) { q->eop_buffer = - allocate_exec_aligned_memory(dev_info->eop_buffer_size, PAGE_SIZE); + allocate_exec_aligned_memory(dev_info->eop_buffer_size, PAGE_SIZE, dev_info->asic_family); if (q->eop_buffer == NULL) { return HSAKMT_STATUS_NO_MEMORY; } @@ -165,7 +261,7 @@ static int handle_concrete_asic(struct device_info *dev_info, struct queue *q, if (dev_info->ctx_save_restore_size > 0) { args->ctx_save_restore_size = dev_info->ctx_save_restore_size; q->ctx_save_restore = - allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, PAGE_SIZE); + allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, PAGE_SIZE, dev_info->asic_family); if (q->ctx_save_restore == NULL) {; return HSAKMT_STATUS_NO_MEMORY; } @@ -201,30 +297,35 @@ hsaKmtCreateQueue( if (result != HSAKMT_STATUS_SUCCESS) return result; - struct queue *q = malloc(sizeof(*q)); + dev_id = get_device_id_by_node(NodeId); + dev_info = get_device_info_by_dev_id(dev_id); + + struct queue *q = allocate_exec_aligned_memory(sizeof (*q), + PAGE_SIZE, dev_info->asic_family); if (q == NULL) return HSAKMT_STATUS_NO_MEMORY; + memset(q, 0, sizeof(*q)); struct kfd_ioctl_create_queue_args args; memset(&args, 0, sizeof(args)); - dev_id = get_device_id_by_node(NodeId); - dev_info = get_device_info_by_dev_id(dev_id); args.gpu_id = gpu_id; + q->type = dev_info->asic_family; + err = handle_concrete_asic(dev_info, q, &args); if (err != HSAKMT_STATUS_SUCCESS) { - free_queue(q); + free_queue(q, dev_info->asic_family); return err; } switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: free_queue(q); return HSAKMT_STATUS_UNAVAILABLE; + case HSA_QUEUE_SDMA: free_queue(q, dev_info->asic_family); return HSAKMT_STATUS_UNAVAILABLE; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; - default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; + default: free_queue(q, dev_info->asic_family); return HSAKMT_STATUS_INVALID_PARAMETER; } if (Type != HSA_QUEUE_COMPUTE_AQL) @@ -244,7 +345,7 @@ hsaKmtCreateQueue( if (err == -1) { - free_queue(q); + free_queue(q, dev_info->asic_family); return HSAKMT_STATUS_ERROR; } @@ -259,7 +360,7 @@ hsaKmtCreateQueue( if (ptr == MAP_FAILED) { pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); hsaKmtDestroyQueue(q->queue_id); - free_queue(q); + free_queue(q, dev_info->asic_family); return HSAKMT_STATUS_ERROR; } @@ -321,7 +422,7 @@ hsaKmtDestroyQueue( struct kfd_ioctl_destroy_queue_args args; if (q == NULL) - return (HSAKMT_STATUS_INVALID_PARAMETER); + return (HSAKMT_STATUS_INVALID_PARAMETER); memset(&args, 0, sizeof(args)); @@ -335,7 +436,7 @@ hsaKmtDestroyQueue( } else { - free_queue(q); + free_queue(q, q->type); return HSAKMT_STATUS_SUCCESS; } } diff --git a/src/topology.c b/src/topology.c index 523df0d8da..608efa2d71 100644 --- a/src/topology.c +++ b/src/topology.c @@ -59,36 +59,40 @@ static struct hsa_gfxip_table { unsigned char major; // GFXIP Major engine version unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info + unsigned char is_dgpu; // Predicat for dGPU devices } gfxip_lookup_table[] = { /* Kaveri Family */ - { 0x1304, 7, 0, 0 }, - { 0x1305, 7, 0, 0 }, - { 0x1306, 7, 0, 0 }, - { 0x1307, 7, 0, 0 }, - { 0x1309, 7, 0, 0 }, - { 0x130A, 7, 0, 0 }, - { 0x130B, 7, 0, 0 }, - { 0x130C, 7, 0, 0 }, - { 0x130D, 7, 0, 0 }, - { 0x130E, 7, 0, 0 }, - { 0x130F, 7, 0, 0 }, - { 0x1310, 7, 0, 0 }, - { 0x1311, 7, 0, 0 }, - { 0x1312, 7, 0, 0 }, - { 0x1313, 7, 0, 0 }, - { 0x1315, 7, 0, 0 }, - { 0x1316, 7, 0, 0 }, - { 0x1317, 7, 0, 0 }, - { 0x1318, 7, 0, 0 }, - { 0x131B, 7, 0, 0 }, - { 0x131C, 7, 0, 0 }, - { 0x131D, 7, 0, 0 }, + { 0x1304, 7, 0, 0, 0 }, + { 0x1305, 7, 0, 0, 0 }, + { 0x1306, 7, 0, 0, 0 }, + { 0x1307, 7, 0, 0, 0 }, + { 0x1309, 7, 0, 0, 0 }, + { 0x130A, 7, 0, 0, 0 }, + { 0x130B, 7, 0, 0, 0 }, + { 0x130C, 7, 0, 0, 0 }, + { 0x130D, 7, 0, 0, 0 }, + { 0x130E, 7, 0, 0, 0 }, + { 0x130F, 7, 0, 0, 0 }, + { 0x1310, 7, 0, 0, 0 }, + { 0x1311, 7, 0, 0, 0 }, + { 0x1312, 7, 0, 0, 0 }, + { 0x1313, 7, 0, 0, 0 }, + { 0x1315, 7, 0, 0, 0 }, + { 0x1316, 7, 0, 0, 0 }, + { 0x1317, 7, 0, 0, 0 }, + { 0x1318, 7, 0, 0, 0 }, + { 0x131B, 7, 0, 0, 0 }, + { 0x131C, 7, 0, 0, 0 }, + { 0x131D, 7, 0, 0, 0 }, /* Carrizo Family */ - { 0x9870, 8, 0, 1 }, - { 0x9874, 8, 0, 1 }, - { 0x9875, 8, 0, 1 }, - { 0x9876, 8, 0, 1 }, - { 0x9877, 8, 0, 1 } + { 0x9870, 8, 0, 1, 0 }, + { 0x9874, 8, 0, 1, 0 }, + { 0x9875, 8, 0, 1, 0 }, + { 0x9876, 8, 0, 1, 0 }, + { 0x9877, 8, 0, 1, 0 }, + /* Tonga Family */ + { 0x6939, 8, 0, 0, 1 }, + { 0x692b, 8, 0, 0, 1 } }; static void @@ -203,7 +207,7 @@ err1: return ret; } -static HSAKMT_STATUS +HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) { FILE *fd; char path[256]; @@ -222,7 +226,25 @@ topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) { return ret; } -static HSAKMT_STATUS +bool topology_is_dgpu(uint16_t gpu_id) +{ + uint32_t i, table_size; + + if (is_dgpu) + return is_dgpu; + + table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); + for (i=0; i Date: Fri, 28 Aug 2015 15:49:24 -0400 Subject: [PATCH 0066/1247] libhsakmt: specify build output via variable Makefile currently sends build output a default location. Allow choice of build output location if so desired using a variable. Signed-off-by: David Ogbeide --- src/Makefile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 5608ab7927..f3bbe91fca 100644 --- a/src/Makefile +++ b/src/Makefile @@ -26,9 +26,16 @@ OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ # Default target all: lnx lnx64a -BUILD_ROOT = ../build -BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) +# To relocate build output, set "O=target/dir/" +# otherwise it defaults to a local directory +ifdef O + BUILD_ROOT := $(O) +else + BUILD_ROOT := ../build +endif + +BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) TARGET = $(addprefix $(BUILDDIR)/,$(OBJS)) $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER): $(TARGET) From ee08f537a7552e92d92da55e5e528af0f157050c Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 14 Sep 2015 12:18:48 -0400 Subject: [PATCH 0067/1247] Topology, memory allocation, cleanup issue for gGPU Patch submitted by Besar Wicaksono 1. Bug on detecting local memory size interpreted as 32 bit value instead of 64. The bug causes thunk to go into an infinite loop trying to reserve virtual address range for dgpu system memory. 2. SIMD count in the node property is 0. Runtime use this attribute to find a gpu device. Regarding other attributes of intel+tonga topology, Harish started a discussion on August iirc, could you please share an update ? This would help me progress with more tests such as scratch memory, which require the scratch aperture information in order to construct a buffer srd in gpuvm space. 3. Bug on releasing memory via fmm_release, where no actual release is being done. The vm_object can't be found because the memory size does not match due to the allocation padded the size with 32KB. 4. Pointer arithmetic on vm_area allocation/release. The value of vm_area_t::end seems to be interpreted inconsistently whether it is (start + size -1) or (start + size). One example of potential issue I see is the logic could report larger size of the hole in the vm area list. 5. Resource cleanup on multiple library load/unload within a single process. - Any memory allocation on subsequent library load will result an error "va above limit". To my understanding this is due to the reserved memory for the system memory not being released on unload. - The static variable events_page needs to be invalidated appropriately on library unload so the next load could reinitialize it. 6. Could you please update if AQL queue is ready to test with the stg kfd/kmt ? 7. The system memory allocation with size larger than 32KB seems to be padded by an extra 32KB. I was wondering if we could remove this overhead. Change-Id: I039988d36637525089c7569dc3b77e58750e2121 --- src/events.c | 7 +++++++ src/fmm.c | 55 +++++++++++++++++++++++++++++++++++++++++++++---- src/fmm.h | 2 ++ src/openclose.c | 4 ++++ src/topology.c | 10 ++++++--- 5 files changed, 71 insertions(+), 7 deletions(-) diff --git a/src/events.c b/src/events.c index 7cedacb459..747534fa78 100644 --- a/src/events.c +++ b/src/events.c @@ -42,6 +42,13 @@ static bool IsSystemEventType(HSA_EVENTTYPE type) return (type != HSA_EVENTTYPE_SIGNAL && type != HSA_EVENTTYPE_DEBUG_EVENT); } +void CleanupEvent(void) +{ + // Release is handled during aperture cleanup. + free_exec_aligned_memory_gpu(events_page, KFD_SIGNAL_EVENT_LIMIT * 8); + events_page = NULL; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent( diff --git a/src/fmm.c b/src/fmm.c index df864779ea..cfd24649e4 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -98,6 +98,7 @@ typedef struct { static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit); +static HSAKMT_STATUS dgpu_mem_release(void); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); static void __fmm_release(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); @@ -309,7 +310,7 @@ static void *aperture_allocate_area(manageble_aperture_t *app, break; /* address space "hole" */ - if ((VOID_PTRS_SUB(next->start, cur->end) >= + if (((VOID_PTRS_SUB(next->start, cur->end) - 1) >= MemorySizeInBytes)) break; @@ -317,7 +318,7 @@ static void *aperture_allocate_area(manageble_aperture_t *app, }; /* If the new range is inside the reserved aperture */ - if (VOID_PTRS_SUB(app->limit, cur->end) + 1 >= + if (VOID_PTRS_SUB(app->limit, cur->end) >= MemorySizeInBytes) { /* * cur points to the last inspected element: the tail @@ -571,6 +572,7 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, #define GPUVM_APP_OFFSET 0x10000 void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) { + MemorySizeInBytes += 0x8000 - (MemorySizeInBytes % 0x8000); manageble_aperture_t *aperture; int32_t gpu_mem_id; @@ -616,6 +618,7 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, manageble_aperture_t *aperture; int32_t gpu_mem_id; uint64_t mmap_offset; + static size_t total_mem = 0; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -637,6 +640,9 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, return NULL; } + total_mem += MemorySizeInBytes; + printf("total_mem: %lu\n", total_mem); + return ret; } @@ -756,6 +762,7 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= gpu_mem[i].dgpu_aperture.base && address <= gpu_mem[i].dgpu_aperture.limit) { found = true; + MemorySizeInBytes += 0x8000 - (MemorySizeInBytes % 0x8000); __fmm_release(gpu_mem[i].gpu_id, address, MemorySizeInBytes, &gpu_mem[i].dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); @@ -817,6 +824,32 @@ HSAKMT_STATUS fmm_init_process_apertures(void) return HSAKMT_STATUS_SUCCESS; } +HSAKMT_STATUS fmm_cleanup_process_apertures(void) +{ + uint32_t i; + struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; + struct kfd_ioctl_free_memory_of_gpu_args free_args; + + for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { + vm_object_t *cur = gpu_mem[i].dgpu_aperture.vm_objects; + pthread_mutex_lock(&gpu_mem[i].dgpu_aperture.fmm_mutex); + while (cur) { + unmap_args.handle = cur->handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); + free_args.handle = cur->handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args); + aperture_release_area(&gpu_mem[i].dgpu_aperture, cur->start, cur->size); + vm_object_t* temp = cur; + cur = temp->next; + vm_remove_object(&gpu_mem[i].dgpu_aperture, temp); + }; + pthread_mutex_unlock(&gpu_mem[i].dgpu_aperture.fmm_mutex); + } + + dgpu_mem_release(); + return HSAKMT_STATUS_SUCCESS; +} + HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id) { int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); @@ -1060,7 +1093,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) bool found; HSAKMT_STATUS ret; void *addr, *ret_addr; - uint32_t max_len; + HSAuint64 max_len; long long unsigned int temp; uint32_t gpu_id; HsaNodeProperties props; @@ -1077,7 +1110,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) if (ret != HSAKMT_STATUS_SUCCESS) return ret; - max_len = (uint32_t)props.LocalMemSize; + max_len = props.LocalMemSize; found = false; for (addr = (void *)PAGE_SIZE, ret_addr = NULL; @@ -1110,6 +1143,20 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) return HSAKMT_STATUS_ERROR; } +static HSAKMT_STATUS dgpu_mem_release(void) +{ + size_t max_len; + if (is_dgpu_mem_init) { + max_len = (size_t)dgpu_shared_aperture_limit - + (size_t)dgpu_shared_aperture_base; + munmap(dgpu_shared_aperture_base, max_len); + is_dgpu_mem_init = false; + dgpu_shared_aperture_base = NULL; + dgpu_shared_aperture_limit = NULL; + } + return HSAKMT_STATUS_SUCCESS; +} + bool fmm_get_handle(void *address, uint64_t *handle) { int32_t i; diff --git a/src/fmm.h b/src/fmm.h index c2ff77e074..30af235170 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -43,7 +43,9 @@ typedef struct { void* start_address; } aperture_properties_t; +void CleanupEvent(void); HSAKMT_STATUS fmm_init_process_apertures(void); +HSAKMT_STATUS fmm_cleanup_process_apertures(void); /* * Memory interface */ diff --git a/src/openclose.c b/src/openclose.c index d5b91e23dd..47aef0df7a 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "fmm.h" static const char kfd_device_name[] = "/dev/kfd"; @@ -90,6 +91,9 @@ hsaKmtCloseKFD(void) { if (--kfd_open_count == 0) { + printf("cleanup aperture\n"); + CleanupEvent(); + fmm_cleanup_process_apertures(); close(kfd_fd); if (amd_hsa_thunk_lock_fd > 0) { diff --git a/src/topology.c b/src/topology.c index 608efa2d71..f367428dd0 100644 --- a/src/topology.c +++ b/src/topology.c @@ -319,8 +319,12 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { if (strcmp(prop_name,"cpu_cores_count") == 0) props->NumCPUCores = (uint32_t)prop_val; - else if (strcmp(prop_name,"simd_count") == 0) - props->NumFComputeCores = (uint32_t)prop_val; + else if (strcmp(prop_name,"simd_count") == 0) { + if ((uint32_t)prop_val == 0) + props->NumFComputeCores = (uint32_t)1; + else + props->NumFComputeCores = (uint32_t)prop_val; + } else if (strcmp(prop_name,"mem_banks_count") == 0) props->NumMemoryBanks = (uint32_t)prop_val; else if (strcmp(prop_name,"caches_count") == 0) @@ -364,7 +368,7 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 else if (strcmp(prop_name,"max_engine_clk_ccompute") == 0) props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; else if (strcmp(prop_name,"local_mem_size") == 0) - props->LocalMemSize = (uint32_t)prop_val; + props->LocalMemSize = (HSAuint64)prop_val; } From 3e9773ff2c9dddc459195e20d6274c843c6483ac Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 21 Sep 2015 10:40:11 -0400 Subject: [PATCH 0068/1247] Revert "Topology, memory allocation, cleanup issue for gGPU" This reverts commit ee08f537a7552e92d92da55e5e528af0f157050c. Change-Id: I92a4ed91bf566259916d1a96207e1fe9a6099c31 Signed-off-by: Harish Kasiviswanathan --- src/events.c | 7 ------- src/fmm.c | 55 ++++--------------------------------------------- src/fmm.h | 2 -- src/openclose.c | 4 ---- src/topology.c | 10 +++------ 5 files changed, 7 insertions(+), 71 deletions(-) diff --git a/src/events.c b/src/events.c index 747534fa78..7cedacb459 100644 --- a/src/events.c +++ b/src/events.c @@ -42,13 +42,6 @@ static bool IsSystemEventType(HSA_EVENTTYPE type) return (type != HSA_EVENTTYPE_SIGNAL && type != HSA_EVENTTYPE_DEBUG_EVENT); } -void CleanupEvent(void) -{ - // Release is handled during aperture cleanup. - free_exec_aligned_memory_gpu(events_page, KFD_SIGNAL_EVENT_LIMIT * 8); - events_page = NULL; -} - HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent( diff --git a/src/fmm.c b/src/fmm.c index cfd24649e4..df864779ea 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -98,7 +98,6 @@ typedef struct { static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit); -static HSAKMT_STATUS dgpu_mem_release(void); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); static void __fmm_release(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); @@ -310,7 +309,7 @@ static void *aperture_allocate_area(manageble_aperture_t *app, break; /* address space "hole" */ - if (((VOID_PTRS_SUB(next->start, cur->end) - 1) >= + if ((VOID_PTRS_SUB(next->start, cur->end) >= MemorySizeInBytes)) break; @@ -318,7 +317,7 @@ static void *aperture_allocate_area(manageble_aperture_t *app, }; /* If the new range is inside the reserved aperture */ - if (VOID_PTRS_SUB(app->limit, cur->end) >= + if (VOID_PTRS_SUB(app->limit, cur->end) + 1 >= MemorySizeInBytes) { /* * cur points to the last inspected element: the tail @@ -572,7 +571,6 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, #define GPUVM_APP_OFFSET 0x10000 void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) { - MemorySizeInBytes += 0x8000 - (MemorySizeInBytes % 0x8000); manageble_aperture_t *aperture; int32_t gpu_mem_id; @@ -618,7 +616,6 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, manageble_aperture_t *aperture; int32_t gpu_mem_id; uint64_t mmap_offset; - static size_t total_mem = 0; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -640,9 +637,6 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, return NULL; } - total_mem += MemorySizeInBytes; - printf("total_mem: %lu\n", total_mem); - return ret; } @@ -762,7 +756,6 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= gpu_mem[i].dgpu_aperture.base && address <= gpu_mem[i].dgpu_aperture.limit) { found = true; - MemorySizeInBytes += 0x8000 - (MemorySizeInBytes % 0x8000); __fmm_release(gpu_mem[i].gpu_id, address, MemorySizeInBytes, &gpu_mem[i].dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); @@ -824,32 +817,6 @@ HSAKMT_STATUS fmm_init_process_apertures(void) return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS fmm_cleanup_process_apertures(void) -{ - uint32_t i; - struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; - struct kfd_ioctl_free_memory_of_gpu_args free_args; - - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { - vm_object_t *cur = gpu_mem[i].dgpu_aperture.vm_objects; - pthread_mutex_lock(&gpu_mem[i].dgpu_aperture.fmm_mutex); - while (cur) { - unmap_args.handle = cur->handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); - free_args.handle = cur->handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args); - aperture_release_area(&gpu_mem[i].dgpu_aperture, cur->start, cur->size); - vm_object_t* temp = cur; - cur = temp->next; - vm_remove_object(&gpu_mem[i].dgpu_aperture, temp); - }; - pthread_mutex_unlock(&gpu_mem[i].dgpu_aperture.fmm_mutex); - } - - dgpu_mem_release(); - return HSAKMT_STATUS_SUCCESS; -} - HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id) { int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); @@ -1093,7 +1060,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) bool found; HSAKMT_STATUS ret; void *addr, *ret_addr; - HSAuint64 max_len; + uint32_t max_len; long long unsigned int temp; uint32_t gpu_id; HsaNodeProperties props; @@ -1110,7 +1077,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) if (ret != HSAKMT_STATUS_SUCCESS) return ret; - max_len = props.LocalMemSize; + max_len = (uint32_t)props.LocalMemSize; found = false; for (addr = (void *)PAGE_SIZE, ret_addr = NULL; @@ -1143,20 +1110,6 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) return HSAKMT_STATUS_ERROR; } -static HSAKMT_STATUS dgpu_mem_release(void) -{ - size_t max_len; - if (is_dgpu_mem_init) { - max_len = (size_t)dgpu_shared_aperture_limit - - (size_t)dgpu_shared_aperture_base; - munmap(dgpu_shared_aperture_base, max_len); - is_dgpu_mem_init = false; - dgpu_shared_aperture_base = NULL; - dgpu_shared_aperture_limit = NULL; - } - return HSAKMT_STATUS_SUCCESS; -} - bool fmm_get_handle(void *address, uint64_t *handle) { int32_t i; diff --git a/src/fmm.h b/src/fmm.h index 30af235170..c2ff77e074 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -43,9 +43,7 @@ typedef struct { void* start_address; } aperture_properties_t; -void CleanupEvent(void); HSAKMT_STATUS fmm_init_process_apertures(void); -HSAKMT_STATUS fmm_cleanup_process_apertures(void); /* * Memory interface */ diff --git a/src/openclose.c b/src/openclose.c index 47aef0df7a..d5b91e23dd 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -30,7 +30,6 @@ #include #include #include -#include #include "fmm.h" static const char kfd_device_name[] = "/dev/kfd"; @@ -91,9 +90,6 @@ hsaKmtCloseKFD(void) { if (--kfd_open_count == 0) { - printf("cleanup aperture\n"); - CleanupEvent(); - fmm_cleanup_process_apertures(); close(kfd_fd); if (amd_hsa_thunk_lock_fd > 0) { diff --git a/src/topology.c b/src/topology.c index f367428dd0..608efa2d71 100644 --- a/src/topology.c +++ b/src/topology.c @@ -319,12 +319,8 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { if (strcmp(prop_name,"cpu_cores_count") == 0) props->NumCPUCores = (uint32_t)prop_val; - else if (strcmp(prop_name,"simd_count") == 0) { - if ((uint32_t)prop_val == 0) - props->NumFComputeCores = (uint32_t)1; - else - props->NumFComputeCores = (uint32_t)prop_val; - } + else if (strcmp(prop_name,"simd_count") == 0) + props->NumFComputeCores = (uint32_t)prop_val; else if (strcmp(prop_name,"mem_banks_count") == 0) props->NumMemoryBanks = (uint32_t)prop_val; else if (strcmp(prop_name,"caches_count") == 0) @@ -368,7 +364,7 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 else if (strcmp(prop_name,"max_engine_clk_ccompute") == 0) props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; else if (strcmp(prop_name,"local_mem_size") == 0) - props->LocalMemSize = (HSAuint64)prop_val; + props->LocalMemSize = (uint32_t)prop_val; } From 692e004047f41284035562f8f6ebe572294cb634 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 9 Sep 2015 14:41:28 +0300 Subject: [PATCH 0069/1247] Adding new memory allocation IOCTL Change-Id: I0eb1924811a2e1e436296ebe632d8f112a61637d Signed-off-by: Ben Goz --- include/linux/kfd_ioctl.h | 23 ++++++++++++++++++++++- src/fmm.c | 21 +++++++++++++-------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 39fd4d5be4..ca86772f33 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -279,6 +279,24 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { uint64_t dgpu_limit; }; +/* + * GPU Memory allocation flags + */ +#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST (1 << 0) +#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE (1 << 1) +#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_SCRATCH (1 << 2) +#define KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE (1 << 3) +#define KFD_IOC_ALLOC_MEM_FLAGS_APU_SCRATCH (1 << 4) + +struct kfd_ioctl_alloc_memory_of_gpu_new_args { + uint64_t va_addr; /* to KFD */ + uint64_t size; /* to KFD */ + uint64_t handle; /* from KFD */ + uint32_t gpu_id; /* to KFD */ + uint64_t mmap_offset; /* from KFD */ + uint32_t flags; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -357,8 +375,11 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { #define AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE \ AMDKFD_IOW(0x18, struct kfd_ioctl_set_process_dgpu_aperture_args) +#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW \ + AMDKFD_IOWR(0x19, struct kfd_ioctl_alloc_memory_of_gpu_new_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x19 +#define AMDKFD_COMMAND_END 0x20 #endif diff --git a/src/fmm.c b/src/fmm.c index df864779ea..15dd536057 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -382,9 +382,10 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture, - uint64_t *mmap_offset) + uint64_t *mmap_offset, + uint32_t flags) { - struct kfd_ioctl_alloc_memory_of_gpu_args args; + struct kfd_ioctl_alloc_memory_of_gpu_new_args args; struct kfd_ioctl_free_memory_of_gpu_args free_args; if (!mem) @@ -394,11 +395,12 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, args.gpu_id = gpu_id; args.size = MemorySizeInBytes; + args.flags = flags; args.va_addr = (uint64_t)mem; - if (!mmap_offset) + if (!mmap_offset && !is_dgpu) args.va_addr = VOID_PTRS_SUB(mem, aperture->base); - if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW, &args)) return -1; /* Allocate object */ @@ -530,7 +532,8 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) } static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, - manageble_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset) + manageble_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset, + uint32_t flags) { void *mem = NULL; /* Check that aperture is properly initialized/supported */ @@ -548,7 +551,7 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, * itself */ if (fmm_allocate_memory_in_device(gpu_id, mem, - MemorySizeInBytes, aperture, mmap_offset)) { + MemorySizeInBytes, aperture, mmap_offset, flags)) { /* * allocation of memory in device failed. * Release region in aperture @@ -582,7 +585,8 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; return __fmm_allocate_device(gpu_id, MemorySizeInBytes, - aperture, GPUVM_APP_OFFSET, NULL); + aperture, GPUVM_APP_OFFSET, NULL, + KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); } static void* fmm_allocate_host_cpu(uint32_t gpu_id, @@ -627,7 +631,8 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, MemorySizeInBytes += 0x8000 - (MemorySizeInBytes % 0x8000); mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, - aperture, 0, &mmap_offset); + aperture, 0, &mmap_offset, + KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST); void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE | PROT_EXEC, From 6170080cf6291dfa4bddfff99f58e0acb9b19111 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 9 Sep 2015 14:41:49 +0300 Subject: [PATCH 0070/1247] Adding support for local memory on dGPU Change-Id: I1a926b11730ba295605eeb37c9b1fc438bed8a64 Signed-off-by: Ben Goz --- src/fmm.c | 47 ++++++++++++++++++++++++++++++++--------------- src/libhsakmt.h | 1 + src/topology.c | 17 ++++++++++++++--- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 15dd536057..ed7acd6399 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -397,7 +397,7 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, args.flags = flags; args.va_addr = (uint64_t)mem; - if (!mmap_offset && !is_dgpu) + if (flags == KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE) args.va_addr = VOID_PTRS_SUB(mem, aperture->base); if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW, &args)) @@ -576,6 +576,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) { manageble_aperture_t *aperture; int32_t gpu_mem_id; + uint32_t flags; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -584,9 +585,16 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; + flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; + if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { + flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; + /* Alignment is needed to match a workaround for a VI HW bug in the kernel */ + MemorySizeInBytes = (MemorySizeInBytes + 0x7fffULL) & ~0x7fffULL; + } + return __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, GPUVM_APP_OFFSET, NULL, - KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); + flags); } static void* fmm_allocate_host_cpu(uint32_t gpu_id, @@ -628,7 +636,8 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; - MemorySizeInBytes += 0x8000 - (MemorySizeInBytes % 0x8000); + /* Alignment is needed to match a workaround for a VI HW bug in the kernel */ + MemorySizeInBytes = (MemorySizeInBytes + 0x7fffULL) & ~0x7fffULL; mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, 0, &mmap_offset, @@ -789,17 +798,6 @@ HSAKMT_STATUS fmm_init_process_apertures(void) gpu_mem[node_id].gpu_id = args.process_apertures[node_id].gpu_id; - - if (topology_sysfs_get_node_props(node_id, &props, &gpu_id) == - HSAKMT_STATUS_SUCCESS) { - if (topology_is_dgpu(props.DeviceId)) { - dgpu_mem_init(node_id, &gpu_mem[node_id].dgpu_aperture.base, - &gpu_mem[node_id].dgpu_aperture.limit); - set_dgpu_aperture(node_id, (uint64_t)gpu_mem[node_id].dgpu_aperture.base, - (uint64_t)gpu_mem[node_id].dgpu_aperture.limit); - } - } - gpu_mem[node_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); @@ -817,6 +815,21 @@ HSAKMT_STATUS fmm_init_process_apertures(void) gpu_mem[node_id].scratch_aperture.limit = PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_limit); + + if (topology_sysfs_get_node_props(node_id, &props, &gpu_id) == + HSAKMT_STATUS_SUCCESS) { + if (topology_is_dgpu(props.DeviceId)) { + dgpu_mem_init(node_id, &gpu_mem[node_id].dgpu_aperture.base, + &gpu_mem[node_id].dgpu_aperture.limit); + set_dgpu_aperture(node_id, (uint64_t)gpu_mem[node_id].dgpu_aperture.base, + (uint64_t)gpu_mem[node_id].dgpu_aperture.limit); + gpu_mem[node_id].gpuvm_aperture.base = gpu_mem[node_id].dgpu_aperture.limit; + gpu_mem[node_id].gpuvm_aperture.limit = (void *)VOID_PTRS_SUB(gpu_mem[node_id].dgpu_aperture.limit, + gpu_mem[node_id].dgpu_aperture.base); + gpu_mem[node_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[node_id].gpuvm_aperture.limit, + (unsigned long)gpu_mem[node_id].gpuvm_aperture.base); + } + } } return HSAKMT_STATUS_SUCCESS; @@ -935,7 +948,11 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, pthread_mutex_unlock(&aperture->fmm_mutex); - *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); + if (gpuvm_address) { + *gpuvm_address = (uint64_t)object->start; + if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) + *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); + } return 0; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 37584ac66d..a1399fc65d 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -65,6 +65,7 @@ extern bool is_dgpu; HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint16_t get_device_id_by_node(HSAuint32 node_id); +uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id); diff --git a/src/topology.c b/src/topology.c index 608efa2d71..7ae6793325 100644 --- a/src/topology.c +++ b/src/topology.c @@ -230,9 +230,6 @@ bool topology_is_dgpu(uint16_t gpu_id) { uint32_t i, table_size; - if (is_dgpu) - return is_dgpu; - table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); for (i=0; iNumNodes; i++) { + if (node[i].gpu_id == gpu_id) + return node[i].node.DeviceId; + } + + return 0; +} + #if 0 static int get_cpu_stepping(uint16_t* stepping) { From f7fffdc2be6e08d4453f40867ca1a4a3a4b02571 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 22 Sep 2015 15:04:43 -0400 Subject: [PATCH 0071/1247] Enable GFXIP version info for dGPU Add GFXIP version 8.0.2(major.minor.stepping) for gfx802 and 8.0.3 for gfx803. Change-Id: Icc7cac6b2e8a78d9cff4105aeb2bfcd2c7759027 --- src/topology.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/topology.c b/src/topology.c index 7ae6793325..c762c70006 100644 --- a/src/topology.c +++ b/src/topology.c @@ -91,8 +91,17 @@ static struct hsa_gfxip_table { { 0x9876, 8, 0, 1, 0 }, { 0x9877, 8, 0, 1, 0 }, /* Tonga Family */ - { 0x6939, 8, 0, 0, 1 }, - { 0x692b, 8, 0, 0, 1 } + { 0x6920, 8, 0, 2, 1 }, + { 0x6921, 8, 0, 2, 1 }, + { 0x6928, 8, 0, 2, 1 }, + { 0x6929, 8, 0, 2, 1 }, + { 0x692B, 8, 0, 2, 1 }, + { 0x692F, 8, 0, 2, 1 }, + { 0x6930, 8, 0, 2, 1 }, + { 0x6938, 8, 0, 2, 1 }, + { 0x6939, 8, 0, 2, 1 }, + /* Fiji */ + { 0x7300, 8, 0, 3, 1 } }; static void From 4b768872c0c06142b9435c580b4b1bcdeab415af Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 23 Sep 2015 13:15:19 -0400 Subject: [PATCH 0072/1247] Fix mem size variable type Memory size is 64-bit. So use HSAuint64 instead of uint32. Change-Id: Iaa607dec9c1a1c5ac46ea442fd482210ea550b45 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 4 ++-- src/topology.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ed7acd6399..61b540a1d8 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1082,7 +1082,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) bool found; HSAKMT_STATUS ret; void *addr, *ret_addr; - uint32_t max_len; + HSAuint64 max_len; long long unsigned int temp; uint32_t gpu_id; HsaNodeProperties props; @@ -1099,7 +1099,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) if (ret != HSAKMT_STATUS_SUCCESS) return ret; - max_len = (uint32_t)props.LocalMemSize; + max_len = props.LocalMemSize; found = false; for (addr = (void *)PAGE_SIZE, ret_addr = NULL; diff --git a/src/topology.c b/src/topology.c index c762c70006..0d08018bb7 100644 --- a/src/topology.c +++ b/src/topology.c @@ -370,7 +370,7 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 else if (strcmp(prop_name,"max_engine_clk_ccompute") == 0) props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; else if (strcmp(prop_name,"local_mem_size") == 0) - props->LocalMemSize = (uint32_t)prop_val; + props->LocalMemSize = prop_val; } From 1438f15fd06f0ee8978a13e7fa24c1795cbfd419 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 23 Sep 2015 15:08:33 -0400 Subject: [PATCH 0073/1247] Fix VM range for dGPU local memory Currently, Kernel imposes a limit on VM. Thunk should be aware of it. This fix is required till Kernel VM limit is sorted. For now both "Host Access" memory and "Local Memory" share the same VM range. Change-Id: I5a9220face20df9ede2b78bd6201a01dd2ea70e0 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 61b540a1d8..652b640482 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -482,6 +482,8 @@ void fmm_print(uint32_t gpu_id) manageble_aperture_print(&gpu_mem[i].gpuvm_aperture); printf("Scratch aperture:\n"); manageble_aperture_print(&gpu_mem[i].scratch_aperture); + printf("dGPU aperture:\n"); + manageble_aperture_print(&gpu_mem[i].dgpu_aperture); } } #else @@ -583,13 +585,18 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) if (gpu_mem_id < 0) return NULL; - aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; - - flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; /* Alignment is needed to match a workaround for a VI HW bug in the kernel */ MemorySizeInBytes = (MemorySizeInBytes + 0x7fffULL) & ~0x7fffULL; + /* + * TODO: Once VA limit is raised from 0x200000000 (8GB) use gpuvm_aperture. + * In that way the host access range won't be used for local memory + */ + aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; + } else { + flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; + aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } return __fmm_allocate_device(gpu_id, MemorySizeInBytes, @@ -1076,13 +1083,36 @@ static void *reserve_address(void *addr, long long unsigned int len) } #define ADDRESS_RANGE_LIMIT_MASK 0xFFFFFFFFFF +#define AMDGPU_SYSFS_VM_SIZE "/sys/module/amdgpu/parameters/vm_size" + +/* + * TODO: Provide a cleaner interface via topology + */ +static HSAKMT_STATUS get_dgpu_vm_limit(uint32_t *vm_size_in_gb) +{ + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + fd = fopen(AMDGPU_SYSFS_VM_SIZE, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + if (fscanf(fd, "%ul", vm_size_in_gb) != 1) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + +err: + fclose(fd); + return ret; +} static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) { bool found; HSAKMT_STATUS ret; void *addr, *ret_addr; - HSAuint64 max_len; + HSAuint64 max_len, max_vm_limit; + uint32_t max_vm_limit_in_gb; long long unsigned int temp; uint32_t gpu_id; HsaNodeProperties props; @@ -1122,9 +1152,19 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) if (base) *base = ret_addr; dgpu_shared_aperture_base = ret_addr; + + ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); + if (ret != HSAKMT_STATUS_SUCCESS) { + printf("Error! Unable to find vm_size for gGPU\n"); + return ret; + } + max_vm_limit = (HSAuint64)max_vm_limit_in_gb << 30; + if (((long long unsigned int)ret_addr + max_len) < max_vm_limit) + max_vm_limit = ((long long unsigned int)ret_addr + max_len); + if (limit) - *limit = (void *)((long long unsigned int)ret_addr + max_len); - dgpu_shared_aperture_limit = (void *)((long long unsigned int)ret_addr + max_len); + *limit = (void *)max_vm_limit; + dgpu_shared_aperture_limit = (void *)max_vm_limit; is_dgpu_mem_init = true; return HSAKMT_STATUS_SUCCESS; } From 082f8314c48c089fed482b23bf2a42112015fdd2 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 24 Sep 2015 00:00:39 -0400 Subject: [PATCH 0074/1247] Sync up HSA_ENGINE_ID type with Windows/Perforce HSA_ENGINE_ID in Perforce added ui32 to the typedef while in Git it doesn't. This causes conflicts to RT applications. Decision being made is to change Git to match Perforce. Change-Id: I7e9c6437b023bb23ec9578737f8534e9453589b9 --- include/hsakmttypes.h | 2 +- src/topology.c | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 0133fdfd9b..25e7ad0832 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -157,7 +157,7 @@ typedef union unsigned int Major : 6; // GFXIP Major engine version unsigned int Minor : 8; // GFXIP Minor engine version unsigned int Stepping : 8; // GFXIP Stepping info - }; + }ui32; } HSA_ENGINE_ID; typedef union diff --git a/src/topology.c b/src/topology.c index 0d08018bb7..1dcbf0cfeb 100644 --- a/src/topology.c +++ b/src/topology.c @@ -375,20 +375,20 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 } // get_cpu_stepping(&stepping); - props->EngineId.uCode = fw_version & 0x3ff; - props->EngineId.Major = 0; - props->EngineId.Minor = 0; - props->EngineId.Stepping = 0; + props->EngineId.ui32.uCode = fw_version & 0x3ff; + props->EngineId.ui32.Major = 0; + props->EngineId.ui32.Minor = 0; + props->EngineId.ui32.Stepping = 0; table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); for (i=0; iDeviceId) { - props->EngineId.Major = gfxip_lookup_table[i].major & 0x3f; - props->EngineId.Minor = gfxip_lookup_table[i].minor; - props->EngineId.Stepping = gfxip_lookup_table[i].stepping; + props->EngineId.ui32.Major = gfxip_lookup_table[i].major & 0x3f; + props->EngineId.ui32.Minor = gfxip_lookup_table[i].minor; + props->EngineId.ui32.Stepping = gfxip_lookup_table[i].stepping; break; } } - assert(props->EngineId.Major); + assert(props->EngineId.ui32.Major); //TODO: error handler when Device ID lookup fails err2: From 2d63ee7b8f45053317bd208cfdb9dcb3039b0adc Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Mon, 21 Sep 2015 17:33:23 -0400 Subject: [PATCH 0075/1247] Initiali support for CWSR on thunk 1. Add IOCTL defines to set trap handler 2. Add control stack size information on create queue argument. 3. Increase the total save&restore area size for carrizo to include the control stack size. Signed-off-by: Shaoyun Liu Change-Id: Iccf15e073b7db2519e96e7f7b46a89d57ab9a4df --- include/linux/kfd_ioctl.h | 15 +++++++++++++-- src/queues.c | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index ca86772f33..4ee1317c1d 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -58,7 +58,8 @@ struct kfd_ioctl_create_queue_args { uint64_t eop_buffer_address; /* to KFD */ uint64_t eop_buffer_size; /* to KFD */ uint64_t ctx_save_restore_address; /* to KFD */ - uint64_t ctx_save_restore_size; /* to KFD */ + uint32_t ctx_save_restore_size; /* to KFD */ + uint32_t ctl_stack_size; /* to KFD */ }; struct kfd_ioctl_destroy_queue_args { @@ -95,6 +96,13 @@ struct kfd_ioctl_set_memory_policy_args { uint32_t pad; }; +struct kfd_ioctl_set_trap_handler_args { + uint64_t tba_addr; + uint64_t tma_addr; + uint32_t gpu_id; /* to KFD */ + uint32_t pad; +}; + /* * All counters are monotonic. They are used for profiling of compute jobs. * The profiling is done by userspace. @@ -379,7 +387,10 @@ struct kfd_ioctl_alloc_memory_of_gpu_new_args { AMDKFD_IOWR(0x19, struct kfd_ioctl_alloc_memory_of_gpu_new_args) +#define AMDKFD_IOC_SET_TRAP_HANDLER \ + AMDKFD_IOW(0x1a, struct kfd_ioctl_set_trap_handler_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x20 +#define AMDKFD_COMMAND_END 0x1b #endif diff --git a/src/queues.c b/src/queues.c index 24b737071d..68aece0694 100644 --- a/src/queues.c +++ b/src/queues.c @@ -49,18 +49,21 @@ struct device_info { enum asic_family_type asic_family; uint32_t ctx_save_restore_size; + uint32_t ctl_stack_size; uint32_t eop_buffer_size; }; struct device_info kaveri_device_info = { .asic_family = CHIP_KAVERI, .ctx_save_restore_size = 0, + .ctl_stack_size = 0, .eop_buffer_size = 0, }; struct device_info carrizo_device_info = { .asic_family = CHIP_CARRIZO, - .ctx_save_restore_size = 2756608, + .ctx_save_restore_size = 2756608 + 4096, + .ctl_stack_size = 4096, .eop_buffer_size = 4096, }; @@ -164,7 +167,6 @@ static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) free(ptr); return NULL; } - memset(ptr, 0, size); return ptr; } @@ -260,6 +262,7 @@ static int handle_concrete_asic(struct device_info *dev_info, struct queue *q, } if (dev_info->ctx_save_restore_size > 0) { args->ctx_save_restore_size = dev_info->ctx_save_restore_size; + args->ctl_stack_size = dev_info->ctl_stack_size; q->ctx_save_restore = allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, PAGE_SIZE, dev_info->asic_family); if (q->ctx_save_restore == NULL) {; @@ -470,3 +473,35 @@ hsaKmtSetQueueCUMask( return HSAKMT_STATUS_SUCCESS; } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetTrapHandler( + HSAuint32 Node, + void *TrapHandlerBaseAddress, + HSAuint64 TrapHandlerSizeInBytes, + void *TrapBufferBaseAddress, + HSAuint64 TrapBufferSizeInBytes +) +{ + struct kfd_ioctl_set_trap_handler_args args; + HSAKMT_STATUS result; + uint32_t gpu_id; + + CHECK_KFD_OPEN(); + + result = validate_nodeid(Node, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + memset(&args, 0, sizeof(args)); + + args.gpu_id = gpu_id; + args.tba_addr = (uintptr_t)TrapHandlerBaseAddress; + args.tma_addr = (uintptr_t)TrapBufferBaseAddress; + + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_TRAP_HANDLER, &args); + + return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; +} + From f2f45cc0e4bb0789121fd70b4fdcf3b71d8da3b0 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 29 Sep 2015 11:35:26 -0400 Subject: [PATCH 0076/1247] Add CoarseGrain memory flag Change-Id: If8ac0339ae8c809c6e6a4f56592a4061d110ea94 --- include/hsakmttypes.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 25e7ad0832..fee460fd05 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -453,7 +453,10 @@ typedef struct _HsaMemFlags unsigned int ExecuteAccess: 1; // default = 0: Identifies if memory is primarily used for data or accessed // for executable code (e.g. queue memory) by the host CPU or the device. // Influences the page attribute setting within the allocation - unsigned int Reserved : 19; + unsigned int CoarseGrain : 1; // default = 0: Memory is assigned explicitly to one agent at a time and does + // not need to be accessed in a cache-coherent way. Memory consistency needs + // to be enforced at synchronization points at dispatch boundaries. + unsigned int Reserved : 18; } ui32; HSAuint32 Value; From f3aaba0621516797dd12f5a7332fff2bc7fa1298 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 29 Sep 2015 11:58:16 -0400 Subject: [PATCH 0077/1247] Fix returning of base and limit on dgpu_mem_init reinitialization Change-Id: I1d1500ee57c3b85fc39c224d233a62097f981719 --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 652b640482..bf9fa07ac0 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1119,9 +1119,9 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) if (is_dgpu_mem_init) { if (base) - base = dgpu_shared_aperture_base; + *base = dgpu_shared_aperture_base; if (limit) - limit = dgpu_shared_aperture_limit; + *limit = dgpu_shared_aperture_limit; return HSAKMT_STATUS_SUCCESS; } From 7505893cc7a6aedf6718c985373ad5e38b94c475 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 2 Oct 2015 12:40:04 -0400 Subject: [PATCH 0078/1247] Add all gfx802 device IDs to supported_devices Without this, queue creation segfaults on unknown devices. Change-Id: Ieea0bc4783e7313b3dcdabf03ab1269e3670b217 --- src/queues.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 68aece0694..bf0f7e0f1f 100644 --- a/src/queues.c +++ b/src/queues.c @@ -107,8 +107,15 @@ struct device_id supported_devices[] = { { 0x9875, &carrizo_device_info }, /* Carrizo */ { 0x9876, &carrizo_device_info }, /* Carrizo */ { 0x9877, &carrizo_device_info }, /* Carrizo */ - { 0x6939, &tonga_device_info }, + { 0x6920, &tonga_device_info }, + { 0x6921, &tonga_device_info }, + { 0x6928, &tonga_device_info }, + { 0x6929, &tonga_device_info }, { 0x692b, &tonga_device_info }, + { 0x692f, &tonga_device_info }, + { 0x6930, &tonga_device_info }, + { 0x6938, &tonga_device_info }, + { 0x6939, &tonga_device_info }, { 0, NULL } }; From 8e836f81833cd4a961f21ecfb5fcf596efd82ba2 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 2 Oct 2015 15:42:02 -0400 Subject: [PATCH 0079/1247] Setup APE1 on dGPU for coherent access The default is non-coherent access for better performance on dGPU. Disabled hsaKmtSetMemoryPolicy function on dGPU to prevent app from overriding the APE1 settings at runtime. Fixed dGPU VM aperture limit to be inclusive. Change-Id: I378ff74a654f533572775c0c97c19779a56bc6d9 --- src/fmm.c | 101 ++++++++++++++++++++++++++++++++++++++++++++++----- src/memory.c | 7 ++++ 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index bf9fa07ac0..3d5d486d3a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -51,7 +51,9 @@ .gpu_id = NON_VALID_GPU_ID, \ .lds_aperture = INIT_APERTURE(0, 0), \ .scratch_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ - .gpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0) \ + .gpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ + .dgpu_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ + .dgpu_alt_aperture = INIT_MANAGEBLE_APERTURE(0, 0) \ } #define INIT_GPUs_MEM {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = INIT_GPU_MEM} @@ -91,8 +93,13 @@ typedef struct { aperture_t lds_aperture; manageble_aperture_t scratch_aperture; manageble_aperture_t scratch_physical; - manageble_aperture_t gpuvm_aperture; - manageble_aperture_t dgpu_aperture; + manageble_aperture_t gpuvm_aperture; /* used for device mem on APU and for Gfx interop, + unusable on dGPU with small-ish VA range */ + manageble_aperture_t dgpu_aperture; /* used for non-coherent system and invisible device mem on dGPU */ + manageble_aperture_t dgpu_alt_aperture; /* used for coherent (fine-grain) system memory on dGPU */ + /* TODO: Merge gpuvm and dgpu apertures. When we have bigger + * VA range, we can add a new invisible aperture for invisible + * device mem on dGPU. */ } gpu_mem_t; static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; @@ -484,6 +491,8 @@ void fmm_print(uint32_t gpu_id) manageble_aperture_print(&gpu_mem[i].scratch_aperture); printf("dGPU aperture:\n"); manageble_aperture_print(&gpu_mem[i].dgpu_aperture); + printf("dGPU alt aperture:\n"); + manageble_aperture_print(&gpu_mem[i].dgpu_alt_aperture); } } #else @@ -641,15 +650,21 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, if (gpu_mem_id < 0) return NULL; - aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; + if (flags.ui32.CoarseGrain) + aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; + else + aperture = &gpu_mem[gpu_mem_id].dgpu_alt_aperture; /* coherent */ /* Alignment is needed to match a workaround for a VI HW bug in the kernel */ + /* FIXME: this breaks fmm_release! */ MemorySizeInBytes = (MemorySizeInBytes + 0x7fffULL) & ~0x7fffULL; mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, 0, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST); + /* FIXME: host memory allocated in this way should be mapped on all GPUs */ + void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); @@ -781,6 +796,14 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) MemorySizeInBytes, &gpu_mem[i].dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); } + + if (address >= gpu_mem[i].dgpu_alt_aperture.base && + address <= gpu_mem[i].dgpu_alt_aperture.limit) { + found = true; + __fmm_release(gpu_mem[i].gpu_id, address, + MemorySizeInBytes, &gpu_mem[i].dgpu_alt_aperture); + fmm_print(gpu_mem[i].gpu_id); + } } /* @@ -791,12 +814,27 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) free(address); } +static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_policy, + uintptr_t alt_base, uint64_t alt_size) +{ + struct kfd_ioctl_set_memory_policy_args args; + + args.gpu_id = gpu_id; + args.default_policy = default_policy; + args.alternate_policy = alt_policy; + args.alternate_aperture_base = alt_base; + args.alternate_aperture_size = alt_size; + + return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args); +} + HSAKMT_STATUS fmm_init_process_apertures(void) { struct kfd_ioctl_get_process_apertures_args args; uint8_t node_id; uint32_t gpu_id; HsaNodeProperties props; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) return HSAKMT_STATUS_ERROR; @@ -826,20 +864,47 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (topology_sysfs_get_node_props(node_id, &props, &gpu_id) == HSAKMT_STATUS_SUCCESS) { if (topology_is_dgpu(props.DeviceId)) { + uintptr_t alt_base; + uint64_t alt_size; + int err; + dgpu_mem_init(node_id, &gpu_mem[node_id].dgpu_aperture.base, &gpu_mem[node_id].dgpu_aperture.limit); set_dgpu_aperture(node_id, (uint64_t)gpu_mem[node_id].dgpu_aperture.base, (uint64_t)gpu_mem[node_id].dgpu_aperture.limit); - gpu_mem[node_id].gpuvm_aperture.base = gpu_mem[node_id].dgpu_aperture.limit; + + /* Place GPUVM aperture after dGPU aperture + * (FK: I think this is broken but leaving it for now) */ + gpu_mem[node_id].gpuvm_aperture.base = VOID_PTR_ADD(gpu_mem[node_id].dgpu_aperture.limit, 1); gpu_mem[node_id].gpuvm_aperture.limit = (void *)VOID_PTRS_SUB(gpu_mem[node_id].dgpu_aperture.limit, gpu_mem[node_id].dgpu_aperture.base); gpu_mem[node_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[node_id].gpuvm_aperture.limit, (unsigned long)gpu_mem[node_id].gpuvm_aperture.base); + + /* Use the first 1/4 of the dGPU aperture as + * alternate aperture for coherent access. + * Base and size must be 64KB aligned. */ + alt_base = (uintptr_t)gpu_mem[node_id].dgpu_aperture.base; + alt_size = (VOID_PTRS_SUB(gpu_mem[node_id].dgpu_aperture.limit, + gpu_mem[node_id].dgpu_aperture.base) + 1) >> 2; + alt_base = (alt_base + 0xffff) & ~0xffffULL; + alt_size = (alt_size + 0xffff) & ~0xffffULL; + gpu_mem[node_id].dgpu_alt_aperture.base = (void *)alt_base; + gpu_mem[node_id].dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); + gpu_mem[node_id].dgpu_aperture.base = VOID_PTR_ADD(gpu_mem[node_id].dgpu_alt_aperture.limit, 1); + err = fmm_set_memory_policy(gpu_id, + KFD_IOC_CACHE_POLICY_NONCOHERENT, + KFD_IOC_CACHE_POLICY_COHERENT, + alt_base, alt_size); + if (err != 0) { + fprintf(stderr, "Error! Failed to set alt aperture for node %d\n", node_id); + ret = HSAKMT_STATUS_ERROR; + } } } } - return HSAKMT_STATUS_SUCCESS; + return ret; } HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id) @@ -992,6 +1057,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, &gpu_mem[i].dgpu_aperture, address, size); + if ((address >= gpu_mem[i].dgpu_alt_aperture.base) && + (address <= gpu_mem[i].dgpu_alt_aperture.limit)) + /* map it */ + return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, + &gpu_mem[i].dgpu_alt_aperture, + address, size); } /* @@ -1046,6 +1117,11 @@ int fmm_unmap_from_gpu(void *address) /* unmap it */ return _fmm_unmap_from_gpu(&gpu_mem[i].dgpu_aperture, address); + else if ((address >= gpu_mem[i].dgpu_alt_aperture.base) && + (address <= gpu_mem[i].dgpu_alt_aperture.limit)) + /* unmap it */ + return _fmm_unmap_from_gpu(&gpu_mem[i].dgpu_alt_aperture, + address); } return 0; @@ -1155,12 +1231,12 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); if (ret != HSAKMT_STATUS_SUCCESS) { - printf("Error! Unable to find vm_size for gGPU\n"); + fprintf(stderr, "Error! Unable to find vm_size for gGPU\n"); return ret; } - max_vm_limit = (HSAuint64)max_vm_limit_in_gb << 30; - if (((long long unsigned int)ret_addr + max_len) < max_vm_limit) - max_vm_limit = ((long long unsigned int)ret_addr + max_len); + max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; + if (((long long unsigned int)ret_addr + max_len - 1) < max_vm_limit) + max_vm_limit = ((long long unsigned int)ret_addr + max_len - 1); if (limit) *limit = (void *)max_vm_limit; @@ -1198,6 +1274,11 @@ bool fmm_get_handle(void *address, uint64_t *handle) aperture = &gpu_mem[i].dgpu_aperture; break; } + else if ((address >= gpu_mem[i].dgpu_alt_aperture.base) && + (address <= gpu_mem[i].dgpu_alt_aperture.limit)) { + aperture = &gpu_mem[i].dgpu_alt_aperture; + break; + } } if (!aperture) diff --git a/src/memory.c b/src/memory.c index 40020f702f..e42848db45 100644 --- a/src/memory.c +++ b/src/memory.c @@ -50,6 +50,13 @@ hsaKmtSetMemoryPolicy( CHECK_KFD_OPEN(); + if (is_dgpu) + /* This is a legacy API useful on Kaveri only. On dGPU + * the alternate aperture is setup and used + * automatically for coherent allocations. Don't let + * app override it. */ + return HSAKMT_STATUS_NOT_IMPLEMENTED; + result = validate_nodeid(Node, &gpu_id); if (result != HSAKMT_STATUS_SUCCESS) return result; From b94ae66c6242a3b096d2c26380c0bbd12c8f5175 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 5 Oct 2015 16:48:50 -0400 Subject: [PATCH 0080/1247] Update HsaMemFlags.ui32.CoarseGrain comment As advised by Paul Blinzer Change-Id: Icabf4acd94866ddbbe53faf48a71e1113f0c76b6 --- include/hsakmttypes.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index fee460fd05..0ffe7663e2 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -453,9 +453,11 @@ typedef struct _HsaMemFlags unsigned int ExecuteAccess: 1; // default = 0: Identifies if memory is primarily used for data or accessed // for executable code (e.g. queue memory) by the host CPU or the device. // Influences the page attribute setting within the allocation - unsigned int CoarseGrain : 1; // default = 0: Memory is assigned explicitly to one agent at a time and does - // not need to be accessed in a cache-coherent way. Memory consistency needs - // to be enforced at synchronization points at dispatch boundaries. + unsigned int CoarseGrain : 1; // default = 0: The memory can be accessed assuming cache + // coherency maintained by link infrastructure and HSA agents. + // 1: memory consistency needs to be enforced at + // synchronization points at dispatch or other software + // enforced synchronization boundaries. unsigned int Reserved : 18; } ui32; From f2a46101d36d9e9daa519833368c984541e99bda Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 8 Oct 2015 11:40:33 -0400 Subject: [PATCH 0081/1247] Refactor hsa_gfxip_table lookup Also fix some formatting Change-Id: Ia04d7a9cd3972cc4d283c576161de639027aac6d --- src/topology.c | 60 ++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/topology.c b/src/topology.c index 1dcbf0cfeb..ce0e34a728 100644 --- a/src/topology.c +++ b/src/topology.c @@ -55,7 +55,7 @@ static HSAKMT_STATUS topology_drop_snapshot(void); //static int get_cpu_stepping(uint16_t* stepping); static struct hsa_gfxip_table { - unsigned short device_id; // Device ID + uint16_t device_id; // Device ID unsigned char major; // GFXIP Major engine version unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info @@ -235,18 +235,27 @@ topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) { return ret; } -bool topology_is_dgpu(uint16_t gpu_id) +static const struct hsa_gfxip_table* find_hsa_gfxip_device(uint16_t device_id) { uint32_t i, table_size; table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); for (i=0; iis_dgpu) { + is_dgpu = true; + return true; + } return false; } @@ -257,9 +266,10 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 char prop_name[256]; char path[256]; long long unsigned int prop_val; - uint32_t i, prog, table_size; + uint32_t i, prog; uint16_t fw_version = 0; - int read_size; + int read_size; + const struct hsa_gfxip_table* hsa_gfxip; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -280,12 +290,12 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 goto err1; } - read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } - p = memchr(read_buf, '\n', read_size); + p = memchr(read_buf, '\n', read_size); if ((!p) || ((p-read_buf) > HSA_PUBLIC_NAME_SIZE)) { ret = HSAKMT_STATUS_ERROR; goto err2; @@ -306,19 +316,19 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 return HSAKMT_STATUS_ERROR; } - read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } - /* Since we're using the buffer as a string, we make sure the string terminates */ - if(read_size >= PAGE_SIZE) - read_size = PAGE_SIZE-1; - read_buf[read_size] = 0; + /* Since we're using the buffer as a string, we make sure the string terminates */ + if(read_size >= PAGE_SIZE) + read_size = PAGE_SIZE-1; + read_buf[read_size] = 0; /* - * Read the node properties + * Read the node properties */ prog = 0; p = read_buf; @@ -379,14 +389,12 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 props->EngineId.ui32.Major = 0; props->EngineId.ui32.Minor = 0; props->EngineId.ui32.Stepping = 0; - table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); - for (i=0; iDeviceId) { - props->EngineId.ui32.Major = gfxip_lookup_table[i].major & 0x3f; - props->EngineId.ui32.Minor = gfxip_lookup_table[i].minor; - props->EngineId.ui32.Stepping = gfxip_lookup_table[i].stepping; - break; - } + + hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); + if (hsa_gfxip) { + props->EngineId.ui32.Major = hsa_gfxip->major & 0x3f; + props->EngineId.ui32.Minor = hsa_gfxip->minor; + props->EngineId.ui32.Stepping = hsa_gfxip->stepping; } assert(props->EngineId.ui32.Major); //TODO: error handler when Device ID lookup fails From 758824db170068fa66c99baa6fe478f323f12ec0 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 8 Oct 2015 11:44:03 -0400 Subject: [PATCH 0082/1247] Fix assert failure for CPU only node hsa_gfxip_table lists only (supported) GPUs. So assert fail only when a non-supported GPU is detected. Change-Id: I6207dc7cd55860c8b3348b6a4ca6102131975722 --- src/topology.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index ce0e34a728..9b6bb9863e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -396,7 +396,8 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 props->EngineId.ui32.Minor = hsa_gfxip->minor; props->EngineId.ui32.Stepping = hsa_gfxip->stepping; } - assert(props->EngineId.ui32.Major); + if (props->NumFComputeCores) + assert(props->EngineId.ui32.Major); //TODO: error handler when Device ID lookup fails err2: From 0fc0a5b526ce999206ee6387c3ad3afa2bde5d22 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 9 Oct 2015 17:40:54 -0400 Subject: [PATCH 0083/1247] Add support for gfx803 Create new device_info and add device ID. Add helper macros to identify chip families (VI, discrete). For now gfx803 behaves like gfx802. But if necessary we can have gfx802 or gfx803-specific code paths or workarounds in the future. Change-Id: I61b4ffef7dd7796bb34cb01fbff0089bd49507bb --- src/queues.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/queues.c b/src/queues.c index bf0f7e0f1f..9702ccd6ab 100644 --- a/src/queues.c +++ b/src/queues.c @@ -42,9 +42,13 @@ enum asic_family_type { CHIP_KAVERI = 0, CHIP_CARRIZO, - CHIP_TONGA + CHIP_TONGA, + CHIP_FIJI }; +#define IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_FIJI) +#define IS_DGPU(chip) ((chip) >= CHIP_TONGA && (chip) <= CHIP_FIJI) + struct device_info { enum asic_family_type asic_family; @@ -73,6 +77,12 @@ struct device_info tonga_device_info = { .eop_buffer_size = TONGA_PAGE_SIZE, }; +struct device_info fiji_device_info = { + .asic_family = CHIP_FIJI, + .ctx_save_restore_size = TONGA_PAGE_SIZE, + .eop_buffer_size = TONGA_PAGE_SIZE, +}; + struct device_id { uint16_t dev_id; @@ -116,6 +126,7 @@ struct device_id supported_devices[] = { { 0x6930, &tonga_device_info }, { 0x6938, &tonga_device_info }, { 0x6939, &tonga_device_info }, + { 0x7300, &fiji_device_info }, { 0, NULL } }; @@ -215,7 +226,7 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size) static void* allocate_exec_aligned_memory(uint32_t size, uint32_t align, enum asic_family_type type) { - if (type == CHIP_TONGA) + if (IS_DGPU(type)) return allocate_exec_aligned_memory_gpu(size, TONGA_PAGE_SIZE); return allocate_exec_aligned_memory_cpu(size, align); } @@ -228,7 +239,7 @@ static void release_exec_aligned_memory_gpu(void *addr, uint32_t size) static void release_exec_aligned_memory(void *addr, uint32_t size, enum asic_family_type type) { - if (type == CHIP_TONGA) + if (IS_DGPU(type)) release_exec_aligned_memory_gpu(addr, TONGA_PAGE_SIZE); else free(addr); @@ -249,7 +260,7 @@ static void free_queue_gpu(struct queue *q) static void free_queue(struct queue *q, enum asic_family_type type) { - if (type == CHIP_TONGA) + if (IS_DGPU(type)) return free_queue_gpu(q); return free_queue_cpu(q); } From 6a5ca4bc5af1bcacd15568883f90c00c3ec1b4c0 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 13 Oct 2015 19:05:39 -0400 Subject: [PATCH 0084/1247] Fix various dgpu memory management issues Fix TONGA_PAGE_SIZE value and move it to libhsakmt.h for usiing it consistently in all places that require the same alignment for the same reason. Create a generic alignment helper macro to replace some incorrect hand-coded size alignments. Move virtual address and size alignments down into aperture management functions. Alignment is a per-aperture property that is set during fmm_init_process_apertures. Doing the alignment there ensures that all allocations in the same aperture are aligned the same way. Finding objects by size and address can take the alignment into account. Also align the size of physical allocations to back aligned virtual address allocations. CPU mappings do not need to be aligned. Map anonymous pages over released memory mappings to allow the backing pages to be released, while keeping the address space reserved. Add alignment parameter to free_exec_aligned_memory_gpu to match the interface of allocate_exec_aligned_memory_cpu. It doesn't make sense to allow an alignment parameter in one but assume a specific alignment in the other. Change-Id: I74226ca6938f4948f643e5aee1d474720cd89e78 --- src/events.c | 2 +- src/fmm.c | 37 ++++++++++++++++++++++++++----------- src/libhsakmt.h | 7 +++++-- src/queues.c | 8 +++----- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/src/events.c b/src/events.c index 7cedacb459..22949d595d 100644 --- a/src/events.c +++ b/src/events.c @@ -74,7 +74,7 @@ hsaKmtCreateEvent( /* dGPU code */ if (is_dgpu && events_page == NULL) { - events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, 0x9000); + events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, TONGA_PAGE_SIZE); if (!events_page) { return HSAKMT_STATUS_ERROR; } diff --git a/src/fmm.c b/src/fmm.c index 3d5d486d3a..318f6497da 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -42,6 +42,7 @@ #define INIT_MANAGEBLE_APERTURE(base_value, limit_value) { \ .base = (void *) base_value, \ .limit = (void *) limit_value, \ + .align = PAGE_SIZE, \ .vm_ranges = NULL, \ .vm_objects = NULL, \ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER \ @@ -78,6 +79,7 @@ typedef struct vm_area vm_area_t; typedef struct { void *base; void *limit; + uint64_t align; vm_area_t *vm_ranges; vm_object_t *vm_objects; pthread_mutex_t fmm_mutex; @@ -103,6 +105,8 @@ typedef struct { } gpu_mem_t; static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; +static void *dgpu_shared_aperture_base = NULL; +static void *dgpu_shared_aperture_limit = NULL; static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); @@ -224,6 +228,8 @@ static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, { vm_object_t *cur = app->vm_objects; + size = ALIGN_UP(size, app->align); + /* Look up the appropriate address range containing the given address */ while (cur) { if (cur->start == address && (cur->size == size || size == 0)) @@ -264,6 +270,8 @@ static void aperture_release_area(manageble_aperture_t *app, void *address, vm_area_t *area; uint64_t SizeOfRegion; + MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); + area = vm_find(app, address); if (!area) return; @@ -302,6 +310,8 @@ static void *aperture_allocate_area(manageble_aperture_t *app, next = NULL; new_area = NULL; + MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); + cur = app->vm_ranges; if (cur) { /* not empty */ /* @@ -358,6 +368,8 @@ static int aperture_allocate_object(manageble_aperture_t *app, { vm_object_t *new_object; + MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); + /* Allocate new object */ new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, @@ -400,7 +412,7 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, /* Allocate memory from amdkfd */ args.gpu_id = gpu_id; - args.size = MemorySizeInBytes; + args.size = ALIGN_UP(MemorySizeInBytes, aperture->align); args.flags = flags; args.va_addr = (uint64_t)mem; @@ -596,8 +608,6 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; - /* Alignment is needed to match a workaround for a VI HW bug in the kernel */ - MemorySizeInBytes = (MemorySizeInBytes + 0x7fffULL) & ~0x7fffULL; /* * TODO: Once VA limit is raised from 0x200000000 (8GB) use gpuvm_aperture. * In that way the host access range won't be used for local memory @@ -655,10 +665,6 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, else aperture = &gpu_mem[gpu_mem_id].dgpu_alt_aperture; /* coherent */ - /* Alignment is needed to match a workaround for a VI HW bug in the kernel */ - /* FIXME: this breaks fmm_release! */ - MemorySizeInBytes = (MemorySizeInBytes + 0x7fffULL) & ~0x7fffULL; - mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, 0, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST); @@ -806,6 +812,14 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) } } + if (found && + address >= dgpu_shared_aperture_base && + address <= dgpu_shared_aperture_limit) { + /* Remove any CPU mapping, but keep the address range reserved */ + mmap(address, MemorySizeInBytes, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); + } + /* * If memory address isn't inside of any defined aperture - it refers * to the system memory @@ -872,6 +886,7 @@ HSAKMT_STATUS fmm_init_process_apertures(void) &gpu_mem[node_id].dgpu_aperture.limit); set_dgpu_aperture(node_id, (uint64_t)gpu_mem[node_id].dgpu_aperture.base, (uint64_t)gpu_mem[node_id].dgpu_aperture.limit); + gpu_mem[node_id].dgpu_aperture.align = TONGA_PAGE_SIZE; /* Place GPUVM aperture after dGPU aperture * (FK: I think this is broken but leaving it for now) */ @@ -880,6 +895,7 @@ HSAKMT_STATUS fmm_init_process_apertures(void) gpu_mem[node_id].dgpu_aperture.base); gpu_mem[node_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[node_id].gpuvm_aperture.limit, (unsigned long)gpu_mem[node_id].gpuvm_aperture.base); + gpu_mem[node_id].gpuvm_aperture.align = TONGA_PAGE_SIZE; /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. @@ -900,6 +916,7 @@ HSAKMT_STATUS fmm_init_process_apertures(void) fprintf(stderr, "Error! Failed to set alt aperture for node %d\n", node_id); ret = HSAKMT_STATUS_ERROR; } + gpu_mem[node_id].dgpu_alt_aperture.align = TONGA_PAGE_SIZE; } } } @@ -1129,8 +1146,6 @@ int fmm_unmap_from_gpu(void *address) /* Tonga dGPU specific functions */ static bool is_dgpu_mem_init = false; -static void *dgpu_shared_aperture_base = NULL; -static void *dgpu_shared_aperture_limit = NULL; static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit) { @@ -1208,9 +1223,9 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) max_len = props.LocalMemSize; found = false; - for (addr = (void *)PAGE_SIZE, ret_addr = NULL; + for (addr = (void *)TONGA_PAGE_SIZE, ret_addr = NULL; ret_addr != addr; - addr = (void *)((unsigned long)addr + 0x8000)) + addr = (void *)((unsigned long)addr + TONGA_PAGE_SIZE)) { ret_addr = reserve_address(addr, max_len); if (!ret_addr) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index a1399fc65d..1ceb583923 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -49,11 +49,14 @@ extern bool is_dgpu; do { if (kfd_open_count == 0) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) #define PAGE_SIZE 4096 +/* VI HW bug requires this virtual address alignment */ +#define TONGA_PAGE_SIZE 0x8000 #define CHECK_PAGE_MULTIPLE(x) \ do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) -#define PAGE_ALIGN_UP(x) (((uint64_t)(x) + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE-1)) +#define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) /* @@ -74,7 +77,7 @@ bool topology_is_dgpu(uint16_t gpu_id); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align); -void free_exec_aligned_memory_gpu(void *addr, uint32_t size); +void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); extern int kmtIoctl(int fd, unsigned long request, void *arg); diff --git a/src/queues.c b/src/queues.c index 9702ccd6ab..7c7c446b81 100644 --- a/src/queues.c +++ b/src/queues.c @@ -34,8 +34,6 @@ #include #include -#define TONGA_PAGE_SIZE 0x9000 - /* 1024 doorbells, 4 bytes each doorbell */ #define DOORBELLS_PAGE_SIZE 1024 * 4 @@ -201,7 +199,7 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align) flags.ui32.ExecuteAccess = 1; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - size += align - (size % align); + size = ALIGN_UP(size, align); ret = hsaKmtAllocMemory(0, size, flags, &mem); if (ret != HSAKMT_STATUS_SUCCESS) { @@ -215,9 +213,9 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align) return mem; } -void free_exec_aligned_memory_gpu(void *addr, uint32_t size) +void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) { - size += TONGA_PAGE_SIZE - (size % TONGA_PAGE_SIZE); + size = ALIGN_UP(size, align); if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(addr, size); From b6c6f791434586e3963cfb16aabe639f336ccc32 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 14 Oct 2015 13:29:33 -0400 Subject: [PATCH 0085/1247] Fix hard-coded usage of Node 0 Use appropriate NodeId instead Change-Id: I46af93b76978fea7bedb34457fcc0864ed4fe2d4 --- src/events.c | 5 ++++- src/libhsakmt.h | 2 +- src/queues.c | 29 ++++++++++++++++++++--------- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/events.c b/src/events.c index 22949d595d..046d02765e 100644 --- a/src/events.c +++ b/src/events.c @@ -70,11 +70,14 @@ hsaKmtCreateEvent( memset(&args, 0, sizeof(args)); args.event_type = EventDesc->EventType; + args.node_id = EventDesc->NodeId; args.auto_reset = !ManualReset; /* dGPU code */ if (is_dgpu && events_page == NULL) { - events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, TONGA_PAGE_SIZE); + events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, + TONGA_PAGE_SIZE, + args.node_id); if (!events_page) { return HSAKMT_STATUS_ERROR; } diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 1ceb583923..64ac2650bc 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -76,7 +76,7 @@ bool topology_is_dgpu(uint16_t gpu_id); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); -void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align); +void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); extern int kmtIoctl(int fd, unsigned long request, void *arg); diff --git a/src/queues.c b/src/queues.c index 7c7c446b81..6604f02889 100644 --- a/src/queues.c +++ b/src/queues.c @@ -187,7 +187,7 @@ static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) return ptr; } -void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align) +void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId) { void *mem; HSAuint64 gpu_va; @@ -201,7 +201,7 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align) size = ALIGN_UP(size, align); - ret = hsaKmtAllocMemory(0, size, flags, &mem); + ret = hsaKmtAllocMemory(NodeId, size, flags, &mem); if (ret != HSAKMT_STATUS_SUCCESS) { return NULL; } @@ -222,10 +222,13 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) } } -static void* allocate_exec_aligned_memory(uint32_t size, uint32_t align, enum asic_family_type type) +static void* allocate_exec_aligned_memory(uint32_t size, + uint32_t align, + enum asic_family_type type, + uint32_t NodeId) { if (IS_DGPU(type)) - return allocate_exec_aligned_memory_gpu(size, TONGA_PAGE_SIZE); + return allocate_exec_aligned_memory_gpu(size, TONGA_PAGE_SIZE, NodeId); return allocate_exec_aligned_memory_cpu(size, align); } @@ -264,12 +267,16 @@ static void free_queue(struct queue *q, enum asic_family_type type) } static int handle_concrete_asic(struct device_info *dev_info, struct queue *q, - struct kfd_ioctl_create_queue_args *args) + struct kfd_ioctl_create_queue_args *args, + uint32_t NodeId) { if (dev_info) { if (dev_info->eop_buffer_size > 0) { q->eop_buffer = - allocate_exec_aligned_memory(dev_info->eop_buffer_size, PAGE_SIZE, dev_info->asic_family); + allocate_exec_aligned_memory(dev_info->eop_buffer_size, + PAGE_SIZE, + dev_info->asic_family, + NodeId); if (q->eop_buffer == NULL) { return HSAKMT_STATUS_NO_MEMORY; } @@ -280,7 +287,10 @@ static int handle_concrete_asic(struct device_info *dev_info, struct queue *q, args->ctx_save_restore_size = dev_info->ctx_save_restore_size; args->ctl_stack_size = dev_info->ctl_stack_size; q->ctx_save_restore = - allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, PAGE_SIZE, dev_info->asic_family); + allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, + PAGE_SIZE, + dev_info->asic_family, + NodeId); if (q->ctx_save_restore == NULL) {; return HSAKMT_STATUS_NO_MEMORY; } @@ -320,7 +330,8 @@ hsaKmtCreateQueue( dev_info = get_device_info_by_dev_id(dev_id); struct queue *q = allocate_exec_aligned_memory(sizeof (*q), - PAGE_SIZE, dev_info->asic_family); + PAGE_SIZE, dev_info->asic_family, + NodeId); if (q == NULL) return HSAKMT_STATUS_NO_MEMORY; @@ -333,7 +344,7 @@ hsaKmtCreateQueue( q->type = dev_info->asic_family; - err = handle_concrete_asic(dev_info, q, &args); + err = handle_concrete_asic(dev_info, q, &args, NodeId); if (err != HSAKMT_STATUS_SUCCESS) { free_queue(q, dev_info->asic_family); return err; From 5cc56a2647c46b753265d886c90f49f65d132aab Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 15 Oct 2015 12:03:31 -0400 Subject: [PATCH 0086/1247] Fix init process apertures Kernel ioctl AMDKFD_IOC_GET_PROCESS_APERTURES returns process apertures only for GPU nodes. The current implementation assumed that this list of GPU nodes returned by the ioctl has one to one correpondence to sysfs topology nodes. This fails when non-GPU nodes exist in topology as in case of Intel + gfx802 Fix this by using gpu_id (./sys/.../kfd/topology/nodes/1/gpu_id) to map information obtained from kernel ioctl call. Change-Id: I4ab8ae5354f12cf0b6609fc4b24182b82eb3677f --- src/fmm.c | 150 +++++++++++++++++++++++++++--------------------- src/libhsakmt.h | 1 + src/topology.c | 2 +- 3 files changed, 88 insertions(+), 65 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 318f6497da..fbea8686e0 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -92,6 +92,9 @@ typedef struct { typedef struct { uint32_t gpu_id; + uint32_t device_id; + uint32_t node_id; + uint64_t local_mem_size; aperture_t lds_aperture; manageble_aperture_t scratch_aperture; manageble_aperture_t scratch_physical; @@ -108,7 +111,7 @@ static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; static void *dgpu_shared_aperture_base = NULL; static void *dgpu_shared_aperture_limit = NULL; -static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit); +static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); static void __fmm_release(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); @@ -845,79 +848,104 @@ static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_po HSAKMT_STATUS fmm_init_process_apertures(void) { struct kfd_ioctl_get_process_apertures_args args; - uint8_t node_id; + uint32_t i = 0; + int32_t gpu_mem_id =0; uint32_t gpu_id; + HsaSystemProperties sys_props; HsaNodeProperties props; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + ret = topology_sysfs_get_system_props(&sys_props); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + /* Initialize gpu_mem[] from sysfs topology. This is necessary because this function + * gets called before hsaKmtAcquireSystemProperties() is called.*/ + while (i < sys_props.NumNodes) { + ret = topology_sysfs_get_node_props(i, &props, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + i++; + /* Skip non-GPU nodes */ + if (gpu_id == 0) + continue; + + gpu_mem[gpu_mem_id].gpu_id = gpu_id; + gpu_mem[gpu_mem_id].local_mem_size = props.LocalMemSize; + gpu_mem[gpu_mem_id].device_id = props.DeviceId; + gpu_mem[gpu_mem_id].node_id = i; + gpu_mem_id++; + } + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) return HSAKMT_STATUS_ERROR; - for (node_id = 0 ; node_id < args.num_of_nodes ; node_id++) { - gpu_mem[node_id].gpu_id = - args.process_apertures[node_id].gpu_id; + for (i = 0 ; i < args.num_of_nodes ; i++) { + /* Map Kernel process device data node i <--> gpu_mem_id which indexes into gpu_mem[] + * based on gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(args.process_apertures[i].gpu_id); + if (gpu_mem_id < 0) + return HSAKMT_STATUS_ERROR; - gpu_mem[node_id].lds_aperture.base = - PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_base); + gpu_mem[gpu_mem_id].lds_aperture.base = + PORT_UINT64_TO_VPTR(args.process_apertures[i].lds_base); - gpu_mem[node_id].lds_aperture.limit = - PORT_UINT64_TO_VPTR(args.process_apertures[node_id].lds_limit); + gpu_mem[gpu_mem_id].lds_aperture.limit = + PORT_UINT64_TO_VPTR(args.process_apertures[i].lds_limit); - gpu_mem[node_id].gpuvm_aperture.base = - PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_base); + gpu_mem[gpu_mem_id].gpuvm_aperture.base = + PORT_UINT64_TO_VPTR(args.process_apertures[i].gpuvm_base); - gpu_mem[node_id].gpuvm_aperture.limit = - PORT_UINT64_TO_VPTR(args.process_apertures[node_id].gpuvm_limit); + gpu_mem[gpu_mem_id].gpuvm_aperture.limit = + PORT_UINT64_TO_VPTR(args.process_apertures[i].gpuvm_limit); - gpu_mem[node_id].scratch_aperture.base = - PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_base); + gpu_mem[gpu_mem_id].scratch_aperture.base = + PORT_UINT64_TO_VPTR(args.process_apertures[i].scratch_base); - gpu_mem[node_id].scratch_aperture.limit = - PORT_UINT64_TO_VPTR(args.process_apertures[node_id].scratch_limit); + gpu_mem[gpu_mem_id].scratch_aperture.limit = + PORT_UINT64_TO_VPTR(args.process_apertures[i].scratch_limit); - if (topology_sysfs_get_node_props(node_id, &props, &gpu_id) == - HSAKMT_STATUS_SUCCESS) { - if (topology_is_dgpu(props.DeviceId)) { - uintptr_t alt_base; - uint64_t alt_size; - int err; + if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + uintptr_t alt_base; + uint64_t alt_size; + int err; - dgpu_mem_init(node_id, &gpu_mem[node_id].dgpu_aperture.base, - &gpu_mem[node_id].dgpu_aperture.limit); - set_dgpu_aperture(node_id, (uint64_t)gpu_mem[node_id].dgpu_aperture.base, - (uint64_t)gpu_mem[node_id].dgpu_aperture.limit); - gpu_mem[node_id].dgpu_aperture.align = TONGA_PAGE_SIZE; + dgpu_mem_init(gpu_mem_id, &gpu_mem[gpu_mem_id].dgpu_aperture.base, + &gpu_mem[gpu_mem_id].dgpu_aperture.limit); + /* Set kernel process dgpu aperture. */ + set_dgpu_aperture(i, (uint64_t)gpu_mem[gpu_mem_id].dgpu_aperture.base, + (uint64_t)gpu_mem[gpu_mem_id].dgpu_aperture.limit); + gpu_mem[gpu_mem_id].dgpu_aperture.align = TONGA_PAGE_SIZE; - /* Place GPUVM aperture after dGPU aperture - * (FK: I think this is broken but leaving it for now) */ - gpu_mem[node_id].gpuvm_aperture.base = VOID_PTR_ADD(gpu_mem[node_id].dgpu_aperture.limit, 1); - gpu_mem[node_id].gpuvm_aperture.limit = (void *)VOID_PTRS_SUB(gpu_mem[node_id].dgpu_aperture.limit, - gpu_mem[node_id].dgpu_aperture.base); - gpu_mem[node_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[node_id].gpuvm_aperture.limit, - (unsigned long)gpu_mem[node_id].gpuvm_aperture.base); - gpu_mem[node_id].gpuvm_aperture.align = TONGA_PAGE_SIZE; + /* Place GPUVM aperture after dGPU aperture + * (FK: I think this is broken but leaving it for now) */ + gpu_mem[gpu_mem_id].gpuvm_aperture.base = VOID_PTR_ADD(gpu_mem[gpu_mem_id].dgpu_aperture.limit, 1); + gpu_mem[gpu_mem_id].gpuvm_aperture.limit = (void *)VOID_PTRS_SUB(gpu_mem[gpu_mem_id].dgpu_aperture.limit, + gpu_mem[gpu_mem_id].dgpu_aperture.base); + gpu_mem[gpu_mem_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[gpu_mem_id].gpuvm_aperture.limit, + (unsigned long)gpu_mem[gpu_mem_id].gpuvm_aperture.base); + gpu_mem[gpu_mem_id].gpuvm_aperture.align = TONGA_PAGE_SIZE; - /* Use the first 1/4 of the dGPU aperture as - * alternate aperture for coherent access. - * Base and size must be 64KB aligned. */ - alt_base = (uintptr_t)gpu_mem[node_id].dgpu_aperture.base; - alt_size = (VOID_PTRS_SUB(gpu_mem[node_id].dgpu_aperture.limit, - gpu_mem[node_id].dgpu_aperture.base) + 1) >> 2; - alt_base = (alt_base + 0xffff) & ~0xffffULL; - alt_size = (alt_size + 0xffff) & ~0xffffULL; - gpu_mem[node_id].dgpu_alt_aperture.base = (void *)alt_base; - gpu_mem[node_id].dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); - gpu_mem[node_id].dgpu_aperture.base = VOID_PTR_ADD(gpu_mem[node_id].dgpu_alt_aperture.limit, 1); - err = fmm_set_memory_policy(gpu_id, - KFD_IOC_CACHE_POLICY_NONCOHERENT, - KFD_IOC_CACHE_POLICY_COHERENT, - alt_base, alt_size); - if (err != 0) { - fprintf(stderr, "Error! Failed to set alt aperture for node %d\n", node_id); - ret = HSAKMT_STATUS_ERROR; - } - gpu_mem[node_id].dgpu_alt_aperture.align = TONGA_PAGE_SIZE; + /* Use the first 1/4 of the dGPU aperture as + * alternate aperture for coherent access. + * Base and size must be 64KB aligned. */ + alt_base = (uintptr_t)gpu_mem[gpu_mem_id].dgpu_aperture.base; + alt_size = (VOID_PTRS_SUB(gpu_mem[gpu_mem_id].dgpu_aperture.limit, + gpu_mem[gpu_mem_id].dgpu_aperture.base) + 1) >> 2; + alt_base = (alt_base + 0xffff) & ~0xffffULL; + alt_size = (alt_size + 0xffff) & ~0xffffULL; + gpu_mem[gpu_mem_id].dgpu_alt_aperture.base = (void *)alt_base; + gpu_mem[gpu_mem_id].dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); + gpu_mem[gpu_mem_id].dgpu_aperture.base = VOID_PTR_ADD(gpu_mem[gpu_mem_id].dgpu_alt_aperture.limit, 1); + err = fmm_set_memory_policy(gpu_mem[gpu_mem_id].gpu_id, + KFD_IOC_CACHE_POLICY_NONCOHERENT, + KFD_IOC_CACHE_POLICY_COHERENT, + alt_base, alt_size); + if (err != 0) { + fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); + ret = HSAKMT_STATUS_ERROR; } + gpu_mem[gpu_mem_id].dgpu_alt_aperture.align = TONGA_PAGE_SIZE; } } @@ -1197,7 +1225,7 @@ err: return ret; } -static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) +static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limit) { bool found; HSAKMT_STATUS ret; @@ -1205,8 +1233,6 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) HSAuint64 max_len, max_vm_limit; uint32_t max_vm_limit_in_gb; long long unsigned int temp; - uint32_t gpu_id; - HsaNodeProperties props; if (is_dgpu_mem_init) { if (base) @@ -1216,11 +1242,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint8_t node_id, void **base, void **limit) return HSAKMT_STATUS_SUCCESS; } - ret = topology_sysfs_get_node_props(node_id, &props, &gpu_id); - if (ret != HSAKMT_STATUS_SUCCESS) - return ret; - - max_len = props.LocalMemSize; + max_len = gpu_mem[gpu_mem_id].local_mem_size; found = false; for (addr = (void *)TONGA_PAGE_SIZE, ret_addr = NULL; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 64ac2650bc..19726784ba 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -72,6 +72,7 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id); +HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); bool topology_is_dgpu(uint16_t gpu_id); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); diff --git a/src/topology.c b/src/topology.c index 9b6bb9863e..87d9ff9330 100644 --- a/src/topology.c +++ b/src/topology.c @@ -139,7 +139,7 @@ err: return ret; } -static HSAKMT_STATUS +HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) { FILE *fd; DIR *dirp; From 0df346aaf94e227a3a341c497bf1170e742e90c0 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Fri, 16 Oct 2015 15:30:15 -0400 Subject: [PATCH 0087/1247] make: add 'deb' target for creating deb packages When 'make deb' is run create a libhsakmt.deb archive that installs libhsakmt into the appropriate folder on the target where the dymanic linker can find it. Change-Id: I32de7198975f7831e509a67371e78456982b5c42 --- DEBIAN/control | 8 ++++++++ Makefile | 3 +++ src/Makefile | 8 ++++++++ 3 files changed, 19 insertions(+) create mode 100644 DEBIAN/control diff --git a/DEBIAN/control b/DEBIAN/control new file mode 100644 index 0000000000..dd99d843b7 --- /dev/null +++ b/DEBIAN/control @@ -0,0 +1,8 @@ +Package: libhsakmt +Architecture: amd64 +Maintainer: Advanced Micro Devices Inc. +Depends: +Priority: optional +Version: 1.6.0 +Description: Thunk library for AMD KFD + diff --git a/Makefile b/Makefile index c899cca992..b3c4b4accf 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ all: lnx64a +deb: + $(MAKE) -C src deb + lnx64a: $(MAKE) -C src lnx64a diff --git a/src/Makefile b/src/Makefile index f3bbe91fca..5c8c080593 100644 --- a/src/Makefile +++ b/src/Makefile @@ -48,6 +48,14 @@ lnx: CFLAGS += -m32 lnx: LDFLAGS += -m32 lnx: $(BUILDDIR)/$(LIB_NAME) +PACKAGE_DIR = $(BUILDDIR)/libhsakmt +INSTALL_DIR = $(PACKAGE_DIR)/usr/lib/x86_64-linux-gnu +deb: lnx64a + @mkdir -p $(INSTALL_DIR) + @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_DIR) + @cp -r ../DEBIAN $(PACKAGE_DIR) + @fakeroot dpkg-deb --build $(PACKAGE_DIR) $(BUILDDIR)/libhsakmt.deb + lnx64a: $(BUILDDIR)/$(LIB_NAME) clean: From 39bde26c9b11b92829626a65e4ebda80a8133332 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 19 Oct 2015 17:08:33 -0400 Subject: [PATCH 0088/1247] Reserve more SVM process address space Try to reserve as much SVM address space as GPUVM can address. Implement a fallback scheme to smaller sizes if larger allocations fail or are not addressable by the GPU, down to an (arbitrary) minimum of 4GB. Change-Id: I770177834cc9e6ddd6ef4f20d789eab63c8055cb --- src/fmm.c | 94 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index fbea8686e0..fb5940259e 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1230,9 +1230,8 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi bool found; HSAKMT_STATUS ret; void *addr, *ret_addr; - HSAuint64 max_len, max_vm_limit; + HSAuint64 len, vm_limit, max_vm_limit, min_vm_size; uint32_t max_vm_limit_in_gb; - long long unsigned int temp; if (is_dgpu_mem_init) { if (base) @@ -1242,47 +1241,72 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi return HSAKMT_STATUS_SUCCESS; } - max_len = gpu_mem[gpu_mem_id].local_mem_size; + ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); + if (ret != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, + "Unable to find vm_size for dGPU, assuming 64GB.\n"); + max_vm_limit_in_gb = 64; + } + max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; + min_vm_size = (HSAuint64)4 << 30; + found = false; - for (addr = (void *)TONGA_PAGE_SIZE, ret_addr = NULL; - ret_addr != addr; - addr = (void *)((unsigned long)addr + TONGA_PAGE_SIZE)) - { - ret_addr = reserve_address(addr, max_len); - if (!ret_addr) + for (len = max_vm_limit+1; !found && len >= min_vm_size; len >>= 1) { + for (addr = (void *)TONGA_PAGE_SIZE, ret_addr = NULL; + (HSAuint64)addr + (len >> 1) < max_vm_limit; + addr = (void *)((HSAuint64)addr + TONGA_PAGE_SIZE)) { + ret_addr = reserve_address(addr, len); + if (!ret_addr) + break; + if ((HSAuint64)ret_addr + (len>>1) < max_vm_limit) + /* At least half the returned address + * space is GPU addressable, we'll + * take it */ + break; + munmap (ret_addr, len); + } + if (!ret_addr) { + fprintf(stderr, + "Failed to reserve %uGB for SVM ...\n", + (unsigned)(len >> 30)); continue; - temp = (long long unsigned int)ret_addr + max_len; - if (temp < ADDRESS_RANGE_LIMIT_MASK) { + } + if ((HSAuint64)ret_addr + min_vm_size - 1 > max_vm_limit) { + /* addressable size is less than the minimum */ + fprintf(stderr, + "Got %uGB for SVM at %p with only %dGB usable ...\n", + (unsigned)(len >> 30), ret_addr, + (int)(((HSAint64)max_vm_limit - + (HSAint64)ret_addr) >> 30)); + munmap(ret_addr, len); + continue; + } else found = true; - break; - } - else - munmap(ret_addr, max_len); } - if (found) { - if (base) - *base = ret_addr; - dgpu_shared_aperture_base = ret_addr; - - ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); - if (ret != HSAKMT_STATUS_SUCCESS) { - fprintf(stderr, "Error! Unable to find vm_size for gGPU\n"); - return ret; - } - max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; - if (((long long unsigned int)ret_addr + max_len - 1) < max_vm_limit) - max_vm_limit = ((long long unsigned int)ret_addr + max_len - 1); - - if (limit) - *limit = (void *)max_vm_limit; - dgpu_shared_aperture_limit = (void *)max_vm_limit; - is_dgpu_mem_init = true; - return HSAKMT_STATUS_SUCCESS; + if (!found) { + fprintf(stderr, + "Failed to reserve SVM address range. Giving up.\n"); + return HSAKMT_STATUS_ERROR; } - return HSAKMT_STATUS_ERROR; + vm_limit = (HSAuint64)ret_addr + len - 1; + if (vm_limit > max_vm_limit) { + /* trim the tail that's not GPU-addressable */ + munmap((void *)(max_vm_limit + 1), vm_limit - max_vm_limit); + vm_limit = max_vm_limit; + } + + if (base) + *base = ret_addr; + dgpu_shared_aperture_base = ret_addr; + if (limit) + *limit = (void *)vm_limit; + dgpu_shared_aperture_limit = (void *)vm_limit; + is_dgpu_mem_init = true; + + return HSAKMT_STATUS_SUCCESS; } bool fmm_get_handle(void *address, uint64_t *handle) From 590c8e522cad5e98a687a46c26fd27fd8b226563 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 21 Oct 2015 20:00:01 -0400 Subject: [PATCH 0089/1247] Fix node 0 system memory allocation for dGPU This is a hack to allow the Runtime to allocate system memory with PreferredNode=0 on a dGPU system. We allocated it from Node 1 instead so that the node 1 GPU can map the memory. A proper fix will be implemented together with multi-GPU support. Change-Id: Ieb52599e5275781c04ee34405ea850bf782c523a --- src/memory.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/memory.c b/src/memory.c index e42848db45..3314c82195 100644 --- a/src/memory.c +++ b/src/memory.c @@ -133,6 +133,21 @@ hsaKmtAllocMemory( } if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { + if (gpu_id == 0 && PreferredNode == 0) { + /* HACK: Currently we need a GPU node for + * system memory allocations on dGPUs and + * MapMemoryToGPU will always map to the same + * GPU used for allocation. Therefore we need + * to allocate system memory from node 1 if + * we're running on a dGPU (indicated by node + * 0 being a CPU with gpu_id==0). This will be + * cleaned up when multi-GPU support is + * implemented. */ + PreferredNode = 1; + result = validate_nodeid(PreferredNode, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + } *MemoryAddress = fmm_allocate_host(gpu_id, SizeInBytes, MemFlags, get_device_id_by_node(PreferredNode)); if (*MemoryAddress == NULL) From e61500c46eb3a1fe2054bb1a3a7fb1596ebf63c2 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Mon, 19 Oct 2015 15:27:36 +0300 Subject: [PATCH 0090/1247] Adding support for new AQL Queue Memory allocation Change-Id: If84fc4b961627dbdd0b77b1c509a3c9a4c709b9f Signed-off-by: Ben Goz --- include/hsakmttypes.h | 8 ++++++-- include/linux/kfd_ioctl.h | 2 ++ src/fmm.c | 14 +++++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 0ffe7663e2..5b4eeba1e1 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -182,7 +182,8 @@ typedef union unsigned int DoorbellType : 2; // 0: This node has pre-1.0 doorbell characteristic // 1: This node has 1.0 doorbell characteristic // 2,3: reserved for future use - unsigned int Reserved : 18; + unsigned int AQLQueueDoubleMap : 1; // The unit needs a VA “double map” + unsigned int Reserved : 17; } ui32; } HSA_CAPABILITY; @@ -458,7 +459,10 @@ typedef struct _HsaMemFlags // 1: memory consistency needs to be enforced at // synchronization points at dispatch or other software // enforced synchronization boundaries. - unsigned int Reserved : 18; + unsigned int AQLQueueMemory: 1; // default = 0; If 1: The caller indicates that the memory will be used as AQL queue memory. + // The KFD will ensure that the memory returned is allocated in the optimal memory location + // and optimal alignment requirements + unsigned int Reserved : 17; } ui32; HSAuint32 Value; diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 4ee1317c1d..c08cea8810 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -296,6 +296,8 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { #define KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE (1 << 3) #define KFD_IOC_ALLOC_MEM_FLAGS_APU_SCRATCH (1 << 4) +#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM (1 << 5) + struct kfd_ioctl_alloc_memory_of_gpu_new_args { uint64_t va_addr; /* to KFD */ uint64_t size; /* to KFD */ diff --git a/src/fmm.c b/src/fmm.c index fb5940259e..ce765d0abd 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -30,6 +30,7 @@ #include #include #include +#include #define NON_VALID_GPU_ID 0 #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) @@ -657,23 +658,30 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, manageble_aperture_t *aperture; int32_t gpu_mem_id; uint64_t mmap_offset; + uint32_t ioc_flags; + uint32_t size; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return NULL; + size = MemorySizeInBytes; + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST; if (flags.ui32.CoarseGrain) aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; else aperture = &gpu_mem[gpu_mem_id].dgpu_alt_aperture; /* coherent */ + if (flags.ui32.AQLQueueMemory) { + size = MemorySizeInBytes * 2; + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; + } - mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, + mem = __fmm_allocate_device(gpu_id, size, aperture, 0, &mmap_offset, - KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST); + ioc_flags); /* FIXME: host memory allocated in this way should be mapped on all GPUs */ - void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); From 55b1a5dc43d4e42ce0a18d9648a736a909937a59 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Mon, 19 Oct 2015 15:49:08 +0300 Subject: [PATCH 0091/1247] Casting local memory size to uint64_t Change-Id: I5c2010056b84ac01bb65361210d2a693e437050a Signed-off-by: Ben Goz --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 87d9ff9330..2282c94977 100644 --- a/src/topology.c +++ b/src/topology.c @@ -447,7 +447,7 @@ topology_sysfs_get_mem_props(uint32_t node_id, uint32_t mem_id, HsaMemoryPropert if (strcmp(prop_name,"heap_type") == 0) props->HeapType = (uint32_t)prop_val; else if (strcmp(prop_name,"size_in_bytes") == 0) - props->SizeInBytes = prop_val; + props->SizeInBytes = (uint64_t)prop_val; else if (strcmp(prop_name,"flags") == 0) props->Flags.MemoryProperty = (uint32_t)prop_val; else if (strcmp(prop_name,"width") == 0) From 149261ba092a20559b9786b1e449749599f60e90 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 17 Oct 2015 02:48:23 -0400 Subject: [PATCH 0092/1247] Allow address space allocations with specific alignment Change-Id: I4bf7f7ac53c3921dd330b9dc7a40582611f88b69 --- src/fmm.c | 92 +++++++++++++++++++++++++++---------------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ce765d0abd..69b68e8aaa 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -304,64 +304,64 @@ static void aperture_release_area(manageble_aperture_t *app, void *address, * returns allocated address or NULL. Assumes, that fmm_mutex is locked * on entry. */ -static void *aperture_allocate_area(manageble_aperture_t *app, - uint64_t MemorySizeInBytes, - uint64_t offset) +static void *aperture_allocate_area_aligned(manageble_aperture_t *app, + uint64_t MemorySizeInBytes, + uint64_t offset, + uint64_t align) { - vm_area_t *cur, *next, *new_area, *start; - void *new_address = NULL; - - next = NULL; - new_area = NULL; + vm_area_t *cur, *next; + void *start; MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); - cur = app->vm_ranges; - if (cur) { /* not empty */ - /* - * Look up the appropriate address space "hole" or end of - * the list - */ - while (cur) { - next = cur->next; + if (align < app->align) + align = app->align; - /* End of the list reached */ - if (!next) - break; + /* Find a big enough "hole" in the address space */ + cur = NULL; + next = app->vm_ranges; + start = (void *)ALIGN_UP((uint64_t)VOID_PTR_ADD(app->base, offset), + align); + while (next) { + if (next->start > start && + VOID_PTRS_SUB(next->start, start) >= MemorySizeInBytes) + break; - /* address space "hole" */ - if ((VOID_PTRS_SUB(next->start, cur->end) >= - MemorySizeInBytes)) - break; + cur = next; + next = next->next; + start = (void *)ALIGN_UP((uint64_t)cur->end + 1, align); + } + if (!next && VOID_PTRS_SUB(app->limit, start) + 1 < MemorySizeInBytes) + /* No hole found and not enough space after the last area */ + return NULL; - cur = next; - }; - - /* If the new range is inside the reserved aperture */ - if (VOID_PTRS_SUB(app->limit, cur->end) + 1 >= - MemorySizeInBytes) { - /* - * cur points to the last inspected element: the tail - * of the list or the found "hole". - * Just extend the existing region - */ - new_address = VOID_PTR_ADD(cur->end, 1); - cur->end = VOID_PTR_ADD(cur->end, MemorySizeInBytes); - } else { - new_address = NULL; - } - } else { /* empty - create the first area */ - /* Some offset from the base */ - start = VOID_PTR_ADD(app->base, offset); + if (cur && VOID_PTR_ADD(cur->end, 1) == start) { + /* extend existing area */ + cur->end = VOID_PTR_ADD(start, MemorySizeInBytes-1); + } else { + vm_area_t *new_area; + /* create a new area between cur and next */ new_area = vm_create_and_init_area(start, VOID_PTR_ADD(start, (MemorySizeInBytes - 1))); - if (new_area) { + if (!new_area) + return NULL; + new_area->next = next; + new_area->prev = cur; + if (cur) + cur->next = new_area; + else app->vm_ranges = new_area; - new_address = new_area->start; - } + if (next) + next->prev = new_area; } - return new_address; + return start; +} +static void *aperture_allocate_area(manageble_aperture_t *app, + uint64_t MemorySizeInBytes, + uint64_t offset) +{ + return aperture_allocate_area_aligned(app, MemorySizeInBytes, offset, app->align); } /* returns 0 on success. Assumes, that fmm_mutex is locked on entry */ From 5131ab4e645f077b1da8e2f9411336f237c6fc6f Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 17 Oct 2015 02:50:50 -0400 Subject: [PATCH 0093/1247] Implement flat scratch support for dGPU hsaKmtAllocMemory only allocates aligned address space and sets up the scratch_physical aperture to match the allocated address space. Actual allocation of backing memory happens in hsaKmtMapMemoryToGPU. Change-Id: Ie709815ab9bedb3d682e096b4005fdfb5e94d3a7 --- src/fmm.c | 216 +++++++++++++++++++++++++++++++++++++++++++++------ src/memory.c | 4 +- 2 files changed, 196 insertions(+), 24 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 69b68e8aaa..b725293859 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -116,6 +116,9 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); static void __fmm_release(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); +static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, + manageble_aperture_t *aperture, + void *address); static vm_area_t *vm_create_and_init_area(void *start, void *end) { @@ -505,6 +508,8 @@ void fmm_print(uint32_t gpu_id) manageble_aperture_print(&gpu_mem[i].gpuvm_aperture); printf("Scratch aperture:\n"); manageble_aperture_print(&gpu_mem[i].scratch_aperture); + printf("Scratch backing memory:\n"); + manageble_aperture_print(&gpu_mem[i].scratch_physical); printf("dGPU aperture:\n"); manageble_aperture_print(&gpu_mem[i].dgpu_aperture); printf("dGPU alt aperture:\n"); @@ -517,45 +522,111 @@ void fmm_print(uint32_t gpu_id) } #endif +static void fmm_release_scratch(uint32_t gpu_id) +{ + int32_t gpu_mem_id; + uint64_t size; + vm_object_t *obj; + manageble_aperture_t *aperture; + + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return; + + aperture = &gpu_mem[gpu_mem_id].scratch_physical; + + size = VOID_PTRS_SUB(aperture->limit, aperture->base) + 1; + + if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + /* unmap and remove all remaining objects */ + pthread_mutex_lock(&aperture->fmm_mutex); + while ((obj = aperture->vm_objects)) { + void *obj_addr = obj->start; + pthread_mutex_unlock(&aperture->fmm_mutex); + + _fmm_unmap_from_gpu_scratch(gpu_id, aperture, obj_addr); + + pthread_mutex_lock(&aperture->fmm_mutex); + } + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* release address space */ + pthread_mutex_lock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); + aperture_release_area(&gpu_mem[gpu_mem_id].dgpu_aperture, + gpu_mem[gpu_mem_id].scratch_physical.base, + size); + pthread_mutex_unlock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); + } else + /* release address space */ + munmap(gpu_mem[gpu_mem_id].scratch_physical.base, size); + + /* invalidate scratch backing aperture */ + gpu_mem[gpu_mem_id].scratch_physical.base = NULL; + gpu_mem[gpu_mem_id].scratch_physical.limit = NULL; +} + +#define SCRATCH_ALIGN 0x10000 void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { - manageble_aperture_t *aperture; manageble_aperture_t *aperture_phy; struct kfd_ioctl_alloc_memory_of_gpu_args args; int32_t gpu_mem_id; void *mem = NULL; + uint64_t aligned_size = ALIGN_UP(MemorySizeInBytes, SCRATCH_ALIGN); /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return NULL; - aperture = &gpu_mem[gpu_mem_id].scratch_aperture; aperture_phy = &gpu_mem[gpu_mem_id].scratch_physical; - - /* Check that aperture is properly initialized/supported */ - if (!aperture_is_valid(aperture->base, aperture->limit)) + if (aperture_phy->base != NULL || aperture_phy->limit != NULL) + /* Scratch was already allocated for this GPU */ return NULL; - /* Allocate address space */ - mem = mmap(0, MemorySizeInBytes + 16 * PAGE_SIZE, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == NULL) - return NULL; + /* Allocate address space for scratch backing, 64KB aligned */ + if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + pthread_mutex_lock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); + mem = aperture_allocate_area_aligned( + &gpu_mem[gpu_mem_id].dgpu_aperture, + aligned_size, 0, SCRATCH_ALIGN); + pthread_mutex_unlock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); + } else { + uint64_t aligned_padded_size = aligned_size + + SCRATCH_ALIGN - PAGE_SIZE; + void *padded_end, *aligned_start, *aligned_end; + mem = mmap(0, aligned_padded_size, + PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, + -1, 0); + if (mem == NULL) + return NULL; + /* align start and unmap padding */ + padded_end = VOID_PTR_ADD(mem, aligned_padded_size); + aligned_start = (void *)ALIGN_UP((uint64_t)mem, SCRATCH_ALIGN); + aligned_end = VOID_PTR_ADD(aligned_start, aligned_size); + if (aligned_start > mem) + munmap(mem, VOID_PTRS_SUB(aligned_start, mem)); + if (aligned_end < padded_end) + munmap(aligned_end, + VOID_PTRS_SUB(padded_end, aligned_end)); + mem = aligned_start; + } - /* Allocate memory from amdkfd */ + /* Remember scratch backing aperture for later */ + aperture_phy->base = mem; + aperture_phy->limit = VOID_PTR_ADD(mem, aligned_size-1); + + /* Allocate memory from amdkfd (just programs SH_HIDDEN_PRIVATE_BASE) */ args.gpu_id = gpu_id; args.size = MemorySizeInBytes; + args.va_addr = ((uint64_t)mem) >> 16; - /* va_addr is 40 bit GPUVM address */ - args.va_addr = (((uint64_t)mem) >> 16) + 1; - - aperture_phy->base = mem; - aperture_phy->limit = (void*)(((uint64_t)mem) + MemorySizeInBytes + 16 * PAGE_SIZE); - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH, &args)) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH, &args)) { + fmm_release_scratch(gpu_id); return NULL; + } - return (void*)(((((uint64_t)mem) >> 16) + 1) << 16); + return mem; } static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, @@ -793,10 +864,10 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if (address >= gpu_mem[i].scratch_physical.base && - address <= gpu_mem[i].scratch_physical.limit){ - munmap(gpu_mem[i].scratch_physical.base,(uint64_t)gpu_mem[i].scratch_physical.limit - (uint64_t)gpu_mem[i].scratch_physical.base); - return; - } + address <= gpu_mem[i].scratch_physical.limit) { + fmm_release_scratch(gpu_mem[i].gpu_id); + return; + } if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { @@ -920,6 +991,10 @@ HSAKMT_STATUS fmm_init_process_apertures(void) dgpu_mem_init(gpu_mem_id, &gpu_mem[gpu_mem_id].dgpu_aperture.base, &gpu_mem[gpu_mem_id].dgpu_aperture.limit); + + /* Set proper alignment for scratch backing aperture */ + gpu_mem[gpu_mem_id].scratch_physical.align = TONGA_PAGE_SIZE; + /* Set kernel process dgpu aperture. */ set_dgpu_aperture(i, (uint64_t)gpu_mem[gpu_mem_id].dgpu_aperture.base, (uint64_t)gpu_mem[gpu_mem_id].dgpu_aperture.limit); @@ -1049,6 +1124,49 @@ err_object_not_found: return -1; } +static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, + void *address, uint64_t size) +{ + int32_t gpu_mem_id; + uint64_t offset; + void *mem; + int ret; + + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return -1; + + if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) + return 0; /* Nothing to do on APU */ + + /* sanity check the address */ + if (address < aperture->base || + VOID_PTR_ADD(address, size -1) > aperture->limit) + return -1; + + /* allocate object within the scratch backing aperture */ + offset = VOID_PTRS_SUB(address, aperture->base); + mem = __fmm_allocate_device(gpu_id, size, aperture, offset, NULL, + KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); + if (mem == NULL) + return -1; + if (mem != address) { + fprintf(stderr, "Got unexpected address for scratch mapping.\n" + " expected: %p\n" + " got: %p\n", address, mem); + __fmm_release(gpu_id, mem, size, aperture); + return -1; + } + + /* map to GPU */ + ret = _fmm_map_to_gpu_gtt(gpu_id, aperture, address, size); + if (ret != 0) + __fmm_release(gpu_id, mem, size, aperture); + + return ret; +} + static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) @@ -1098,6 +1216,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; + if ((address >= gpu_mem[i].scratch_physical.base) && + (address <= gpu_mem[i].scratch_physical.limit)) + return _fmm_map_to_gpu_scratch(gpu_mem[i].gpu_id, + &gpu_mem[i].scratch_physical, + address, size); + if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* map it */ @@ -1151,6 +1275,48 @@ err: return -1; } +static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, + manageble_aperture_t *aperture, + void *address) +{ + int32_t gpu_mem_id; + vm_object_t *object; + uint64_t size; + struct kfd_ioctl_unmap_memory_from_gpu_args args; + + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return -1; + + if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) + return 0; /* Nothing to do on APU */ + + pthread_mutex_lock(&aperture->fmm_mutex); + + /* Find the object to retrieve the handle and size */ + object = vm_find_object_by_address(aperture, address, 0); + if (!object) + goto err; + + size = object->size; + + /* unmap from GPU */ + args.handle = object->handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* free object in scratch backing aperture */ + __fmm_release(gpu_id, address, size, aperture); + + return 0; + +err: + pthread_mutex_unlock(&aperture->fmm_mutex); + return -1; +} + int fmm_unmap_from_gpu(void *address) { int32_t i; @@ -1160,6 +1326,12 @@ int fmm_unmap_from_gpu(void *address) if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; + if ((address >= gpu_mem[i].scratch_physical.base) && + (address <= gpu_mem[i].scratch_physical.limit)) + return _fmm_unmap_from_gpu_scratch(gpu_mem[i].gpu_id, + &gpu_mem[i].gpuvm_aperture, + address); + if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* unmap it */ diff --git a/src/memory.c b/src/memory.c index 3314c82195..8052fcd3bd 100644 --- a/src/memory.c +++ b/src/memory.c @@ -155,7 +155,7 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_SUCCESS; } - if (!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged) { + if (!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); if (*MemoryAddress == NULL) @@ -163,7 +163,7 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_SUCCESS; } - if (MemFlags.ui32.HostAccess && MemFlags.ui32.Scratch ) { + if (MemFlags.ui32.Scratch ) { *MemoryAddress = fmm_allocate_scratch(gpu_id, SizeInBytes); if (*MemoryAddress == NULL) From cb53548c89109f54cc1d73e60a6c36605266dae3 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 22 Oct 2015 21:32:41 -0400 Subject: [PATCH 0094/1247] Unify fmm_get_aperture_xxx functions Unify fmm_get_aperture_base and fmm_get_aperture_limit into one function. Make the return value to HSAKMT_STATUS. Change-Id: I0b3f563ffb268947ab891f4935f61788d0af0e01 --- src/fmm.c | 60 +++++++++++++++++--------------------------------- src/fmm.h | 4 ++-- src/topology.c | 27 +++++++++++++---------- 3 files changed, 37 insertions(+), 54 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index b725293859..0dd26c46f2 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1035,8 +1035,10 @@ HSAKMT_STATUS fmm_init_process_apertures(void) return ret; } -HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id) +HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, + HSAuint64 *aperture_base, HSAuint64 *aperture_limit) { + HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); if (slot < 0) @@ -1044,56 +1046,34 @@ HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id switch (aperture_type) { case FMM_GPUVM: - return aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, - gpu_mem[slot].gpuvm_aperture.limit) ? - PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.limit) : 0; + if (aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, + gpu_mem[slot].gpuvm_aperture.limit)) { + *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.base); + *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.limit); + } break; case FMM_SCRATCH: - return aperture_is_valid(gpu_mem[slot].scratch_aperture.base, - gpu_mem[slot].scratch_aperture.limit) ? - PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.limit) : 0; + if (aperture_is_valid(gpu_mem[slot].scratch_aperture.base, + gpu_mem[slot].scratch_aperture.limit)) { + *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.base); + *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.limit); + } break; case FMM_LDS: - return aperture_is_valid(gpu_mem[slot].lds_aperture.base, - gpu_mem[slot].lds_aperture.limit) ? - PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.limit) : 0; + if (aperture_is_valid(gpu_mem[slot].lds_aperture.base, + gpu_mem[slot].lds_aperture.limit)) { + *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.base); + *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.limit); + } break; default: - return 0; + err = HSAKMT_STATUS_ERROR; } -} -HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id) -{ - int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); - if (slot < 0) - return HSAKMT_STATUS_INVALID_PARAMETER; - - switch (aperture_type) { - case FMM_GPUVM: - return aperture_is_valid(gpu_mem[slot].gpuvm_aperture.base, - gpu_mem[slot].gpuvm_aperture.limit) ? - PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.base) : 0; - break; - - case FMM_SCRATCH: - return aperture_is_valid(gpu_mem[slot].scratch_aperture.base, - gpu_mem[slot].scratch_aperture.limit) ? - PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.base) : 0; - break; - - case FMM_LDS: - return aperture_is_valid(gpu_mem[slot].lds_aperture.base, - gpu_mem[slot].lds_aperture.limit) ? - PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.base) : 0; - break; - - default: - return 0; - } + return err; } static int _fmm_map_to_gpu_gtt(uint32_t gpu_id, manageble_aperture_t *aperture, diff --git a/src/fmm.h b/src/fmm.h index c2ff77e074..10a56ff54a 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -65,6 +65,6 @@ bool fmm_get_handle(void *address, uint64_t *handle); /* Topology interface*/ HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); -HSAuint64 fmm_get_aperture_base(aperture_type_e aperture_type, HSAuint32 gpu_id); -HSAuint64 fmm_get_aperture_limit(aperture_type_e aperture_type, HSAuint32 gpu_id); +HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, + HSAuint64 *aperture_base, HSAuint64 *aperture_limit); #endif /* FMM_H_ */ diff --git a/src/topology.c b/src/topology.c index 2282c94977..2fca87a016 100644 --- a/src/topology.c +++ b/src/topology.c @@ -894,8 +894,9 @@ hsaKmtGetNodeMemoryProperties( HsaMemoryProperties* MemoryProperties //OUT ) { - HSAKMT_STATUS err; + HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; uint32_t i, gpu_id; + HSAuint64 aperture_limit; if (!MemoryProperties) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -918,7 +919,7 @@ hsaKmtGetNodeMemoryProperties( err = validate_nodeid(NodeId, &gpu_id); if (err != HSAKMT_STATUS_SUCCESS) - return err; + goto out; for (i = 0; i < MIN(node[NodeId].node.NumMemoryBanks, NumBanks); i++) { assert(node[NodeId].mem); @@ -926,31 +927,33 @@ hsaKmtGetNodeMemoryProperties( } /*Add LDS*/ - if (i < NumBanks){ + if (i < NumBanks && + fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id, + &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_LDS; MemoryProperties[i].SizeInBytes = node[NodeId].node.LDSSizeInKB * 1024; - MemoryProperties[i].VirtualBaseAddress = fmm_get_aperture_base(FMM_LDS, gpu_id); i++; } - /*Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE*/ - if ((i < NumBanks) && (node[NodeId].node.LocalMemSize > 0)) { + /* Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE*/ + if (i < NumBanks && + node[NodeId].node.LocalMemSize > 0 && + fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, + &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; MemoryProperties[i].SizeInBytes = node[NodeId].node.LocalMemSize; - MemoryProperties[i].VirtualBaseAddress = fmm_get_aperture_base(FMM_GPUVM, gpu_id); i++; } /*Add SCRATCH*/ - if (i < NumBanks){ + if (i < NumBanks && + fmm_get_aperture_base_and_limit(FMM_SCRATCH, gpu_id, + &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_SCRATCH; - MemoryProperties[i].VirtualBaseAddress = fmm_get_aperture_base(FMM_SCRATCH, gpu_id); - MemoryProperties[i].SizeInBytes = fmm_get_aperture_limit(FMM_SCRATCH, gpu_id) - MemoryProperties[i].VirtualBaseAddress; + MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1; i++; } - err = HSAKMT_STATUS_SUCCESS; - out: pthread_mutex_unlock(&hsakmt_mutex); return err; From 69662da3dc88c42b22ece5116ef80f08510817e2 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 23 Oct 2015 16:19:53 -0400 Subject: [PATCH 0095/1247] Correct parameter name for topology_is_dgpu() The function expects device_id and not gpu_id. Change-Id: I79794fd4e58e6e6adb26659da30f3e4d8e108434 --- src/libhsakmt.h | 2 +- src/topology.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 19726784ba..4ec1acafaa 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -73,7 +73,7 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); -bool topology_is_dgpu(uint16_t gpu_id); +bool topology_is_dgpu(uint16_t device_id); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); diff --git a/src/topology.c b/src/topology.c index 2fca87a016..e1ca33c7a6 100644 --- a/src/topology.c +++ b/src/topology.c @@ -247,10 +247,10 @@ static const struct hsa_gfxip_table* find_hsa_gfxip_device(uint16_t device_id) return NULL; } -bool topology_is_dgpu(uint16_t gpu_id) +bool topology_is_dgpu(uint16_t device_id) { const struct hsa_gfxip_table* hsa_gfxip = - find_hsa_gfxip_device(gpu_id); + find_hsa_gfxip_device(device_id); if (hsa_gfxip && hsa_gfxip->is_dgpu) { is_dgpu = true; From f885e551aa47aea5f01aa17f243c4f70323bf3f8 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 23 Oct 2015 17:35:50 -0400 Subject: [PATCH 0096/1247] Remove erroneous and redundant memory banks reported hsaKmtGetNodeMemoryProperties - - Return only HSA_HEAPTYPE_SYSTEM memory for CPU only node. - For dGPU remove redundant HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE entry. Change-Id: I0349be39b8409a0fd64a038b8b2956191356d937 --- src/topology.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/topology.c b/src/topology.c index e1ca33c7a6..39db33f1b8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -33,7 +33,9 @@ #include "fmm.h" #define PAGE_SIZE 4096 #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) -#define NUM_OF_HEAPS 3 +/* Number of memory banks added by thunk on top of topology */ +#define NUM_OF_IGPU_HEAPS 3 +#define NUM_OF_DGPU_HEAPS 2 /* SYSFS related */ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" @@ -877,8 +879,13 @@ hsaKmtGetNodeProperties( return err; *NodeProperties = node[NodeId].node; - NodeProperties->NumMemoryBanks += NUM_OF_HEAPS; - + /* For CPU only node don't add any additional GPU memory banks. */ + if (gpu_id) { + if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) + NodeProperties->NumMemoryBanks += NUM_OF_DGPU_HEAPS; + else + NodeProperties->NumMemoryBanks += NUM_OF_IGPU_HEAPS; + } err = HSAKMT_STATUS_SUCCESS; out: @@ -921,11 +928,17 @@ hsaKmtGetNodeMemoryProperties( if (err != HSAKMT_STATUS_SUCCESS) goto out; + memset(MemoryProperties, 0, NumBanks * sizeof(HsaMemoryProperties)); + for (i = 0; i < MIN(node[NodeId].node.NumMemoryBanks, NumBanks); i++) { assert(node[NodeId].mem); MemoryProperties[i] = node[NodeId].mem[i]; } + /* The following memory banks does not apply to CPU only node */ + if (gpu_id == 0) + goto out; + /*Add LDS*/ if (i < NumBanks && fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id, @@ -935,8 +948,10 @@ hsaKmtGetNodeMemoryProperties( i++; } - /* Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE*/ - if (i < NumBanks && + /* Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE. + * For dGPU the topology node contains Local Memory and it is added by the for loop above */ + if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && + i < NumBanks && node[NodeId].node.LocalMemSize > 0 && fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { From 6ceed7def39b281388ee50b9b8d0ea9b4b47a3d8 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 28 Oct 2015 14:37:11 -0400 Subject: [PATCH 0097/1247] libhsakmt - Add make option to package thunk as RPM Add an option to libhsakmt to allow the thunk to be packaged as an RPM. The default will remain being built as-is, but this can now be packaged as an RPM by using "make src rpm" . build_thunk.sh will be modified to reflect this new option. Change-Id: I38e03d10cfb5035bdf0a87635a784c47a709a5b6 --- Makefile | 3 +++ RPM/libhsakmt.spec | 30 ++++++++++++++++++++++++++++++ src/Makefile | 13 ++++++++++--- 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 RPM/libhsakmt.spec diff --git a/Makefile b/Makefile index b3c4b4accf..c306e90756 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,9 @@ all: lnx64a deb: $(MAKE) -C src deb +rpm: + $(MAKE) -C src rpm + lnx64a: $(MAKE) -C src lnx64a diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec new file mode 100644 index 0000000000..106c876b3a --- /dev/null +++ b/RPM/libhsakmt.spec @@ -0,0 +1,30 @@ +%define name libhsakmt +%define version 1.6.0 +%define thunkroot %{getenv:THUNK_ROOT} + +Name: %{name} +Version: %{version} +Release: 1 +Summary: Thunk libraries for AMD KFD + +Group: System Environment/Libraries +License: Advanced Micro Devices Inc. + +%description +This package includes the libhsakmt (Thunk) libraries +for AMD KFD + +%prep +%setup -T -D -c -n %{name} + +%install +mkdir -p $RPM_BUILD_ROOT +cp -R %{thunkroot}/build/rpm/libhsakmt/usr $RPM_BUILD_ROOT + +find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> fw.list + +%clean +rm -rf $RPM_BUILD_ROOT + +%files -f fw.list +%defattr(-,root,root,-) diff --git a/src/Makefile b/src/Makefile index 5c8c080593..c65c41aacf 100644 --- a/src/Makefile +++ b/src/Makefile @@ -49,13 +49,20 @@ lnx: LDFLAGS += -m32 lnx: $(BUILDDIR)/$(LIB_NAME) PACKAGE_DIR = $(BUILDDIR)/libhsakmt -INSTALL_DIR = $(PACKAGE_DIR)/usr/lib/x86_64-linux-gnu +INSTALL_DIR_DEB = $(PACKAGE_DIR)/usr/lib/x86_64-linux-gnu +INSTALL_DIR_RPM = $(PACKAGE_DIR)/usr/lib64 + deb: lnx64a - @mkdir -p $(INSTALL_DIR) - @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_DIR) + @mkdir -p $(INSTALL_DIR_DEB) + @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_DIR_DEB) @cp -r ../DEBIAN $(PACKAGE_DIR) @fakeroot dpkg-deb --build $(PACKAGE_DIR) $(BUILDDIR)/libhsakmt.deb +rpm: lnx64a + mkdir -p $(INSTALL_DIR_RPM) + @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_DIR_RPM) + @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec + lnx64a: $(BUILDDIR)/$(LIB_NAME) clean: From 0c234c7ef311e9fdaa51698ee2f9f28275597d8d Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Sun, 25 Oct 2015 10:21:57 -0400 Subject: [PATCH 0098/1247] Add SDMA IOCTL type to Create Queue function. Change-Id: I7e31507b761ca388b2cac93f994f6106de962f17 --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 6604f02889..a6025bb5cc 100644 --- a/src/queues.c +++ b/src/queues.c @@ -353,7 +353,7 @@ hsaKmtCreateQueue( switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: free_queue(q, dev_info->asic_family); return HSAKMT_STATUS_UNAVAILABLE; + case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; default: free_queue(q, dev_info->asic_family); return HSAKMT_STATUS_INVALID_PARAMETER; } From bd93eecc6484461dddbed5da8e7c5e4ac25bd0fa Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 27 Oct 2015 15:30:53 -0400 Subject: [PATCH 0099/1247] Use correct aperture for _fmm_unmap_from_gpu_scratch Passing in the wrong aperture resulted in failure to unmap scratch. Change-Id: Icd7423abfb1bcc773b33becffcbefc233f4ff340 --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 0dd26c46f2..ce4a40b8e6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1309,7 +1309,7 @@ int fmm_unmap_from_gpu(void *address) if ((address >= gpu_mem[i].scratch_physical.base) && (address <= gpu_mem[i].scratch_physical.limit)) return _fmm_unmap_from_gpu_scratch(gpu_mem[i].gpu_id, - &gpu_mem[i].gpuvm_aperture, + &gpu_mem[i].scratch_physical, address); if ((address >= gpu_mem[i].gpuvm_aperture.base) && From cabbcbabffd83fbf1ba70c4edcbb40040a7bda4c Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 29 Oct 2015 08:54:38 -0400 Subject: [PATCH 0100/1247] Cleanup RPM build of thunk Change-Id: Ib437a3ec7be9f5aa7d3ef9e53c13e3c5e7b7382e --- RPM/libhsakmt.spec | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index 106c876b3a..3171c9ac02 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -18,13 +18,12 @@ for AMD KFD %setup -T -D -c -n %{name} %install -mkdir -p $RPM_BUILD_ROOT -cp -R %{thunkroot}/build/rpm/libhsakmt/usr $RPM_BUILD_ROOT - -find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> fw.list +cp -R %{thunkroot}/build/rpm/libhsakmt $RPM_BUILD_ROOT +find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> thunk.list %clean rm -rf $RPM_BUILD_ROOT -%files -f fw.list +%files -f thunk.list + %defattr(-,root,root,-) From cb3a6640655e10fcbf48e2622ac14702188b1794 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 30 Oct 2015 14:05:40 -0400 Subject: [PATCH 0101/1247] Add option to create release build for Thunk By adding REL=1 to the make command line (e.g. make REL=1 deb), we can create a release build of the Thunk. This will not affect existing functionality, and will only have an effect if REL=1 is specified on the command line, or in the build_thunk.sh script. Change-Id: Iedc3b6094e70a4ebd726499eda56013cc254b83d --- src/Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index c65c41aacf..03ce54e07e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -13,7 +13,12 @@ CFLAGS += -Wformat-security -Wswitch-default -Wundef \ -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes \ -Wmissing-declarations -Wredundant-decls \ -Wunreachable-code -CFLAGS += -std=gnu99 -ggdb -pthread -fvisibility=hidden -O2 +CFLAGS += -std=gnu99 -pthread -fvisibility=hidden -O2 + +# If this is not a release build, add debug-specific flags +ifneq ($(REL),1) + CFLAGS += -ggdb +endif LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) From b6f65f98498563c8e18d4a9758d1fff40b3f889c Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 5 Nov 2015 15:37:06 -0500 Subject: [PATCH 0102/1247] Add CPU cache information Fill up cache properties of CPU node by reading data from /proc/cpuinfo and /sys/devices/system/cpu/cpuX/cache/indexY Change-Id: I0a96760575e504e38962554f192c3fe66bea3c15 --- src/topology.c | 281 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 231 insertions(+), 50 deletions(-) diff --git a/src/topology.c b/src/topology.c index 39db33f1b8..f3c7cc804f 100644 --- a/src/topology.c +++ b/src/topology.c @@ -25,9 +25,12 @@ #include #include +#include #include #include #include +#include +#include #include "libhsakmt.h" #include "fmm.h" @@ -40,6 +43,8 @@ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" #define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" +#define MAX_CPU_CORES 32 +#define MAX_CACHES 128 typedef struct { uint32_t gpu_id; @@ -49,7 +54,7 @@ typedef struct { HsaIoLinkProperties *link; } node_t; -static HsaSystemProperties *system = NULL; +static HsaSystemProperties *_system = NULL; static node_t *node = NULL; static HSAKMT_STATUS topology_take_snapshot(void); @@ -122,6 +127,57 @@ free_node(node_t *n) free((n)->link); } +/* num_subdirs - find the number of sub-directories in the specified path + * @dirpath - directory path to find sub-directories underneath + * @prefix - only count sub-directory names starting with prefix. + * Use blank string, "", to count all. + * Return - number of sub-directories + */ +static int num_subdirs(char *dirpath, char *prefix) +{ + int count = 0; + DIR *dirp; + struct dirent *dir; + int prefix_len = strlen(prefix); + + dirp = opendir(dirpath); + if(dirp) { + while ((dir = readdir(dirp)) != 0) { + if ((strcmp(dir->d_name, ".") == 0) || + (strcmp(dir->d_name, "..") == 0)) + continue; + if (prefix_len && + strncmp(dir->d_name, prefix, prefix_len)) + continue; + count++; + } + closedir(dirp); + } + + return count; +} + +/* read_file - Read the content of a file + * @file - file to read + * @buf - [OUT] buffer containing data read from the file + * @buf_sz - buffer size + * Return - data size in the returning buffer + */ +static size_t read_file(char *file, char *buf, size_t buf_sz) +{ + int fd; + size_t len = 0; + + memset(buf, 0, buf_sz); + + if ((fd = open(file, O_RDONLY)) < 0) + return 0; + len = read(fd, buf, buf_sz); + close(fd); + + return len; +} + static HSAKMT_STATUS topology_sysfs_get_generation(uint32_t *gen) { FILE *fd; @@ -144,13 +200,11 @@ err: HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) { FILE *fd; - DIR *dirp; char *read_buf, *p; char prop_name[256]; long long unsigned int prop_val; - uint32_t node_count, prog; - struct dirent *dir; - int read_size; + uint32_t prog; + int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -191,25 +245,11 @@ topology_sysfs_get_system_props(HsaSystemProperties *props) { } /* - * Discover the number of nodes + * Discover the number of nodes: + * Assuming that inside nodes folder there are only folders + * which represent the node numbers */ - node_count = 0; - dirp = opendir(KFD_SYSFS_PATH_NODES); - if(dirp) { - /* - * Assuming that inside nodes folder there are only folders - * which represent the node numbers - */ - while ((dir = readdir(dirp)) != 0) { - if ((strcmp(dir->d_name, ".") == 0) || - (strcmp(dir->d_name, "..") == 0)) - continue; - node_count++; - } - closedir(dirp); - } - props->NumNodes = node_count; - + props->NumNodes = num_subdirs(KFD_SYSFS_PATH_NODES, ""); err2: free(read_buf); @@ -465,6 +505,137 @@ err1: return ret; } +/* topology_get_cpu_cache_props - get CPU cache properties and fill in the + * cache entry of the node's table + * @tbl - the node table to fill up + * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure + */ +static HSAKMT_STATUS +topology_get_cpu_cache_props(node_t *tbl) +{ + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + char *token, *str; + uint32_t apicid[MAX_CPU_CORES]; + int num_cpus = 0, num_caches = 0; + char path[256], buf[256], cache_paths[MAX_CACHES][256]; + int i, j, n; + const char SYSDIR[] = "/sys/devices/system/cpu"; + unsigned long ul; + + if (tbl == NULL) + return HSAKMT_STATUS_ERROR; + + /* Get apicid info from /proc/cpuinfo for ProcessorIdLow */ + if (!(fd = fopen("/proc/cpuinfo", "r"))) + return HSAKMT_STATUS_ERROR; + + while (fgets(buf, 256, fd) != NULL) { + /* /proc/cpuinfo lists in format - property : value */ + token = strtok_r(buf, ":", &str); + if (strncmp(token, "apicid", 6) == 0) { + if (num_cpus >= MAX_CPU_CORES) { + printf("MAX_CPU_CORES %d is not enough.", MAX_CPU_CORES); + fclose(fd); + return HSAKMT_STATUS_ERROR; + } + apicid[num_cpus++] = atoi(str); + } + } + fclose(fd); + + /* Get cache data from /sys/devices/system/cpu/cpuX/cache/indexY + * 1. Calculate how many caches + * 2. Fill up cache properties + */ + + for (i=0; i= MAX_CACHES) { + printf("MAX_CACHES %d is not enough!\n", MAX_CACHES); + return HSAKMT_STATUS_ERROR; + } + snprintf(cache_paths[num_caches++], 256, + "%s/cpu%d/cache/index%d", SYSDIR, i, j); + } + } + + tbl->node.NumCaches = num_caches; + tbl->cache = calloc(tbl->node.NumCaches * sizeof(HsaCacheProperties), 1); + if (!tbl->cache) + return HSAKMT_STATUS_NO_MEMORY; + + for (i=0; icache[i].CacheLevel = atoi(buf); + /* cache size */ + snprintf(path, 256, "%s/size", cache_paths[i]); + read_file(path, buf, sizeof(buf)); + tbl->cache[i].CacheSize = (atoi(buf)); + /* cache line size in bytes */ + snprintf(path, 256, "%s/coherency_line_size", cache_paths[i]); + read_file(path, buf, sizeof(buf)); + tbl->cache[i].CacheLineSize = (atoi(buf)); + /* cache lines per tag */ + snprintf(path, 256, "%s/physical_line_partition", cache_paths[i]); + read_file(path, buf, sizeof(buf)); + tbl->cache[i].CacheLinesPerTag = (atoi(buf)); + /* cache associativity */ + snprintf(path, 256, "%s/ways_of_associativity", cache_paths[i]); + read_file(path, buf, sizeof(buf)); + tbl->cache[i].CacheAssociativity = (atoi(buf)); + /* cache type */ + tbl->cache[i].CacheType.ui32.CPU = 1; + snprintf(path, 256, "%s/type", cache_paths[i]); + read_file(path, buf, sizeof(buf)); + if (buf[0] == 'D') + tbl->cache[i].CacheType.ui32.Data = 1; + else if (buf[0] == 'I') + tbl->cache[i].CacheType.ui32.Instruction = 1; + /* sibling map */ + snprintf(path, 256, "%s/shared_cpu_map", cache_paths[i]); + read_file(path, buf, sizeof(buf)); + ul = strtol(buf, NULL, 16); + j = 0; + while (!(ul & (1 << j))) + j++; // the lowest processor id in shared map + tbl->cache[i].ProcessorIdLow = apicid[j]; + j = num_cpus; + while (j-- > 0) { + if (ul & (1<cache[i].ProcessorIdLow; + /* Use array instead of bitmask so the software + * is endian-worry free + */ + tbl->cache[i].SiblingMap[n] = 1; + } + } + } + + return ret; +} + static HSAKMT_STATUS topology_sysfs_get_cache_props(uint32_t node_id, uint32_t cache_id, HsaCacheProperties *props) { FILE *fd; @@ -676,6 +847,16 @@ retry: } } } + else if (!temp_nodes[i].gpu_id) { /* This is a CPU node */ + /* Get info from /proc/cpuinfo and /sys/devices/system */ + ret = topology_get_cpu_cache_props(&temp_nodes[i]); + if (ret != HSAKMT_STATUS_SUCCESS) { + for (j=0; j <= i; j++) + free_node(&temp_nodes[j]); + free(temp_nodes); + goto err; + } + } if (temp_nodes[i].node.NumIOLinks) { temp_nodes[i].link = calloc(temp_nodes[i].node.NumIOLinks * sizeof(HsaIoLinkProperties), 1); @@ -720,9 +901,9 @@ retry: goto retry; } - if (!system) { - system = malloc(sizeof(HsaSystemProperties)); - if (!system) { + if (!_system) { + _system = malloc(sizeof(HsaSystemProperties)); + if (!_system) { if (temp_nodes) { for (j=0; j < sys_props.NumNodes; j++) free_node(&temp_nodes[j]); @@ -732,7 +913,7 @@ retry: } } - *system = sys_props; + *_system = sys_props; if (node) free(node); node = temp_nodes; @@ -750,7 +931,7 @@ topology_drop_snapshot(void) { HSAKMT_STATUS err; - if (!!system != !!node) { + if (!!_system != !!node) { printf("Probable inconsistency?\n"); err = HSAKMT_STATUS_SUCCESS; goto out; @@ -760,7 +941,7 @@ topology_drop_snapshot(void) uint64_t nodeid; /* Remove state */ - for (nodeid = 0; nodeid < system->NumNodes; nodeid++) { + for (nodeid = 0; nodeid < _system->NumNodes; nodeid++) { free_node(&node[nodeid]); } @@ -768,8 +949,8 @@ topology_drop_snapshot(void) node = NULL; } - free(system); - system = NULL; + free(_system); + _system = NULL; err = HSAKMT_STATUS_SUCCESS; out: @@ -779,7 +960,7 @@ out: HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) { - if (nodeid >= MAX_NODES || !node || !system || system->NumNodes <= nodeid) + if (nodeid >= MAX_NODES || !node || !_system || _system->NumNodes <= nodeid) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (gpu_id) *gpu_id = node[nodeid].gpu_id; @@ -790,7 +971,7 @@ validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id){ uint64_t node_idx; - for(node_idx = 0; node_idx < system->NumNodes; node_idx++){ + for(node_idx = 0; node_idx < _system->NumNodes; node_idx++){ if (node[node_idx].gpu_id == gpu_id){ *node_id = node_idx; return HSAKMT_STATUS_SUCCESS; @@ -819,9 +1000,9 @@ hsaKmtAcquireSystemProperties( if (err != HSAKMT_STATUS_SUCCESS) goto out; - assert(system); + assert(_system); - *SystemProperties = *system; + *SystemProperties = *_system; err = HSAKMT_STATUS_SUCCESS; out: @@ -863,13 +1044,13 @@ hsaKmtGetNodeProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (system == NULL) { + if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(system); + assert(_system); goto out; } - if (NodeId >= system->NumNodes) { + if (NodeId >= _system->NumNodes) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -912,14 +1093,14 @@ hsaKmtGetNodeMemoryProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (system == NULL) { + if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(system); + assert(_system); goto out; } /* Check still necessary */ - if (NodeId >= system->NumNodes ) { + if (NodeId >= _system->NumNodes ) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -993,13 +1174,13 @@ hsaKmtGetNodeCacheProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (system == NULL) { + if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(system); + assert(_system); goto out; } - if (NodeId >= system->NumNodes || NumCaches > node[NodeId].node.NumCaches) { + if (NodeId >= _system->NumNodes || NumCaches > node[NodeId].node.NumCaches) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -1035,13 +1216,13 @@ hsaKmtGetNodeIoLinkProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (system == NULL) { + if (_system == NULL) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(system); + assert(_system); goto out; } - if (NodeId >= system->NumNodes || NumIoLinks > node[NodeId].node.NumIOLinks) { + if (NodeId >= _system->NumNodes || NumIoLinks > node[NodeId].node.NumIOLinks) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -1060,7 +1241,7 @@ out: uint16_t get_device_id_by_node(HSAuint32 node_id) { - if (!node || !system || system->NumNodes <= node_id) + if (!node || !_system || _system->NumNodes <= node_id) return 0; return node[node_id].node.DeviceId; @@ -1069,10 +1250,10 @@ uint16_t get_device_id_by_node(HSAuint32 node_id) uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; - if (!node || !system) + if (!node || !_system) return 0; - for (i = 0; i < system->NumNodes; i++) { + for (i = 0; i < _system->NumNodes; i++) { if (node[i].gpu_id == gpu_id) return node[i].node.DeviceId; } From 4e4d4a81e17db2722cdd3aa512c1e5ef91dd1785 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Sun, 8 Nov 2015 22:57:30 -0500 Subject: [PATCH 0103/1247] Fix variable for RPM build Certain versions of rpmbuild need the variable to be outside of curly braces. This addresses that issue in that situation. Change-Id: Iff7200b332b9d8e41a4d7676ca14c5a32c075beb --- RPM/libhsakmt.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index 3171c9ac02..15e78e3361 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -18,7 +18,7 @@ for AMD KFD %setup -T -D -c -n %{name} %install -cp -R %{thunkroot}/build/rpm/libhsakmt $RPM_BUILD_ROOT +cp -R %thunkroot/build/rpm/libhsakmt $RPM_BUILD_ROOT find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> thunk.list %clean From 3786e18d990e0baae5c0b95ed8585dbb8da72177 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 9 Nov 2015 16:13:21 -0500 Subject: [PATCH 0104/1247] Use OUT_DIR for thunkroot variable Pick up the thunk from the correct location. It is no longer inside THUNK_ROOT, but instead part of the OUT folder. Change-Id: I41dd7dae243e66270d0ea7182f1ba119b18a1cfb --- RPM/libhsakmt.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index 15e78e3361..afce127bc8 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -1,6 +1,6 @@ %define name libhsakmt %define version 1.6.0 -%define thunkroot %{getenv:THUNK_ROOT} +%define thunkroot %{getenv:OUT_DIR} Name: %{name} Version: %{version} @@ -18,7 +18,7 @@ for AMD KFD %setup -T -D -c -n %{name} %install -cp -R %thunkroot/build/rpm/libhsakmt $RPM_BUILD_ROOT +cp -R %thunkroot/build/build_thunk.sh/rpm/libhsakmt $RPM_BUILD_ROOT find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> thunk.list %clean From 62337b6c0aeac43762b9d2116bb9c9dbbf71c7e0 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 10 Nov 2015 18:19:56 -0500 Subject: [PATCH 0105/1247] Reserve address space with PROT_NONE Access to reserved address space that has not been allocated should result in a segfault. Use PROT_NONE to ensure that. Change-Id: Ic5da9392fabbe78c9ec14f98e8b7b47e5267a98a --- src/fmm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ce4a40b8e6..80d52aa8e0 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -754,7 +754,7 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, /* FIXME: host memory allocated in this way should be mapped on all GPUs */ void *ret = mmap(mem, MemorySizeInBytes, - PROT_READ | PROT_WRITE | PROT_EXEC, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); if (ret == MAP_FAILED) { __fmm_release(gpu_id, mem, MemorySizeInBytes, aperture); @@ -898,7 +898,7 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { /* Remove any CPU mapping, but keep the address range reserved */ - mmap(address, MemorySizeInBytes, PROT_READ | PROT_WRITE, + mmap(address, MemorySizeInBytes, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); } @@ -1353,7 +1353,7 @@ static void *reserve_address(void *addr, long long unsigned int len) if (len <= 0) return NULL; - ret_addr = mmap(addr, len, PROT_READ | PROT_WRITE, + ret_addr = mmap(addr, len, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); if (addr == MAP_FAILED) return NULL; From a5bc8360e899ed4086e4c94858d5be88e86f6bb0 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 11 Nov 2015 17:46:55 -0500 Subject: [PATCH 0106/1247] Fix sibling map in CPU cache properties Previous code only works for systems where shared_cpu_map lists 32 or less bits. Some systems list more than 32 bits and express them as XXXXXXXX,XXXXXXXX,.... This patch adds that calculation. Also increase MAX_CPU_CORES and MAX_CACHES to accommodate more advanced systems. Change-Id: Ia5c7041866456a6aa3b66f8f0f951022d7c51028 --- src/topology.c | 179 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 126 insertions(+), 53 deletions(-) diff --git a/src/topology.c b/src/topology.c index f3c7cc804f..8adc52a450 100644 --- a/src/topology.c +++ b/src/topology.c @@ -43,8 +43,8 @@ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" #define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" -#define MAX_CPU_CORES 32 -#define MAX_CACHES 128 +#define MAX_CPU_CORES 128 +#define MAX_CACHES 256 typedef struct { uint32_t gpu_id; @@ -505,6 +505,119 @@ err1: return ret; } + +/* parse_sysfs_cache - + * @sys_path - cache path in sysfs + * @prop - [OUT] HSA cache property to fill up + * @apicid - an array where contains each processor's apicid + * Return - HSAKMT_STATUS_SUCCESS in success or an error number in failure + */ +static HSAKMT_STATUS +parse_sysfs_cache(char *sys_path, HsaCacheProperties *prop, uint32_t *apicid) +{ + char file[256], buf[256]; + int i, j, n, cpu; + int last; /* the last valid entry in array, which is n-1 if n items */ + unsigned long map[32]; + char *token, *str; + + /* cache level */ + snprintf(file, 256, "%s/level", sys_path); + read_file(file, buf, sizeof(buf)); + prop->CacheLevel = atoi(buf); + + /* cache size */ + snprintf(file, 256, "%s/size", sys_path); + read_file(file, buf, sizeof(buf)); + prop->CacheSize = (atoi(buf)); + + /* cache line size in bytes */ + snprintf(file, 256, "%s/coherency_line_size", sys_path); + read_file(file, buf, sizeof(buf)); + prop->CacheLineSize = (atoi(buf)); + + /* cache lines per tag */ + snprintf(file, 256, "%s/physical_line_partition", sys_path); + read_file(file, buf, sizeof(buf)); + prop->CacheLinesPerTag = (atoi(buf)); + + /* cache associativity */ + snprintf(file, 256, "%s/ways_of_associativity", sys_path); + read_file(file, buf, sizeof(buf)); + prop->CacheAssociativity = (atoi(buf)); + + /* cache type */ + prop->CacheType.ui32.CPU = 1; + snprintf(file, 256, "%s/type", sys_path); + read_file(file, buf, sizeof(buf)); + if (buf[0] == 'D') + prop->CacheType.ui32.Data = 1; + else if (buf[0] == 'I') + prop->CacheType.ui32.Instruction = 1; + + /* sibling map */ + snprintf(file, 256, "%s/shared_cpu_map", sys_path); + read_file(file, buf, sizeof(buf)); + /* Data in shared_cpu_map can be XXXXXXXX when the system doesn't have + * more than 32 processors; it also can be XXXXXXXX,XXXXXXXX,XX .... to + * represent more than 32 processors. We'll parse each XXXXXXXX and + * store them into map[]. + * Say shared_cpu_map is "Nn-1,Nn-2,...,N2,N1,N0\n". Because strtok_r() + * parses Nn-1 first, map[] will store data in a reversed order: + * map[0]=Nn-1, map[1]=Nn-2, ... map[n-2]=N1, map[n-1]=N0 + */ + str = (char *)&buf[0]; + for (last = 0; last < 32; last++) { /* declared map[32] */ + token = strtok_r(str, ",", &str); + map[last] = strtol(token, NULL, 16); + if (token[strlen(token)-1] == '\n') /* this is N0 */ + break; + } + if (last >= 32) { + printf("Fail to parse shared_cpu_map. Increase map[].\n"); + return HSAKMT_STATUS_ERROR; + } + + /* Lower processor ID doesn't always have lower apicid. + * Search the lowest apicid for ProcIdLow + */ + prop->ProcessorIdLow = 0xffffffff; + for (i = last; i >= 0; i--) { /* N0 is stored in map[count] */ + if (!map[i]) + continue; + j = 32; + while (j-- > 0) { + if (map[i] & (1<ProcessorIdLow) + prop->ProcessorIdLow = apicid[cpu]; + } + } + } + /* Now fill in SiblingMap using ProcIdLow as the offset */ + for (i = last; i >= 0; i--) { + j = 32; + while (j-- > 0) { + if (map[i] & (1<ProcessorIdLow; + /* Use array instead of bitmask so the software + * is endian-worry free + */ + if (n < HSA_CPU_SIBLINGS) + prop->SiblingMap[n] = 1; + else { + printf("Increase HSA_CPU_SIBLINGS.\n"); + return HSAKMT_STATUS_ERROR; + } + } + } + } + + return HSAKMT_STATUS_SUCCESS; +} + /* topology_get_cpu_cache_props - get CPU cache properties and fill in the * cache entry of the node's table * @tbl - the node table to fill up @@ -521,7 +634,6 @@ topology_get_cpu_cache_props(node_t *tbl) char path[256], buf[256], cache_paths[MAX_CACHES][256]; int i, j, n; const char SYSDIR[] = "/sys/devices/system/cpu"; - unsigned long ul; if (tbl == NULL) return HSAKMT_STATUS_ERROR; @@ -544,11 +656,9 @@ topology_get_cpu_cache_props(node_t *tbl) } fclose(fd); - /* Get cache data from /sys/devices/system/cpu/cpuX/cache/indexY - * 1. Calculate how many caches - * 2. Fill up cache properties - */ + /* Get cache data from /sys/devices/system/cpu/cpuX/cache/indexY */ + /* 1. Calculate how many caches */ for (i=0; inode.NumCaches = num_caches; tbl->cache = calloc(tbl->node.NumCaches * sizeof(HsaCacheProperties), 1); if (!tbl->cache) return HSAKMT_STATUS_NO_MEMORY; + /* 3. Fill up cache properties */ for (i=0; icache[i].CacheLevel = atoi(buf); - /* cache size */ - snprintf(path, 256, "%s/size", cache_paths[i]); - read_file(path, buf, sizeof(buf)); - tbl->cache[i].CacheSize = (atoi(buf)); - /* cache line size in bytes */ - snprintf(path, 256, "%s/coherency_line_size", cache_paths[i]); - read_file(path, buf, sizeof(buf)); - tbl->cache[i].CacheLineSize = (atoi(buf)); - /* cache lines per tag */ - snprintf(path, 256, "%s/physical_line_partition", cache_paths[i]); - read_file(path, buf, sizeof(buf)); - tbl->cache[i].CacheLinesPerTag = (atoi(buf)); - /* cache associativity */ - snprintf(path, 256, "%s/ways_of_associativity", cache_paths[i]); - read_file(path, buf, sizeof(buf)); - tbl->cache[i].CacheAssociativity = (atoi(buf)); - /* cache type */ - tbl->cache[i].CacheType.ui32.CPU = 1; - snprintf(path, 256, "%s/type", cache_paths[i]); - read_file(path, buf, sizeof(buf)); - if (buf[0] == 'D') - tbl->cache[i].CacheType.ui32.Data = 1; - else if (buf[0] == 'I') - tbl->cache[i].CacheType.ui32.Instruction = 1; - /* sibling map */ - snprintf(path, 256, "%s/shared_cpu_map", cache_paths[i]); - read_file(path, buf, sizeof(buf)); - ul = strtol(buf, NULL, 16); - j = 0; - while (!(ul & (1 << j))) - j++; // the lowest processor id in shared map - tbl->cache[i].ProcessorIdLow = apicid[j]; - j = num_cpus; - while (j-- > 0) { - if (ul & (1<cache[i].ProcessorIdLow; - /* Use array instead of bitmask so the software - * is endian-worry free - */ - tbl->cache[i].SiblingMap[n] = 1; - } + ret = parse_sysfs_cache(cache_paths[i], + &tbl->cache[i], + &apicid[0]); + if (ret != HSAKMT_STATUS_SUCCESS) { + printf("Failed to parse cache properties.\n"); + free(tbl->cache); + return ret; } } From 5a55383baf6075e6a5637fbc3493cbb022130bf7 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 23 Nov 2015 11:49:11 -0500 Subject: [PATCH 0107/1247] Fix dgpu_vm_limit Break from the for-loop once dgpu VM range is found, otherwise the length is reduced by half Change-Id: Ie602054c16ea69ea1cbb75e804ead551bc3615c0 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 80d52aa8e0..5d13f57069 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1441,8 +1441,10 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi (HSAint64)ret_addr) >> 30)); munmap(ret_addr, len); continue; - } else + } else { found = true; + break; + } } if (!found) { From 2903a610e1dc4334bd67fb126bd03aadc0f9e8e6 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 25 Nov 2015 18:00:42 -0500 Subject: [PATCH 0108/1247] Use same VM range for all dGPUs dgpu_aperture and dgpu_alt_aperture will be shared by all dGPUs. Change-Id: I814495e43b51acabdc6266cfa8d83db5a062e20d Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 192 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 109 insertions(+), 83 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 5d13f57069..0bd5e46966 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -52,10 +52,9 @@ #define INIT_GPU_MEM { \ .gpu_id = NON_VALID_GPU_ID, \ .lds_aperture = INIT_APERTURE(0, 0), \ + .scratch_physical = INIT_MANAGEBLE_APERTURE(0, 0), \ .scratch_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ .gpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ - .dgpu_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ - .dgpu_alt_aperture = INIT_MANAGEBLE_APERTURE(0, 0) \ } #define INIT_GPUs_MEM {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = INIT_GPU_MEM} @@ -77,6 +76,7 @@ struct vm_area { }; typedef struct vm_area vm_area_t; +/* Memory manager for an aperture */ typedef struct { void *base; void *limit; @@ -98,20 +98,39 @@ typedef struct { uint64_t local_mem_size; aperture_t lds_aperture; manageble_aperture_t scratch_aperture; - manageble_aperture_t scratch_physical; + manageble_aperture_t scratch_physical; /* For dGPU, scratch physical + is allocated from dgpu_aperture. When requested by RT, each + GPU will get a differnt range */ manageble_aperture_t gpuvm_aperture; /* used for device mem on APU and for Gfx interop, unusable on dGPU with small-ish VA range */ - manageble_aperture_t dgpu_aperture; /* used for non-coherent system and invisible device mem on dGPU */ - manageble_aperture_t dgpu_alt_aperture; /* used for coherent (fine-grain) system memory on dGPU */ /* TODO: Merge gpuvm and dgpu apertures. When we have bigger * VA range, we can add a new invisible aperture for invisible * device mem on dGPU. */ } gpu_mem_t; +/* The main structure for GPU Memory Management */ +typedef struct { + /* used for non-coherent system and invisible device mem on dGPU. + * This aperture is shared by all dGPUs */ + manageble_aperture_t dgpu_aperture; + + /* used for coherent (fine-grain) system memory on dGPU, + * This aperture is shared by all dGPUs */ + manageble_aperture_t dgpu_alt_aperture; +} svm_t; + +/* The other apertures are specific to each GPU. gpu_mem_t manages GPU +* specific memory apertures. */ static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; + static void *dgpu_shared_aperture_base = NULL; static void *dgpu_shared_aperture_limit = NULL; +static svm_t svm = { + INIT_MANAGEBLE_APERTURE(0, 0), + INIT_MANAGEBLE_APERTURE(0, 0) +}; + static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); static void __fmm_release(uint32_t gpu_id, void *address, @@ -499,22 +518,24 @@ static void manageble_aperture_print(manageble_aperture_t *app) void fmm_print(uint32_t gpu_id) { - int32_t i = gpu_mem_find_by_gpu_id(gpu_id); + int32_t gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); - if (i >= 0) { /* Found */ + if (gpu_mem_id >= 0) { /* Found */ printf("LDS aperture:\n"); - aperture_print(&gpu_mem[i].lds_aperture); + aperture_print(&gpu_mem[gpu_mem_id].lds_aperture); printf("GPUVM aperture:\n"); - manageble_aperture_print(&gpu_mem[i].gpuvm_aperture); + manageble_aperture_print(&gpu_mem[gpu_mem_id].gpuvm_aperture); printf("Scratch aperture:\n"); - manageble_aperture_print(&gpu_mem[i].scratch_aperture); + manageble_aperture_print(&gpu_mem[gpu_mem_id].scratch_aperture); printf("Scratch backing memory:\n"); - manageble_aperture_print(&gpu_mem[i].scratch_physical); - printf("dGPU aperture:\n"); - manageble_aperture_print(&gpu_mem[i].dgpu_aperture); - printf("dGPU alt aperture:\n"); - manageble_aperture_print(&gpu_mem[i].dgpu_alt_aperture); + manageble_aperture_print(&gpu_mem[gpu_mem_id].scratch_physical); } + + printf("dGPU aperture:\n"); + manageble_aperture_print(&svm.dgpu_aperture); + printf("dGPU alt aperture:\n"); + manageble_aperture_print(&svm.dgpu_alt_aperture); + } #else void fmm_print(uint32_t gpu_id) @@ -551,11 +572,11 @@ static void fmm_release_scratch(uint32_t gpu_id) pthread_mutex_unlock(&aperture->fmm_mutex); /* release address space */ - pthread_mutex_lock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); - aperture_release_area(&gpu_mem[gpu_mem_id].dgpu_aperture, + pthread_mutex_lock(&svm.dgpu_aperture.fmm_mutex); + aperture_release_area(&svm.dgpu_aperture, gpu_mem[gpu_mem_id].scratch_physical.base, size); - pthread_mutex_unlock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); + pthread_mutex_unlock(&svm.dgpu_aperture.fmm_mutex); } else /* release address space */ munmap(gpu_mem[gpu_mem_id].scratch_physical.base, size); @@ -586,11 +607,11 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) /* Allocate address space for scratch backing, 64KB aligned */ if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { - pthread_mutex_lock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); + pthread_mutex_lock(&svm.dgpu_aperture.fmm_mutex); mem = aperture_allocate_area_aligned( - &gpu_mem[gpu_mem_id].dgpu_aperture, + &svm.dgpu_aperture, aligned_size, 0, SCRATCH_ALIGN); - pthread_mutex_unlock(&gpu_mem[gpu_mem_id].dgpu_aperture.fmm_mutex); + pthread_mutex_unlock(&svm.dgpu_aperture.fmm_mutex); } else { uint64_t aligned_padded_size = aligned_size + SCRATCH_ALIGN - PAGE_SIZE; @@ -687,7 +708,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) * TODO: Once VA limit is raised from 0x200000000 (8GB) use gpuvm_aperture. * In that way the host access range won't be used for local memory */ - aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; + aperture = &svm.dgpu_aperture; } else { flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; @@ -740,9 +761,9 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, size = MemorySizeInBytes; ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST; if (flags.ui32.CoarseGrain) - aperture = &gpu_mem[gpu_mem_id].dgpu_aperture; + aperture = &svm.dgpu_aperture; else - aperture = &gpu_mem[gpu_mem_id].dgpu_alt_aperture; /* coherent */ + aperture = &svm.dgpu_alt_aperture; /* coherent */ if (flags.ui32.AQLQueueMemory) { size = MemorySizeInBytes * 2; ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; @@ -860,36 +881,37 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) uint32_t i; bool found = false; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS && !found ; i++) { + for (i = 0; i < NUM_OF_SUPPORTED_GPUS && !found; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; - if (address >= gpu_mem[i].scratch_physical.base && + if (address >= gpu_mem[i].scratch_physical.base && address <= gpu_mem[i].scratch_physical.limit) { fmm_release_scratch(gpu_mem[i].gpu_id); return; } - if (address >= gpu_mem[i].gpuvm_aperture.base && + if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; __fmm_release(gpu_mem[i].gpu_id, address, MemorySizeInBytes, &gpu_mem[i].gpuvm_aperture); fmm_print(gpu_mem[i].gpu_id); } + } - if (address >= gpu_mem[i].dgpu_aperture.base && - address <= gpu_mem[i].dgpu_aperture.limit) { + if (!found) { + if (address >= svm.dgpu_aperture.base && + address <= svm.dgpu_aperture.limit) { found = true; __fmm_release(gpu_mem[i].gpu_id, address, - MemorySizeInBytes, &gpu_mem[i].dgpu_aperture); + MemorySizeInBytes, &svm.dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); } - - if (address >= gpu_mem[i].dgpu_alt_aperture.base && - address <= gpu_mem[i].dgpu_alt_aperture.limit) { + else if (address >= svm.dgpu_alt_aperture.base && + address <= svm.dgpu_alt_aperture.limit) { found = true; __fmm_release(gpu_mem[i].gpu_id, address, - MemorySizeInBytes, &gpu_mem[i].dgpu_alt_aperture); + MemorySizeInBytes, &svm.dgpu_alt_aperture); fmm_print(gpu_mem[i].gpu_id); } } @@ -989,22 +1011,22 @@ HSAKMT_STATUS fmm_init_process_apertures(void) uint64_t alt_size; int err; - dgpu_mem_init(gpu_mem_id, &gpu_mem[gpu_mem_id].dgpu_aperture.base, - &gpu_mem[gpu_mem_id].dgpu_aperture.limit); + dgpu_mem_init(gpu_mem_id, &svm.dgpu_aperture.base, + &svm.dgpu_aperture.limit); /* Set proper alignment for scratch backing aperture */ gpu_mem[gpu_mem_id].scratch_physical.align = TONGA_PAGE_SIZE; /* Set kernel process dgpu aperture. */ - set_dgpu_aperture(i, (uint64_t)gpu_mem[gpu_mem_id].dgpu_aperture.base, - (uint64_t)gpu_mem[gpu_mem_id].dgpu_aperture.limit); - gpu_mem[gpu_mem_id].dgpu_aperture.align = TONGA_PAGE_SIZE; + set_dgpu_aperture(i, (uint64_t)svm.dgpu_aperture.base, + (uint64_t)svm.dgpu_aperture.limit); + svm.dgpu_aperture.align = TONGA_PAGE_SIZE; /* Place GPUVM aperture after dGPU aperture * (FK: I think this is broken but leaving it for now) */ - gpu_mem[gpu_mem_id].gpuvm_aperture.base = VOID_PTR_ADD(gpu_mem[gpu_mem_id].dgpu_aperture.limit, 1); - gpu_mem[gpu_mem_id].gpuvm_aperture.limit = (void *)VOID_PTRS_SUB(gpu_mem[gpu_mem_id].dgpu_aperture.limit, - gpu_mem[gpu_mem_id].dgpu_aperture.base); + gpu_mem[gpu_mem_id].gpuvm_aperture.base = VOID_PTR_ADD(svm.dgpu_aperture.limit, 1); + gpu_mem[gpu_mem_id].gpuvm_aperture.limit = (void *)VOID_PTRS_SUB(svm.dgpu_aperture.limit, + svm.dgpu_aperture.base); gpu_mem[gpu_mem_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[gpu_mem_id].gpuvm_aperture.limit, (unsigned long)gpu_mem[gpu_mem_id].gpuvm_aperture.base); gpu_mem[gpu_mem_id].gpuvm_aperture.align = TONGA_PAGE_SIZE; @@ -1012,14 +1034,14 @@ HSAKMT_STATUS fmm_init_process_apertures(void) /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. * Base and size must be 64KB aligned. */ - alt_base = (uintptr_t)gpu_mem[gpu_mem_id].dgpu_aperture.base; - alt_size = (VOID_PTRS_SUB(gpu_mem[gpu_mem_id].dgpu_aperture.limit, - gpu_mem[gpu_mem_id].dgpu_aperture.base) + 1) >> 2; + alt_base = (uintptr_t)svm.dgpu_aperture.base; + alt_size = (VOID_PTRS_SUB(svm.dgpu_aperture.limit, + svm.dgpu_aperture.base) + 1) >> 2; alt_base = (alt_base + 0xffff) & ~0xffffULL; alt_size = (alt_size + 0xffff) & ~0xffffULL; - gpu_mem[gpu_mem_id].dgpu_alt_aperture.base = (void *)alt_base; - gpu_mem[gpu_mem_id].dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); - gpu_mem[gpu_mem_id].dgpu_aperture.base = VOID_PTR_ADD(gpu_mem[gpu_mem_id].dgpu_alt_aperture.limit, 1); + svm.dgpu_alt_aperture.base = (void *)alt_base; + svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); + svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); err = fmm_set_memory_policy(gpu_mem[gpu_mem_id].gpu_id, KFD_IOC_CACHE_POLICY_NONCOHERENT, KFD_IOC_CACHE_POLICY_COHERENT, @@ -1028,7 +1050,7 @@ HSAKMT_STATUS fmm_init_process_apertures(void) fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); ret = HSAKMT_STATUS_ERROR; } - gpu_mem[gpu_mem_id].dgpu_alt_aperture.align = TONGA_PAGE_SIZE; + svm.dgpu_alt_aperture.align = TONGA_PAGE_SIZE; } } @@ -1199,8 +1221,8 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= gpu_mem[i].scratch_physical.base) && (address <= gpu_mem[i].scratch_physical.limit)) return _fmm_map_to_gpu_scratch(gpu_mem[i].gpu_id, - &gpu_mem[i].scratch_physical, - address, size); + &gpu_mem[i].scratch_physical, + address, size); if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) @@ -1208,20 +1230,22 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) return _fmm_map_to_gpu(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); - if ((address >= gpu_mem[i].dgpu_aperture.base) && - (address <= gpu_mem[i].dgpu_aperture.limit)) - /* map it */ - return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, - &gpu_mem[i].dgpu_aperture, - address, size); - if ((address >= gpu_mem[i].dgpu_alt_aperture.base) && - (address <= gpu_mem[i].dgpu_alt_aperture.limit)) - /* map it */ - return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, - &gpu_mem[i].dgpu_alt_aperture, - address, size); } + if ((address >= svm.dgpu_aperture.base) && + (address <= svm.dgpu_aperture.limit)) + /* map it */ + return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, + &svm.dgpu_aperture, + address, size); + else if ((address >= svm.dgpu_alt_aperture.base) && + (address <= svm.dgpu_alt_aperture.limit)) + /* map it */ + return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, + &svm.dgpu_alt_aperture, + address, size); + + /* * If address isn't Local memory address, we assume that this is * system memory address accessed through IOMMU. Thus we "prefetch" it @@ -1309,26 +1333,28 @@ int fmm_unmap_from_gpu(void *address) if ((address >= gpu_mem[i].scratch_physical.base) && (address <= gpu_mem[i].scratch_physical.limit)) return _fmm_unmap_from_gpu_scratch(gpu_mem[i].gpu_id, - &gpu_mem[i].scratch_physical, - address); + &gpu_mem[i].scratch_physical, + address); if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* unmap it */ return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, address); - else if ((address >= gpu_mem[i].dgpu_aperture.base) && - (address <= gpu_mem[i].dgpu_aperture.limit)) - /* unmap it */ - return _fmm_unmap_from_gpu(&gpu_mem[i].dgpu_aperture, - address); - else if ((address >= gpu_mem[i].dgpu_alt_aperture.base) && - (address <= gpu_mem[i].dgpu_alt_aperture.limit)) - /* unmap it */ - return _fmm_unmap_from_gpu(&gpu_mem[i].dgpu_alt_aperture, - address); } + if ((address >= svm.dgpu_aperture.base) && + (address <= svm.dgpu_aperture.limit)) + /* unmap it */ + return _fmm_unmap_from_gpu(&svm.dgpu_aperture, + address); + else if ((address >= svm.dgpu_alt_aperture.base) && + (address <= svm.dgpu_alt_aperture.limit)) + /* unmap it */ + return _fmm_unmap_from_gpu(&svm.dgpu_alt_aperture, + address); + + return 0; } @@ -1491,16 +1517,16 @@ bool fmm_get_handle(void *address, uint64_t *handle) aperture = &gpu_mem[i].gpuvm_aperture; break; } + } - else if ((address >= gpu_mem[i].dgpu_aperture.base) && - (address <= gpu_mem[i].dgpu_aperture.limit)) { - aperture = &gpu_mem[i].dgpu_aperture; - break; + if (!aperture) { + if ((address >= svm.dgpu_aperture.base) && + (address <= svm.dgpu_aperture.limit)) { + aperture = &svm.dgpu_aperture; } - else if ((address >= gpu_mem[i].dgpu_alt_aperture.base) && - (address <= gpu_mem[i].dgpu_alt_aperture.limit)) { - aperture = &gpu_mem[i].dgpu_alt_aperture; - break; + else if ((address >= svm.dgpu_alt_aperture.base) && + (address <= svm.dgpu_alt_aperture.limit)) { + aperture = &svm.dgpu_alt_aperture; } } From a4cf02d7978d8c42f2645a8a091c1f0a57b2f843 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 30 Nov 2015 11:43:09 -0500 Subject: [PATCH 0109/1247] Remove unused parameter gpu_id from few functions This will also fix out of bound access in functions fmm_get_aperture_base_and_limit and fmm_release Change-Id: Icf064c46647e69a069126171dbacdf3d5b27f972 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 0bd5e46966..aca71907af 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -133,7 +133,7 @@ static svm_t svm = { static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); -static void __fmm_release(uint32_t gpu_id, void *address, +static void __fmm_release(void *address, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, @@ -719,8 +719,8 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) flags); } -static void* fmm_allocate_host_cpu(uint32_t gpu_id, - uint64_t MemorySizeInBytes, HsaMemFlags flags) +static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, + HsaMemFlags flags) { int err; HSAuint64 page_size; @@ -778,7 +778,7 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); if (ret == MAP_FAILED) { - __fmm_release(gpu_id, mem, MemorySizeInBytes, aperture); + __fmm_release(mem, MemorySizeInBytes, aperture); return NULL; } @@ -789,7 +789,7 @@ void* fmm_allocate_host(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags { if (topology_is_dgpu(dev_id)) return fmm_allocate_host_gpu(gpu_id, MemorySizeInBytes, flags); - return fmm_allocate_host_cpu(gpu_id, MemorySizeInBytes, flags); + return fmm_allocate_host_cpu(MemorySizeInBytes, flags); } void *fmm_open_graphic_handle(uint32_t gpu_id, @@ -849,7 +849,7 @@ out: return NULL; } -static void __fmm_release(uint32_t gpu_id, void *address, +static void __fmm_release(void *address, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args; @@ -893,7 +893,7 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; - __fmm_release(gpu_mem[i].gpu_id, address, + __fmm_release(address, MemorySizeInBytes, &gpu_mem[i].gpuvm_aperture); fmm_print(gpu_mem[i].gpu_id); } @@ -903,14 +903,14 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { found = true; - __fmm_release(gpu_mem[i].gpu_id, address, + __fmm_release(address, MemorySizeInBytes, &svm.dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { found = true; - __fmm_release(gpu_mem[i].gpu_id, address, + __fmm_release(address, MemorySizeInBytes, &svm.dgpu_alt_aperture); fmm_print(gpu_mem[i].gpu_id); } @@ -1098,7 +1098,7 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA return err; } -static int _fmm_map_to_gpu_gtt(uint32_t gpu_id, manageble_aperture_t *aperture, +static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, void *address, uint64_t size) { struct kfd_ioctl_map_memory_to_gpu_args args; @@ -1157,14 +1157,14 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu fprintf(stderr, "Got unexpected address for scratch mapping.\n" " expected: %p\n" " got: %p\n", address, mem); - __fmm_release(gpu_id, mem, size, aperture); + __fmm_release(mem, size, aperture); return -1; } /* map to GPU */ - ret = _fmm_map_to_gpu_gtt(gpu_id, aperture, address, size); + ret = _fmm_map_to_gpu_gtt(aperture, address, size); if (ret != 0) - __fmm_release(gpu_id, mem, size, aperture); + __fmm_release(mem, size, aperture); return ret; } @@ -1235,14 +1235,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) /* map it */ - return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, - &svm.dgpu_aperture, + return _fmm_map_to_gpu_gtt(&svm.dgpu_aperture, address, size); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* map it */ - return _fmm_map_to_gpu_gtt(gpu_mem[i].gpu_id, - &svm.dgpu_alt_aperture, + return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture, address, size); @@ -1312,7 +1310,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ - __fmm_release(gpu_id, address, size, aperture); + __fmm_release(address, size, aperture); return 0; From 8f529e3c72a2e55f2befcbc00a9730905e194e48 Mon Sep 17 00:00:00 2001 From: Yair Shachar Date: Mon, 7 Dec 2015 21:09:44 +0200 Subject: [PATCH 0110/1247] Add support for per device debug register state tracking Change-Id: I8d51670f5de8d379ead898d484f668a8034f9878 Signed-off-by: Yair Shachar --- src/debug.c | 21 ++++++++++++++++----- src/fmm.c | 1 + 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/debug.c b/src/debug.c index eff9023d90..00cbbeea9d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -28,6 +28,9 @@ #include #include +static bool is_device_debugged[MAX_NODES] = {false}; +int debug_get_reg_status(uint32_t node_id, bool* is_debugged); + HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister( @@ -47,9 +50,10 @@ hsaKmtDbgRegister( args.gpu_id = gpu_id; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER, &args); - if (err == 0) + if (err == 0) { + is_device_debugged[NodeId] = true; result = HSAKMT_STATUS_SUCCESS; - else + } else result = HSAKMT_STATUS_ERROR; return (result); @@ -75,9 +79,10 @@ hsaKmtDbgUnregister( memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER, &args); - if (err == 0) + if (err == 0) { + is_device_debugged[NodeId] = false; result = HSAKMT_STATUS_SUCCESS; - else + } else result = HSAKMT_STATUS_ERROR; return (result); @@ -276,4 +281,10 @@ hsaKmtDbgAddressWatch( } /* =============================================================================== */ - +int debug_get_reg_status(uint32_t node_id, bool* is_debugged) +{ + if ( node_id >= MAX_NODES) + return -1; + else + return (is_device_debugged[node_id]); +} diff --git a/src/fmm.c b/src/fmm.c index aca71907af..af4840268d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -131,6 +131,7 @@ static svm_t svm = { INIT_MANAGEBLE_APERTURE(0, 0) }; +extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); static void __fmm_release(void *address, From 582b70f9c323daf9b2cf89bb5393a7eb3994b4b6 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 1 Dec 2015 14:41:59 -0500 Subject: [PATCH 0111/1247] Free resources when dlclose is called When the Thunk is initialized multiple times in the lifetime of a single process , some global resources are leaked. This can happen when dlopen and dlclose are used to load the library at runtime, rather than linking the runtime against the Thunk. This patch adds the destructor to release global resources when dlclose is called. Change-Id: Ia00da0d41f095d0b2706f98c0e75effedd596f49 --- src/fmm.c | 14 ++++++++++++++ src/perfctr.c | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index af4840268d..cf1be8ac45 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -139,6 +139,7 @@ static void __fmm_release(void *address, static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, void *address); +void __attribute__ ((destructor)) fmm_release_global_resources(void); static vm_area_t *vm_create_and_init_area(void *start, void *end) { @@ -1544,3 +1545,16 @@ bool fmm_get_handle(void *address, uint64_t *handle) return found; } + +void fmm_release_global_resources(void) +{ + uint64_t len; + + if (dgpu_shared_aperture_base) { + len = (uint64_t)dgpu_shared_aperture_limit + - (uint64_t)dgpu_shared_aperture_base + 1; + munmap(dgpu_shared_aperture_base, len); + } + dgpu_shared_aperture_base = NULL; + dgpu_shared_aperture_limit = NULL; +} diff --git a/src/perfctr.c b/src/perfctr.c index 64ab168fda..403ff4ad3e 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -48,6 +48,8 @@ extern int amd_hsa_thunk_lock_fd; static HsaCounterProperties *counter_props[MAX_NODES] = {NULL}; +void __attribute__ ((destructor)) perfctr_release_global_resources(void); + static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) { int rc = 0; @@ -368,3 +370,14 @@ hsaKmtPmcStopTrace( return HSAKMT_STATUS_SUCCESS; } + +void perfctr_release_global_resources(void) +{ + int i; + + for (i=0; i Date: Mon, 14 Dec 2015 16:25:18 -0500 Subject: [PATCH 0112/1247] Fix node_id in gpu_mem[] array Change-Id: I4897623612e1749e275fb97ce1603dc5130fc9ce --- src/fmm.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index cf1be8ac45..2852192a50 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -968,16 +968,16 @@ HSAKMT_STATUS fmm_init_process_apertures(void) ret = topology_sysfs_get_node_props(i, &props, &gpu_id); if (ret != HSAKMT_STATUS_SUCCESS) return ret; - i++; - /* Skip non-GPU nodes */ - if (gpu_id == 0) - continue; - gpu_mem[gpu_mem_id].gpu_id = gpu_id; - gpu_mem[gpu_mem_id].local_mem_size = props.LocalMemSize; - gpu_mem[gpu_mem_id].device_id = props.DeviceId; - gpu_mem[gpu_mem_id].node_id = i; - gpu_mem_id++; + /* Skip non-GPU nodes */ + if (gpu_id != 0) { + gpu_mem[gpu_mem_id].gpu_id = gpu_id; + gpu_mem[gpu_mem_id].local_mem_size = props.LocalMemSize; + gpu_mem[gpu_mem_id].device_id = props.DeviceId; + gpu_mem[gpu_mem_id].node_id = i; + gpu_mem_id++; + } + i++; } if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) From 681f4dcecc58aece5402cc1e741e6835a894c13f Mon Sep 17 00:00:00 2001 From: Yair Shachar Date: Sun, 20 Dec 2015 15:01:13 +0200 Subject: [PATCH 0113/1247] Add support for scratch GPUVM on host memory This is required when we have a debug session Change-Id: If9d6d2d23a9016b6ca9562e02a91fc16e0354ee4 Signed-off-by: Yair Shachar --- src/debug.c | 6 ++++-- src/fmm.c | 48 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/debug.c b/src/debug.c index 00cbbeea9d..bb04cfbc2c 100644 --- a/src/debug.c +++ b/src/debug.c @@ -285,6 +285,8 @@ int debug_get_reg_status(uint32_t node_id, bool* is_debugged) { if ( node_id >= MAX_NODES) return -1; - else - return (is_device_debugged[node_id]); + else { + *is_debugged = is_device_debugged[node_id]; + return 0; + } } diff --git a/src/fmm.c b/src/fmm.c index 2852192a50..ed2857fe44 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1133,9 +1133,11 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu { int32_t gpu_mem_id; uint64_t offset; - void *mem; + void *mem = NULL; int ret; - + bool is_debugger = 0; + void *mmap_ret = NULL; + uint64_t mmap_offset = 0; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) @@ -1149,18 +1151,38 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu VOID_PTR_ADD(address, size -1) > aperture->limit) return -1; + debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); /* allocate object within the scratch backing aperture */ - offset = VOID_PTRS_SUB(address, aperture->base); - mem = __fmm_allocate_device(gpu_id, size, aperture, offset, NULL, - KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); - if (mem == NULL) - return -1; - if (mem != address) { - fprintf(stderr, "Got unexpected address for scratch mapping.\n" - " expected: %p\n" - " got: %p\n", address, mem); - __fmm_release(mem, size, aperture); - return -1; + if (!is_debugger) { + offset = VOID_PTRS_SUB(address, aperture->base); + mem = __fmm_allocate_device(gpu_id, size, aperture, offset, + NULL, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); + if (mem == NULL) + return -1; + + if (mem != address) { + fprintf(stderr, + "Got unexpected address for scratch mapping.\n" + " expected: %p\n" + " got: %p\n", address, mem); + __fmm_release(mem, size, aperture); + return -1; + } + } else { + fmm_allocate_memory_in_device(gpu_id, + address, + size, + aperture, + &mmap_offset, + KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST); + mmap_ret = mmap(address, size, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, + kfd_fd, mmap_offset); + if (mmap_ret == MAP_FAILED) { + __fmm_release(mem, size, aperture); + return -1; + } } /* map to GPU */ From 53b208adf292b0471da97eb1b59769e18240c5c3 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 30 Dec 2015 13:30:23 +0200 Subject: [PATCH 0114/1247] Fix AQL Double buffer allocation mode Change-Id: I5162ffd89416d317fd0ca0fc51da523298488922 Signed-off-by: Ben Goz --- src/fmm.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ed2857fe44..ab500d075c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -28,6 +28,7 @@ #include "libhsakmt.h" #include #include +#include #include #include #include @@ -768,7 +769,7 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, aperture = &svm.dgpu_alt_aperture; /* coherent */ if (flags.ui32.AQLQueueMemory) { size = MemorySizeInBytes * 2; - ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; } mem = __fmm_allocate_device(gpu_id, size, @@ -784,6 +785,14 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, return NULL; } + if (flags.ui32.AQLQueueMemory) { + uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; + memset(ret, 0, MemorySizeInBytes); + mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + } + return ret; } @@ -863,7 +872,7 @@ static void __fmm_release(void *address, pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, MemorySizeInBytes); + object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return; @@ -930,8 +939,9 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) * If memory address isn't inside of any defined aperture - it refers * to the system memory */ - if (!found) + if (!found) { free(address); + } } static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_policy, From ea0f9d2a0b7dd16a54fdb337d6554fdf47d3da0a Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 23 Dec 2015 17:23:25 +0200 Subject: [PATCH 0115/1247] Adding support for mGPU Change-Id: I5ed184e6a58b38d9dde48867f14513d161cf41a9 Signed-off-by: Ben Goz --- include/linux/kfd_ioctl.h | 23 +++++- src/events.c | 3 +- src/fmm.c | 147 ++++++++++++++++++++++++++++++++++---- src/fmm.h | 5 ++ src/libhsakmt.h | 3 +- src/memory.c | 15 ++++ src/queues.c | 15 +++- 7 files changed, 192 insertions(+), 19 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index c08cea8810..8b41710aa2 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -265,13 +265,27 @@ struct kfd_ioctl_free_memory_of_gpu_args { }; struct kfd_ioctl_map_memory_to_gpu_args { - uint64_t handle; /* to KFD */ + uint64_t handle; /* to KFD */ +}; + +struct kfd_ioctl_map_memory_to_gpu_new_args { + uint64_t handle; /* to KFD */ + uint32_t *device_ids_array; /* to KFD */ + uint32_t device_ids_array_size; /* to KFD */ + uint32_t pad; }; struct kfd_ioctl_unmap_memory_from_gpu_args { uint64_t handle; /* to KFD */ }; +struct kfd_ioctl_unmap_memory_from_gpu_new_args { + uint64_t handle; /* to KFD */ + uint32_t *device_ids_array; /* to KFD */ + uint32_t device_ids_array_size; /* to KFD */ + uint32_t pad; +}; + struct kfd_ioctl_open_graphic_handle_args { uint64_t va_addr; /* to KFD */ uint64_t handle; /* from KFD */ @@ -392,7 +406,12 @@ struct kfd_ioctl_alloc_memory_of_gpu_new_args { #define AMDKFD_IOC_SET_TRAP_HANDLER \ AMDKFD_IOW(0x1a, struct kfd_ioctl_set_trap_handler_args) +#define AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW \ + AMDKFD_IOWR(0x1b, struct kfd_ioctl_map_memory_to_gpu_new_args) +#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW \ + AMDKFD_IOWR(0x1c, struct kfd_ioctl_unmap_memory_from_gpu_new_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1b +#define AMDKFD_COMMAND_END 0x1d #endif diff --git a/src/events.c b/src/events.c index 046d02765e..e8024b95ad 100644 --- a/src/events.c +++ b/src/events.c @@ -77,7 +77,7 @@ hsaKmtCreateEvent( if (is_dgpu && events_page == NULL) { events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, TONGA_PAGE_SIZE, - args.node_id); + args.node_id, true); if (!events_page) { return HSAKMT_STATUS_ERROR; } @@ -143,7 +143,6 @@ hsaKmtDestroyEvent( } free(Event); - return HSAKMT_STATUS_SUCCESS; } diff --git a/src/fmm.c b/src/fmm.c index ab500d075c..2e593de2a2 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -66,6 +66,11 @@ struct vm_object { uint64_t handle; /* opaque */ struct vm_object *next; struct vm_object *prev; + /* + * Nodes to map on SVM mGPU + */ + uint32_t *device_ids_array; + uint32_t device_ids_array_size; }; typedef struct vm_object vm_object_t; @@ -165,6 +170,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->size = size; object->handle = handle; object->next = object->prev = NULL; + object->device_ids_array_size = 0; } return object; @@ -698,7 +704,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) { manageble_aperture_t *aperture; int32_t gpu_mem_id; - uint32_t flags; + uint32_t flags, offset; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -712,13 +718,15 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) * In that way the host access range won't be used for local memory */ aperture = &svm.dgpu_aperture; + offset = 0; } else { flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; + offset = GPUVM_APP_OFFSET; } return __fmm_allocate_device(gpu_id, MemorySizeInBytes, - aperture, GPUVM_APP_OFFSET, NULL, + aperture, offset, NULL, flags); } @@ -812,7 +820,7 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, void *mem = NULL; int32_t i = gpu_mem_find_by_gpu_id(gpu_id); struct kfd_ioctl_open_graphic_handle_args open_graphic_handle_args; - struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; + struct kfd_ioctl_unmap_memory_from_gpu_new_args unmap_args; /* If not found or aperture isn't properly initialized/supported */ if (i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, @@ -850,7 +858,9 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, release_mem: unmap_args.handle = open_graphic_handle_args.handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); + unmap_args.device_ids_array = NULL; + unmap_args.device_ids_array_size = 0; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &unmap_args); release_area: aperture_release_area(&gpu_mem[i].gpuvm_aperture, mem, MemorySizeInBytes); @@ -1113,7 +1123,7 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, void *address, uint64_t size) { - struct kfd_ioctl_map_memory_to_gpu_args args; + struct kfd_ioctl_map_memory_to_gpu_new_args args; vm_object_t *object; pthread_mutex_lock(&aperture->fmm_mutex); @@ -1125,7 +1135,9 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, } args.handle = object->handle; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) goto err_map_ioctl_failed; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1207,7 +1219,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) { - struct kfd_ioctl_map_memory_to_gpu_args args; + struct kfd_ioctl_map_memory_to_gpu_new_args args; vm_object_t *object; /* Check that address space was previously reserved */ @@ -1222,7 +1234,9 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, goto err_object_not_found; args.handle = object->handle; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) goto err_map_ioctl_failed; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1291,7 +1305,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) { vm_object_t *object; - struct kfd_ioctl_unmap_memory_from_gpu_args args; + struct kfd_ioctl_unmap_memory_from_gpu_new_args args; pthread_mutex_lock(&aperture->fmm_mutex); @@ -1301,7 +1315,9 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) goto err; args.handle = object->handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1318,7 +1334,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, int32_t gpu_mem_id; vm_object_t *object; uint64_t size; - struct kfd_ioctl_unmap_memory_from_gpu_args args; + struct kfd_ioctl_unmap_memory_from_gpu_new_args args; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -1339,7 +1355,9 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, /* unmap from GPU */ args.handle = object->handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1590,3 +1608,108 @@ void fmm_release_global_resources(void) dgpu_shared_aperture_base = NULL; dgpu_shared_aperture_limit = NULL; } + +int fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *nodes_arr, uint32_t nodes_arr_size) +{ + bool found = false; + manageble_aperture_t *aperture; + vm_object_t *object; + + /* + * Object can be found only on SVM aperture as you can't map + * non SVM object on different device. + */ + aperture = &svm.dgpu_aperture; + pthread_mutex_lock(&aperture->fmm_mutex); + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (object) + found = true; + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (!found) { + aperture = &svm.dgpu_alt_aperture; + + pthread_mutex_lock(&aperture->fmm_mutex); + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (object) + found = true; + pthread_mutex_unlock(&aperture->fmm_mutex); + } + + if (!object) + return 1; + + object->device_ids_array = nodes_arr; + object->device_ids_array_size = nodes_arr_size * sizeof(uint32_t); + + return 0; +} + +void fmm_deregister_memory(void *address) +{ + bool found = false; + manageble_aperture_t *aperture; + vm_object_t *object; + + /* + * Object can be found only on SVM aperture as you can't map + * non SVM object on different device. + */ + aperture = &svm.dgpu_aperture; + pthread_mutex_lock(&aperture->fmm_mutex); + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (object) + found = true; + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (!found) { + aperture = &svm.dgpu_alt_aperture; + pthread_mutex_lock(&aperture->fmm_mutex); + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (object) + found = true; + pthread_mutex_unlock(&aperture->fmm_mutex); + } + + if (!object || object->device_ids_array_size <= 0) + return; + + free(object->device_ids_array); + object->device_ids_array = NULL; + object->device_ids_array_size = 0; +} + +int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num) +{ + uint32_t i, *arr; + if (!nodes) { + nodes_num = 0; + for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { + if (gpu_mem[i].gpu_id == 0) + continue; + nodes_num++; + } + } + + arr = (uint32_t *)malloc(sizeof(uint32_t) * nodes_num); + if (!array) + return 1; + + memset(arr, 0, sizeof(uint32_t) * nodes_num); + + nodes_num = 0; + for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { + if (gpu_mem[i].gpu_id == 0) + continue; + arr[nodes_num] = gpu_mem[i].gpu_id; + nodes_num++; + } + + *array = arr; + return nodes_num; +} diff --git a/src/fmm.h b/src/fmm.h index 10a56ff54a..37bdcb3b65 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -67,4 +67,9 @@ HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit); + +int fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *nodes_arr, uint32_t nodes_arr_size); +void fmm_deregister_memory(void *address); +int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num); #endif /* FMM_H_ */ diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 4ec1acafaa..c44bb528d0 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -77,7 +77,8 @@ bool topology_is_dgpu(uint16_t device_id); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); -void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId); +void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, + uint32_t NodeId, bool peer_to_peer); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); extern int kmtIoctl(int fd, unsigned long request, void *arg); diff --git a/src/memory.c b/src/memory.c index 8052fcd3bd..1b043b60c2 100644 --- a/src/memory.c +++ b/src/memory.c @@ -195,8 +195,21 @@ hsaKmtRegisterMemory( HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ ) { + uint32_t *NodesArray; + uint32_t NodesArraySize; + CHECK_KFD_OPEN(); + /* + * Build NodesArray from all dGPU nodes. + */ + NodesArraySize = fmm_build_nodes_array(&NodesArray, NULL, 0); + if (!NodesArray) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (fmm_register_memory(MemoryAddress, MemorySizeInBytes, + NodesArray, NodesArraySize) != 0) + return HSAKMT_STATUS_ERROR; return HSAKMT_STATUS_SUCCESS; } @@ -208,6 +221,8 @@ hsaKmtDeregisterMemory( { CHECK_KFD_OPEN(); + fmm_deregister_memory(MemoryAddress); + return HSAKMT_STATUS_SUCCESS; } diff --git a/src/queues.c b/src/queues.c index a6025bb5cc..10604befa7 100644 --- a/src/queues.c +++ b/src/queues.c @@ -187,7 +187,8 @@ static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) return ptr; } -void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId) +void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, + uint32_t NodeId, bool peer_to_peer) { void *mem; HSAuint64 gpu_va; @@ -205,6 +206,14 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t N if (ret != HSAKMT_STATUS_SUCCESS) { return NULL; } + + if (peer_to_peer) { + if (hsaKmtRegisterMemory(mem, size) != HSAKMT_STATUS_SUCCESS) { + hsaKmtFreeMemory(mem, size); + return NULL; + } + } + if (hsaKmtMapMemoryToGPU(mem, size, &gpu_va) != HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(mem, size); return NULL; @@ -220,6 +229,7 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(addr, size); } + hsaKmtDeregisterMemory(addr); } static void* allocate_exec_aligned_memory(uint32_t size, @@ -228,7 +238,7 @@ static void* allocate_exec_aligned_memory(uint32_t size, uint32_t NodeId) { if (IS_DGPU(type)) - return allocate_exec_aligned_memory_gpu(size, TONGA_PAGE_SIZE, NodeId); + return allocate_exec_aligned_memory_gpu(size, TONGA_PAGE_SIZE, NodeId, false); return allocate_exec_aligned_memory_cpu(size, align); } @@ -236,6 +246,7 @@ static void release_exec_aligned_memory_gpu(void *addr, uint32_t size) { if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) hsaKmtFreeMemory(addr, (HSAuint64)size); + hsaKmtDeregisterMemory(addr); } static void release_exec_aligned_memory(void *addr, uint32_t size, enum asic_family_type type) From 063ad3ad9e120c49f03918cc82f1da6a1493ac7b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 7 Jan 2016 16:28:20 -0500 Subject: [PATCH 0116/1247] Implement hsaKmtRegisterMemoryToNodes Fix hsaKmtRegisterMemory to be a no-op for now and move the multi-GPU implementation to hsaKmtRegisterMemoryToNodes. Make GPU memory mappings of host memory visible to all GPUs by default. Device memory is still visible to the allocating GPU only by default (but can be overridden with hsaKmtRegisterMemoryToNodes for experimenting with P2P). Change-Id: I73408afbe3b10c8dad2ab3a780f58413249692e6 --- include/hsakmt.h | 16 +++++- src/fmm.c | 142 ++++++++++++++++++++++++---------------------- src/fmm.h | 8 +-- src/libhsakmt.ver | 1 + src/memory.c | 54 +++++++++++++----- src/queues.c | 8 +-- 6 files changed, 137 insertions(+), 92 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index bdb8f2bdd3..5fadf78577 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -369,7 +369,6 @@ hsaKmtFreeMemory( /** Registers with KFD a memory buffer that may be accessed by the GPU - This function will never be required for Linux */ HSAKMT_STATUS @@ -380,9 +379,22 @@ hsaKmtRegisterMemory( ); +/** + Registers with KFD a memory buffer that may be accessed by specific GPUs +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemoryToNodes( + void *MemoryAddress, // IN (page-aligned) + HSAuint64 MemorySizeInBytes, // IN (page-aligned) + HSAuint64 NumberOfNodes, // IN + HSAuint32* NodeArray // IN + ); + + /** Unregisters with KFD a memory buffer - This function will never be required for Linux */ HSAKMT_STATUS diff --git a/src/fmm.c b/src/fmm.c index 2e593de2a2..c8cb4c62bd 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -66,6 +66,7 @@ struct vm_object { uint64_t handle; /* opaque */ struct vm_object *next; struct vm_object *prev; + uint32_t flags; /* memory allocation flags */ /* * Nodes to map on SVM mGPU */ @@ -137,6 +138,10 @@ static svm_t svm = { INIT_MANAGEBLE_APERTURE(0, 0) }; +/* GPU node array for default mappings */ +static uint32_t all_gpu_id_array_size = 0; +static uint32_t *all_gpu_id_array = NULL; + extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); @@ -161,7 +166,7 @@ static vm_area_t *vm_create_and_init_area(void *start, void *end) } static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, - uint64_t handle) + uint64_t handle, uint32_t flags) { vm_object_t *object = (vm_object_t *) malloc(sizeof(vm_object_t)); @@ -171,6 +176,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->handle = handle; object->next = object->prev = NULL; object->device_ids_array_size = 0; + object->flags = flags; } return object; @@ -399,7 +405,8 @@ static void *aperture_allocate_area(manageble_aperture_t *app, static int aperture_allocate_object(manageble_aperture_t *app, void *new_address, uint64_t handle, - uint64_t MemorySizeInBytes) + uint64_t MemorySizeInBytes, + uint32_t flags) { vm_object_t *new_object; @@ -407,8 +414,8 @@ static int aperture_allocate_object(manageble_aperture_t *app, /* Allocate new object */ new_object = vm_create_and_init_object(new_address, - MemorySizeInBytes, - handle); + MemorySizeInBytes, + handle, flags); if (!new_object) return -1; @@ -460,7 +467,7 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); if (aperture_allocate_object(aperture, mem, args.handle, - MemorySizeInBytes)) + MemorySizeInBytes, flags)) goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -848,7 +855,7 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, /* Allocate object */ if (aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, open_graphic_handle_args.handle, - MemorySizeInBytes)) + MemorySizeInBytes, 0)) goto release_mem; pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); @@ -888,6 +895,9 @@ static void __fmm_release(void *address, return; } + if (object->device_ids_array_size > 0) + free(object->device_ids_array); + args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); @@ -1003,6 +1013,13 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) return HSAKMT_STATUS_ERROR; + all_gpu_id_array_size = 0; + if (args.num_of_nodes > 0) { + all_gpu_id_array = malloc(sizeof(uint32_t) * args.num_of_nodes); + if (all_gpu_id_array == NULL) + return HSAKMT_STATUS_NO_MEMORY; + } + for (i = 0 ; i < args.num_of_nodes ; i++) { /* Map Kernel process device data node i <--> gpu_mem_id which indexes into gpu_mem[] * based on gpu_id */ @@ -1010,6 +1027,9 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (gpu_mem_id < 0) return HSAKMT_STATUS_ERROR; + all_gpu_id_array[i] = args.process_apertures[i].gpu_id; + all_gpu_id_array_size += sizeof(uint32_t); + gpu_mem[gpu_mem_id].lds_aperture.base = PORT_UINT64_TO_VPTR(args.process_apertures[i].lds_base); @@ -1135,8 +1155,17 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, } args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + if (object->device_ids_array_size > 0) { + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + /* Only enable multi-GPU mapping on host memory for now */ + args.device_ids_array = all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } else { + args.device_ids_array = NULL; + args.device_ids_array_size = 0; + } if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) goto err_map_ioctl_failed; @@ -1315,8 +1344,17 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) goto err; args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + if (object->device_ids_array_size > 0) { + args.device_ids_array = object->device_ids_array; + args.device_ids_array_size = object->device_ids_array_size; + } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + /* Only enable multi-GPU mapping on host memory for now */ + args.device_ids_array = all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } else { + args.device_ids_array = NULL; + args.device_ids_array_size = 0; + } kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1607,14 +1645,22 @@ void fmm_release_global_resources(void) } dgpu_shared_aperture_base = NULL; dgpu_shared_aperture_limit = NULL; + + if (all_gpu_id_array != NULL) + free(all_gpu_id_array); + all_gpu_id_array = NULL; + all_gpu_id_array_size = 0; } -int fmm_register_memory(void *address, uint32_t size_in_bytes, - uint32_t *nodes_arr, uint32_t nodes_arr_size) +HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size) { - bool found = false; manageble_aperture_t *aperture; - vm_object_t *object; + vm_object_t *object = NULL; + + if (gpu_id_array_size > 0 && gpu_id_array == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; /* * Object can be found only on SVM aperture as you can't map @@ -1622,37 +1668,33 @@ int fmm_register_memory(void *address, uint32_t size_in_bytes, */ aperture = &svm.dgpu_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); - if (!found) { + if (!object) { aperture = &svm.dgpu_alt_aperture; - pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); } if (!object) - return 1; + return HSAKMT_STATUS_NOT_SUPPORTED; + if (object->device_ids_array_size > 0) + return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; - object->device_ids_array = nodes_arr; - object->device_ids_array_size = nodes_arr_size * sizeof(uint32_t); + if (gpu_id_array_size > 0) { + object->device_ids_array = gpu_id_array; + object->device_ids_array_size = gpu_id_array_size; + } - return 0; + return HSAKMT_STATUS_SUCCESS; } -void fmm_deregister_memory(void *address) +HSAKMT_STATUS fmm_deregister_memory(void *address) { - bool found = false; manageble_aperture_t *aperture; - vm_object_t *object; + vm_object_t *object = NULL; /* * Object can be found only on SVM aperture as you can't map @@ -1660,56 +1702,22 @@ void fmm_deregister_memory(void *address) */ aperture = &svm.dgpu_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); - if (!found) { + if (!object) { aperture = &svm.dgpu_alt_aperture; pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (object) - found = true; pthread_mutex_unlock(&aperture->fmm_mutex); } if (!object || object->device_ids_array_size <= 0) - return; + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; free(object->device_ids_array); object->device_ids_array = NULL; object->device_ids_array_size = 0; -} - -int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num) -{ - uint32_t i, *arr; - if (!nodes) { - nodes_num = 0; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { - if (gpu_mem[i].gpu_id == 0) - continue; - nodes_num++; - } - } - - arr = (uint32_t *)malloc(sizeof(uint32_t) * nodes_num); - if (!array) - return 1; - - memset(arr, 0, sizeof(uint32_t) * nodes_num); - - nodes_num = 0; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS; i++) { - if (gpu_mem[i].gpu_id == 0) - continue; - arr[nodes_num] = gpu_mem[i].gpu_id; - nodes_num++; - } - - *array = arr; - return nodes_num; + + return HSAKMT_STATUS_SUCCESS; } diff --git a/src/fmm.h b/src/fmm.h index 37bdcb3b65..5bdbf5bbba 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -68,8 +68,8 @@ HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit); -int fmm_register_memory(void *address, uint32_t size_in_bytes, - uint32_t *nodes_arr, uint32_t nodes_arr_size); -void fmm_deregister_memory(void *address); -int fmm_build_nodes_array(uint32_t **array, uint32_t *nodes, uint32_t nodes_num); +HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size); +HSAKMT_STATUS fmm_deregister_memory(void *address); #endif /* FMM_H_ */ diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 3c8064d43c..9a8a14835b 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -25,6 +25,7 @@ hsaKmtSetMemoryPolicy; hsaKmtAllocMemory; hsaKmtFreeMemory; hsaKmtRegisterMemory; +hsaKmtRegisterMemoryToNodes; hsaKmtDeregisterMemory; hsaKmtMapMemoryToGPU; hsaKmtUnmapMemoryToGPU; diff --git a/src/memory.c b/src/memory.c index 1b043b60c2..c0161b36dc 100644 --- a/src/memory.c +++ b/src/memory.c @@ -195,22 +195,48 @@ hsaKmtRegisterMemory( HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ ) { - uint32_t *NodesArray; - uint32_t NodesArraySize; - CHECK_KFD_OPEN(); - /* - * Build NodesArray from all dGPU nodes. - */ - NodesArraySize = fmm_build_nodes_array(&NodesArray, NULL, 0); - if (!NodesArray) + /* No-op for APU, TODO for dGPU */ + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemoryToNodes( + void *MemoryAddress, /* IN (page-aligned) */ + HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ + HSAuint64 NumberOfNodes, /* IN */ + HSAuint32* NodeArray /* IN */ +) +{ + CHECK_KFD_OPEN(); + uint32_t *gpu_id_array; + unsigned i; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + if (NumberOfNodes == 0 || NodeArray == NULL) return HSAKMT_STATUS_INVALID_PARAMETER; - if (fmm_register_memory(MemoryAddress, MemorySizeInBytes, - NodesArray, NodesArraySize) != 0) - return HSAKMT_STATUS_ERROR; - return HSAKMT_STATUS_SUCCESS; + /* Translate Node IDs to gpu_ids */ + gpu_id_array = malloc(NumberOfNodes * sizeof(uint32_t)); + if (gpu_id_array == NULL) + return HSAKMT_STATUS_NO_MEMORY; + for (i = 0; i < NumberOfNodes; i++) { + ret = validate_nodeid(NodeArray[i], &gpu_id_array[i]); + if (ret != HSAKMT_STATUS_SUCCESS) + break; + } + + if (ret == HSAKMT_STATUS_SUCCESS) + ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, + gpu_id_array, + NumberOfNodes*sizeof(uint32_t)); + + if (ret != HSAKMT_STATUS_SUCCESS) + free(gpu_id_array); + + return ret; } HSAKMT_STATUS @@ -221,9 +247,7 @@ hsaKmtDeregisterMemory( { CHECK_KFD_OPEN(); - fmm_deregister_memory(MemoryAddress); - - return HSAKMT_STATUS_SUCCESS; + return fmm_deregister_memory(MemoryAddress); } HSAKMT_STATUS diff --git a/src/queues.c b/src/queues.c index 10604befa7..247172ef4f 100644 --- a/src/queues.c +++ b/src/queues.c @@ -207,8 +207,10 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, return NULL; } - if (peer_to_peer) { - if (hsaKmtRegisterMemory(mem, size) != HSAKMT_STATUS_SUCCESS) { + if (!peer_to_peer) { + uint32_t nodes_array[1] = {NodeId}; + if (hsaKmtRegisterMemoryToNodes(mem, size, 1, nodes_array) + != HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(mem, size); return NULL; } @@ -229,7 +231,6 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(addr, size); } - hsaKmtDeregisterMemory(addr); } static void* allocate_exec_aligned_memory(uint32_t size, @@ -246,7 +247,6 @@ static void release_exec_aligned_memory_gpu(void *addr, uint32_t size) { if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) hsaKmtFreeMemory(addr, (HSAuint64)size); - hsaKmtDeregisterMemory(addr); } static void release_exec_aligned_memory(void *addr, uint32_t size, enum asic_family_type type) From e7e1361c3da53af7f11fa901a0bd9ff2340e9295 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 21 Dec 2015 15:42:52 -0500 Subject: [PATCH 0117/1247] Use new ioctl for getting process apertures Change-Id: I73678744ad73942edec442ad9c6d38637f7e1235 --- include/linux/kfd_ioctl.h | 14 +++++++++-- src/fmm.c | 51 +++++++++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 17 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 8b41710aa2..f12e96e46b 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -321,6 +321,14 @@ struct kfd_ioctl_alloc_memory_of_gpu_new_args { uint32_t flags; }; +struct kfd_ioctl_get_process_apertures_new_args { + uint64_t kfd_process_device_apertures_ptr; /* pointer to struct + struct kfd_process_device_apertures. User allocated */ + uint32_t num_of_nodes; /* Number of entries in + kfd_process_device_apertures_ptr */ + uint32_t pad; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -402,7 +410,6 @@ struct kfd_ioctl_alloc_memory_of_gpu_new_args { #define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW \ AMDKFD_IOWR(0x19, struct kfd_ioctl_alloc_memory_of_gpu_new_args) - #define AMDKFD_IOC_SET_TRAP_HANDLER \ AMDKFD_IOW(0x1a, struct kfd_ioctl_set_trap_handler_args) @@ -411,7 +418,10 @@ struct kfd_ioctl_alloc_memory_of_gpu_new_args { #define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW \ AMDKFD_IOWR(0x1c, struct kfd_ioctl_unmap_memory_from_gpu_new_args) +#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ + AMDKFD_IOWR(0x1d, struct kfd_ioctl_get_process_apertures_new_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1d +#define AMDKFD_COMMAND_END 0x1e #endif diff --git a/src/fmm.c b/src/fmm.c index c8cb4c62bd..99c5551816 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -980,12 +980,13 @@ static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_po HSAKMT_STATUS fmm_init_process_apertures(void) { - struct kfd_ioctl_get_process_apertures_args args; + struct kfd_ioctl_get_process_apertures_new_args args; uint32_t i = 0; int32_t gpu_mem_id =0; uint32_t gpu_id; HsaSystemProperties sys_props; HsaNodeProperties props; + struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; ret = topology_sysfs_get_system_props(&sys_props); @@ -1010,43 +1011,60 @@ HSAKMT_STATUS fmm_init_process_apertures(void) i++; } - if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *) &args)) - return HSAKMT_STATUS_ERROR; + /* The ioctl will also return Number of Nodes if args.kfd_process_device_apertures_ptr + * is set to NULL. This is not required since Number of nodes is already known. Kernel + * will fill in the apertures in kfd_process_device_apertures_ptr */ + process_apertures = malloc(gpu_mem_id * sizeof(struct kfd_process_device_apertures)); + if (process_apertures == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + args.kfd_process_device_apertures_ptr = (uintptr_t)process_apertures; + args.num_of_nodes = gpu_mem_id; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW, (void *)&args)) { + ret = HSAKMT_STATUS_ERROR; + goto get_aperture_ioctl_failed; + } all_gpu_id_array_size = 0; + all_gpu_id_array = NULL; if (args.num_of_nodes > 0) { all_gpu_id_array = malloc(sizeof(uint32_t) * args.num_of_nodes); - if (all_gpu_id_array == NULL) - return HSAKMT_STATUS_NO_MEMORY; + if (all_gpu_id_array == NULL) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto get_aperture_ioctl_failed; + } } for (i = 0 ; i < args.num_of_nodes ; i++) { /* Map Kernel process device data node i <--> gpu_mem_id which indexes into gpu_mem[] * based on gpu_id */ - gpu_mem_id = gpu_mem_find_by_gpu_id(args.process_apertures[i].gpu_id); - if (gpu_mem_id < 0) - return HSAKMT_STATUS_ERROR; + gpu_mem_id = gpu_mem_find_by_gpu_id(process_apertures[i].gpu_id); + if (gpu_mem_id < 0) { + ret = HSAKMT_STATUS_ERROR; + goto invalid_gpu_id; + } - all_gpu_id_array[i] = args.process_apertures[i].gpu_id; + all_gpu_id_array[i] = process_apertures[i].gpu_id; all_gpu_id_array_size += sizeof(uint32_t); gpu_mem[gpu_mem_id].lds_aperture.base = - PORT_UINT64_TO_VPTR(args.process_apertures[i].lds_base); + PORT_UINT64_TO_VPTR(process_apertures[i].lds_base); gpu_mem[gpu_mem_id].lds_aperture.limit = - PORT_UINT64_TO_VPTR(args.process_apertures[i].lds_limit); + PORT_UINT64_TO_VPTR(process_apertures[i].lds_limit); gpu_mem[gpu_mem_id].gpuvm_aperture.base = - PORT_UINT64_TO_VPTR(args.process_apertures[i].gpuvm_base); + PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_base); gpu_mem[gpu_mem_id].gpuvm_aperture.limit = - PORT_UINT64_TO_VPTR(args.process_apertures[i].gpuvm_limit); + PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_limit); gpu_mem[gpu_mem_id].scratch_aperture.base = - PORT_UINT64_TO_VPTR(args.process_apertures[i].scratch_base); + PORT_UINT64_TO_VPTR(process_apertures[i].scratch_base); gpu_mem[gpu_mem_id].scratch_aperture.limit = - PORT_UINT64_TO_VPTR(args.process_apertures[i].scratch_limit); + PORT_UINT64_TO_VPTR(process_apertures[i].scratch_limit); if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { uintptr_t alt_base; @@ -1096,6 +1114,9 @@ HSAKMT_STATUS fmm_init_process_apertures(void) } } +get_aperture_ioctl_failed: +invalid_gpu_id : + free(process_apertures); return ret; } From ce83dc623f21c799868b46130db3bf0908889734 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 13 Jan 2016 17:27:31 -0500 Subject: [PATCH 0118/1247] Don't limit number of supported GPUs Stop using NUM_OF_SUPPORTED_GPUS. For now the definitions itself cannot be removed as ioctl code is in upstream Kernel. Change-Id: If846625a8ad5062d5483e762850c793d3c00b9d0 --- src/fmm.c | 93 +++++++++++++++++++++++++++++-------------------- src/fmm.h | 2 ++ src/libhsakmt.h | 2 ++ src/openclose.c | 33 ++++++++++++------ src/queues.c | 34 +++++++++++++++++- 5 files changed, 115 insertions(+), 49 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 99c5551816..b615311c65 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -36,11 +36,6 @@ #define NON_VALID_GPU_ID 0 #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) -#define INIT_APERTURE(base_value, limit_value) { \ - .base = (void *) base_value, \ - .limit = (void *) limit_value \ - } - #define INIT_MANAGEBLE_APERTURE(base_value, limit_value) { \ .base = (void *) base_value, \ .limit = (void *) limit_value, \ @@ -50,16 +45,6 @@ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER \ } -#define INIT_GPU_MEM { \ - .gpu_id = NON_VALID_GPU_ID, \ - .lds_aperture = INIT_APERTURE(0, 0), \ - .scratch_physical = INIT_MANAGEBLE_APERTURE(0, 0), \ - .scratch_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ - .gpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0), \ -} - -#define INIT_GPUs_MEM {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = INIT_GPU_MEM} - struct vm_object { void *start; uint64_t size; @@ -128,7 +113,8 @@ typedef struct { /* The other apertures are specific to each GPU. gpu_mem_t manages GPU * specific memory apertures. */ -static gpu_mem_t gpu_mem[] = INIT_GPUs_MEM; +static gpu_mem_t *gpu_mem; +static unsigned int gpu_mem_count; static void *dgpu_shared_aperture_base = NULL; static void *dgpu_shared_aperture_limit = NULL; @@ -431,9 +417,9 @@ static int aperture_allocate_object(manageble_aperture_t *app, static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) { - int32_t i; + uint32_t i; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS ; i++) + for (i = 0 ; i < gpu_mem_count ; i++) if (gpu_mem[i].gpu_id == gpu_id) return i; @@ -486,9 +472,9 @@ err_object_allocation_failed: bool fmm_is_inside_some_aperture(void *address) { - int32_t i; + uint32_t i; - for (i = 0 ; i < NUM_OF_SUPPORTED_GPUS ; i++) { + for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if ((address >= gpu_mem[i].lds_aperture.base) && @@ -912,7 +898,7 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) uint32_t i; bool found = false; - for (i = 0; i < NUM_OF_SUPPORTED_GPUS && !found; i++) { + for (i = 0; i < gpu_mem_count && !found; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if (address >= gpu_mem[i].scratch_physical.base && @@ -993,20 +979,35 @@ HSAKMT_STATUS fmm_init_process_apertures(void) if (ret != HSAKMT_STATUS_SUCCESS) return ret; - /* Initialize gpu_mem[] from sysfs topology. This is necessary because this function + /* Trade off - sys_props.NumNodes includes GPU nodes + CPU Node. So in + * systems with CPU node, slightly more memory is allocated than + * necessary*/ + gpu_mem = (gpu_mem_t *)calloc(sys_props.NumNodes * sizeof(gpu_mem_t), 1); + if (gpu_mem == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + /* Initialize gpu_mem[] from sysfs topology. Rest of the members are set to + * 0 by calloc. This is necessary because this function * gets called before hsaKmtAcquireSystemProperties() is called.*/ + gpu_mem_count = 0; while (i < sys_props.NumNodes) { ret = topology_sysfs_get_node_props(i, &props, &gpu_id); if (ret != HSAKMT_STATUS_SUCCESS) - return ret; + goto sysfs_parse_failed; /* Skip non-GPU nodes */ if (gpu_id != 0) { - gpu_mem[gpu_mem_id].gpu_id = gpu_id; - gpu_mem[gpu_mem_id].local_mem_size = props.LocalMemSize; - gpu_mem[gpu_mem_id].device_id = props.DeviceId; - gpu_mem[gpu_mem_id].node_id = i; - gpu_mem_id++; + gpu_mem[gpu_mem_count].gpu_id = gpu_id; + gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; + gpu_mem[gpu_mem_count].device_id = props.DeviceId; + gpu_mem[gpu_mem_count].node_id = i; + gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; + pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_physical.fmm_mutex, NULL); + gpu_mem[gpu_mem_count].scratch_aperture.align = PAGE_SIZE; + pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_aperture.fmm_mutex, NULL); + gpu_mem[gpu_mem_count].gpuvm_aperture.align = PAGE_SIZE; + pthread_mutex_init(&gpu_mem[gpu_mem_count].gpuvm_aperture.fmm_mutex, NULL); + gpu_mem_count++; } i++; } @@ -1014,12 +1015,14 @@ HSAKMT_STATUS fmm_init_process_apertures(void) /* The ioctl will also return Number of Nodes if args.kfd_process_device_apertures_ptr * is set to NULL. This is not required since Number of nodes is already known. Kernel * will fill in the apertures in kfd_process_device_apertures_ptr */ - process_apertures = malloc(gpu_mem_id * sizeof(struct kfd_process_device_apertures)); - if (process_apertures == NULL) - return HSAKMT_STATUS_NO_MEMORY; + process_apertures = malloc(gpu_mem_count * sizeof(struct kfd_process_device_apertures)); + if (process_apertures == NULL) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto sysfs_parse_failed; + } args.kfd_process_device_apertures_ptr = (uintptr_t)process_apertures; - args.num_of_nodes = gpu_mem_id; + args.num_of_nodes = gpu_mem_count; if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW, (void *)&args)) { ret = HSAKMT_STATUS_ERROR; @@ -1114,12 +1117,26 @@ HSAKMT_STATUS fmm_init_process_apertures(void) } } + free(process_apertures); + return ret; + get_aperture_ioctl_failed: invalid_gpu_id : free(process_apertures); +sysfs_parse_failed: + fmm_destroy_process_apertures(); return ret; } +void fmm_destroy_process_apertures(void) +{ + if (gpu_mem) { + free(gpu_mem); + gpu_mem = NULL; + } + gpu_mem_count = 0; +} + HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit) { @@ -1308,11 +1325,11 @@ err_object_not_found: int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) { - int32_t i; + uint32_t i; uint64_t pi; /* Find an aperture the requested address belongs to */ - for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; @@ -1432,10 +1449,10 @@ err: int fmm_unmap_from_gpu(void *address) { - int32_t i; + uint32_t i; /* Find the aperture the requested address belongs to */ - for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; @@ -1608,7 +1625,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi bool fmm_get_handle(void *address, uint64_t *handle) { - int32_t i; + uint32_t i; manageble_aperture_t *aperture; vm_object_t *object; bool found; @@ -1617,7 +1634,7 @@ bool fmm_get_handle(void *address, uint64_t *handle) aperture = NULL; /* Find the aperture the requested address belongs to */ - for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; diff --git a/src/fmm.h b/src/fmm.h index 5bdbf5bbba..45fedb4d21 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -44,6 +44,8 @@ typedef struct { } aperture_properties_t; HSAKMT_STATUS fmm_init_process_apertures(void); +void fmm_destroy_process_apertures(void); + /* * Memory interface */ diff --git a/src/libhsakmt.h b/src/libhsakmt.h index c44bb528d0..b976d5a3e5 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -80,6 +80,8 @@ HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId, bool peer_to_peer); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); +HSAKMT_STATUS init_process_doorbells(void); +void destroy_process_doorbells(void); extern int kmtIoctl(int fd, unsigned long request, void *arg); diff --git a/src/openclose.c b/src/openclose.c index d5b91e23dd..620db3debf 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -41,27 +41,30 @@ HSAKMTAPI hsaKmtOpenKFD(void) { HSAKMT_STATUS result; + int fd; pthread_mutex_lock(&hsakmt_mutex); if (kfd_open_count == 0) { - int fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); + fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); - if (fd != -1) - { + if (fd != -1) { kfd_fd = fd; kfd_open_count = 1; - - result = fmm_init_process_apertures(); - if (result != HSAKMT_STATUS_SUCCESS) - close(fd); - } - else - { + } else { result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; + goto open_failed; } + result = fmm_init_process_apertures(); + if (result != HSAKMT_STATUS_SUCCESS) + goto init_process_aperture_failed; + + result = init_process_doorbells(); + if (result != HSAKMT_STATUS_SUCCESS) + goto init_doorbell_failed; + amd_hsa_thunk_lock_fd = open(tmp_file, O_CREAT | //create the file if it's not present. O_RDWR, //only need write access for the internal locking semantics. @@ -73,6 +76,14 @@ hsaKmtOpenKFD(void) result = HSAKMT_STATUS_SUCCESS; } + pthread_mutex_unlock(&hsakmt_mutex); + return result; + +init_doorbell_failed: + fmm_destroy_process_apertures(); +init_process_aperture_failed: + close(fd); +open_failed: pthread_mutex_unlock(&hsakmt_mutex); return result; @@ -90,6 +101,8 @@ hsaKmtCloseKFD(void) { if (--kfd_open_count == 0) { + destroy_process_doorbells(); + fmm_destroy_process_apertures(); close(kfd_fd); if (amd_hsa_thunk_lock_fd > 0) { diff --git a/src/queues.c b/src/queues.c index 247172ef4f..8567e6ce26 100644 --- a/src/queues.c +++ b/src/queues.c @@ -145,7 +145,39 @@ struct process_doorbells pthread_mutex_t doorbells_mutex; }; -struct process_doorbells doorbells[] = {[0 ... (NUM_OF_SUPPORTED_GPUS-1)] = {.need_mmap = true, .doorbells = NULL, .doorbells_mutex = PTHREAD_MUTEX_INITIALIZER}}; +static struct process_doorbells *doorbells; + +HSAKMT_STATUS init_process_doorbells(void) +{ + HsaSystemProperties sys_props; + unsigned int i; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + ret = topology_sysfs_get_system_props(&sys_props); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + /* doorbells[] is accessed using Topology NodeId. This means doorbells[0], + * which corresponds to CPU only Node, might not be used */ + doorbells = malloc(sys_props.NumNodes * sizeof(struct process_doorbells)); + if (doorbells == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + for (i = 0; i < sys_props.NumNodes; i++) { + doorbells[i].need_mmap = true; + doorbells[i].doorbells = NULL; + pthread_mutex_init(&doorbells[i].doorbells_mutex, NULL); + } + return ret; +} + +void destroy_process_doorbells(void) +{ + if (doorbells) { + free(doorbells); + doorbells = NULL; + } +} static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) { From 5e53205b9e581380fbe0270a8bc864039ebf69ad Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 14 Jan 2016 17:07:28 -0500 Subject: [PATCH 0119/1247] Don't limit number of supported HSA Nodes Remove #define MAX_NODES 8 Change-Id: I756cadc652543dd17ea48a1c956adc08c3d2631a --- src/debug.c | 31 +++++++++++++++++++++++++++++-- src/fmm.c | 17 ++++++----------- src/fmm.h | 2 +- src/libhsakmt.h | 12 +++++------- src/openclose.c | 19 +++++++++++++++++-- src/perfctr.c | 46 +++++++++++++++++++++++++++++++++------------- src/queues.c | 11 +++-------- src/topology.c | 2 +- 8 files changed, 95 insertions(+), 45 deletions(-) diff --git a/src/debug.c b/src/debug.c index bb04cfbc2c..ff7ec7fb33 100644 --- a/src/debug.c +++ b/src/debug.c @@ -28,9 +28,29 @@ #include #include -static bool is_device_debugged[MAX_NODES] = {false}; +static bool *is_device_debugged; int debug_get_reg_status(uint32_t node_id, bool* is_debugged); +HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes) +{ + unsigned int i; + + is_device_debugged = malloc(NumNodes * sizeof(bool)); + if (is_device_debugged == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + for (i = 0; i < NumNodes; i++) + is_device_debugged[i] = false; + + return HSAKMT_STATUS_SUCCESS; +} + +void destroy_device_debugging_memory(void) +{ + if (is_device_debugged) + free(is_device_debugged); +} + HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister( @@ -41,6 +61,9 @@ hsaKmtDbgRegister( uint32_t gpu_id; CHECK_KFD_OPEN(); + if (is_device_debugged == NULL) + return HSAKMT_STATUS_NO_MEMORY; + result = validate_nodeid(NodeId, &gpu_id); if (result != HSAKMT_STATUS_SUCCESS) return result; @@ -71,6 +94,9 @@ hsaKmtDbgUnregister( uint32_t gpu_id; CHECK_KFD_OPEN(); + if (is_device_debugged == NULL) + return HSAKMT_STATUS_NO_MEMORY; + result = validate_nodeid(NodeId, &gpu_id); if (result != HSAKMT_STATUS_SUCCESS) return result; @@ -283,7 +309,8 @@ hsaKmtDbgAddressWatch( /* =============================================================================== */ int debug_get_reg_status(uint32_t node_id, bool* is_debugged) { - if ( node_id >= MAX_NODES) + *is_debugged = NULL; + if (is_device_debugged == NULL) return -1; else { *is_debugged = is_device_debugged[node_id]; diff --git a/src/fmm.c b/src/fmm.c index b615311c65..056daba975 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -964,25 +964,20 @@ static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_po return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args); } -HSAKMT_STATUS fmm_init_process_apertures(void) +HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { struct kfd_ioctl_get_process_apertures_new_args args; uint32_t i = 0; int32_t gpu_mem_id =0; uint32_t gpu_id; - HsaSystemProperties sys_props; HsaNodeProperties props; struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - ret = topology_sysfs_get_system_props(&sys_props); - if (ret != HSAKMT_STATUS_SUCCESS) - return ret; - - /* Trade off - sys_props.NumNodes includes GPU nodes + CPU Node. So in + /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than * necessary*/ - gpu_mem = (gpu_mem_t *)calloc(sys_props.NumNodes * sizeof(gpu_mem_t), 1); + gpu_mem = (gpu_mem_t *)calloc(NumNodes, sizeof(gpu_mem_t)); if (gpu_mem == NULL) return HSAKMT_STATUS_NO_MEMORY; @@ -990,7 +985,7 @@ HSAKMT_STATUS fmm_init_process_apertures(void) * 0 by calloc. This is necessary because this function * gets called before hsaKmtAcquireSystemProperties() is called.*/ gpu_mem_count = 0; - while (i < sys_props.NumNodes) { + while (i < NumNodes) { ret = topology_sysfs_get_node_props(i, &props, &gpu_id); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; @@ -1240,9 +1235,9 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu VOID_PTR_ADD(address, size -1) > aperture->limit) return -1; - debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); + ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); /* allocate object within the scratch backing aperture */ - if (!is_debugger) { + if (!ret && !is_debugger) { offset = VOID_PTRS_SUB(address, aperture->base); mem = __fmm_allocate_device(gpu_id, size, aperture, offset, NULL, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); diff --git a/src/fmm.h b/src/fmm.h index 45fedb4d21..881413bbdf 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -43,7 +43,7 @@ typedef struct { void* start_address; } aperture_properties_t; -HSAKMT_STATUS fmm_init_process_apertures(void); +HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes); void fmm_destroy_process_apertures(void); /* diff --git a/src/libhsakmt.h b/src/libhsakmt.h index b976d5a3e5..e1d52b7efe 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -59,12 +59,6 @@ extern bool is_dgpu; #define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) -/* - * Even though the toplogy code doesn't limit us to maximum number of nodes, - * the current HSA spec says the maximum is 8 nodes - */ -#define MAX_NODES 8 - HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint16_t get_device_id_by_node(HSAuint32 node_id); @@ -80,8 +74,12 @@ HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId, bool peer_to_peer); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); -HSAKMT_STATUS init_process_doorbells(void); +HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes); void destroy_process_doorbells(void); +HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes); +void destroy_device_debugging_memory(void); +HSAKMT_STATUS init_counter_props(unsigned int NumNodes); +void destroy_counter_props(void); extern int kmtIoctl(int fd, unsigned long request, void *arg); diff --git a/src/openclose.c b/src/openclose.c index 620db3debf..72baf7c997 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "fmm.h" static const char kfd_device_name[] = "/dev/kfd"; @@ -42,6 +43,7 @@ hsaKmtOpenKFD(void) { HSAKMT_STATUS result; int fd; + HsaSystemProperties sys_props; pthread_mutex_lock(&hsakmt_mutex); @@ -57,14 +59,24 @@ hsaKmtOpenKFD(void) goto open_failed; } - result = fmm_init_process_apertures(); + result = topology_sysfs_get_system_props(&sys_props); + if (result != HSAKMT_STATUS_SUCCESS) + goto topology_sysfs_failed; + + result = fmm_init_process_apertures(sys_props.NumNodes); if (result != HSAKMT_STATUS_SUCCESS) goto init_process_aperture_failed; - result = init_process_doorbells(); + result = init_process_doorbells(sys_props.NumNodes); if (result != HSAKMT_STATUS_SUCCESS) goto init_doorbell_failed; + if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) + printf("Insufficient Memory. Debugging unavailable\n"); + + if (init_counter_props(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) + printf("Insufficient Memory. Performance Counter information unavailable\n"); + amd_hsa_thunk_lock_fd = open(tmp_file, O_CREAT | //create the file if it's not present. O_RDWR, //only need write access for the internal locking semantics. @@ -82,6 +94,7 @@ hsaKmtOpenKFD(void) init_doorbell_failed: fmm_destroy_process_apertures(); init_process_aperture_failed: +topology_sysfs_failed: close(fd); open_failed: pthread_mutex_unlock(&hsakmt_mutex); @@ -101,6 +114,8 @@ hsaKmtCloseKFD(void) { if (--kfd_open_count == 0) { + destroy_counter_props(); + destroy_device_debugging_memory(); destroy_process_doorbells(); fmm_destroy_process_apertures(); close(kfd_fd); diff --git a/src/perfctr.c b/src/perfctr.c index 403ff4ad3e..95102a19ed 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -46,9 +46,34 @@ struct perf_trace { extern int amd_hsa_thunk_lock_fd; -static HsaCounterProperties *counter_props[MAX_NODES] = {NULL}; +static HsaCounterProperties **counter_props; +static unsigned int counter_props_count; -void __attribute__ ((destructor)) perfctr_release_global_resources(void); +HSAKMT_STATUS init_counter_props(unsigned int NumNodes) +{ + counter_props = calloc(NumNodes, sizeof(struct HsaCounterProperties*)); + if (counter_props == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + counter_props_count = NumNodes; + return HSAKMT_STATUS_SUCCESS; +} + +void destroy_counter_props(void) +{ + unsigned int i; + + if (counter_props == NULL) + return; + + for (i = 0; i= MAX_NODES || !node || !_system || _system->NumNodes <= nodeid) + if (!node || !_system || _system->NumNodes <= nodeid) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (gpu_id) *gpu_id = node[nodeid].gpu_id; From 47cef87a343a0a4d9a7c49282751707908e4ac1f Mon Sep 17 00:00:00 2001 From: Serguei Sagalovitch Date: Fri, 8 Jan 2016 16:48:40 -0500 Subject: [PATCH 0120/1247] Skeleton for RDMA unit test v4 Added application and driver to serve as the starting point for RDMA unit test uility. v2: Added initial mmap support v3: Fixed logic to find correct ioctl handler v4: Fixed logic in mmap to find correct pages table Change-Id: Iaf97c0eb2acef2160d542c71afed58cf400414f7 Signed-off-by: Serguei Sagalovitch --- tests/rdma/simple/app/CMakeLists.txt | 12 + tests/rdma/simple/app/rdma_test.cpp | 264 +++++++++++++++ tests/rdma/simple/drv/Makefile | 54 +++ tests/rdma/simple/drv/amdp2ptest.c | 480 +++++++++++++++++++++++++++ tests/rdma/simple/drv/amdp2ptest.h | 71 ++++ 5 files changed, 881 insertions(+) create mode 100644 tests/rdma/simple/app/CMakeLists.txt create mode 100644 tests/rdma/simple/app/rdma_test.cpp create mode 100644 tests/rdma/simple/drv/Makefile create mode 100644 tests/rdma/simple/drv/amdp2ptest.c create mode 100644 tests/rdma/simple/drv/amdp2ptest.h diff --git a/tests/rdma/simple/app/CMakeLists.txt b/tests/rdma/simple/app/CMakeLists.txt new file mode 100644 index 0000000000..762420d461 --- /dev/null +++ b/tests/rdma/simple/app/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required (VERSION 2.6) + +project (rdma_test) + +link_directories($ENV{ROOT_OF_ROOTS}/out/lib) + +include_directories($ENV{LIBHSAKMT_ROOT}/include) +include_directories(../drv) + +add_executable(rdma_test rdma_test.cpp) +target_link_libraries(rdma_test libhsakmt.so) + diff --git a/tests/rdma/simple/app/rdma_test.cpp b/tests/rdma/simple/app/rdma_test.cpp new file mode 100644 index 0000000000..79aaf8ae33 --- /dev/null +++ b/tests/rdma/simple/app/rdma_test.cpp @@ -0,0 +1,264 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hsakmt.h" +#include "amdp2ptest.h" + +int rdma_fd = -1; + +void rdma_open() +{ + rdma_fd = open(AMDP2PTEST_DEVICE_PATH, O_RDWR); + + if (-1 == rdma_fd ) { + int ret = errno; + fprintf(stderr, "error opening driver (errno=%d/%s)\n", ret, strerror(ret)); + exit(EXIT_FAILURE); + } +} + +void rdma_close() +{ + int retcode = close(rdma_fd); + + if (-1 == retcode) { + fprintf(stderr, "error closing driver (errno=%d/%s)\n", retcode, strerror(retcode)); + exit(EXIT_FAILURE); + } + + rdma_fd = -1; +} + +int rdma_map(uint64_t gpu_ptr, size_t size, void **cpu_ptr) +{ + int ret = 0; + + *cpu_ptr = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, rdma_fd, gpu_ptr); + + if (*cpu_ptr == NULL) { + int __errno = errno; + *cpu_ptr = NULL; + fprintf(stderr, "Can't BAR, error=%s(%d) size=%zu offset=%llx\n", + strerror(__errno), __errno, size, (long long unsigned)gpu_ptr); + ret = __errno; + } + + return ret; +} + +int rdma_unmap(void *cpu_ptr, size_t size) +{ + int ret = 0; + + int retcode = munmap(cpu_ptr, size); + + if (-1 == retcode) { + int __errno = errno; + fprintf(stderr, "can't unmap BAR, error=%s(%d) size=%zu\n", + strerror(__errno), __errno, size); + ret = __errno; + } + + return ret; +} + +void run_rdma_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty) +{ + printf("Size 0x%lx (%ld MB)\n", MemoryProperty->SizeInBytes, + MemoryProperty->SizeInBytes / (1024 * 1024)); + printf("VirtualBaseAddress 0x%lx\n", MemoryProperty->VirtualBaseAddress); + + + void *cpu_ptr; + int ret = 0; + void *MemoryAddress = 0; + HSAuint64 SizeInBytes = 4096; + HsaMemFlags memFlags = {0}; + + memFlags.ui32.NonPaged = 1; + memFlags.ui32.CachePolicy = HSA_CACHING_WRITECOMBINED; + memFlags.ui32.NoSubstitute = 1; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; +// memFlags.ui32.HostAccess = 1; + memFlags.ui32.CoarseGrain = 1; + + HSAKMT_STATUS status = hsaKmtAllocMemory(Node, + SizeInBytes, + memFlags, + &MemoryAddress); + + if (status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "Failure to allocate memory. Status %d\n", status); + exit(EXIT_FAILURE); + } + + printf("Memory allocated. Address 0x%p\n", MemoryAddress); + + struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM get_page_size = {0}; + get_page_size.addr = (uint64_t) MemoryAddress; + get_page_size.length = SizeInBytes; + + ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGE_SIZE, &get_page_size); + + if (ret < 0) + { + fprintf(stderr, + "AMD2P2PTEST_IOCTL_GET_PAGE_SIZE error (errno=%d/%s)\n", + ret, strerror(ret)); + exit(EXIT_FAILURE); + } + + printf("GPU Page size: 0x%ld\n", get_page_size.page_size); + + struct AMDRDMA_IOCTL_GET_PAGES_PARAM get_cpu_ptr = {0}; + get_cpu_ptr.addr = (uint64_t) MemoryAddress; + get_cpu_ptr.length = SizeInBytes; + + ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGES, &get_cpu_ptr); + + if (ret < 0) + { + fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGES error (errno=%d/%s)\n", + ret, strerror(ret)); + exit(EXIT_FAILURE); + } + + + ret = rdma_map((uint64_t)MemoryAddress, 4096, &cpu_ptr); + + if (ret < 0) + { + exit(EXIT_FAILURE); + } + + printf("CPU Virtual address 0x%p\n", cpu_ptr); + + hsaKmtFreeMemory(MemoryAddress, SizeInBytes); +} + +int main(void) +{ + HsaVersionInfo VersionInfo; + + HSAKMT_STATUS status = hsaKmtOpenKFD(); + + if( status == HSAKMT_STATUS_SUCCESS) + { + status = hsaKmtGetVersion(&VersionInfo); + + if(status == HSAKMT_STATUS_SUCCESS) + { + printf("Kernel Interface Major Version: %d\n", VersionInfo.KernelInterfaceMajorVersion); + printf("Kernel Interface Minor Version: %d\n", VersionInfo.KernelInterfaceMinorVersion); + } + } + + rdma_open(); + + HsaSystemProperties SystemProperties = {0}; + status = hsaKmtAcquireSystemProperties(&SystemProperties); + + if(status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "hsaKmtAcquireSystemProperties call failed. Error: %d\n", status); + exit(EXIT_FAILURE); + } + + printf("System properties: Number of nodes: %d\n", SystemProperties.NumNodes); + + for (HSAuint32 iNode = 0; iNode < SystemProperties.NumNodes; iNode++) + { + HsaNodeProperties NodeProperties = {0}; + status = hsaKmtGetNodeProperties(iNode, &NodeProperties); + + if(status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "hsaKmtGetNodeProperties (Node = %d) call failed. Error: %d\n", + iNode, status); + exit(EXIT_FAILURE); + } + + printf("Node %d -> Number of Memory Banks = %d\n", iNode, + NodeProperties.NumMemoryBanks); + + HsaMemoryProperties* MemoryProperties = + new HsaMemoryProperties[NodeProperties.NumMemoryBanks]; + + status = hsaKmtGetNodeMemoryProperties(iNode, + NodeProperties.NumMemoryBanks, + MemoryProperties); + + if(status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "hsaKmtGetNodeMemoryProperties (Node = %d) call failed. Error: %d\n", + iNode, status); + exit(EXIT_FAILURE); + } + + for (HSAuint32 iMemBank = 0; iMemBank < NodeProperties.NumMemoryBanks; iMemBank++) + { + printf("Heap type: %d\n", MemoryProperties[iMemBank].HeapType); + + if (MemoryProperties[iMemBank].HeapType == HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC) + { + // We found local memory available for RDMA operation. + // Run some tests on it. + run_rdma_tests(iNode, &MemoryProperties[iMemBank]); + } + } + } + + + status = hsaKmtReleaseSystemProperties(); + + if(status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "hsaKmtReleaseSystemProperties call failed. Error: %d\n", + status); + exit(EXIT_FAILURE); + } + + rdma_close(); + + status = hsaKmtCloseKFD(); + + if(status != HSAKMT_STATUS_SUCCESS) + { + fprintf(stderr, "hsaKmtCloseKFD call failed. Error: %d\n", status); + exit(EXIT_FAILURE); + } + + return EXIT_SUCCESS; +} + + diff --git a/tests/rdma/simple/drv/Makefile b/tests/rdma/simple/drv/Makefile new file mode 100644 index 0000000000..e6bea3edda --- /dev/null +++ b/tests/rdma/simple/drv/Makefile @@ -0,0 +1,54 @@ + +ifeq ($(src),) + src := $$PWD +endif + + + +ifneq ($(KERNELRELEASE),) +# Specify default location of RDMA header +RDMA_HEADER_DIR := $(ROOT_OF_ROOTS)/kernel/drivers/gpu/drm/amd/include +ifeq ("$(wildcard $(RDMA_HEADER_DIR)/amd_rdma.h)","") +RDMA_HEADER_DIR := /usr/src/linux-headers-$(shell uname -r)/drivers/gpu/drm/amd/include +ifeq ("$(wildcard $(RDMA_HEADER_DIR)/amd_rdma.h)","") +$(error amd_rdma.h header is not found) +endif +endif + +$(info INFO: RDMA Header directory is $(RDMA_HEADER_DIR)) + +kver_major:=$(shell echo $(KERNELRELEASE) | awk -F '.' '// { print $$2;}' ) + +obj-m += amdp2ptest.o + +ccflags-y += -I $(RDMA_HEADER_DIR) + +else + +KDIR ?= /lib/modules/$(shell uname -r)/build +REL := $(subst ., , $(subst -, , $(shell uname -r))) +REL_MAJOR := $(word 1,$(REL)) +REL_MEDIUM := $(word 2,$(REL)) +REL_MINOR := $(word 3,$(REL)) + +all: default + + +default: + @ $(MAKE) -C $(KDIR) M=$$PWD modules + +install: + $(MAKE) -C $(KDIR) M=$$PWD modules_install + +help: + $(MAKE) -C $(KDIR) M=$$PWD help + +clean: + rm -rf *.o *.ko* *.mod.* .*.cmd Module.symvers modules.order .tmp_versions/ *~ core .depend TAGS + +TAGS: + find $(KERNELDIR) -follow -name \*.h -o -name \*.c |xargs etags + +.PHONY: clean all help install default linksyms + +endif diff --git a/tests/rdma/simple/drv/amdp2ptest.c b/tests/rdma/simple/drv/amdp2ptest.c new file mode 100644 index 0000000000..788185ae73 --- /dev/null +++ b/tests/rdma/simple/drv/amdp2ptest.c @@ -0,0 +1,480 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "amd_rdma.h" +#include "amdp2ptest.h" + + +MODULE_AUTHOR("serguei.sagalovitch@amd.com"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD RDMA basic API test kernel-mode driver"); +MODULE_VERSION("1.0"); + + +const struct amd_rdma_interface *rdma_interface; + + +struct va_pages_node { + struct list_head node; + struct amd_p2p_page_table *pages; +}; + + +struct amdp2ptest_pages_list { + struct list_head head; + struct mutex lock; +}; + + +#define MSG_INFO(fmt, args ...) \ + pr_info(AMDP2PTEST_DEVICE_NAME ": " fmt, ## args) +#define MSG_ERR(fmt, args ...) \ + pr_err(AMDP2PTEST_DEVICE_NAME ": " fmt, ## args) +#define MSG_warn(fmt, args ...) \ + pr_warn(AMDP2PTEST_DEVICE_NAME ": " fmt, ## args) + + + +static void free_callback(struct amd_p2p_page_table *page_table, + void *client_priv) +{ + struct va_pages_node *va_pages = NULL; + struct amdp2ptest_pages_list *list = client_priv; + struct list_head *p, *n; + + MSG_ERR("Free callback is called on va 0x%llx\n", page_table->va); + + list_for_each_safe(p, n, &list->head) { + va_pages = list_entry(p, struct va_pages_node, node); + + if (va_pages->pages == page_table) { + MSG_INFO("Found free page table to free\n"); + + mutex_lock(&list->lock); + list_del(&va_pages->node); + mutex_unlock(&list->lock); + kfree(va_pages); + + /* Note: Do not break from loop to allow test + * situation when "get_pages" would be called + * on the same memory several times + **/ + } + } +} + + + +static int amdp2ptest_open(struct inode *inode, struct file *filp) +{ + struct amdp2ptest_pages_list *list; + + MSG_INFO("Open driver\n"); + + list = kmalloc(sizeof(struct amdp2ptest_pages_list), GFP_KERNEL); + + if (!list) { + MSG_ERR("Can't alloc kernel memory to store list stucture\n"); + return -ENOMEM; + } + + INIT_LIST_HEAD(&list->head); + mutex_init(&list->lock); + + filp->private_data = list; + + return 0; +} + + +static int amdp2ptest_release(struct inode *inode, struct file *filp) +{ + struct va_pages_node *va_pages = NULL; + int retcode; + struct amdp2ptest_pages_list *list = filp->private_data; + struct list_head *p, *n; + + MSG_INFO("Close driver\n"); + + list_for_each_safe(p, n, &list->head) { + va_pages = list_entry(p, struct va_pages_node, node); + MSG_INFO("Free pages: VA 0x%llx\n", va_pages->pages->va); + retcode = rdma_interface->put_pages(va_pages->pages); + + if (retcode != 0) + MSG_ERR("Could not put pages back: %d\n", retcode); + + mutex_lock(&list->lock); + list_del(&va_pages->node); + mutex_unlock(&list->lock); + kfree(va_pages); + } + + filp->private_data = NULL; + kfree(list); + return 0; +} + + +static int ioctl_get_page_size(struct file *filp, unsigned long arg) +{ + struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM params = {0}; + unsigned long page_size; + int result; + + MSG_INFO("AMD2P2PTEST_IOCTL_GET_PAGE_SIZE"); + + if (copy_from_user(¶ms, (void *)arg, sizeof(params))) { + MSG_ERR("copy_from_user failed on pointer %p\n", + (void *)arg); + return -EFAULT; + } + + MSG_INFO("addr %llx, length %llx\n", params.addr, + params.length); + result = rdma_interface->get_page_size(params.addr, + params.length, + get_task_pid(current, PIDTYPE_PID), + &page_size); + + if (result) { + MSG_ERR("Could not get page size. %d", result); + return -EFAULT; + } + + params.page_size = page_size; + MSG_INFO("Page size %llx\n", params.page_size); + + if (result && copy_to_user((void *)arg, ¶ms, + sizeof(params))) { + MSG_ERR("copy_to_user failed on user pointer %p\n", + (void *)arg); + + return -EFAULT; + } + + return 0; +} + +static int ioctl_get_pages(struct file *filp, unsigned long arg) +{ + struct va_pages_node *va_pages = NULL; + struct amdp2ptest_pages_list *list = filp->private_data; + struct AMDRDMA_IOCTL_GET_PAGES_PARAM params = {0}; + int result; + struct amd_p2p_page_table *pages; + + MSG_INFO("AMD2P2PTEST_IOCTL_GET_PAGES"); + + if (copy_from_user(¶ms, (void *)arg, sizeof(params))) { + MSG_ERR("copy_from_user failed on pointer %p\n", + (void *)arg); + return -EFAULT; + } + + + MSG_INFO("addr %llx, length %llx\n", params.addr, params.length); + + result = rdma_interface->get_pages(params.addr, params.length, + get_task_pid(current, PIDTYPE_PID), + 0, /* There is no dma_device for which + to get pages -> no IOMMU support + is needed */ + &pages, + free_callback, + list /* Pointer to the list */ + ); + + if (result) { + MSG_ERR("Could not get pages table. %d", result); + return -EFAULT; + } + + if (copy_to_user((void *)arg, ¶ms, sizeof(params))) { + MSG_ERR("copy_to_user failed on user pointer %p\n", + (void *)arg); + rdma_interface->put_pages(pages); + return -EFAULT; + } + + + va_pages = kmalloc(sizeof(struct va_pages_node), GFP_KERNEL); + + if (va_pages == 0) { + MSG_ERR("Can't alloc kernel memory\n"); + rdma_interface->put_pages(pages); + return -ENOMEM; + } + + memset(va_pages, 0, sizeof(struct va_pages_node)); + va_pages->pages = pages; + + mutex_lock(&list->lock); + list_add(&va_pages->node, &list->head); + mutex_unlock(&list->lock); + + return 0; +} + + +static int ioctl_put_pages(struct file *filp, unsigned long arg) +{ + struct va_pages_node *va_pages = NULL; + struct amdp2ptest_pages_list *list = filp->private_data; + struct AMDRDMA_IOCTL_PUT_PAGES_PARAM params = {0}; + struct list_head *p, *n; + int retcode; + + MSG_INFO("AMD2P2PTEST_IOCTL_PUT_PAGES"); + + if (copy_from_user(¶ms, (void *)arg, sizeof(params))) { + MSG_ERR("copy_from_user failed on pointer %p\n", + (void *)arg); + return -EFAULT; + } + + MSG_INFO("addr %llx, length %llx\n", params.addr, params.length); + + + list_for_each_safe(p, n, &list->head) { + va_pages = list_entry(p, struct va_pages_node, node); + + if (va_pages->pages->va == params.addr && + va_pages->pages->size == params.length) { + + retcode = rdma_interface->put_pages(va_pages->pages); + + if (retcode != 0) { + MSG_ERR("Could not put pages back: %d\n", + retcode); + } + + mutex_lock(&list->lock); + list_del(&va_pages->node); + mutex_unlock(&list->lock); + kfree(va_pages); + /* Note: Do not break from loop to allow test + * situation when "get_pages" would be called + * on the same memory several times + **/ + } + } + + return 0; +} + + +static const struct ioctl_handler_map { + int (*handler)(struct file *filp, unsigned long arg); + unsigned int cmd; +} handlers[] = { + { ioctl_get_page_size, AMD2P2PTEST_IOCTL_GET_PAGE_SIZE }, + { ioctl_get_pages, AMD2P2PTEST_IOCTL_GET_PAGES }, + { ioctl_put_pages, AMD2P2PTEST_IOCTL_PUT_PAGES }, + { NULL, 0 } +}; + + + +static long amdp2ptest_unlocked_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + int result = -EINVAL; + int i; + + for (i = 0; handlers[i].handler != NULL; i++) + if (cmd == handlers[i].cmd) { + result = handlers[i].handler(filp, arg); + break; + } + + return result; +} + + +static int amdp2ptest_mmap(struct file *filp, struct vm_area_struct *vma) +{ + int i; + struct scatterlist *sg; + struct va_pages_node *va_pages = NULL; + struct amdp2ptest_pages_list *list = filp->private_data; + size_t size = vma->vm_end - vma->vm_start; + struct list_head *p, *n; + uint64_t gpu_va = vma->vm_pgoff << PAGE_SHIFT; + + MSG_INFO("Mapping to CPU user space\n"); + MSG_INFO("Begin vm_end 0x%lx, vm_start 0x%lx\n", vma->vm_end, + vma->vm_start); + MSG_INFO("vm_pgoff / pfn 0x%lx\n", vma->vm_pgoff); + MSG_INFO("gpu_va / phys. address 0x%llx\n", gpu_va); + + if (size != PAGE_SIZE) { + MSG_ERR("Mapping works now only per page size=%ld", PAGE_SIZE); + return -EINVAL; + } + + /* This is the first very simple version of getting CPU pointer for + * the single page. + * The logic is the following: + * - We get GPU VA address and enumerate list to find "get_pages" + * node for such range + * - Then we enumerate sg table to find the correct dma_address + * + * NOTE/TODO: Assumption is that the page size is 4KB to allow testing + * of the basic logic. Eventually more complex logic must be added. + */ + list_for_each_safe(p, n, &list->head) { + va_pages = list_entry(p, struct va_pages_node, node); + + if (va_pages->pages->va >= gpu_va && + (va_pages->pages->va + va_pages->pages->size) + < vma->vm_end) { + + MSG_INFO("Found node: va=0x%llx,size=0x%llx,nents %d\n", + va_pages->pages->va, + va_pages->pages->size, + va_pages->pages->pages->nents); + + for_each_sg(va_pages->pages->pages->sgl, sg, + va_pages->pages->pages->nents, i) { + if (va_pages->pages->va + (i * sg->length) == + gpu_va) { + + MSG_INFO("Found page[%d]: dma 0x%llx\n", + i, sg->dma_address); + + if (remap_pfn_range(vma, + vma->vm_start, + sg->dma_address, + size, + vma->vm_page_prot)) { + MSG_ERR("Failed remap_pfn()\n"); + return -EINVAL; + } + return 0; + } + } + } + } + + return -EINVAL; +} + + +/*---------------------------------------------------------------------------*/ + +static const struct file_operations amdp2ptest_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = amdp2ptest_unlocked_ioctl, + .open = amdp2ptest_open, + .release = amdp2ptest_release, + .mmap = amdp2ptest_mmap, +}; + + + +static struct miscdevice amdp2ptest_dev = { + /* + * We don't care what minor number we end up with, so tell the + * kernel to just pick one. + */ + .minor = MISC_DYNAMIC_MINOR, + /* + * Name ourselves /dev/hello. + */ + .name = AMDP2PTEST_DEVICE_NAME, + /* + * What functions to call when a program performs file + * operations on the device. + */ + .fops = &amdp2ptest_fops, + + /* Security attribute / access */ + .mode = S_IRWXU | S_IRWXG | S_IRWXO +}; + +static int __init amdp2ptest_init(void) +{ + int result; + + result = amdkfd_query_rdma_interface(&rdma_interface); + + if (result < 0) { + MSG_ERR("Can not get RDMA Interface (result = %d)\n", result); + return result; + } + + MSG_INFO("RDMA Interface %p\n", rdma_interface); + MSG_INFO(" get_pages %p\n", rdma_interface->get_pages); + MSG_INFO(" put_pages %p\n", rdma_interface->put_pages); + MSG_INFO(" is_gpu_address %p\n", rdma_interface->is_gpu_address); + MSG_INFO(" get_page_size %p\n", rdma_interface->get_page_size); + + + /* + * Create the device in the /sys/class/misc directory. + * Udev will automatically create the /dev/xxxxx device using + * the default rules. + */ + result = misc_register(&amdp2ptest_dev); + + if (result < 0) { + MSG_ERR("Can not register device (result = %d)\n", result); + return result; + } + + return 0; +} + + +/* Note: cleanup_module is never called if registering failed */ +static void __exit amdp2ptest_cleanup(void) +{ + MSG_INFO("Unregistering\n"); + + misc_deregister(&amdp2ptest_dev); +} + + +module_init(amdp2ptest_init); +module_exit(amdp2ptest_cleanup); + + diff --git a/tests/rdma/simple/drv/amdp2ptest.h b/tests/rdma/simple/drv/amdp2ptest.h new file mode 100644 index 0000000000..e37a1caf2e --- /dev/null +++ b/tests/rdma/simple/drv/amdp2ptest.h @@ -0,0 +1,71 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef AMDP2PTEST_H_ +#define AMDP2PTEST_H_ + +#include + +#define AMDP2PTEST_IOCTL_MAGIC 'A' + + +#define AMDP2PTEST_DEVICE_NAME "amdp2ptest" +#define AMDP2PTEST_DEVICE_PATH "/dev/amdp2ptest" + +struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM { + /* Input parameters */ + uint64_t addr; + uint64_t length; + + /* Output parameters */ + uint64_t page_size; +}; + +struct AMDRDMA_IOCTL_GET_PAGES_PARAM { + /* Input parameters */ + uint64_t addr; + uint64_t length; + uint64_t is_local; /* 1 if this is the pointer to local + allocation */ + + /* Output parameters */ + uint64_t cpu_ptr; +}; + + +struct AMDRDMA_IOCTL_PUT_PAGES_PARAM { + /* Input parameters */ + uint64_t addr; + uint64_t length; +}; + + +#define AMD2P2PTEST_IOCTL_GET_PAGE_SIZE \ +_IOWR(AMDP2PTEST_IOCTL_MAGIC, 1, struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM *) + +#define AMD2P2PTEST_IOCTL_GET_PAGES \ +_IOWR(AMDP2PTEST_IOCTL_MAGIC, 2, struct AMDRDMA_IOCTL_GET_PAGES_PARAM *) + +#define AMD2P2PTEST_IOCTL_PUT_PAGES \ +_IOW(AMDP2PTEST_IOCTL_MAGIC, 3, struct AMDRDMA_IOCTL_PUT_PAGES_PARAM *) + + +#endif /* AMDP2PTEST_H */ From f44982a7ca2c8fb636a75cd1fa70c9b4c1001e02 Mon Sep 17 00:00:00 2001 From: Serguei Sagalovitch Date: Fri, 22 Jan 2016 14:37:20 -0500 Subject: [PATCH 0121/1247] Fixed logic to return data back to user Change-Id: I324d07c38e8d7eb202d4dccfed6e62006cf9cd29 Signed-off-by: Serguei Sagalovitch --- tests/rdma/simple/app/rdma_test.cpp | 4 ++-- tests/rdma/simple/drv/amdp2ptest.c | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/rdma/simple/app/rdma_test.cpp b/tests/rdma/simple/app/rdma_test.cpp index 79aaf8ae33..211299fd10 100644 --- a/tests/rdma/simple/app/rdma_test.cpp +++ b/tests/rdma/simple/app/rdma_test.cpp @@ -130,7 +130,7 @@ void run_rdma_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty) ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGE_SIZE, &get_page_size); - if (ret < 0) + if (ret != 0) { fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGE_SIZE error (errno=%d/%s)\n", @@ -146,7 +146,7 @@ void run_rdma_tests(HSAuint32 Node, HsaMemoryProperties *MemoryProperty) ret = ioctl(rdma_fd, AMD2P2PTEST_IOCTL_GET_PAGES, &get_cpu_ptr); - if (ret < 0) + if (ret != 0) { fprintf(stderr, "AMD2P2PTEST_IOCTL_GET_PAGES error (errno=%d/%s)\n", ret, strerror(ret)); diff --git a/tests/rdma/simple/drv/amdp2ptest.c b/tests/rdma/simple/drv/amdp2ptest.c index 788185ae73..c504393c10 100644 --- a/tests/rdma/simple/drv/amdp2ptest.c +++ b/tests/rdma/simple/drv/amdp2ptest.c @@ -183,8 +183,7 @@ static int ioctl_get_page_size(struct file *filp, unsigned long arg) params.page_size = page_size; MSG_INFO("Page size %llx\n", params.page_size); - if (result && copy_to_user((void *)arg, ¶ms, - sizeof(params))) { + if (copy_to_user((void *)arg, ¶ms, sizeof(params))) { MSG_ERR("copy_to_user failed on user pointer %p\n", (void *)arg); From 7416805a445f43eda892fe14ceb257cfa6a6b953 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 22 Jan 2016 15:31:43 -0500 Subject: [PATCH 0122/1247] Revert "Free resources when dlclose is called" This reverts commit 582b70f9c323daf9b2cf89bb5393a7eb3994b4b6. Conflicts: src/fmm.c src/perfctr.c Change-Id: Ib6113c2dd3962c72100c7f74cdef6897e1df40b3 --- src/fmm.c | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 056daba975..c95835cc29 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -136,7 +136,6 @@ static void __fmm_release(void *address, static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, void *address); -void __attribute__ ((destructor)) fmm_release_global_resources(void); static vm_area_t *vm_create_and_init_area(void *start, void *end) { @@ -1667,24 +1666,6 @@ bool fmm_get_handle(void *address, uint64_t *handle) return found; } -void fmm_release_global_resources(void) -{ - uint64_t len; - - if (dgpu_shared_aperture_base) { - len = (uint64_t)dgpu_shared_aperture_limit - - (uint64_t)dgpu_shared_aperture_base + 1; - munmap(dgpu_shared_aperture_base, len); - } - dgpu_shared_aperture_base = NULL; - dgpu_shared_aperture_limit = NULL; - - if (all_gpu_id_array != NULL) - free(all_gpu_id_array); - all_gpu_id_array = NULL; - all_gpu_id_array_size = 0; -} - HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) From cc7491ec7191562d93e102a2b6b76a29a956e0ae Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 22 Jan 2016 18:08:19 -0500 Subject: [PATCH 0123/1247] Link libhsakmt with -z nodelete This prevents the library from being unloaded at runtime, even when dlclose is called. This preserves global variables, such as state about the SVM address space and avoids catastrophic leaks on dlclose. Change-Id: I34f1d19a450835200e9d4815458e8d1b3045053c --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 03ce54e07e..143385b5f6 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,7 +20,7 @@ ifneq ($(REL),1) CFLAGS += -ggdb endif -LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) +LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) -Wl,-z,nodelete OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ events.o openclose.o queues.o topology.o fmm.o pmc_table.o \ From 641bfd2cd569a7ef4944f37b787bc45bef6057d7 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 22 Jan 2016 18:32:04 -0500 Subject: [PATCH 0124/1247] Add simple test for unloading and reloading Thunk Change-Id: I4ca95dee8a180023d1de5f69161607dd368164de --- tests/reopen/CMakeLists.txt | 11 ++++ tests/reopen/kmtreopen.c | 105 ++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 tests/reopen/CMakeLists.txt create mode 100644 tests/reopen/kmtreopen.c diff --git a/tests/reopen/CMakeLists.txt b/tests/reopen/CMakeLists.txt new file mode 100644 index 0000000000..4229ee56ff --- /dev/null +++ b/tests/reopen/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required (VERSION 2.6) + +project (kmtreopen) + +link_directories($ENV{ROOT_OF_ROOTS}/out/lib) + +include_directories($ENV{LIBHSAKMT_ROOT}/include) + +add_executable(kmtreopen kmtreopen.c) +target_link_libraries(kmtreopen libdl.so) + diff --git a/tests/reopen/kmtreopen.c b/tests/reopen/kmtreopen.c new file mode 100644 index 0000000000..1f6332767a --- /dev/null +++ b/tests/reopen/kmtreopen.c @@ -0,0 +1,105 @@ +#include +#include +#include +#include + +HSAKMT_STATUS HSAKMTAPI (*pfn_hsaKmtOpenKFD)(void); +HSAKMT_STATUS HSAKMTAPI (*pfn_hsaKmtCloseKFD)(void); +HSAKMT_STATUS HSAKMTAPI (*pfn_hsaKmtGetVersion)(HsaVersionInfo* VersionInfo); +HSAKMT_STATUS HSAKMTAPI (*pfn_hsaKmtAcquireSystemProperties)(HsaSystemProperties* SystemProperties); +HSAKMT_STATUS HSAKMTAPI (*pfn_hsaKmtReleaseSystemProperties)(void); + +HsaVersionInfo g_versionInfo; +HsaSystemProperties g_systemProperties; + +static void hsa_perror(const char *s, HSAKMT_STATUS status) +{ + static const char *errorStrings[] = { + [HSAKMT_STATUS_SUCCESS] = "Success", + [HSAKMT_STATUS_ERROR] = "General error", + [HSAKMT_STATUS_DRIVER_MISMATCH] = "Driver mismatch", + [HSAKMT_STATUS_INVALID_PARAMETER] = "Invalid parameter", + [HSAKMT_STATUS_INVALID_HANDLE] = "Invalid handle", + [HSAKMT_STATUS_INVALID_NODE_UNIT] = "Invalid node or unit", + [HSAKMT_STATUS_NO_MEMORY] = "No memory", + [HSAKMT_STATUS_BUFFER_TOO_SMALL] = "Buffer too small", + [HSAKMT_STATUS_NOT_IMPLEMENTED] = "Not implemented", + [HSAKMT_STATUS_NOT_SUPPORTED] = "Not supported", + [HSAKMT_STATUS_UNAVAILABLE] = "Unavailable", + [HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED] = "Kernel IO channel not opened", + [HSAKMT_STATUS_KERNEL_COMMUNICATION_ERROR] = "Kernel communication error", + [HSAKMT_STATUS_KERNEL_ALREADY_OPENED] = "Kernel already opened", + [HSAKMT_STATUS_HSAMMU_UNAVAILABLE] = "HSA MMU unavailable", + [HSAKMT_STATUS_WAIT_FAILURE] = "Wait failure", + [HSAKMT_STATUS_WAIT_TIMEOUT] = "Wait timeout", + [HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED] = "Memory already registered", + [HSAKMT_STATUS_MEMORY_NOT_REGISTERED] = "Memory not registered", + [HSAKMT_STATUS_MEMORY_ALIGNMENT] = "Memory alignment error" + }; + + if (status >= 0 && status <= HSAKMT_STATUS_MEMORY_ALIGNMENT) + fprintf(stderr, "%s: %s\n", s, errorStrings[status]); + else + fprintf(stderr, "%s: Unknown error %d\n", s, status); +} + +#define HSA_CHECK_RETURN(call) do { \ + HSAKMT_STATUS __ret; \ + printf(" Calling %s\n", #call); \ + __ret = pfn_##call; \ + if (__ret != HSAKMT_STATUS_SUCCESS) { \ + hsa_perror(#call, __ret); \ + return __ret; \ + } \ + } while(0) + +#define HSA_DLSYM(handle, func) do { \ + pfn_##func = dlsym(handle, #func); \ + if (pfn_##func == NULL) { \ + fprintf(stderr, "dlsym failed: %s\n", dlerror()); \ + return HSAKMT_STATUS_ERROR; \ + } \ + } while(0) + +static int runTest(void *handle) +{ + HSA_DLSYM(handle, hsaKmtOpenKFD); + HSA_DLSYM(handle, hsaKmtCloseKFD); + HSA_DLSYM(handle, hsaKmtGetVersion); + HSA_DLSYM(handle, hsaKmtAcquireSystemProperties); + HSA_DLSYM(handle, hsaKmtReleaseSystemProperties); + + HSA_CHECK_RETURN(hsaKmtOpenKFD()); + HSA_CHECK_RETURN(hsaKmtGetVersion(&g_versionInfo)); + HSA_CHECK_RETURN(hsaKmtAcquireSystemProperties(&g_systemProperties)); + + HSA_CHECK_RETURN(hsaKmtReleaseSystemProperties()); + HSA_CHECK_RETURN(hsaKmtCloseKFD()); + + return HSAKMT_STATUS_SUCCESS; +} + +int main(int argc, char *argv[]) +{ + void *handle; + int i; + + for (i = 0; i < 5; i++) { + printf("Iteration %d:\n Loading libhsakmt.so\n", i+1); + + handle = dlopen("libhsakmt.so", RTLD_LAZY); + if (handle == NULL) { + fprintf(stderr, "dlopen failed: %s\n", dlerror()); + exit(1); + } + + if (runTest(handle) != HSAKMT_STATUS_SUCCESS) + exit(1); + + printf(" Unloading libhsakmt.so\n"); + if (dlclose(handle) != 0) { + fprintf(stderr, "dlclose failed: %s\n", dlerror()); + exit(1); + } + } +} From 4e4a881940eee301d15f7b966456944cbf1b266c Mon Sep 17 00:00:00 2001 From: David Ogbeide Date: Thu, 14 Jan 2016 14:25:20 -0500 Subject: [PATCH 0125/1247] libhsakmt: Add marketing names for GPU nodes HSA thunk API returns null when querying for GPU node marketing names due to empty system topology file. - Add marketing names to device GFX IP data structs. - Modify name retrieval to pull from data structs instead of file. Signed-off by: David Ogbeide Change-Id: I30ea04111be7e0df2e93894f801fbeb414ffa790 --- src/topology.c | 123 +++++++++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 70 deletions(-) diff --git a/src/topology.c b/src/topology.c index c5cbf5ccb7..0c0ce576d0 100644 --- a/src/topology.c +++ b/src/topology.c @@ -66,49 +66,50 @@ static struct hsa_gfxip_table { unsigned char major; // GFXIP Major engine version unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info - unsigned char is_dgpu; // Predicat for dGPU devices + unsigned char is_dgpu; // Predicate for dGPU devices + const char* marketing_name; // Marketing Name of the device } gfxip_lookup_table[] = { /* Kaveri Family */ - { 0x1304, 7, 0, 0, 0 }, - { 0x1305, 7, 0, 0, 0 }, - { 0x1306, 7, 0, 0, 0 }, - { 0x1307, 7, 0, 0, 0 }, - { 0x1309, 7, 0, 0, 0 }, - { 0x130A, 7, 0, 0, 0 }, - { 0x130B, 7, 0, 0, 0 }, - { 0x130C, 7, 0, 0, 0 }, - { 0x130D, 7, 0, 0, 0 }, - { 0x130E, 7, 0, 0, 0 }, - { 0x130F, 7, 0, 0, 0 }, - { 0x1310, 7, 0, 0, 0 }, - { 0x1311, 7, 0, 0, 0 }, - { 0x1312, 7, 0, 0, 0 }, - { 0x1313, 7, 0, 0, 0 }, - { 0x1315, 7, 0, 0, 0 }, - { 0x1316, 7, 0, 0, 0 }, - { 0x1317, 7, 0, 0, 0 }, - { 0x1318, 7, 0, 0, 0 }, - { 0x131B, 7, 0, 0, 0 }, - { 0x131C, 7, 0, 0, 0 }, - { 0x131D, 7, 0, 0, 0 }, + { 0x1304, 7, 0, 0, 0, "Spectre" }, + { 0x1305, 7, 0, 0, 0, "Spectre" }, + { 0x1306, 7, 0, 0, 0, "Spectre" }, + { 0x1307, 7, 0, 0, 0, "Spectre" }, + { 0x1309, 7, 0, 0, 0, "Spectre" }, + { 0x130A, 7, 0, 0, 0, "Spectre" }, + { 0x130B, 7, 0, 0, 0, "Spectre" }, + { 0x130C, 7, 0, 0, 0, "Spectre" }, + { 0x130D, 7, 0, 0, 0, "Spectre" }, + { 0x130E, 7, 0, 0, 0, "Spectre" }, + { 0x130F, 7, 0, 0, 0, "Spectre" }, + { 0x1310, 7, 0, 0, 0, "Spectre" }, + { 0x1311, 7, 0, 0, 0, "Spectre" }, + { 0x1312, 7, 0, 0, 0, "Spooky" }, + { 0x1313, 7, 0, 0, 0, "Spectre" }, + { 0x1315, 7, 0, 0, 0, "Spectre" }, + { 0x1316, 7, 0, 0, 0, "Spooky" }, + { 0x1317, 7, 0, 0, 0, "Spooky" }, + { 0x1318, 7, 0, 0, 0, "Spectre" }, + { 0x131B, 7, 0, 0, 0, "Spectre" }, + { 0x131C, 7, 0, 0, 0, "Spectre" }, + { 0x131D, 7, 0, 0, 0, "Spectre" }, /* Carrizo Family */ - { 0x9870, 8, 0, 1, 0 }, - { 0x9874, 8, 0, 1, 0 }, - { 0x9875, 8, 0, 1, 0 }, - { 0x9876, 8, 0, 1, 0 }, - { 0x9877, 8, 0, 1, 0 }, + { 0x9870, 8, 0, 1, 0, "Carrizo" }, + { 0x9874, 8, 0, 1, 0, "Carrizo" }, + { 0x9875, 8, 0, 1, 0, "Carrizo" }, + { 0x9876, 8, 0, 1, 0, "Carrizo" }, + { 0x9877, 8, 0, 1, 0, "Carrizo" }, /* Tonga Family */ - { 0x6920, 8, 0, 2, 1 }, - { 0x6921, 8, 0, 2, 1 }, - { 0x6928, 8, 0, 2, 1 }, - { 0x6929, 8, 0, 2, 1 }, - { 0x692B, 8, 0, 2, 1 }, - { 0x692F, 8, 0, 2, 1 }, - { 0x6930, 8, 0, 2, 1 }, - { 0x6938, 8, 0, 2, 1 }, - { 0x6939, 8, 0, 2, 1 }, + { 0x6920, 8, 0, 2, 1, "Tonga" }, + { 0x6921, 8, 0, 2, 1, "Tonga" }, + { 0x6928, 8, 0, 2, 1, "Tonga" }, + { 0x6929, 8, 0, 2, 1, "Tonga" }, + { 0x692B, 8, 0, 2, 1, "Tonga" }, + { 0x692F, 8, 0, 2, 1, "Tonga" }, + { 0x6930, 8, 0, 2, 1, "Tonga" }, + { 0x6938, 8, 0, 2, 1, "Tonga" }, + { 0x6939, 8, 0, 2, 1, "Tonga" }, /* Fiji */ - { 0x7300, 8, 0, 3, 1 } + { 0x7300, 8, 0, 3, 1, "Fiji" } }; static void @@ -320,35 +321,9 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 /* Retrieve the GPU ID */ ret = topology_sysfs_get_gpu_id(node_id, gpu_id); - /* Retrieve the marketing name of the node */ - snprintf(path, 256, "%s/%d/name", KFD_SYSFS_PATH_NODES, node_id); - fd = fopen(path, "r"); - if (!fd) - return HSAKMT_STATUS_ERROR; - read_buf = malloc(PAGE_SIZE); - if (!read_buf) { - ret = HSAKMT_STATUS_NO_MEMORY; - goto err1; - } - - read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { - ret = HSAKMT_STATUS_ERROR; - goto err2; - } - p = memchr(read_buf, '\n', read_size); - if ((!p) || ((p-read_buf) > HSA_PUBLIC_NAME_SIZE)) { - ret = HSAKMT_STATUS_ERROR; - goto err2; - } - /* - * Convert UTF8 to UTF16 - */ - for (i = 0; (i < HSA_PUBLIC_NAME_SIZE) && (read_buf[i] != '\n'); i++) - props->MarketingName[i] = read_buf[i]; - props->MarketingName[i] = 0; - fclose(fd); + if (!read_buf) + return HSAKMT_STATUS_NO_MEMORY; /* Retrieve the node properties */ snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, node_id); @@ -361,7 +336,7 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; - goto err2; + goto err; } /* Since we're using the buffer as a string, we make sure the string terminates */ @@ -437,14 +412,22 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 props->EngineId.ui32.Major = hsa_gfxip->major & 0x3f; props->EngineId.ui32.Minor = hsa_gfxip->minor; props->EngineId.ui32.Stepping = hsa_gfxip->stepping; + + if (!hsa_gfxip->marketing_name) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + + /* Retrieve the marketing name of the node, convert UTF8 to UTF16 */ + for (i = 0; hsa_gfxip->marketing_name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + props->MarketingName[i] = hsa_gfxip->marketing_name[i]; + props->MarketingName[i] = 0; } if (props->NumFComputeCores) assert(props->EngineId.ui32.Major); - //TODO: error handler when Device ID lookup fails -err2: +err: free(read_buf); -err1: fclose(fd); return ret; } From cc9fc386bdd459b8f6ed5058155c324bf12fdd65 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 13 Jan 2016 14:24:42 -0500 Subject: [PATCH 0126/1247] Remove gfx802 page size workaround on gfx803 All tonga page size alignment is done in the memory management functions in fmm.c. All other code only specifies the minimum alignment it needs and lets fmm.c handle the HW-specific alignment. Clean up aligned-exec memory allocation in queue.c to remove hard-coded TONGA_PAGE_SIZE alignments and remove code duplication. Make sure alignments are consistent between allocate and free. Change-Id: Ia8923448173d1cef315af24cebff12adef385cb0 --- src/events.c | 2 +- src/fmm.c | 14 ++++++++++---- src/queues.c | 48 ++++++++++++++++++++++-------------------------- 3 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/events.c b/src/events.c index e8024b95ad..50c3e1d3bf 100644 --- a/src/events.c +++ b/src/events.c @@ -76,7 +76,7 @@ hsaKmtCreateEvent( /* dGPU code */ if (is_dgpu && events_page == NULL) { events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, - TONGA_PAGE_SIZE, + PAGE_SIZE, args.node_id, true); if (!events_page) { return HSAKMT_STATUS_ERROR; diff --git a/src/fmm.c b/src/fmm.c index c95835cc29..825d76b8c7 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1067,17 +1067,23 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uintptr_t alt_base; uint64_t alt_size; int err; + uint64_t vm_alignment = PAGE_SIZE; + + if (gpu_mem[gpu_mem_id].device_id >= 0x6920 && + gpu_mem[gpu_mem_id].device_id <= 0x6939) + /* Workaround for Tonga GPUVM HW bug */ + vm_alignment = TONGA_PAGE_SIZE; dgpu_mem_init(gpu_mem_id, &svm.dgpu_aperture.base, &svm.dgpu_aperture.limit); /* Set proper alignment for scratch backing aperture */ - gpu_mem[gpu_mem_id].scratch_physical.align = TONGA_PAGE_SIZE; + gpu_mem[gpu_mem_id].scratch_physical.align = vm_alignment; /* Set kernel process dgpu aperture. */ set_dgpu_aperture(i, (uint64_t)svm.dgpu_aperture.base, (uint64_t)svm.dgpu_aperture.limit); - svm.dgpu_aperture.align = TONGA_PAGE_SIZE; + svm.dgpu_aperture.align = vm_alignment; /* Place GPUVM aperture after dGPU aperture * (FK: I think this is broken but leaving it for now) */ @@ -1086,7 +1092,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) svm.dgpu_aperture.base); gpu_mem[gpu_mem_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[gpu_mem_id].gpuvm_aperture.limit, (unsigned long)gpu_mem[gpu_mem_id].gpuvm_aperture.base); - gpu_mem[gpu_mem_id].gpuvm_aperture.align = TONGA_PAGE_SIZE; + gpu_mem[gpu_mem_id].gpuvm_aperture.align = vm_alignment; /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. @@ -1107,7 +1113,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); ret = HSAKMT_STATUS_ERROR; } - svm.dgpu_alt_aperture.align = TONGA_PAGE_SIZE; + svm.dgpu_alt_aperture.align = vm_alignment; } } diff --git a/src/queues.c b/src/queues.c index bd0fd79eef..b6f1205329 100644 --- a/src/queues.c +++ b/src/queues.c @@ -135,7 +135,7 @@ struct queue uint32_t rptr; void *eop_buffer; void *ctx_save_restore; - enum asic_family_type type; + const struct device_info *dev_info; }; struct process_doorbells @@ -266,20 +266,15 @@ static void* allocate_exec_aligned_memory(uint32_t size, uint32_t NodeId) { if (IS_DGPU(type)) - return allocate_exec_aligned_memory_gpu(size, TONGA_PAGE_SIZE, NodeId, false); + return allocate_exec_aligned_memory_gpu(size, align, NodeId, false); return allocate_exec_aligned_memory_cpu(size, align); } -static void release_exec_aligned_memory_gpu(void *addr, uint32_t size) -{ - if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) - hsaKmtFreeMemory(addr, (HSAuint64)size); -} - -static void release_exec_aligned_memory(void *addr, uint32_t size, enum asic_family_type type) +static void free_exec_aligned_memory(void *addr, uint32_t size, uint32_t align, + enum asic_family_type type) { if (IS_DGPU(type)) - release_exec_aligned_memory_gpu(addr, TONGA_PAGE_SIZE); + free_exec_aligned_memory_gpu(addr, size, align); else free(addr); } @@ -288,30 +283,31 @@ static void free_queue_gpu(struct queue *q) { if (q->eop_buffer) { hsaKmtUnmapMemoryToGPU(q->eop_buffer); - hsaKmtFreeMemory(q->eop_buffer, TONGA_PAGE_SIZE); + hsaKmtFreeMemory(q->eop_buffer, q->dev_info->eop_buffer_size); } if (q->ctx_save_restore) { hsaKmtUnmapMemoryToGPU(q->ctx_save_restore); - hsaKmtFreeMemory(q->ctx_save_restore, TONGA_PAGE_SIZE); + hsaKmtFreeMemory(q->ctx_save_restore, q->dev_info->ctx_save_restore_size); } - release_exec_aligned_memory((void *)q, sizeof(*q), q->type); + free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } -static void free_queue(struct queue *q, enum asic_family_type type) +static void free_queue(struct queue *q) { - if (IS_DGPU(type)) + if (IS_DGPU(q->dev_info->asic_family)) return free_queue_gpu(q); return free_queue_cpu(q); } -static int handle_concrete_asic(struct device_info *dev_info, struct queue *q, - struct kfd_ioctl_create_queue_args *args, - uint32_t NodeId) +static int handle_concrete_asic(struct queue *q, + struct kfd_ioctl_create_queue_args *args, + uint32_t NodeId) { + const struct device_info *dev_info = q->dev_info; if (dev_info) { if (dev_info->eop_buffer_size > 0) { q->eop_buffer = - allocate_exec_aligned_memory(dev_info->eop_buffer_size, + allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, PAGE_SIZE, dev_info->asic_family, NodeId); @@ -380,11 +376,11 @@ hsaKmtCreateQueue( args.gpu_id = gpu_id; - q->type = dev_info->asic_family; + q->dev_info = dev_info; - err = handle_concrete_asic(dev_info, q, &args, NodeId); + err = handle_concrete_asic(q, &args, NodeId); if (err != HSAKMT_STATUS_SUCCESS) { - free_queue(q, dev_info->asic_family); + free_queue(q); return err; } @@ -393,7 +389,7 @@ hsaKmtCreateQueue( case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; - default: free_queue(q, dev_info->asic_family); return HSAKMT_STATUS_INVALID_PARAMETER; + default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; } if (Type != HSA_QUEUE_COMPUTE_AQL) @@ -413,7 +409,7 @@ hsaKmtCreateQueue( if (err == -1) { - free_queue(q, dev_info->asic_family); + free_queue(q); return HSAKMT_STATUS_ERROR; } @@ -428,7 +424,7 @@ hsaKmtCreateQueue( if (ptr == MAP_FAILED) { pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); hsaKmtDestroyQueue(q->queue_id); - free_queue(q, dev_info->asic_family); + free_queue(q); return HSAKMT_STATUS_ERROR; } @@ -504,7 +500,7 @@ hsaKmtDestroyQueue( } else { - free_queue(q, q->type); + free_queue(q); return HSAKMT_STATUS_SUCCESS; } } From e37863d7f2aba4659cb4ed1eb84dfcc05fec338a Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Thu, 28 Jan 2016 14:47:02 +0200 Subject: [PATCH 0127/1247] Adding HsaMemMapFlags struct Change-Id: Ib0ee6dede1169582fd58bfca648347c3f8aa0b54 Signed-off-by: Ben Goz --- include/hsakmttypes.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 5b4eeba1e1..75fbcb7a58 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -469,6 +469,36 @@ typedef struct _HsaMemFlags }; } HsaMemFlags; +typedef struct _HsaMemMapFlags +{ + union + { + struct + { + unsigned int Reserved1 : 1; // + unsigned int CachePolicy : 4; // see HSA_CACHING_TYPE + unsigned int ReadOnly : 1; // memory is not modified while mapped + // allows migration scale-out + unsigned int PageSize : 2; // see HSA_PAGE_SIZE, hint to use + // this page size if possible and + // smaller than default + unsigned int HostAccess : 1; // default = 0: GPU access only + unsigned int Migrate : 1; // Hint: Allows migration to local mem + // of mapped GPU(s), instead of mapping + // physical location + unsigned int Probe : 1; // default = 0: Indicates that a range + // will be mapped by the process soon, + // but does not initiate a map operation + // may trigger eviction of nonessential + // data from the memory, reduces latency + // “cleanup hint” only, may be ignored + unsigned int Reserved : 21; + }; + HSAuint32 Value; + }; +} HsaMemMapFlags; + + typedef enum _HSA_CACHING_TYPE { HSA_CACHING_CACHED = 0, From 3797b56ec91e31494e060355641afe1198871c7d Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Fri, 29 Jan 2016 17:06:52 -0500 Subject: [PATCH 0128/1247] Bump version for bugfix release 1.8.1 Change-Id: I06701905592594221d26c075a8fe370b4cc92aff --- DEBIAN/control | 2 +- RPM/libhsakmt.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DEBIAN/control b/DEBIAN/control index dd99d843b7..563a85813d 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -3,6 +3,6 @@ Architecture: amd64 Maintainer: Advanced Micro Devices Inc. Depends: Priority: optional -Version: 1.6.0 +Version: 1.8.1 Description: Thunk library for AMD KFD diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index afce127bc8..3640f53ba8 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -1,5 +1,5 @@ %define name libhsakmt -%define version 1.6.0 +%define version 1.8.1 %define thunkroot %{getenv:OUT_DIR} Name: %{name} From 7e40877e81fc433421b890f87f8b61227d81fb8e Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Mon, 25 Jan 2016 18:14:50 -0500 Subject: [PATCH 0129/1247] libhsakmt: Use GPU ID instead of Node ID in set_process_dgpu_aperture Change-Id: I0e66ca4a018c15c009a3516d250f0044a4407878 --- include/linux/kfd_ioctl.h | 2 +- src/fmm.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index f12e96e46b..8e9c5c1604 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -296,7 +296,7 @@ struct kfd_ioctl_open_graphic_handle_args { }; struct kfd_ioctl_set_process_dgpu_aperture_args { - uint32_t node_id; + uint32_t gpu_id; uint64_t dgpu_base; uint64_t dgpu_limit; }; diff --git a/src/fmm.c b/src/fmm.c index 825d76b8c7..7c6837843b 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -130,7 +130,7 @@ static uint32_t *all_gpu_id_array = NULL; extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); -static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit); +static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit); static void __fmm_release(void *address, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, @@ -1081,7 +1081,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_id].scratch_physical.align = vm_alignment; /* Set kernel process dgpu aperture. */ - set_dgpu_aperture(i, (uint64_t)svm.dgpu_aperture.base, + set_dgpu_aperture(process_apertures[i].gpu_id, + (uint64_t)svm.dgpu_aperture.base, (uint64_t)svm.dgpu_aperture.limit); svm.dgpu_aperture.align = vm_alignment; @@ -1487,11 +1488,11 @@ int fmm_unmap_from_gpu(void *address) /* Tonga dGPU specific functions */ static bool is_dgpu_mem_init = false; -static int set_dgpu_aperture(uint32_t node_id, uint64_t base, uint64_t limit) +static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit) { struct kfd_ioctl_set_process_dgpu_aperture_args args; - args.node_id = node_id; + args.gpu_id = gpu_id; args.dgpu_base = base; args.dgpu_limit = limit; From 4e6c25e55bc3b2210ba0f5bfc09d07a6e393dd4c Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Fri, 22 Jan 2016 17:32:59 -0500 Subject: [PATCH 0130/1247] libhsaKmt: Add CWSR support on dGPU This is thunk part of the CWSR support. 1. SDMA queue don't support CWSR , no necessary to allocate the context save/restore memory 2. Allocate the context save/restore memory in local frame buffer for dGPU Change-Id: Ie83506f0cced2a5a537c49d68125796d831c2764 --- src/queues.c | 48 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/queues.c b/src/queues.c index b6f1205329..e7e26b87ca 100644 --- a/src/queues.c +++ b/src/queues.c @@ -64,20 +64,23 @@ struct device_info kaveri_device_info = { struct device_info carrizo_device_info = { .asic_family = CHIP_CARRIZO, - .ctx_save_restore_size = 2756608 + 4096, + .ctx_save_restore_size = PAGE_ALIGN_UP(2756608 + 4096), .ctl_stack_size = 4096, .eop_buffer_size = 4096, }; struct device_info tonga_device_info = { .asic_family = CHIP_TONGA, - .ctx_save_restore_size = TONGA_PAGE_SIZE, + .ctx_save_restore_size = PAGE_ALIGN_UP(8269824 + PAGE_ALIGN_UP(6152)), + .ctl_stack_size = PAGE_ALIGN_UP(6152), .eop_buffer_size = TONGA_PAGE_SIZE, }; struct device_info fiji_device_info = { .asic_family = CHIP_FIJI, - .ctx_save_restore_size = TONGA_PAGE_SIZE, + /*SR size = work group ctx data + ctl stack size*/ + .ctx_save_restore_size = PAGE_ALIGN_UP(20674560 + PAGE_ALIGN_UP(15368)), + .ctl_stack_size = PAGE_ALIGN_UP(15368), .eop_buffer_size = TONGA_PAGE_SIZE, }; @@ -317,10 +320,30 @@ static int handle_concrete_asic(struct queue *q, args->eop_buffer_address = (uintptr_t)q->eop_buffer; args->eop_buffer_size = dev_info->eop_buffer_size; } - if (dev_info->ctx_save_restore_size > 0) { + if (args->queue_type != KFD_IOC_QUEUE_TYPE_SDMA && + dev_info->ctx_save_restore_size > 0) { args->ctx_save_restore_size = dev_info->ctx_save_restore_size; args->ctl_stack_size = dev_info->ctl_stack_size; - q->ctx_save_restore = + if (IS_DGPU(dev_info->asic_family)) { + void *mem; + HsaMemFlags flags; + HSAKMT_STATUS ret; + HSAuint64 size = dev_info->ctx_save_restore_size; + flags.Value = 0; + flags.ui32.NonPaged = 1; /* device memory*/ + + ret = hsaKmtAllocMemory(NodeId, size, flags, &mem); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = hsaKmtMapMemoryToGPU(mem, size, NULL); + if (ret != HSAKMT_STATUS_SUCCESS) { + hsaKmtFreeMemory(mem, size); + return ret; + } + q->ctx_save_restore = mem; + + } else + q->ctx_save_restore = allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, PAGE_SIZE, dev_info->asic_family, @@ -378,18 +401,12 @@ hsaKmtCreateQueue( q->dev_info = dev_info; - err = handle_concrete_asic(q, &args, NodeId); - if (err != HSAKMT_STATUS_SUCCESS) { - free_queue(q); - return err; - } - switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; - default: free_queue(q); return HSAKMT_STATUS_INVALID_PARAMETER; + default: return HSAKMT_STATUS_INVALID_PARAMETER; } if (Type != HSA_QUEUE_COMPUTE_AQL) @@ -398,6 +415,13 @@ hsaKmtCreateQueue( QueueResource->QueueWptrValue = (uintptr_t)&q->wptr; } + err = handle_concrete_asic(q, &args, NodeId); + if (err != HSAKMT_STATUS_SUCCESS) { + free_queue(q); + return err; + } + + args.read_pointer_address = QueueResource->QueueRptrValue; args.write_pointer_address = QueueResource->QueueWptrValue; args.ring_base_address = (uintptr_t)QueueAddress; From 7070f7ec5eeaca911d7fc31f600e520042bf53d4 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Thu, 28 Jan 2016 15:33:56 +0200 Subject: [PATCH 0131/1247] Adding support to hsaKmtMapMemoryToGPUNodes Change-Id: Iab6222402a43c3cd31b0efc5a316a6482986258e Signed-off-by: Ben Goz --- include/hsakmt.h | 15 +++++++++++++++ include/hsakmttypes.h | 4 ++-- src/libhsakmt.h | 2 ++ src/libhsakmt.ver | 1 + src/memory.c | 45 ++++++++++++++++++++++++++++++------------- src/topology.c | 24 +++++++++++++++++++++++ 6 files changed, 76 insertions(+), 15 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 5fadf78577..55199bb659 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -416,6 +416,21 @@ hsaKmtMapMemoryToGPU( HSAuint64* AlternateVAGPU //OUT (page-aligned) ); +/** + Ensures that the memory is resident and can be accessed by GPUs +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtMapMemoryToGPUNodes( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 MemorySizeInBytes, //IN (page-aligned) + HSAuint64* AlternateVAGPU, //OUT (page-aligned) + HsaMemMapFlags MemMapFlags, //IN + HSAuint64 NumberOfNodes, //IN + HSAuint32* NodeArray //IN + ); + /** Releases the residency of the memory */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 75fbcb7a58..e9885398e5 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -476,7 +476,7 @@ typedef struct _HsaMemMapFlags struct { unsigned int Reserved1 : 1; // - unsigned int CachePolicy : 4; // see HSA_CACHING_TYPE + unsigned int CachePolicy : 2; // see HSA_CACHING_TYPE unsigned int ReadOnly : 1; // memory is not modified while mapped // allows migration scale-out unsigned int PageSize : 2; // see HSA_PAGE_SIZE, hint to use @@ -492,7 +492,7 @@ typedef struct _HsaMemMapFlags // may trigger eviction of nonessential // data from the memory, reduces latency // “cleanup hint” only, may be ignored - unsigned int Reserved : 21; + unsigned int Reserved : 23; }; HSAuint32 Value; }; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index e1d52b7efe..d277da07b0 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -63,6 +63,8 @@ HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint16_t get_device_id_by_node(HSAuint32 node_id); uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); +HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, + uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id); diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 9a8a14835b..95790923dd 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -28,6 +28,7 @@ hsaKmtRegisterMemory; hsaKmtRegisterMemoryToNodes; hsaKmtDeregisterMemory; hsaKmtMapMemoryToGPU; +hsaKmtMapMemoryToGPUNodes; hsaKmtUnmapMemoryToGPU; hsaKmtDbgRegister; hsaKmtDbgUnregister; diff --git a/src/memory.c b/src/memory.c index c0161b36dc..514673d003 100644 --- a/src/memory.c +++ b/src/memory.c @@ -212,21 +212,10 @@ hsaKmtRegisterMemoryToNodes( { CHECK_KFD_OPEN(); uint32_t *gpu_id_array; - unsigned i; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - if (NumberOfNodes == 0 || NodeArray == NULL) - return HSAKMT_STATUS_INVALID_PARAMETER; - - /* Translate Node IDs to gpu_ids */ - gpu_id_array = malloc(NumberOfNodes * sizeof(uint32_t)); - if (gpu_id_array == NULL) - return HSAKMT_STATUS_NO_MEMORY; - for (i = 0; i < NumberOfNodes; i++) { - ret = validate_nodeid(NodeArray[i], &gpu_id_array[i]); - if (ret != HSAKMT_STATUS_SUCCESS) - break; - } + ret = validate_nodeid_array(&gpu_id_array, + NumberOfNodes, NodeArray); if (ret == HSAKMT_STATUS_SUCCESS) ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, @@ -269,6 +258,36 @@ hsaKmtMapMemoryToGPU( return HSAKMT_STATUS_ERROR; } +HSAKMT_STATUS +HSAKMTAPI +hsaKmtMapMemoryToGPUNodes( + void* MemoryAddress, //IN (page-aligned) + HSAuint64 MemorySizeInBytes, //IN (page-aligned) + HSAuint64* AlternateVAGPU, //OUT (page-aligned) + HsaMemMapFlags MemMapFlags, //IN + HSAuint64 NumberOfNodes, //IN + HSAuint32* NodeArray //IN +) +{ + uint32_t *gpu_id_array; + HSAKMT_STATUS ret; + + ret = validate_nodeid_array(&gpu_id_array, + NumberOfNodes, NodeArray); + if (ret != HSAKMT_STATUS_SUCCESS) { + free(gpu_id_array); + return ret; + } + + ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, + gpu_id_array, NumberOfNodes); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + return fmm_map_to_gpu(MemoryAddress, + MemorySizeInBytes, AlternateVAGPU); +} + HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU( diff --git a/src/topology.c b/src/topology.c index 0c0ce576d0..c6e55f09c3 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1317,6 +1317,30 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) return 0; } +HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, + uint32_t NumberOfNodes, uint32_t *NodeArray) +{ + HSAKMT_STATUS ret; + unsigned int i; + + if (NumberOfNodes == 0 || NodeArray == NULL || gpu_id_array == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + /* Translate Node IDs to gpu_ids */ + *gpu_id_array = malloc(NumberOfNodes * sizeof(uint32_t)); + if (*gpu_id_array == NULL) + return HSAKMT_STATUS_NO_MEMORY; + for (i = 0; i < NumberOfNodes; i++) { + ret = validate_nodeid(NodeArray[i], gpu_id_array[i]); + if (ret != HSAKMT_STATUS_SUCCESS) { + free(gpu_id_array); + break; + } + } + + return ret; +} + #if 0 static int get_cpu_stepping(uint16_t* stepping) { From a815a4337f89769a6bf5d43cf26caf16b2ae4f50 Mon Sep 17 00:00:00 2001 From: Yair Shachar Date: Sun, 7 Feb 2016 17:56:40 +0200 Subject: [PATCH 0132/1247] Disable scratch Host allocation - via debug registration flags. Change-Id: Ia6e5f86ec3979c4a49800f7af4509442a4e5be27 Signed-off-by: Yair Shachar --- src/debug.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/debug.c b/src/debug.c index ff7ec7fb33..327fbbcacb 100644 --- a/src/debug.c +++ b/src/debug.c @@ -73,10 +73,9 @@ hsaKmtDbgRegister( args.gpu_id = gpu_id; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER, &args); - if (err == 0) { - is_device_debugged[NodeId] = true; + if (err == 0) result = HSAKMT_STATUS_SUCCESS; - } else + else result = HSAKMT_STATUS_ERROR; return (result); @@ -105,10 +104,9 @@ hsaKmtDbgUnregister( memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER, &args); - if (err == 0) { - is_device_debugged[NodeId] = false; + if (err == 0) result = HSAKMT_STATUS_SUCCESS; - } else + else result = HSAKMT_STATUS_ERROR; return (result); From 887b32fe86d37a76680d90ec3d798b84d9b4ebea Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 6 Feb 2016 18:47:40 -0500 Subject: [PATCH 0133/1247] Make hsaKmtAllocMemory more compliant with the Thunk spec Allocations from GPU nodes will return VRAM, not system memory. Only non-paged allocation from GPU nodes is supported. System memory can only be allocated from CPU nodes (usually node 0). The HostAccess flag is no longer used to distinguish the memory type. It only indicates, whether the memory is mapped for CPU access. Maintain compatibility with broken KfdTests by returning system memory for paged-memory requested from GPU nodes. Change-Id: I514defede735f55e6de436f41944125b6f2c4ccf --- src/events.c | 5 ++-- src/fmm.c | 68 ++++++++++++++++++++++++++++++++----------------- src/fmm.h | 3 +-- src/libhsakmt.h | 2 +- src/memory.c | 35 ++++++++++++------------- src/queues.c | 8 +++--- 6 files changed, 68 insertions(+), 53 deletions(-) diff --git a/src/events.c b/src/events.c index 50c3e1d3bf..a1359d6647 100644 --- a/src/events.c +++ b/src/events.c @@ -75,9 +75,8 @@ hsaKmtCreateEvent( /* dGPU code */ if (is_dgpu && events_page == NULL) { - events_page = allocate_exec_aligned_memory_gpu(KFD_SIGNAL_EVENT_LIMIT * 8, - PAGE_SIZE, - args.node_id, true); + events_page = allocate_exec_aligned_memory_gpu( + KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0); if (!events_page) { return HSAKMT_STATUS_ERROR; } diff --git a/src/fmm.c b/src/fmm.c index 7c6837843b..6976a44723 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -137,6 +137,23 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, void *address); +static int32_t find_first_dgpu(HSAuint32 *gpu_id) { + int32_t i; + + *gpu_id = NON_VALID_GPU_ID; + + for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + continue; + if (!topology_is_dgpu(gpu_mem[i].device_id)) + continue; + *gpu_id = gpu_mem[i].gpu_id; + return i; + } + + return -1; +} + static vm_area_t *vm_create_and_init_area(void *start, void *end) { vm_area_t *area = (vm_area_t *) malloc(sizeof(vm_area_t)); @@ -720,6 +737,8 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) return __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, offset, NULL, flags); + /* TODO: honor host access mem flag and map to user mode VM if + * needed */ } static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, @@ -746,19 +765,19 @@ static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, return mem; } -static void* fmm_allocate_host_gpu(uint32_t gpu_id, - uint64_t MemorySizeInBytes, HsaMemFlags flags) +static void* fmm_allocate_host_gpu(uint64_t MemorySizeInBytes, + HsaMemFlags flags) { void *mem; manageble_aperture_t *aperture; - int32_t gpu_mem_id; uint64_t mmap_offset; uint32_t ioc_flags; uint32_t size; + int32_t i; + uint32_t gpu_id; - /* Retrieve gpu_mem id according to gpu_id */ - gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); - if (gpu_mem_id < 0) + i = find_first_dgpu(&gpu_id); + if (i < 0) return NULL; size = MemorySizeInBytes; @@ -776,30 +795,31 @@ static void* fmm_allocate_host_gpu(uint32_t gpu_id, aperture, 0, &mmap_offset, ioc_flags); - /* FIXME: host memory allocated in this way should be mapped on all GPUs */ - void *ret = mmap(mem, MemorySizeInBytes, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); - if (ret == MAP_FAILED) { - __fmm_release(mem, MemorySizeInBytes, aperture); - return NULL; + if (flags.ui32.HostAccess) { + void *ret = mmap(mem, MemorySizeInBytes, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + if (ret == MAP_FAILED) { + __fmm_release(mem, MemorySizeInBytes, aperture); + return NULL; + } + if (flags.ui32.AQLQueueMemory) { + uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; + memset(ret, 0, MemorySizeInBytes); + mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + } } - if (flags.ui32.AQLQueueMemory) { - uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; - memset(ret, 0, MemorySizeInBytes); - mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); - } - return ret; + return mem; } -void* fmm_allocate_host(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags, uint16_t dev_id) +void* fmm_allocate_host(uint64_t MemorySizeInBytes, HsaMemFlags flags) { - if (topology_is_dgpu(dev_id)) - return fmm_allocate_host_gpu(gpu_id, MemorySizeInBytes, flags); + if (is_dgpu) + return fmm_allocate_host_gpu(MemorySizeInBytes, flags); return fmm_allocate_host_cpu(MemorySizeInBytes, flags); } diff --git a/src/fmm.h b/src/fmm.h index 881413bbdf..75b5cac4a7 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -51,8 +51,7 @@ void fmm_destroy_process_apertures(void); */ void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes); -void* fmm_allocate_host(uint32_t gpu_id, uint64_t MemorySizeInBytes, - HsaMemFlags flags, uint16_t dev_id); +void* fmm_allocate_host(uint64_t MemorySizeInBytes, HsaMemFlags flags); void* fmm_open_graphic_handle(uint32_t gpu_id, int32_t graphic_device_handle, uint32_t graphic_handle, diff --git a/src/libhsakmt.h b/src/libhsakmt.h index d277da07b0..e7c3975646 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -74,7 +74,7 @@ bool topology_is_dgpu(uint16_t device_id); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId, bool peer_to_peer); + uint32_t NodeId); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes); void destroy_process_doorbells(void); diff --git a/src/memory.c b/src/memory.c index 514673d003..08b08bd734 100644 --- a/src/memory.c +++ b/src/memory.c @@ -132,30 +132,16 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_INVALID_PARAMETER; } - if (MemFlags.ui32.HostAccess && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { - if (gpu_id == 0 && PreferredNode == 0) { - /* HACK: Currently we need a GPU node for - * system memory allocations on dGPUs and - * MapMemoryToGPU will always map to the same - * GPU used for allocation. Therefore we need - * to allocate system memory from node 1 if - * we're running on a dGPU (indicated by node - * 0 being a CPU with gpu_id==0). This will be - * cleaned up when multi-GPU support is - * implemented. */ - PreferredNode = 1; - result = validate_nodeid(PreferredNode, &gpu_id); - if (result != HSAKMT_STATUS_SUCCESS) - return result; - } - *MemoryAddress = fmm_allocate_host(gpu_id, SizeInBytes, MemFlags, - get_device_id_by_node(PreferredNode)); + if (gpu_id == 0 && !MemFlags.ui32.Scratch) { + *MemoryAddress = fmm_allocate_host(SizeInBytes, MemFlags); + if (*MemoryAddress == NULL) return HSAKMT_STATUS_ERROR; + return HSAKMT_STATUS_SUCCESS; } - if (!MemFlags.ui32.HostAccess && MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { + if (gpu_id && MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); if (*MemoryAddress == NULL) @@ -172,6 +158,17 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_SUCCESS; } + /* Backwards compatibility hack: Allocate system memory if app + * asks for paged memory from a GPU node. */ + if (gpu_id && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { + *MemoryAddress = fmm_allocate_host(SizeInBytes, MemFlags); + + if (*MemoryAddress == NULL) + return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; + } + return HSAKMT_STATUS_INVALID_PARAMETER; } diff --git a/src/queues.c b/src/queues.c index e7e26b87ca..0867548e7d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -218,7 +218,7 @@ static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) } void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId, bool peer_to_peer) + uint32_t NodeId) { void *mem; HSAuint64 gpu_va; @@ -232,12 +232,12 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, size = ALIGN_UP(size, align); - ret = hsaKmtAllocMemory(NodeId, size, flags, &mem); + ret = hsaKmtAllocMemory(0, size, flags, &mem); if (ret != HSAKMT_STATUS_SUCCESS) { return NULL; } - if (!peer_to_peer) { + if (NodeId != 0) { uint32_t nodes_array[1] = {NodeId}; if (hsaKmtRegisterMemoryToNodes(mem, size, 1, nodes_array) != HSAKMT_STATUS_SUCCESS) { @@ -269,7 +269,7 @@ static void* allocate_exec_aligned_memory(uint32_t size, uint32_t NodeId) { if (IS_DGPU(type)) - return allocate_exec_aligned_memory_gpu(size, align, NodeId, false); + return allocate_exec_aligned_memory_gpu(size, align, NodeId); return allocate_exec_aligned_memory_cpu(size, align); } From cd6d75880f2ea4146c1d223d87d3ce32587b652c Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 12 Feb 2016 08:28:57 -0500 Subject: [PATCH 0134/1247] Fix build location for thunk RPM Change-Id: I4f5c7688a3e9b4dd31d8d72cae3adf9a796e38f9 --- RPM/libhsakmt.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index 3640f53ba8..b9855fcd24 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -18,7 +18,7 @@ for AMD KFD %setup -T -D -c -n %{name} %install -cp -R %thunkroot/build/build_thunk.sh/rpm/libhsakmt $RPM_BUILD_ROOT +cp -R %thunkroot/build/thunk/rpm/libhsakmt $RPM_BUILD_ROOT find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> thunk.list %clean From b37f99a01ea47062bb501f4cdd9b9b6539ec162c Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Thu, 11 Feb 2016 17:53:00 +0200 Subject: [PATCH 0135/1247] Fix double free issue and pointer alignment Change-Id: Id5bab454d53d404883a92282168b3f6cbc468cbb Signed-off-by: Ben Goz --- src/memory.c | 4 +--- src/topology.c | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/memory.c b/src/memory.c index 08b08bd734..ee60f6d775 100644 --- a/src/memory.c +++ b/src/memory.c @@ -271,10 +271,8 @@ hsaKmtMapMemoryToGPUNodes( ret = validate_nodeid_array(&gpu_id_array, NumberOfNodes, NodeArray); - if (ret != HSAKMT_STATUS_SUCCESS) { - free(gpu_id_array); + if (ret != HSAKMT_STATUS_SUCCESS) return ret; - } ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, gpu_id_array, NumberOfNodes); diff --git a/src/topology.c b/src/topology.c index c6e55f09c3..f930c7daf9 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1331,9 +1331,9 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, if (*gpu_id_array == NULL) return HSAKMT_STATUS_NO_MEMORY; for (i = 0; i < NumberOfNodes; i++) { - ret = validate_nodeid(NodeArray[i], gpu_id_array[i]); + ret = validate_nodeid(NodeArray[i], *gpu_id_array + i); if (ret != HSAKMT_STATUS_SUCCESS) { - free(gpu_id_array); + free(*gpu_id_array); break; } } From 1a0f915957e4bc45d6b987cd5c0c46af9ae04968 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 11 Feb 2016 15:10:27 -0500 Subject: [PATCH 0136/1247] Adding missing performance counters for gfx801 Few more counters are now available in GFX8 register specs. So adding them. Also for gfx700 and gfx801 report correct number of SQ perf counter slots Change-Id: I9e6b4b10238230aabeccbfaa5e491a28b5e54f2d --- src/pmc_table.c | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/pmc_table.c b/src/pmc_table.c index 0390639928..63767b9b3f 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -44,26 +44,30 @@ static uint32_t kaveri_sq_counter_ids[] = { 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 }; +/* Unused counters - 166, 292 - 297 */ static uint32_t carrizo_sq_counter_ids[] = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { - .num_of_slots = 16, + .num_of_slots = 8, .num_of_counters = sizeof(kaveri_sq_counter_ids)/sizeof(*kaveri_sq_counter_ids), .counter_ids = kaveri_sq_counter_ids, .counter_size_in_bits = 64, @@ -73,7 +77,7 @@ static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { - .num_of_slots = 16, + .num_of_slots = 8, .num_of_counters = sizeof(carrizo_sq_counter_ids)/sizeof(*carrizo_sq_counter_ids), .counter_ids = carrizo_sq_counter_ids, .counter_size_in_bits = 64, From 04b92b8e0509d6719f84a1c4f27335de6e6c5bca Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 11 Feb 2016 16:43:30 -0500 Subject: [PATCH 0137/1247] gfx803: Add performance counter information Change-Id: Id81b43e90029306f03c84752cef06dc336e3a4a9 --- src/pmc_table.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 63767b9b3f..38a2013474 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -65,6 +65,27 @@ static uint32_t carrizo_sq_counter_ids[] = { 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; +/* Unused counters - 166, 292 - 297 */ +static uint32_t fiji_sq_counter_ids[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, + 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, + 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, + 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, + 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, + 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, + 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, + 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, + 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -85,6 +106,16 @@ static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = sizeof(fiji_sq_counter_ids) / sizeof(*fiji_sq_counter_ids), + .counter_ids = fiji_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + HSAKMT_STATUS get_block_properties(uint16_t dev_id, enum perf_block_id block_id, @@ -128,6 +159,9 @@ get_block_properties(uint16_t dev_id, *block = carrizo_blocks[block_id]; break; + case 0x7300: + *block = fiji_blocks[block_id]; + break; default: rc = HSAKMT_STATUS_INVALID_PARAMETER; } From 00386734b1209b597824b563605e93435305c8d4 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Mon, 15 Feb 2016 17:15:45 +0200 Subject: [PATCH 0138/1247] Align gpu-id-array size to multiple of sizeof(uint32_t) Change-Id: I9f46b6a331a8d928ef570b420fb60b99b2edfdd1 Signed-off-by: Ben Goz --- src/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memory.c b/src/memory.c index ee60f6d775..55a67ff757 100644 --- a/src/memory.c +++ b/src/memory.c @@ -275,7 +275,7 @@ hsaKmtMapMemoryToGPUNodes( return ret; ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, - gpu_id_array, NumberOfNodes); + gpu_id_array, NumberOfNodes*sizeof(uint32_t)); if (ret != HSAKMT_STATUS_SUCCESS) return ret; From 85f9efb1a055c84e1f9b4fb8dc05be406629f3fc Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 6 Nov 2015 18:28:30 -0500 Subject: [PATCH 0139/1247] Add support for register/deregister memory for dGPU Allocate SVM address space for the registered memory and use new userptr support in KFD to create a system memory BO associated with the given user pointer. Map this BO at the SVM address for CPU access. MapMemoryToGPU can be used with the registered user pointer and will return the SVM address as alternate GPUVA. Change-Id: I4886e193c51fb6870a567878870c36bf8b5c3748 --- include/hsakmt.h | 8 +- include/linux/kfd_ioctl.h | 4 +- src/fmm.c | 233 +++++++++++++++++++++++++++++++++----- src/fmm.h | 2 +- src/memory.c | 28 +++-- 5 files changed, 233 insertions(+), 42 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 55199bb659..e43ebecd9e 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -374,8 +374,8 @@ hsaKmtFreeMemory( HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory( - void* MemoryAddress, //IN (page-aligned) - HSAuint64 MemorySizeInBytes //IN (page-aligned) + void* MemoryAddress, //IN (cache-aligned) + HSAuint64 MemorySizeInBytes //IN (cache-aligned) ); @@ -386,8 +386,8 @@ hsaKmtRegisterMemory( HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes( - void *MemoryAddress, // IN (page-aligned) - HSAuint64 MemorySizeInBytes, // IN (page-aligned) + void *MemoryAddress, // IN (cache-aligned) + HSAuint64 MemorySizeInBytes, // IN (cache-aligned) HSAuint64 NumberOfNodes, // IN HSAuint32* NodeArray // IN ); diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 8e9c5c1604..e3c96b3b78 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -312,12 +312,14 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { #define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM (1 << 5) +#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 6) + struct kfd_ioctl_alloc_memory_of_gpu_new_args { uint64_t va_addr; /* to KFD */ uint64_t size; /* to KFD */ uint64_t handle; /* from KFD */ uint32_t gpu_id; /* to KFD */ - uint64_t mmap_offset; /* from KFD */ + uint64_t mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ uint32_t flags; }; diff --git a/src/fmm.c b/src/fmm.c index 6976a44723..6b3926aa75 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -47,6 +47,7 @@ struct vm_object { void *start; + void *userptr; uint64_t size; uint64_t handle; /* opaque */ struct vm_object *next; @@ -174,6 +175,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, if (object) { object->start = start; + object->userptr = NULL; object->size = size; object->handle = handle; object->next = object->prev = NULL; @@ -283,6 +285,21 @@ static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, return cur; /* NULL if not found */ } +static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, + void *address) +{ + vm_object_t *cur = app->vm_objects; + + /* Look up the appropriate address range containing the given address */ + while (cur) { + if (cur->userptr == address) + break; + cur = cur->next; + }; + + return cur; /* NULL if not found */ +} + static vm_area_t *vm_find(manageble_aperture_t *app, void *address) { vm_area_t *cur = app->vm_ranges; @@ -462,6 +479,8 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, args.va_addr = (uint64_t)mem; if (flags == KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE) args.va_addr = VOID_PTRS_SUB(mem, aperture->base); + if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) + args.mmap_offset = *mmap_offset; if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW, &args)) return -1; @@ -1217,7 +1236,8 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, if (object->device_ids_array_size > 0) { args.device_ids_array = object->device_ids_array; args.device_ids_array_size = object->device_ids_array_size; - } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + } else if ((object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) || + (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) { /* Only enable multi-GPU mapping on host memory for now */ args.device_ids_array = all_gpu_id_array; args.device_ids_array_size = all_gpu_id_array_size; @@ -1344,6 +1364,38 @@ err_object_not_found: return -1; } +static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, + uint64_t *gpuvm_addr) +{ + manageble_aperture_t *aperture; + vm_object_t *obj; + void *svm_addr; + HSAuint64 svm_size; + HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); + int ret; + + aperture = &svm.dgpu_aperture; + + /* Find the start address in SVM space for GPU mapping */ + pthread_mutex_lock(&aperture->fmm_mutex); + obj = vm_find_object_by_userptr(aperture, addr); + if (obj == NULL) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_ERROR; + } + svm_addr = obj->start; + svm_size = obj->size; + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* Map and return the GPUVM address adjusted by the offset + * from the start of the page */ + ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size); + if (ret == 0 && gpuvm_addr) + *gpuvm_addr = (uint64_t)svm_addr + page_offset; + + return ret; +} + int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) { uint32_t i; @@ -1379,10 +1431,18 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture, address, size); + /* + * If address isn't an SVM memory address, we assume that this + * is system memory address. On dGPU we need to map it, + * assuming it was previously registered. + */ + if (is_dgpu) + /* TODO: support mixed APU and dGPU configurations */ + return _fmm_map_to_gpu_userptr(address, size, gpuvm_address); /* - * If address isn't Local memory address, we assume that this is - * system memory address accessed through IOMMU. Thus we "prefetch" it + * On an APU a system memory address is accessed through + * IOMMU. Thus we "prefetch" it. */ for (pi = 0; pi < size / PAGE_SIZE; pi++) ((char *) address)[pi * PAGE_SIZE] = 0; @@ -1406,7 +1466,8 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) if (object->device_ids_array_size > 0) { args.device_ids_array = object->device_ids_array; args.device_ids_array_size = object->device_ids_array_size; - } else if (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) { + } else if ((object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) || + (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) { /* Only enable multi-GPU mapping on host memory for now */ args.device_ids_array = all_gpu_id_array; args.device_ids_array_size = all_gpu_id_array_size; @@ -1468,6 +1529,28 @@ err: return -1; } +static int _fmm_unmap_from_gpu_userptr(void *addr) +{ + manageble_aperture_t *aperture; + vm_object_t *obj; + void *svm_addr; + + aperture = &svm.dgpu_aperture; + + /* Find the start address in SVM space for GPU unmapping */ + pthread_mutex_lock(&aperture->fmm_mutex); + obj = vm_find_object_by_userptr(aperture, addr); + if (obj == NULL) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_ERROR; + } + svm_addr = obj->start; + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* Unmap */ + return _fmm_unmap_from_gpu(aperture, svm_addr); +} + int fmm_unmap_from_gpu(void *address) { uint32_t i; @@ -1501,6 +1584,13 @@ int fmm_unmap_from_gpu(void *address) return _fmm_unmap_from_gpu(&svm.dgpu_alt_aperture, address); + /* + * If address isn't an SVM address, we assume that this is + * system memory address. + */ + if (is_dgpu) + /* TODO: support mixed APU and dGPU configurations */ + return _fmm_unmap_from_gpu_userptr(address); return 0; } @@ -1693,27 +1783,83 @@ bool fmm_get_handle(void *address, uint64_t *handle) return found; } -HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, +static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_object_t **obj_ret) +{ + int32_t i; + HSAuint32 gpu_id; + manageble_aperture_t *aperture; + void *svm_addr = NULL; + vm_object_t *obj; + HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); + HSAuint64 aligned_addr = (HSAuint64)addr - page_offset; + HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); + + /* Find first dGPU for creating the userptr BO */ + i = find_first_dgpu(&gpu_id); + if (i < 0) + return HSAKMT_STATUS_ERROR; + aperture = &svm.dgpu_aperture; + + /* Check if this address was already registered */ + pthread_mutex_lock(&aperture->fmm_mutex); + obj = vm_find_object_by_userptr(aperture, addr); + pthread_mutex_unlock(&aperture->fmm_mutex); + if (obj != NULL) + return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; + + /* Allocate BO, userptr address is passed in mmap_offset */ + svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, + &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); + if (svm_addr == NULL) + return HSAKMT_STATUS_ERROR; + + /* Find the object and set its userptr address */ + pthread_mutex_lock(&aperture->fmm_mutex); + obj = vm_find_object_by_address(aperture, svm_addr, size); + if (obj == NULL) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_ERROR; + } + obj->userptr = addr; + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (obj_ret) + *obj_ret = obj; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) { manageble_aperture_t *aperture; vm_object_t *object = NULL; + HSAKMT_STATUS ret; if (gpu_id_array_size > 0 && gpu_id_array == NULL) return HSAKMT_STATUS_INVALID_PARAMETER; - /* - * Object can be found only on SVM aperture as you can't map - * non SVM object on different device. - */ - aperture = &svm.dgpu_aperture; - pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_address(aperture, address, 0); - pthread_mutex_unlock(&aperture->fmm_mutex); + if ((address >= svm.dgpu_aperture.base) && + (address <= svm.dgpu_aperture.limit)) + aperture = &svm.dgpu_aperture; + else if ((address >= svm.dgpu_alt_aperture.base) && + (address <= svm.dgpu_alt_aperture.limit)) + aperture = &svm.dgpu_alt_aperture; + else { + /* + * If address isn't SVM address, we assume that this + * is system memory address. + */ + ret = fmm_register_user_memory(address, size_in_bytes, &object); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + if (gpu_id_array_size == 0) + return HSAKMT_STATUS_SUCCESS; + aperture = &svm.dgpu_aperture; + /* fall through */ + } if (!object) { - aperture = &svm.dgpu_alt_aperture; pthread_mutex_lock(&aperture->fmm_mutex); object = vm_find_object_by_address(aperture, address, 0); pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1732,30 +1878,65 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, return HSAKMT_STATUS_SUCCESS; } +static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) +{ + manageble_aperture_t *aperture; + vm_object_t *obj; + void *svm_addr; + HSAuint64 size; + + aperture = &svm.dgpu_aperture; + + /* Find the size and start address in SVM space */ + pthread_mutex_lock(&aperture->fmm_mutex); + obj = vm_find_object_by_userptr(aperture, addr); + if (obj == NULL) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + } + svm_addr = obj->start; + size = obj->size; + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* Destroy BO */ + __fmm_release(svm_addr, size, aperture); + + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS fmm_deregister_memory(void *address) { manageble_aperture_t *aperture; vm_object_t *object = NULL; - /* - * Object can be found only on SVM aperture as you can't map - * non SVM object on different device. - */ - aperture = &svm.dgpu_aperture; + if ((address >= svm.dgpu_aperture.base) && + (address <= svm.dgpu_aperture.limit)) + aperture = &svm.dgpu_aperture; + else if ((address >= svm.dgpu_alt_aperture.base) && + (address <= svm.dgpu_alt_aperture.limit)) + aperture = &svm.dgpu_alt_aperture; + else { + /* + * If address isn't SVM address, we assume that this + * is system memory address. If the userptr object had + * a device_ids_array, it will be freed by + * __fmm_release. Also the object will be + * removed. Therefore we can short-circuit the rest of + * the function below. + */ + return fmm_deregister_user_memory(address); + } + pthread_mutex_lock(&aperture->fmm_mutex); object = vm_find_object_by_address(aperture, address, 0); pthread_mutex_unlock(&aperture->fmm_mutex); - if (!object) { - aperture = &svm.dgpu_alt_aperture; - pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_address(aperture, address, 0); - pthread_mutex_unlock(&aperture->fmm_mutex); - } - if (!object || object->device_ids_array_size <= 0) return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + if (object->userptr) + return fmm_deregister_user_memory(object->userptr); + free(object->device_ids_array); object->device_ids_array = NULL; object->device_ids_array_size = 0; diff --git a/src/fmm.h b/src/fmm.h index 75b5cac4a7..9a106f90b4 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -69,7 +69,7 @@ HSAKMT_STATUS fmm_node_removed(HSAuint32 gpu_id); HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit); -HSAKMT_STATUS fmm_register_memory(void *address, uint32_t size_in_bytes, +HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_deregister_memory(void *address); diff --git a/src/memory.c b/src/memory.c index 55a67ff757..a616cd413f 100644 --- a/src/memory.c +++ b/src/memory.c @@ -188,21 +188,25 @@ hsaKmtFreeMemory( HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory( - void *MemoryAddress, /* IN (page-aligned) */ - HSAuint64 MemorySizeInBytes /* IN (page-aligned) */ + void *MemoryAddress, /* IN (cache-aligned) */ + HSAuint64 MemorySizeInBytes /* IN (cache-aligned) */ ) { CHECK_KFD_OPEN(); - /* No-op for APU, TODO for dGPU */ - return HSAKMT_STATUS_SUCCESS; + if (!is_dgpu) + /* TODO: support mixed APU and dGPU configurations */ + return HSAKMT_STATUS_SUCCESS; + + return fmm_register_memory(MemoryAddress, MemorySizeInBytes, + NULL, 0); } HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes( - void *MemoryAddress, /* IN (page-aligned) */ - HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ + void *MemoryAddress, /* IN (cache-aligned) */ + HSAuint64 MemorySizeInBytes, /* IN (cache-aligned) */ HSAuint64 NumberOfNodes, /* IN */ HSAuint32* NodeArray /* IN */ ) @@ -211,16 +215,20 @@ hsaKmtRegisterMemoryToNodes( uint32_t *gpu_id_array; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + if (!is_dgpu) + /* TODO: support mixed APU and dGPU configurations */ + return HSAKMT_STATUS_NOT_SUPPORTED; + ret = validate_nodeid_array(&gpu_id_array, NumberOfNodes, NodeArray); - if (ret == HSAKMT_STATUS_SUCCESS) + if (ret == HSAKMT_STATUS_SUCCESS) { ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, gpu_id_array, NumberOfNodes*sizeof(uint32_t)); - - if (ret != HSAKMT_STATUS_SUCCESS) - free(gpu_id_array); + if (ret != HSAKMT_STATUS_SUCCESS) + free(gpu_id_array); + } return ret; } From 7a383f9d8861b2c4fdb986354a10a730a0227a73 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 15 Jan 2016 22:05:28 -0500 Subject: [PATCH 0140/1247] Fix memory leaks due to stale CPU mappings Use the aligned size of the buffer objects for CPU unmapping in __fmm_release instead of relying on the unaligned size passed in by the caller. Change-Id: If986ec24e9a05d32981549fddbf143221fc40bac --- src/fmm.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 6b3926aa75..9e9b0a05e7 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -922,6 +922,13 @@ static void __fmm_release(void *address, if (object->device_ids_array_size > 0) free(object->device_ids_array); + if (address >= dgpu_shared_aperture_base && + address <= dgpu_shared_aperture_limit) { + /* Remove any CPU mapping, but keep the address range reserved */ + mmap(address, object->size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); + } + args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); @@ -971,14 +978,6 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) } } - if (found && - address >= dgpu_shared_aperture_base && - address <= dgpu_shared_aperture_limit) { - /* Remove any CPU mapping, but keep the address range reserved */ - mmap(address, MemorySizeInBytes, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); - } - /* * If memory address isn't inside of any defined aperture - it refers * to the system memory From 3f02a3cf0bbc4e5530a3f36648be58bc7ad53d04 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 17 Feb 2016 17:52:25 +0200 Subject: [PATCH 0141/1247] Mapping public VRAM BO to cpu Change-Id: I2ff62ff0784f8ce556ad80739a177b90d866f1b4 Signed-off-by: Ben Goz --- src/fmm.c | 36 +++++++++++++++++++++++++++--------- src/fmm.h | 2 +- src/memory.c | 2 +- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 9e9b0a05e7..a9be3ecd17 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -728,36 +728,54 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, * (after base subtraction) won't be used */ #define GPUVM_APP_OFFSET 0x10000 -void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes) +void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { manageble_aperture_t *aperture; int32_t gpu_mem_id; - uint32_t flags, offset; + uint32_t ioc_flags, offset; + uint64_t size, mmap_offset; + void *mem; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) return NULL; + size = MemorySizeInBytes; + if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { - flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; /* * TODO: Once VA limit is raised from 0x200000000 (8GB) use gpuvm_aperture. * In that way the host access range won't be used for local memory */ aperture = &svm.dgpu_aperture; offset = 0; + if (flags.ui32.AQLQueueMemory) { + size = MemorySizeInBytes * 2; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; + } } else { - flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; offset = GPUVM_APP_OFFSET; } - return __fmm_allocate_device(gpu_id, MemorySizeInBytes, - aperture, offset, NULL, - flags); - /* TODO: honor host access mem flag and map to user mode VM if - * needed */ + mem = __fmm_allocate_device(gpu_id, size, + aperture, offset, &mmap_offset, + ioc_flags); + + if (mem && flags.ui32.HostAccess) { + void *ret = mmap(mem, MemorySizeInBytes, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + if (ret == MAP_FAILED) { + __fmm_release(mem, MemorySizeInBytes, aperture); + return NULL; + } + } + + return mem; } static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, diff --git a/src/fmm.h b/src/fmm.h index 9a106f90b4..792be36b56 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -50,7 +50,7 @@ void fmm_destroy_process_apertures(void); * Memory interface */ void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); -void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes); +void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); void* fmm_allocate_host(uint64_t MemorySizeInBytes, HsaMemFlags flags); void* fmm_open_graphic_handle(uint32_t gpu_id, int32_t graphic_device_handle, diff --git a/src/memory.c b/src/memory.c index a616cd413f..e45dd67ffe 100644 --- a/src/memory.c +++ b/src/memory.c @@ -142,7 +142,7 @@ hsaKmtAllocMemory( } if (gpu_id && MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { - *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes); + *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes, MemFlags); if (*MemoryAddress == NULL) return HSAKMT_STATUS_NO_MEMORY; From 8a0161d6bb23b220397e19f248ee5c4382a0e9e7 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 19 Feb 2016 17:05:13 -0500 Subject: [PATCH 0142/1247] Use aligned size for looking up userptr object after allocation The alignment performed in vm_find_object_by_address isn't sufficient because it doesn't take into account the offset from the start of the page. This fixes a bug where certain unaligned userpointers and sizes fail to register correctly. Change-Id: I17872e264467a619f5e1bedb7e1ed3d994a856bf --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index a9be3ecd17..bd6c96978f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1832,7 +1832,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Find the object and set its userptr address */ pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_address(aperture, svm_addr, size); + obj = vm_find_object_by_address(aperture, svm_addr, aligned_size); if (obj == NULL) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; From 006f3ee41bea18c4ae4ed0706bd31e0110b85367 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 23 Feb 2016 19:01:51 -0500 Subject: [PATCH 0143/1247] Fix address space leak in __fmm_release Use the object size when freeing address space, instead of the parameter passed in by the caller. The parameter may be incorrect due to app or runtime bugs, or when the buffers is an AQL ring buffer with double mapping workaround. Change-Id: I00bb31d4520ef969a49d6d5ea723e8a33418acc3 --- src/fmm.c | 38 ++++++++++++++------------------------ src/fmm.h | 2 +- src/memory.c | 2 +- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index bd6c96978f..89592b6d93 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -132,8 +132,7 @@ static uint32_t *all_gpu_id_array = NULL; extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit); -static void __fmm_release(void *address, - uint64_t MemorySizeInBytes, manageble_aperture_t *aperture); +static void __fmm_release(void *address, manageble_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, void *address); @@ -770,7 +769,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); if (ret == MAP_FAILED) { - __fmm_release(mem, MemorySizeInBytes, aperture); + __fmm_release(mem, aperture); return NULL; } } @@ -837,7 +836,7 @@ static void* fmm_allocate_host_gpu(uint64_t MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); if (ret == MAP_FAILED) { - __fmm_release(mem, MemorySizeInBytes, aperture); + __fmm_release(mem, aperture); return NULL; } if (flags.ui32.AQLQueueMemory) { @@ -919,8 +918,7 @@ out: return NULL; } -static void __fmm_release(void *address, - uint64_t MemorySizeInBytes, manageble_aperture_t *aperture) +static void __fmm_release(void *address, manageble_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args; vm_object_t *object; @@ -950,13 +948,13 @@ static void __fmm_release(void *address, args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); + aperture_release_area(aperture, address, object->size); vm_remove_object(aperture, object); - aperture_release_area(aperture, address, MemorySizeInBytes); pthread_mutex_unlock(&aperture->fmm_mutex); } -void fmm_release(void *address, uint64_t MemorySizeInBytes) +void fmm_release(void *address) { uint32_t i; bool found = false; @@ -973,8 +971,7 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { found = true; - __fmm_release(address, - MemorySizeInBytes, &gpu_mem[i].gpuvm_aperture); + __fmm_release(address, &gpu_mem[i].gpuvm_aperture); fmm_print(gpu_mem[i].gpu_id); } } @@ -983,15 +980,13 @@ void fmm_release(void *address, uint64_t MemorySizeInBytes) if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { found = true; - __fmm_release(address, - MemorySizeInBytes, &svm.dgpu_aperture); + __fmm_release(address, &svm.dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { found = true; - __fmm_release(address, - MemorySizeInBytes, &svm.dgpu_alt_aperture); + __fmm_release(address, &svm.dgpu_alt_aperture); fmm_print(gpu_mem[i].gpu_id); } } @@ -1312,7 +1307,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu "Got unexpected address for scratch mapping.\n" " expected: %p\n" " got: %p\n", address, mem); - __fmm_release(mem, size, aperture); + __fmm_release(mem, aperture); return -1; } } else { @@ -1327,7 +1322,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu MAP_SHARED | MAP_FIXED, kfd_fd, mmap_offset); if (mmap_ret == MAP_FAILED) { - __fmm_release(mem, size, aperture); + __fmm_release(mem, aperture); return -1; } } @@ -1335,7 +1330,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu /* map to GPU */ ret = _fmm_map_to_gpu_gtt(aperture, address, size); if (ret != 0) - __fmm_release(mem, size, aperture); + __fmm_release(mem, aperture); return ret; } @@ -1508,7 +1503,6 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, { int32_t gpu_mem_id; vm_object_t *object; - uint64_t size; struct kfd_ioctl_unmap_memory_from_gpu_new_args args; /* Retrieve gpu_mem id according to gpu_id */ @@ -1526,8 +1520,6 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, if (!object) goto err; - size = object->size; - /* unmap from GPU */ args.handle = object->handle; args.device_ids_array = object->device_ids_array; @@ -1537,7 +1529,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ - __fmm_release(address, size, aperture); + __fmm_release(address, aperture); return 0; @@ -1900,7 +1892,6 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) manageble_aperture_t *aperture; vm_object_t *obj; void *svm_addr; - HSAuint64 size; aperture = &svm.dgpu_aperture; @@ -1912,11 +1903,10 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } svm_addr = obj->start; - size = obj->size; pthread_mutex_unlock(&aperture->fmm_mutex); /* Destroy BO */ - __fmm_release(svm_addr, size, aperture); + __fmm_release(svm_addr, aperture); return HSAKMT_STATUS_SUCCESS; } diff --git a/src/fmm.h b/src/fmm.h index 792be36b56..b0a18e1743 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -58,7 +58,7 @@ void* fmm_open_graphic_handle(uint32_t gpu_id, uint64_t MemorySizeInBytes); void fmm_print(uint32_t node); bool fmm_is_inside_some_aperture(void* address); -void fmm_release(void* address, HSAuint64 MemorySizeInBytes); +void fmm_release(void* address); int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); int fmm_unmap_from_gpu(void *address); bool fmm_get_handle(void *address, uint64_t *handle); diff --git a/src/memory.c b/src/memory.c index e45dd67ffe..d4c1cfa05f 100644 --- a/src/memory.c +++ b/src/memory.c @@ -181,7 +181,7 @@ hsaKmtFreeMemory( { CHECK_KFD_OPEN(); - fmm_release(MemoryAddress, SizeInBytes); + fmm_release(MemoryAddress); return HSAKMT_STATUS_SUCCESS; } From 3a662ac712c64e2780957b53dbfa66d7a62e9487 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Mon, 15 Feb 2016 12:02:30 -0600 Subject: [PATCH 0144/1247] Fix race in dGPU event page setup events_page is unprotected from multiple allocation. The first event creation ioctl is unprotected from a race with args.event_page_offset being set (for page setup) and null (all subsequent invocations). Change-Id: I40ba712a17e9eff257785f90c553a74ad09c661d Signed-off-by: Yair Shachar --- src/events.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/events.c b/src/events.c index a1359d6647..b31353f49c 100644 --- a/src/events.c +++ b/src/events.c @@ -74,10 +74,13 @@ hsaKmtCreateEvent( args.auto_reset = !ManualReset; /* dGPU code */ + pthread_mutex_lock(&hsakmt_mutex); + if (is_dgpu && events_page == NULL) { events_page = allocate_exec_aligned_memory_gpu( KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0); if (!events_page) { + pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; } fmm_get_handle(events_page, &args.event_page_offset); @@ -86,6 +89,7 @@ hsaKmtCreateEvent( if (kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) != 0) { free(e); *Event = NULL; + pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; } @@ -93,12 +97,15 @@ hsaKmtCreateEvent( events_page = mmap(NULL, KFD_SIGNAL_EVENT_LIMIT * 8, PROT_WRITE | PROT_READ, MAP_SHARED, kfd_fd, args.event_page_offset); if (events_page == MAP_FAILED) { - hsaKmtDestroyEvent(e); events_page = NULL; + pthread_mutex_unlock(&hsakmt_mutex); + hsaKmtDestroyEvent(e); return HSAKMT_STATUS_ERROR; } } + pthread_mutex_unlock(&hsakmt_mutex); + if (args.event_page_offset > 0 && args.event_slot_index < KFD_SIGNAL_EVENT_LIMIT) e->EventData.HWData2 = (HSAuint64)&events_page[args.event_slot_index]; From bf030581127724786678c1f5f5a91ce590a5ff70 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 26 Feb 2016 17:07:58 -0500 Subject: [PATCH 0145/1247] Fix io_links sysfs directory name typo Change-Id: I4f6fb43c4a038b94c0f94f66ee383e83ad0ffa62 --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index f930c7daf9..a3fda87686 100644 --- a/src/topology.c +++ b/src/topology.c @@ -776,7 +776,7 @@ topology_sysfs_get_iolink_props(uint32_t node_id, uint32_t iolink_id, HsaIoLinkP HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); - snprintf(path, 256, "%s/%d/io_link/%d/properties", KFD_SYSFS_PATH_NODES, node_id, iolink_id); + snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, node_id, iolink_id); fd = fopen(path, "r"); if (!fd) { return HSAKMT_STATUS_ERROR; From fea5ab9114a27b0fcaac0d2dd008404c53bc7723 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Mon, 29 Feb 2016 10:49:36 -0500 Subject: [PATCH 0146/1247] Export libKmtSetTrapHandler symbol as global Change-Id: I065dbecd05e992bc528128d893edaf636c1beff7 --- src/libhsakmt.ver | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 95790923dd..d520144cbc 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -45,6 +45,7 @@ hsaKmtPmcQueryTrace; hsaKmtPmcStopTrace; hsaKmtMapGraphicHandle; hsaKmtUnmapGraphicHandle; +hsaKmtSetTrapHandler; local: *; }; From e2fb4bc31256a7ead86b1cd243a12e1e13610ab6 Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 24 Feb 2016 17:30:48 +0200 Subject: [PATCH 0147/1247] Align hsaKmtMapMemoryToGPUNodes according thunk spec Change-Id: I507ba5c6029ca5e7088c25930d46f5221679ace4 Signed-off-by: Ben Goz --- src/fmm.c | 396 ++++++++++++++++++++++++++++++++++++++++++--------- src/fmm.h | 2 + src/memory.c | 9 +- 3 files changed, 331 insertions(+), 76 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 89592b6d93..c4e29e5ed5 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -54,10 +54,15 @@ struct vm_object { struct vm_object *prev; uint32_t flags; /* memory allocation flags */ /* - * Nodes to map on SVM mGPU + * Registered nodes to map on SVM mGPU */ - uint32_t *device_ids_array; - uint32_t device_ids_array_size; + uint32_t *registered_device_id_array; + uint32_t registered_device_id_array_size; + /* + * Nodes that mapped already + */ + uint32_t *mapped_device_id_array; + uint32_t mapped_device_id_array_size; }; typedef struct vm_object vm_object_t; @@ -136,6 +141,7 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, void *address); +static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_array_size); static int32_t find_first_dgpu(HSAuint32 *gpu_id) { int32_t i; @@ -178,7 +184,8 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->size = size; object->handle = handle; object->next = object->prev = NULL; - object->device_ids_array_size = 0; + object->registered_device_id_array_size = 0; + object->mapped_device_id_array_size = 0; object->flags = flags; } @@ -935,8 +942,23 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) return; } - if (object->device_ids_array_size > 0) - free(object->device_ids_array); + if (object->registered_device_id_array_size > 0) { + if (object->mapped_device_id_array == + object->registered_device_id_array) { + object->mapped_device_id_array_size = 0; + object->mapped_device_id_array = NULL; + } + free(object->registered_device_id_array); + object->registered_device_id_array_size = 0; + } + if (object->mapped_device_id_array != NULL && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + { + free(object->mapped_device_id_array); + object->mapped_device_id_array_size = 0; + } if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { @@ -1231,42 +1253,61 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA } static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, - void *address, uint64_t size) + void *address, uint64_t size, vm_object_t *obj) { struct kfd_ioctl_map_memory_to_gpu_new_args args; vm_object_t *object; + void *temp_mapped_id_array = NULL; - pthread_mutex_lock(&aperture->fmm_mutex); + if (!obj) + pthread_mutex_lock(&aperture->fmm_mutex); - /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); + object = obj; if (!object) { - goto err_object_not_found; + /* Find the object to retrieve the handle */ + object = vm_find_object_by_address(aperture, address, 0); + if (!object) + goto err_object_not_found; } args.handle = object->handle; - if (object->device_ids_array_size > 0) { - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; - } else if ((object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) || - (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) { - /* Only enable multi-GPU mapping on host memory for now */ + if (object->registered_device_id_array_size > 0) { + args.device_ids_array = object->registered_device_id_array; + args.device_ids_array_size = object->registered_device_id_array_size; + } else { args.device_ids_array = all_gpu_id_array; args.device_ids_array_size = all_gpu_id_array_size; - } else { - args.device_ids_array = NULL; - args.device_ids_array_size = 0; } + + temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); + if (!temp_mapped_id_array) + goto err_object_not_found; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) goto err_map_ioctl_failed; - pthread_mutex_unlock(&aperture->fmm_mutex); + print_device_id_array(args.device_ids_array, args.device_ids_array_size); + + if (object->mapped_device_id_array != NULL && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + + memcpy(temp_mapped_id_array, args.device_ids_array, args.device_ids_array_size); + object->mapped_device_id_array = temp_mapped_id_array; + object->mapped_device_id_array_size = args.device_ids_array_size; + + if (!obj) + pthread_mutex_unlock(&aperture->fmm_mutex); return 0; err_map_ioctl_failed: + free(temp_mapped_id_array); err_object_not_found: - pthread_mutex_unlock(&aperture->fmm_mutex); + if (!obj) + pthread_mutex_unlock(&aperture->fmm_mutex); return -1; } @@ -1327,8 +1368,9 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu } } + /* map to GPU */ - ret = _fmm_map_to_gpu_gtt(aperture, address, size); + ret = _fmm_map_to_gpu_gtt(aperture, address, size, NULL); if (ret != 0) __fmm_release(mem, aperture); @@ -1341,6 +1383,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_new_args args; vm_object_t *object; + void *temp_mapped_id_array = NULL; /* Check that address space was previously reserved */ if (vm_find(aperture, address) == NULL) @@ -1354,11 +1397,32 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, goto err_object_not_found; args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + if (object->registered_device_id_array_size > 0 && + object->registered_device_id_array) { + args.device_ids_array = object->registered_device_id_array; + args.device_ids_array_size = object->registered_device_id_array_size; + } else { + args.device_ids_array = all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } + + temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); + if (!temp_mapped_id_array) + goto err_object_not_found; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) goto err_map_ioctl_failed; + if (object->mapped_device_id_array != NULL && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + + memcpy(temp_mapped_id_array, args.device_ids_array, args.device_ids_array_size); + object->mapped_device_id_array = temp_mapped_id_array; + object->mapped_device_id_array_size = args.device_ids_array_size; + pthread_mutex_unlock(&aperture->fmm_mutex); if (gpuvm_address) { @@ -1377,7 +1441,7 @@ err_object_not_found: } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, - uint64_t *gpuvm_addr) + uint64_t *gpuvm_addr, vm_object_t *object) { manageble_aperture_t *aperture; vm_object_t *obj; @@ -1389,22 +1453,29 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, aperture = &svm.dgpu_aperture; /* Find the start address in SVM space for GPU mapping */ - pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_userptr(aperture, addr); - if (obj == NULL) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_ERROR; + if (!object) + pthread_mutex_lock(&aperture->fmm_mutex); + + obj = object; + if (!obj) { + obj = vm_find_object_by_userptr(aperture, addr); + if (obj == NULL) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_ERROR; + } } svm_addr = obj->start; svm_size = obj->size; - pthread_mutex_unlock(&aperture->fmm_mutex); /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size); + ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size, obj); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; + if (!object) + pthread_mutex_unlock(&aperture->fmm_mutex); + return ret; } @@ -1436,12 +1507,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) (address <= svm.dgpu_aperture.limit)) /* map it */ return _fmm_map_to_gpu_gtt(&svm.dgpu_aperture, - address, size); + address, size, NULL); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* map it */ return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture, - address, size); + address, size, NULL); /* * If address isn't an SVM memory address, we assume that this @@ -1450,7 +1521,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) */ if (is_dgpu) /* TODO: support mixed APU and dGPU configurations */ - return _fmm_map_to_gpu_userptr(address, size, gpuvm_address); + return _fmm_map_to_gpu_userptr(address, size, gpuvm_address, NULL); /* * On an APU a system memory address is accessed through @@ -1462,39 +1533,80 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) return 0; } -static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address) +static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_array_size) +{ +#ifdef DEBUG_PRINT_APERTURE + device_id_array_size /= sizeof(uint32_t); + + printf("device id array size %d\n", device_id_array_size); + + for (uint32_t i = 0 ; i < device_id_array_size; i++) + printf("%d . 0x%x\n", (i+1), device_id_array[i]); +#endif +} + +static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, + uint32_t *device_ids_array, uint32_t device_ids_array_size, + vm_object_t *obj) { vm_object_t *object; + int ret; struct kfd_ioctl_unmap_memory_from_gpu_new_args args; - pthread_mutex_lock(&aperture->fmm_mutex); + if (!obj) + pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); - if (!object) - goto err; + object = obj; + if (!object) { + object = vm_find_object_by_address(aperture, address, 0); + if (!object) { + ret = -1; + goto err; + } + } args.handle = object->handle; - if (object->device_ids_array_size > 0) { - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; - } else if ((object->flags & KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST) || - (object->flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR)) { - /* Only enable multi-GPU mapping on host memory for now */ - args.device_ids_array = all_gpu_id_array; - args.device_ids_array_size = all_gpu_id_array_size; + if (device_ids_array && device_ids_array_size > 0) { + args.device_ids_array = device_ids_array; + args.device_ids_array_size = device_ids_array_size; + } else if (object->mapped_device_id_array_size > 0) { + args.device_ids_array = object->mapped_device_id_array; + args.device_ids_array_size = object->mapped_device_id_array_size; } else { - args.device_ids_array = NULL; - args.device_ids_array_size = 0; + /* + * When unmap exits here it should return failing error code as the user tried to + * unmap already unmapped buffer. Currently we returns success as KFDTEST and RT + * need to deploy the change on there side before thunk fails on this case. + */ + ret = 0; + goto err; } - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); - pthread_mutex_unlock(&aperture->fmm_mutex); + print_device_id_array(args.device_ids_array, args.device_ids_array_size); + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); + if (ret != 0) + goto err; + + /* Clearing all mapped nodes list */ + if (object->mapped_device_id_array != NULL && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + + object->mapped_device_id_array = NULL; + object->mapped_device_id_array_size = 0; + + if (!obj) + pthread_mutex_unlock(&aperture->fmm_mutex); return 0; err: - pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; + if (!obj) + pthread_mutex_unlock(&aperture->fmm_mutex); + return ret; } static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, @@ -1520,12 +1632,29 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, if (!object) goto err; + if (object->mapped_device_id_array == NULL || + object->mapped_device_id_array_size == 0) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return 0; + } + + /* unmap from GPU */ args.handle = object->handle; - args.device_ids_array = object->device_ids_array; - args.device_ids_array_size = object->device_ids_array_size; + args.device_ids_array = object->mapped_device_id_array; + args.device_ids_array_size = object->mapped_device_id_array_size; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); + /* Clearing all mapped nodes list */ + if (object->mapped_device_id_array != NULL && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + + object->mapped_device_id_array = NULL; + object->mapped_device_id_array_size = 0; + pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ @@ -1557,7 +1686,7 @@ static int _fmm_unmap_from_gpu_userptr(void *addr) pthread_mutex_unlock(&aperture->fmm_mutex); /* Unmap */ - return _fmm_unmap_from_gpu(aperture, svm_addr); + return _fmm_unmap_from_gpu(aperture, svm_addr, NULL, 0, NULL); } int fmm_unmap_from_gpu(void *address) @@ -1579,19 +1708,19 @@ int fmm_unmap_from_gpu(void *address) (address <= gpu_mem[i].gpuvm_aperture.limit)) /* unmap it */ return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, - address); + address, NULL, 0, NULL); } if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) /* unmap it */ return _fmm_unmap_from_gpu(&svm.dgpu_aperture, - address); + address, NULL, 0, NULL); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* unmap it */ return _fmm_unmap_from_gpu(&svm.dgpu_alt_aperture, - address); + address, NULL, 0, NULL); /* * If address isn't an SVM address, we assume that this is @@ -1876,12 +2005,12 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (!object) return HSAKMT_STATUS_NOT_SUPPORTED; - if (object->device_ids_array_size > 0) + if (object->registered_device_id_array_size > 0) return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; if (gpu_id_array_size > 0) { - object->device_ids_array = gpu_id_array; - object->device_ids_array_size = gpu_id_array_size; + object->registered_device_id_array = gpu_id_array; + object->registered_device_id_array_size = gpu_id_array_size; } return HSAKMT_STATUS_SUCCESS; @@ -1926,7 +2055,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) /* * If address isn't SVM address, we assume that this * is system memory address. If the userptr object had - * a device_ids_array, it will be freed by + * a registered_device_id_array, it will be freed by * __fmm_release. Also the object will be * removed. Therefore we can short-circuit the rest of * the function below. @@ -1938,15 +2067,144 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) object = vm_find_object_by_address(aperture, address, 0); pthread_mutex_unlock(&aperture->fmm_mutex); - if (!object || object->device_ids_array_size <= 0) + if (!object || object->registered_device_id_array_size <= 0) return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; if (object->userptr) return fmm_deregister_user_memory(object->userptr); - free(object->device_ids_array); - object->device_ids_array = NULL; - object->device_ids_array_size = 0; + free(object->registered_device_id_array); + object->registered_device_id_array = NULL; + object->registered_device_id_array_size = 0; return HSAKMT_STATUS_SUCCESS; } + +/* + * This function unmaps all nodes on current mapped nodes list that are not included on nodes_to_map + * and maps nodes_to_map + */ + +HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, + uint32_t *nodes_to_map, uint32_t nodes_to_map_size, + uint64_t *gpuvm_address) +{ + manageble_aperture_t *aperture; + vm_object_t *object = NULL; + uint32_t i, j, temp_node; + bool found, userptr = false; + uint32_t *temp_node_id_array, temp_node_id_array_size; + uint32_t *registered_node_id_array, registered_node_id_array_size; + HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; + int retcode = 0; + + if ((nodes_to_map_size > 0 && nodes_to_map == NULL) || address == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + + /* Find object by address */ + if ((address >= svm.dgpu_aperture.base) && + (address <= svm.dgpu_aperture.limit)) + aperture = &svm.dgpu_aperture; + else if ((address >= svm.dgpu_alt_aperture.base) && + (address <= svm.dgpu_alt_aperture.limit)) + aperture = &svm.dgpu_alt_aperture; + else { + aperture = &svm.dgpu_aperture; + userptr = true; + + } + + pthread_mutex_lock(&aperture->fmm_mutex); + if (userptr && is_dgpu) + object = vm_find_object_by_userptr(aperture, address); + else + object = vm_find_object_by_address(aperture, address, 0); + + if (!object) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_ERROR; + } + + /* Verify that all nodes to map are registered already */ + registered_node_id_array = all_gpu_id_array; + registered_node_id_array_size = all_gpu_id_array_size; + if (object->registered_device_id_array_size > 0 && + object->registered_device_id_array != NULL) { + registered_node_id_array = object->registered_device_id_array; + registered_node_id_array_size = object->registered_device_id_array_size; + } + for (i = 0 ; i < nodes_to_map_size / sizeof(uint32_t); i++) { + temp_node = nodes_to_map[i]; + found = false; + for (j = 0 ; j < registered_node_id_array_size / sizeof(uint32_t); j++) { + if (temp_node == registered_node_id_array[j]) { + found = true; + break; + } + } + if (!found) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_ERROR; + } + } + + /* Unmap buffer from all nodes that have this buffer mapped that are not included on nodes_to_map array */ + if (object->mapped_device_id_array_size > 0) { + temp_node_id_array = (uint32_t *)malloc(object->registered_device_id_array_size); + if (!temp_node_id_array) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_NO_MEMORY; + } + temp_node_id_array_size = 0; + for (i = 0 ; i < object->mapped_device_id_array_size / sizeof(uint32_t); i++) { + temp_node = object->mapped_device_id_array[i]; + found = false; + for (j = 0 ; j < nodes_to_map_size / sizeof(uint32_t); j++) { + if (temp_node == nodes_to_map[j]) { + found = true; + break; + } + } + if (!found) + temp_node_id_array[temp_node_id_array_size++] = temp_node; + } + temp_node_id_array_size *= sizeof(uint32_t); + + ret = _fmm_unmap_from_gpu(aperture, address, + temp_node_id_array, temp_node_id_array_size, + object); + free(temp_node_id_array); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + } + + /* Keep registered device id array and size */ + temp_node_id_array = object->registered_device_id_array; + temp_node_id_array_size = object->registered_device_id_array_size; + /* Change registered device id array and size to nodes array/size that we want to map */ + object->registered_device_id_array = nodes_to_map; + object->registered_device_id_array_size = nodes_to_map_size; + + if (nodes_to_map_size > 0) { + if (userptr && is_dgpu) + retcode = _fmm_map_to_gpu_userptr(address, size, + gpuvm_address, + object); + else + retcode = _fmm_map_to_gpu_gtt(aperture, address, + size, + object); + } + + /* Restore old registered device id array */ + object->registered_device_id_array = temp_node_id_array; + object->registered_device_id_array_size = temp_node_id_array_size; + + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (retcode != 0) + return HSAKMT_STATUS_ERROR; + + return 0; +} diff --git a/src/fmm.h b/src/fmm.h index b0a18e1743..2f27d0cf60 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -73,4 +73,6 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_deregister_memory(void *address); +HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, + uint32_t *nodes_to_map, uint32_t nodes_to_map_size, uint64_t *gpuvm_address); #endif /* FMM_H_ */ diff --git a/src/memory.c b/src/memory.c index d4c1cfa05f..bbdf2ca32e 100644 --- a/src/memory.c +++ b/src/memory.c @@ -282,13 +282,8 @@ hsaKmtMapMemoryToGPUNodes( if (ret != HSAKMT_STATUS_SUCCESS) return ret; - ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, - gpu_id_array, NumberOfNodes*sizeof(uint32_t)); - if (ret != HSAKMT_STATUS_SUCCESS) - return ret; - - return fmm_map_to_gpu(MemoryAddress, - MemorySizeInBytes, AlternateVAGPU); + return fmm_map_to_gpu_nodes(MemoryAddress, MemorySizeInBytes, + gpu_id_array, NumberOfNodes * sizeof(uint32_t), AlternateVAGPU); } HSAKMT_STATUS From 35e8fc6b1559df9e82ad0c9fbe49b8ad5c0a9d23 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Tue, 26 Jan 2016 12:55:15 -0500 Subject: [PATCH 0148/1247] readme: add an initial README.md file This is a simple README.md since most of the details should be in the ROCK project. Change-Id: I3175e2a5ade0f9ecb913076a4842b528f14947f0 --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000000..127b6a638d --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +### Boltzmann Thunk Library + +This repository includes the user-mode API interfaces used to interact with the Boltzmann KFD driver. Currently supported agents include only the AMD/ATI Fiji family of discrete GPUs. + +#### Boltzmann Kernel Driver + +The thunk is not a standalone product and requires the you have the correct KFD installed. We recommend reading the full comptaibility and installation details which are available in the ROCK github: +https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver + +#### Binaries for Ubuntu and Fedora + +For deb and rpm binaries, please check the packages/ folder in the ROCK repository: +https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver + +#### Building the Thunk + +A simple make based system is available for building thunk. The following are the supported targets: + +```bash + make + make deb + make rpm + make clean +``` + +#### Disclaimer + +The information contained herein is for informational purposes only, and is subject to change without notice. While every precaution has been taken in the preparation of this document, it may contain technical inaccuracies, omissions and typographical errors, and AMD is under no obligation to update or otherwise correct this information. Advanced Micro Devices, Inc. makes no representations or warranties with respect to the accuracy or completeness of the contents of this document, and assumes no liability of any kind, including the implied warranties of noninfringement, merchantability or fitness for particular purposes, with respect to the operation or use of AMD hardware, software or other products described herein. No license, including implied or arising by estoppel, to any intellectual property rights is granted by this document. Terms and limitations applicable to the purchase or use of AMD's products are as set forth in a signed agreement between the parties or in AMD's Standard Terms and Conditions of Sale. + +AMD, the AMD Arrow logo, and combinations thereof are trademarks of Advanced Micro Devices, Inc. Other product names used in this publication are for identification purposes only and may be trademarks of their respective companies. + +Copyright (c) 2014-2015 Advanced Micro Devices, Inc. All rights reserved. From 7c376247b53cb0ea50754c41c647b7c47d716ffb Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Wed, 2 Mar 2016 12:15:18 -0500 Subject: [PATCH 0149/1247] README: spelling and date fixes Change-Id: I51fa196971b91ea71fd8b0abe169fe23502ebb96 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 127b6a638d..304a7f929a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repository includes the user-mode API interfaces used to interact with the #### Boltzmann Kernel Driver -The thunk is not a standalone product and requires the you have the correct KFD installed. We recommend reading the full comptaibility and installation details which are available in the ROCK github: +The thunk is not a standalone product and requires that you have the correct KFD installed. We recommend reading the full compatibility and installation details which are available in the ROCK github: https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver #### Binaries for Ubuntu and Fedora @@ -14,7 +14,7 @@ https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver #### Building the Thunk -A simple make based system is available for building thunk. The following are the supported targets: +A simple make-based system is available for building thunk. The following are the supported targets: ```bash make @@ -29,4 +29,4 @@ The information contained herein is for informational purposes only, and is subj AMD, the AMD Arrow logo, and combinations thereof are trademarks of Advanced Micro Devices, Inc. Other product names used in this publication are for identification purposes only and may be trademarks of their respective companies. -Copyright (c) 2014-2015 Advanced Micro Devices, Inc. All rights reserved. +Copyright (c) 2014-2016 Advanced Micro Devices, Inc. All rights reserved. From a80d2f23033f94faf79831bb06f0aef01d1e31bb Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 29 Feb 2016 15:59:28 -0500 Subject: [PATCH 0150/1247] Add free_nodes() helper function Change-Id: I18ae0ac91b05275d7ad9d93175bae06870080844 --- src/topology.c | 72 +++++++++++++++++--------------------------------- 1 file changed, 24 insertions(+), 48 deletions(-) diff --git a/src/topology.c b/src/topology.c index a3fda87686..f22586e198 100644 --- a/src/topology.c +++ b/src/topology.c @@ -128,6 +128,16 @@ free_node(node_t *n) free((n)->link); } +static void free_nodes(node_t *temp_nodes, int size) +{ + int i; + if (temp_nodes) { + for (i = 0; i < size; i++) + free_node(&temp_nodes[i]); + free(temp_nodes); + } +} + /* num_subdirs - find the number of sub-directories in the specified path * @dirpath - directory path to find sub-directories underneath * @prefix - only count sub-directory names starting with prefix. @@ -838,7 +848,7 @@ err1: HSAKMT_STATUS topology_take_snapshot(void) { - uint32_t gen_start, gen_end, i, j, mem_id, cache_id, link_id; + uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; HsaSystemProperties sys_props; node_t *temp_nodes = 0; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -859,26 +869,20 @@ retry: &temp_nodes[i].node, &temp_nodes[i].gpu_id); if (ret != HSAKMT_STATUS_SUCCESS) { - for (j=0; j < i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i); goto err; } if (temp_nodes[i].node.NumMemoryBanks) { temp_nodes[i].mem = calloc(temp_nodes[i].node.NumMemoryBanks * sizeof(HsaMemoryProperties), 1); if (!temp_nodes[i].mem) { ret = HSAKMT_STATUS_NO_MEMORY; - for (j=0; j <= i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i + 1); goto err; } for (mem_id = 0; mem_id < temp_nodes[i].node.NumMemoryBanks; mem_id++) { ret = topology_sysfs_get_mem_props(i, mem_id, &temp_nodes[i].mem[mem_id]); if (ret != HSAKMT_STATUS_SUCCESS) { - for (j=0; j <= i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i + 1); goto err; } } @@ -888,17 +892,13 @@ retry: temp_nodes[i].cache = calloc(temp_nodes[i].node.NumCaches * sizeof(HsaCacheProperties), 1); if (!temp_nodes[i].cache) { ret = HSAKMT_STATUS_NO_MEMORY; - for (j=0; j <= i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i + 1); goto err; } for (cache_id = 0; cache_id < temp_nodes[i].node.NumCaches; cache_id++) { ret = topology_sysfs_get_cache_props(i, cache_id, &temp_nodes[i].cache[cache_id]); if (ret != HSAKMT_STATUS_SUCCESS) { - for (j=0; j <= i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i + 1); goto err; } } @@ -907,9 +907,7 @@ retry: /* Get info from /proc/cpuinfo and /sys/devices/system */ ret = topology_get_cpu_cache_props(&temp_nodes[i]); if (ret != HSAKMT_STATUS_SUCCESS) { - for (j=0; j <= i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i + 1); goto err; } } @@ -918,17 +916,13 @@ retry: temp_nodes[i].link = calloc(temp_nodes[i].node.NumIOLinks * sizeof(HsaIoLinkProperties), 1); if (!temp_nodes[i].link) { ret = HSAKMT_STATUS_NO_MEMORY; - for (j=0; j <= i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i + 1); goto err; } for (link_id = 0; link_id < temp_nodes[i].node.NumIOLinks; link_id++) { ret = topology_sysfs_get_iolink_props(i, link_id, &temp_nodes[i].link[link_id]); if (ret != HSAKMT_STATUS_SUCCESS) { - for (j=0; j <= i; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); + free_nodes(temp_nodes, i+1); goto err; } } @@ -939,32 +933,20 @@ retry: ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { - if (temp_nodes) { - for (j=0; j < sys_props.NumNodes; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); - } + free_nodes(temp_nodes, sys_props.NumNodes); goto err; } if (gen_start != gen_end) { - if (temp_nodes) { - for (j=0; j < sys_props.NumNodes; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); - temp_nodes = 0; - } + free_nodes(temp_nodes, sys_props.NumNodes); + temp_nodes = 0; goto retry; } if (!_system) { _system = malloc(sizeof(HsaSystemProperties)); if (!_system) { - if (temp_nodes) { - for (j=0; j < sys_props.NumNodes; j++) - free_node(&temp_nodes[j]); - free(temp_nodes); - } + free_nodes(temp_nodes, sys_props.NumNodes); return HSAKMT_STATUS_NO_MEMORY; } } @@ -994,14 +976,8 @@ topology_drop_snapshot(void) } if (node) { - uint64_t nodeid; - /* Remove state */ - for (nodeid = 0; nodeid < _system->NumNodes; nodeid++) { - free_node(&node[nodeid]); - } - - free(node); + free_nodes(node, _system->NumNodes); node = NULL; } From 1d1c30db7cde97fba7f3b72b10708f3c1c5d85d9 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 3 Mar 2016 18:31:17 -0500 Subject: [PATCH 0151/1247] Add reverse direct io_links The Kernel only creates one way direct link - GPU(PCI_BUS) --> [Parent NUMA Node] Create the reverse direct io_link here - [Parent NUMA Node] --> GPU(PCI_BUS) Change-Id: I829a1b1b7f34bda42871ede3472d60915e88418c --- src/topology.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/topology.c b/src/topology.c index f22586e198..b1510ae0ab 100644 --- a/src/topology.c +++ b/src/topology.c @@ -845,6 +845,29 @@ err1: return ret; } +/* topology_create_reverse_io_link - Create io_links from the given CPU + * NUMA node to all the GPUs attached to that node + */ +static void topology_create_reverse_io_link(uint32_t cpu_node, + const HsaSystemProperties *sys_props, node_t *temp_nodes) +{ + unsigned int gpu_node; + HsaIoLinkProperties *props = temp_nodes[cpu_node].link; + + for (gpu_node = 0; gpu_node < sys_props->NumNodes; gpu_node++) { + if (temp_nodes[gpu_node].gpu_id != 0) { + /* Check if this GPU is connected to the give cpu_node, + * if so create an io_link */ + if (temp_nodes[gpu_node].link->NodeTo == cpu_node) { + props->NodeFrom = cpu_node; + props->NodeTo = gpu_node; + props->Weight = temp_nodes[gpu_node].link->Weight; + props++; + } + } + } +} + HSAKMT_STATUS topology_take_snapshot(void) { @@ -931,6 +954,47 @@ retry: } } + /* The Kernel only creates one way direct link - + * GPU(PCI_BUS) --> Parent NUMA Node. Create the reverse direct + * io_link here. [NUMA node] --> GPU */ + + for (i = 0; i < sys_props.NumNodes; i++) { + /* For each CPU Node, compute the number of direct io_links it has. + * For that, parse all the GPU Nodes, find the CPU Parent node to + * which it has a direct link to. And increment NumIOLinks for that + * CPU node */ + if (temp_nodes[i].gpu_id != 0) { + if (temp_nodes[i].link) { + if (temp_nodes[i].link->NodeTo < sys_props.NumNodes) + temp_nodes[temp_nodes[i].link->NodeTo].node.NumIOLinks++; + else + printf("Node [%d] has io_link to invalid Node [%d]\n", + i, temp_nodes[i].link->NodeTo); + } + else + printf("GPU [0x%x] is missing its direct IO LINK\n", + temp_nodes[i].gpu_id); + } + } + + /* Create the reverse io_link for all the CPU nodes */ + for (i = 0; i < sys_props.NumNodes; i++) { + if (temp_nodes[i].gpu_id == 0) { + if (temp_nodes[i].link) { + printf("Node [%d] has unexpected io_link. Skipping.\n", i); + continue; + } + temp_nodes[i].link = calloc(temp_nodes[i].node.NumIOLinks, + sizeof(HsaIoLinkProperties)); + if (!temp_nodes[i].link) { + ret = HSAKMT_STATUS_NO_MEMORY; + free_nodes(temp_nodes, i + 1); + goto err; + } + topology_create_reverse_io_link(i, &sys_props, temp_nodes); + } + } + ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, sys_props.NumNodes); From c42ec0b82c2f368bd5da7ccd46a938ac5be3ee32 Mon Sep 17 00:00:00 2001 From: Yair Shachar Date: Mon, 7 Mar 2016 18:41:34 +0200 Subject: [PATCH 0152/1247] name unnamed struct within HsaMemMapFlagd union For aligning with RT definitions Change-Id: I4dca0c5818fdcea6c596a48c7516835fc595a289 Signed-off-by: Yair Shachar --- include/hsakmttypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index e9885398e5..ea4a1f4de5 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -493,7 +493,7 @@ typedef struct _HsaMemMapFlags // data from the memory, reduces latency // “cleanup hint” only, may be ignored unsigned int Reserved : 23; - }; + } ui32; HSAuint32 Value; }; } HsaMemMapFlags; From b837c3e7b066000f9d4e10d7e7ceb7a6c903f3a2 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 1 Mar 2016 17:50:46 -0500 Subject: [PATCH 0153/1247] Clean up GPUVM aperture management Non-canonical GPUVM aperture doesn't exist on dGPUs. Remove comments and code that say otherwise. Fix alignment of GPUVM aperture for gfx801. Requires the same workaround as gfx802. It's not used for anything on gfx801 yet, but will be soon. Change-Id: I88607fe7b340081cc0715b85f28fdbf5f1bb0ad7 --- src/fmm.c | 45 ++++++++++++++++++++------------------------- src/memory.c | 6 ++++++ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index c4e29e5ed5..098426848c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -99,14 +99,11 @@ typedef struct { manageble_aperture_t scratch_physical; /* For dGPU, scratch physical is allocated from dgpu_aperture. When requested by RT, each GPU will get a differnt range */ - manageble_aperture_t gpuvm_aperture; /* used for device mem on APU and for Gfx interop, - unusable on dGPU with small-ish VA range */ - /* TODO: Merge gpuvm and dgpu apertures. When we have bigger - * VA range, we can add a new invisible aperture for invisible - * device mem on dGPU. */ + manageble_aperture_t gpuvm_aperture; /* used for GPUVM on APU, outside + * the canonical address range */ } gpu_mem_t; -/* The main structure for GPU Memory Management */ +/* The main structure for dGPU Shared Virtual Memory Management */ typedef struct { /* used for non-coherent system and invisible device mem on dGPU. * This aperture is shared by all dGPUs */ @@ -751,10 +748,6 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; - /* - * TODO: Once VA limit is raised from 0x200000000 (8GB) use gpuvm_aperture. - * In that way the host access range won't be used for local memory - */ aperture = &svm.dgpu_aperture; offset = 0; if (flags.ui32.AQLQueueMemory) { @@ -1036,6 +1029,15 @@ static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_po return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_MEMORY_POLICY, &args); } +static uint32_t get_vm_alignment(uint32_t device_id) +{ + if (device_id >= 0x6920 && device_id <= 0x6939) /* Tonga */ + return TONGA_PAGE_SIZE; + if (device_id >= 0x9870 && device_id <= 0x9877) /* Carrizo */ + return TONGA_PAGE_SIZE; + return PAGE_SIZE; +} + HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { struct kfd_ioctl_get_process_apertures_new_args args; @@ -1072,7 +1074,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_physical.fmm_mutex, NULL); gpu_mem[gpu_mem_count].scratch_aperture.align = PAGE_SIZE; pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_aperture.fmm_mutex, NULL); - gpu_mem[gpu_mem_count].gpuvm_aperture.align = PAGE_SIZE; + gpu_mem[gpu_mem_count].gpuvm_aperture.align = + get_vm_alignment(props.DeviceId); pthread_mutex_init(&gpu_mem[gpu_mem_count].gpuvm_aperture.fmm_mutex, NULL); gpu_mem_count++; } @@ -1140,12 +1143,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uintptr_t alt_base; uint64_t alt_size; int err; - uint64_t vm_alignment = PAGE_SIZE; - - if (gpu_mem[gpu_mem_id].device_id >= 0x6920 && - gpu_mem[gpu_mem_id].device_id <= 0x6939) - /* Workaround for Tonga GPUVM HW bug */ - vm_alignment = TONGA_PAGE_SIZE; + uint64_t vm_alignment = get_vm_alignment( + gpu_mem[gpu_mem_id].device_id); dgpu_mem_init(gpu_mem_id, &svm.dgpu_aperture.base, &svm.dgpu_aperture.limit); @@ -1159,14 +1158,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) (uint64_t)svm.dgpu_aperture.limit); svm.dgpu_aperture.align = vm_alignment; - /* Place GPUVM aperture after dGPU aperture - * (FK: I think this is broken but leaving it for now) */ - gpu_mem[gpu_mem_id].gpuvm_aperture.base = VOID_PTR_ADD(svm.dgpu_aperture.limit, 1); - gpu_mem[gpu_mem_id].gpuvm_aperture.limit = (void *)VOID_PTRS_SUB(svm.dgpu_aperture.limit, - svm.dgpu_aperture.base); - gpu_mem[gpu_mem_id].gpuvm_aperture.limit = VOID_PTR_ADD(gpu_mem[gpu_mem_id].gpuvm_aperture.limit, - (unsigned long)gpu_mem[gpu_mem_id].gpuvm_aperture.base); - gpu_mem[gpu_mem_id].gpuvm_aperture.align = vm_alignment; + /* Non-canonical per-ASIC GPUVM aperture does + * not exist on dGPUs in GPUVM64 address mode */ + gpu_mem[gpu_mem_id].gpuvm_aperture.base = NULL; + gpu_mem[gpu_mem_id].gpuvm_aperture.limit = NULL; /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. diff --git a/src/memory.c b/src/memory.c index bbdf2ca32e..50d4011788 100644 --- a/src/memory.c +++ b/src/memory.c @@ -294,6 +294,12 @@ hsaKmtUnmapMemoryToGPU( { CHECK_KFD_OPEN(); + if (MemoryAddress == NULL) { + /* Workaround for runtime bug */ + fprintf(stderr, "FIXME: Unmapping NULL pointer\n"); + return HSAKMT_STATUS_SUCCESS; + } + if (!fmm_unmap_from_gpu(MemoryAddress)) return HSAKMT_STATUS_SUCCESS; else From cb0315d31d644e5f4425e5d778abb6db1f5816cf Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 4 Mar 2016 21:00:37 -0500 Subject: [PATCH 0154/1247] Update kfd_ioctl.h from kernel Change-Id: I9852ef2e33e1f3b24343747e3c1c09b0050ffdc1 --- include/linux/kfd_ioctl.h | 69 +++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index e3c96b3b78..ffa68a3555 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -59,7 +59,7 @@ struct kfd_ioctl_create_queue_args { uint64_t eop_buffer_size; /* to KFD */ uint64_t ctx_save_restore_address; /* to KFD */ uint32_t ctx_save_restore_size; /* to KFD */ - uint32_t ctl_stack_size; /* to KFD */ + uint32_t ctl_stack_size; /* to KFD */ }; struct kfd_ioctl_destroy_queue_args { @@ -301,18 +301,28 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { uint64_t dgpu_limit; }; +struct kfd_ioctl_eviction_args { + uint64_t size; + uint64_t type; +}; + +enum evict_type { + EVICT_FIRST_PDD = 0, + EVICT_BIGGEST_PDD +}; + /* * GPU Memory allocation flags */ -#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST (1 << 0) -#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE (1 << 1) +#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST (1 << 0) +#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE (1 << 1) #define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_SCRATCH (1 << 2) -#define KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE (1 << 3) -#define KFD_IOC_ALLOC_MEM_FLAGS_APU_SCRATCH (1 << 4) +#define KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE (1 << 3) +#define KFD_IOC_ALLOC_MEM_FLAGS_APU_SCRATCH (1 << 4) #define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM (1 << 5) -#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 6) +#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 6) struct kfd_ioctl_alloc_memory_of_gpu_new_args { uint64_t va_addr; /* to KFD */ @@ -324,13 +334,36 @@ struct kfd_ioctl_alloc_memory_of_gpu_new_args { }; struct kfd_ioctl_get_process_apertures_new_args { - uint64_t kfd_process_device_apertures_ptr; /* pointer to struct - struct kfd_process_device_apertures. User allocated */ - uint32_t num_of_nodes; /* Number of entries in - kfd_process_device_apertures_ptr */ + /* User allocated. Pointer to struct kfd_process_device_apertures + * filled in by Kernel */ + uint64_t kfd_process_device_apertures_ptr; + + /* to KFD - indicates amount of memory present in + * kfd_process_device_apertures_ptr + * from KFD - Number of entries filled by KFD. + */ + uint32_t num_of_nodes; + uint32_t pad; }; +struct kfd_ioctl_get_dmabuf_info_args { + uint64_t size; /* from KFD */ + uint64_t metadata_ptr; /* to KFD */ + uint32_t metadata_size; /* to KFD (space allocated by user) + * from KFD (actual metadata size) */ + uint32_t gpu_id; /* from KFD */ + uint32_t flags; /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */ + uint32_t dmabuf_fd; /* to KFD */ +}; + +struct kfd_ioctl_import_dmabuf_args { + uint64_t va_addr; /* to KFD */ + uint64_t handle; /* from KFD */ + uint32_t gpu_id; /* to KFD */ + uint32_t dmabuf_fd; /* to KFD */ +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -407,7 +440,8 @@ struct kfd_ioctl_get_process_apertures_new_args { AMDKFD_IOW(0x17, struct kfd_ioctl_set_cu_mask_args) #define AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE \ - AMDKFD_IOW(0x18, struct kfd_ioctl_set_process_dgpu_aperture_args) + AMDKFD_IOW(0x18, \ + struct kfd_ioctl_set_process_dgpu_aperture_args) #define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW \ AMDKFD_IOWR(0x19, struct kfd_ioctl_alloc_memory_of_gpu_new_args) @@ -421,9 +455,18 @@ struct kfd_ioctl_get_process_apertures_new_args { AMDKFD_IOWR(0x1c, struct kfd_ioctl_unmap_memory_from_gpu_new_args) #define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ - AMDKFD_IOWR(0x1d, struct kfd_ioctl_get_process_apertures_new_args) + AMDKFD_IOWR(0x1d, struct kfd_ioctl_get_process_apertures_new_args) + +#define AMDKFD_IOC_EVICT_MEMORY \ + AMDKFD_IOWR(0x1e, struct kfd_ioctl_eviction_args) + +#define AMDKFD_IOC_GET_DMABUF_INFO \ + AMDKFD_IOWR(0x1f, struct kfd_ioctl_get_dmabuf_info_args) + +#define AMDKFD_IOC_IMPORT_DMABUF \ + AMDKFD_IOWR(0x20, struct kfd_ioctl_import_dmabuf_args) #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1e +#define AMDKFD_COMMAND_END 0x21 #endif From b1393f822438747f5ac57a38609b732bfe2e46ec Mon Sep 17 00:00:00 2001 From: Ben Goz Date: Wed, 9 Mar 2016 12:05:52 +0200 Subject: [PATCH 0155/1247] Support MapMemoryToGPUNodes on APU Change-Id: Ie77a2eb23cd9fe6671ff9e0630977220218e55dd Signed-off-by: Ben Goz --- src/memory.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/memory.c b/src/memory.c index 50d4011788..de0b0aaf0b 100644 --- a/src/memory.c +++ b/src/memory.c @@ -277,6 +277,11 @@ hsaKmtMapMemoryToGPUNodes( uint32_t *gpu_id_array; HSAKMT_STATUS ret; + if (!is_dgpu && NumberOfNodes == 1) + return hsaKmtMapMemoryToGPU(MemoryAddress, + MemorySizeInBytes, + AlternateVAGPU); + ret = validate_nodeid_array(&gpu_id_array, NumberOfNodes, NodeArray); if (ret != HSAKMT_STATUS_SUCCESS) From 61ec3df2f9d5d8cc707f181286198e10cf1a7c4a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 5 Mar 2016 01:12:40 -0500 Subject: [PATCH 0156/1247] Add support for hsaKmtRegisterGraphicsHandleToNodes Change-Id: I6fd7154dea78188480d5cb89ac237bad572356c4 --- include/hsakmt.h | 14 ++++ include/hsakmttypes.h | 7 ++ src/fmm.c | 175 +++++++++++++++++++++++++++++++++++++----- src/fmm.h | 4 + src/libhsakmt.ver | 1 + src/memory.c | 27 +++++++ 6 files changed, 208 insertions(+), 20 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index e43ebecd9e..9d8b9d5225 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -393,6 +393,20 @@ hsaKmtRegisterMemoryToNodes( ); +/** + Registers with KFD a graphics buffer and returns graphics metadata +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterGraphicsHandleToNodes( + HSAuint64 GraphicsResourceHandle, //IN + HsaGraphicsResourceInfo *GraphicsResourceInfo, //OUT + HSAuint64 NumberOfNodes, //IN + HSAuint32* NodeArray //IN + ); + + /** Unregisters with KFD a memory buffer */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index ea4a1f4de5..b49ccf9541 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -498,6 +498,13 @@ typedef struct _HsaMemMapFlags }; } HsaMemMapFlags; +typedef struct _HsaGraphicsResourceInfo { + void *MemoryAddress; // For use in hsaKmtMapMemoryToGPU(Nodes) + HSAuint64 SizeInBytes; // Buffer size + const void *Metadata; // Pointer to metadata owned by Thunk + HSAuint32 MetadataSizeInBytes; // Size of metadata + HSAuint32 Reserved; // Reserved for future use, will be set to 0 +} HsaGraphicsResourceInfo; typedef enum _HSA_CACHING_TYPE { diff --git a/src/fmm.c b/src/fmm.c index 098426848c..a997becb29 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -32,6 +32,7 @@ #include #include #include +#include #define NON_VALID_GPU_ID 0 #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) @@ -63,6 +64,8 @@ struct vm_object { */ uint32_t *mapped_device_id_array; uint32_t mapped_device_id_array_size; + /* Metadata of imported graphics buffers */ + void *metadata; }; typedef struct vm_object vm_object_t; @@ -184,6 +187,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->registered_device_id_array_size = 0; object->mapped_device_id_array_size = 0; object->flags = flags; + object->metadata = NULL; } return object; @@ -424,11 +428,11 @@ static void *aperture_allocate_area(manageble_aperture_t *app, } /* returns 0 on success. Assumes, that fmm_mutex is locked on entry */ -static int aperture_allocate_object(manageble_aperture_t *app, - void *new_address, - uint64_t handle, - uint64_t MemorySizeInBytes, - uint32_t flags) +static vm_object_t *aperture_allocate_object(manageble_aperture_t *app, + void *new_address, + uint64_t handle, + uint64_t MemorySizeInBytes, + uint32_t flags) { vm_object_t *new_object; @@ -439,7 +443,7 @@ static int aperture_allocate_object(manageble_aperture_t *app, MemorySizeInBytes, handle, flags); if (!new_object) - return -1; + return NULL; /* check for non-empty list */ if (app->vm_objects != NULL) @@ -448,7 +452,7 @@ static int aperture_allocate_object(manageble_aperture_t *app, app->vm_objects = new_object; /* Update head */ - return 0; + return new_object; } static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) @@ -490,8 +494,8 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); - if (aperture_allocate_object(aperture, mem, args.handle, - MemorySizeInBytes, flags)) + if (!aperture_allocate_object(aperture, mem, args.handle, + MemorySizeInBytes, flags)) goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -894,7 +898,7 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, goto release_area; /* Allocate object */ - if (aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, + if (!aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, open_graphic_handle_args.handle, MemorySizeInBytes, 0)) goto release_mem; @@ -952,6 +956,8 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) free(object->mapped_device_id_array); object->mapped_device_id_array_size = 0; } + if (object->metadata) + free(object->metadata); if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { @@ -2011,6 +2017,104 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, return HSAKMT_STATUS_SUCCESS; } +#define GRAPHICS_METADATA_DEFAULT_SIZE 64 +HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, + HsaGraphicsResourceInfo *GraphicsResourceInfo, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size) +{ + struct kfd_ioctl_get_dmabuf_info_args infoArgs; + struct kfd_ioctl_import_dmabuf_args importArgs; + struct kfd_ioctl_free_memory_of_gpu_args freeArgs; + manageble_aperture_t *aperture; + vm_object_t *obj; + void *metadata; + void *mem, *aperture_base; + int32_t gpu_mem_id; + uint64_t offset; + int r; + HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; + + infoArgs.dmabuf_fd = GraphicsResourceHandle; + infoArgs.metadata_size = GRAPHICS_METADATA_DEFAULT_SIZE; + metadata = calloc(infoArgs.metadata_size, 1); + if (!metadata) + return HSAKMT_STATUS_NO_MEMORY; + infoArgs.metadata_ptr = (uint64_t)metadata; + r = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs); + if (r && infoArgs.metadata_size > GRAPHICS_METADATA_DEFAULT_SIZE) { + /* Try again with bigger metadata */ + free(metadata); + metadata = calloc(infoArgs.metadata_size, 1); + if (!metadata) + return HSAKMT_STATUS_NO_MEMORY; + infoArgs.metadata_ptr = (uint64_t)metadata; + r = kmtIoctl(kfd_fd, AMDKFD_IOC_GET_DMABUF_INFO, (void *)&infoArgs); + } + + if (r) + goto error_free_metadata; + + /* Choose aperture based on GPU and allocate virtual address */ + gpu_mem_id = gpu_mem_find_by_gpu_id(infoArgs.gpu_id); + if (gpu_mem_id < 0) + goto error_free_metadata; + if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + aperture = &svm.dgpu_aperture; + aperture_base = NULL; + offset = 0; + } else { + aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; + aperture_base = aperture->base; + offset = GPUVM_APP_OFFSET; + } + if (!aperture_is_valid(aperture->base, aperture->limit)) + goto error_free_metadata; + pthread_mutex_lock(&aperture->fmm_mutex); + mem = aperture_allocate_area(aperture, infoArgs.size, offset); + pthread_mutex_unlock(&aperture->fmm_mutex); + if (mem == NULL) + goto error_free_metadata; + + /* Import DMA buffer */ + importArgs.va_addr = VOID_PTRS_SUB(mem, aperture_base); + importArgs.gpu_id = infoArgs.gpu_id; + importArgs.dmabuf_fd = GraphicsResourceHandle; + r = kmtIoctl(kfd_fd, AMDKFD_IOC_IMPORT_DMABUF, (void *)&importArgs); + if (r) + goto error_release_aperture; + + pthread_mutex_lock(&aperture->fmm_mutex); + obj = aperture_allocate_object(aperture, mem, importArgs.handle, + infoArgs.size, infoArgs.flags); + if (obj) { + obj->metadata = metadata; + obj->registered_device_id_array = gpu_id_array; + obj->registered_device_id_array_size = gpu_id_array_size; + } + pthread_mutex_unlock(&aperture->fmm_mutex); + if (!obj) + goto error_release_buffer; + + GraphicsResourceInfo->MemoryAddress = mem; + GraphicsResourceInfo->SizeInBytes = infoArgs.size; + GraphicsResourceInfo->Metadata = (void *)(unsigned long)infoArgs.metadata_ptr; + GraphicsResourceInfo->MetadataSizeInBytes = infoArgs.metadata_size; + GraphicsResourceInfo->Reserved = 0; + + return HSAKMT_STATUS_SUCCESS; + +error_release_buffer: + freeArgs.handle = importArgs.handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs); +error_release_aperture: + aperture_release_area(aperture, mem, infoArgs.size); +error_free_metadata: + free(metadata); + + return status; +} + static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) { manageble_aperture_t *aperture; @@ -2037,8 +2141,9 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) HSAKMT_STATUS fmm_deregister_memory(void *address) { - manageble_aperture_t *aperture; + manageble_aperture_t *aperture = NULL; vm_object_t *object = NULL; + unsigned i; if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) @@ -2046,11 +2151,25 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) aperture = &svm.dgpu_alt_aperture; - else { - /* - * If address isn't SVM address, we assume that this - * is system memory address. If the userptr object had - * a registered_device_id_array, it will be freed by + else + for (i = 0; i < gpu_mem_count; i++) { + if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && + address >= gpu_mem[i].gpuvm_aperture.base && + address <= gpu_mem[i].gpuvm_aperture.limit) { + aperture = &gpu_mem[i].gpuvm_aperture; + break; + } + } + + if (!aperture) { + /* If address isn't found in any aperture, we assume + * that this is system memory address. On APUs, there + * is nothing to do (for now). + */ + if (!is_dgpu) + return HSAKMT_STATUS_SUCCESS; + /* If the userptr object had a + * registered_device_id_array, it will be freed by * __fmm_release. Also the object will be * removed. Therefore we can short-circuit the rest of * the function below. @@ -2059,19 +2178,35 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) } pthread_mutex_lock(&aperture->fmm_mutex); + object = vm_find_object_by_address(aperture, address, 0); - pthread_mutex_unlock(&aperture->fmm_mutex); - - if (!object || object->registered_device_id_array_size <= 0) + if (!object) { + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + } - if (object->userptr) + if (object->userptr) { + pthread_mutex_unlock(&aperture->fmm_mutex); return fmm_deregister_user_memory(object->userptr); + } else if (object->metadata) { + /* An object with metadata is an imported graphics + * buffer. Deregistering it means releasing the buffer. */ + pthread_mutex_unlock(&aperture->fmm_mutex); + __fmm_release(address, aperture); + return HSAKMT_STATUS_SUCCESS; + } + + if (object->registered_device_id_array_size <= 0) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + } free(object->registered_device_id_array); object->registered_device_id_array = NULL; object->registered_device_id_array_size = 0; + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_SUCCESS; } diff --git a/src/fmm.h b/src/fmm.h index 2f27d0cf60..b1c5ec28a3 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -72,6 +72,10 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size); +HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, + HsaGraphicsResourceInfo *GraphicsResourceInfo, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_deregister_memory(void *address); HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint32_t nodes_to_map_size, uint64_t *gpuvm_address); diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index d520144cbc..2d3d5db4f1 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -26,6 +26,7 @@ hsaKmtAllocMemory; hsaKmtFreeMemory; hsaKmtRegisterMemory; hsaKmtRegisterMemoryToNodes; +hsaKmtRegisterGraphicsHandleToNodes; hsaKmtDeregisterMemory; hsaKmtMapMemoryToGPU; hsaKmtMapMemoryToGPUNodes; diff --git a/src/memory.c b/src/memory.c index de0b0aaf0b..1ef6a83748 100644 --- a/src/memory.c +++ b/src/memory.c @@ -233,6 +233,33 @@ hsaKmtRegisterMemoryToNodes( return ret; } +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterGraphicsHandleToNodes( + HSAuint64 GraphicsResourceHandle, /* IN */ + HsaGraphicsResourceInfo *GraphicsResourceInfo, /* OUT */ + HSAuint64 NumberOfNodes, /* IN */ + HSAuint32* NodeArray /* IN */ +) +{ + CHECK_KFD_OPEN(); + uint32_t *gpu_id_array; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + ret = validate_nodeid_array(&gpu_id_array, + NumberOfNodes, NodeArray); + + if (ret == HSAKMT_STATUS_SUCCESS) { + ret = fmm_register_graphics_handle( + GraphicsResourceHandle, GraphicsResourceInfo, + gpu_id_array,NumberOfNodes*sizeof(uint32_t)); + if (ret != HSAKMT_STATUS_SUCCESS) + free(gpu_id_array); + } + + return ret; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory( From 1e729510d2971819f1f6308a62d4f5e2e497596c Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 4 Mar 2016 16:21:05 -0500 Subject: [PATCH 0157/1247] Allocate memory for indirect io_links To simplify, allocate maximum needed memory for node_t->link array. No need for realloc when indirect links are added. Trade off - for some nodes more memory than required will be allocated. This means the loop to compute the number of direct (reverse) io_links for a CPU node is not necessary. Change-Id: I2b2559142cbec3b262d0b4ea5fdebfd8f36c28fc --- src/topology.c | 79 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/topology.c b/src/topology.c index b1510ae0ab..0f21fc7e7e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -845,6 +845,34 @@ err1: return ret; } +/* topology_get_free_io_link_slot_for_node - For the given node_id, find the next + * available free slot to add an io_link + */ +static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node(uint32_t node_id, + const HsaSystemProperties *sys_props, node_t *temp_nodes) +{ + HsaIoLinkProperties *props; + + if (node_id >= sys_props->NumNodes) { + printf("Invalid node [%d]\n", node_id); + return NULL; + } + + props = temp_nodes[node_id].link; + if (!props) { + printf("Error. No io_link reported for Node [%d]\n", node_id); + return NULL; + } + + if (temp_nodes[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { + printf("Error. No more space for io_link for Node [%d]\n", node_id); + return NULL; + } + + return &props[temp_nodes[node_id].node.NumIOLinks]; +} + + /* topology_create_reverse_io_link - Create io_links from the given CPU * NUMA node to all the GPUs attached to that node */ @@ -852,17 +880,21 @@ static void topology_create_reverse_io_link(uint32_t cpu_node, const HsaSystemProperties *sys_props, node_t *temp_nodes) { unsigned int gpu_node; - HsaIoLinkProperties *props = temp_nodes[cpu_node].link; + HsaIoLinkProperties *props; for (gpu_node = 0; gpu_node < sys_props->NumNodes; gpu_node++) { if (temp_nodes[gpu_node].gpu_id != 0) { /* Check if this GPU is connected to the give cpu_node, * if so create an io_link */ if (temp_nodes[gpu_node].link->NodeTo == cpu_node) { + props = topology_get_free_io_link_slot_for_node(cpu_node, + sys_props, temp_nodes); + if (!props) + return; props->NodeFrom = cpu_node; props->NodeTo = gpu_node; props->Weight = temp_nodes[gpu_node].link->Weight; - props++; + temp_nodes[cpu_node].node.NumIOLinks++; } } } @@ -935,12 +967,19 @@ retry: } } + /* To simplify, allocate maximum needed memory for io_links for each node. This + * removes the need for realloc when indirect and QPI links are added later */ + temp_nodes[i].link = calloc(sys_props.NumNodes - 1, sizeof(HsaIoLinkProperties)); + if (!temp_nodes[i].link) { + ret = HSAKMT_STATUS_NO_MEMORY; + free_nodes(temp_nodes, i + 1); + goto err; + } + if (temp_nodes[i].node.NumIOLinks) { - temp_nodes[i].link = calloc(temp_nodes[i].node.NumIOLinks * sizeof(HsaIoLinkProperties), 1); - if (!temp_nodes[i].link) { - ret = HSAKMT_STATUS_NO_MEMORY; - free_nodes(temp_nodes, i + 1); - goto err; + if (temp_nodes[i].gpu_id == 0) { + printf("Warning. Not expecting CPU Node [%d] to have [%d] io_links.\n", + i, temp_nodes[i].node.NumIOLinks); } for (link_id = 0; link_id < temp_nodes[i].node.NumIOLinks; link_id++) { ret = topology_sysfs_get_iolink_props(i, link_id, &temp_nodes[i].link[link_id]); @@ -958,35 +997,11 @@ retry: * GPU(PCI_BUS) --> Parent NUMA Node. Create the reverse direct * io_link here. [NUMA node] --> GPU */ - for (i = 0; i < sys_props.NumNodes; i++) { - /* For each CPU Node, compute the number of direct io_links it has. - * For that, parse all the GPU Nodes, find the CPU Parent node to - * which it has a direct link to. And increment NumIOLinks for that - * CPU node */ - if (temp_nodes[i].gpu_id != 0) { - if (temp_nodes[i].link) { - if (temp_nodes[i].link->NodeTo < sys_props.NumNodes) - temp_nodes[temp_nodes[i].link->NodeTo].node.NumIOLinks++; - else - printf("Node [%d] has io_link to invalid Node [%d]\n", - i, temp_nodes[i].link->NodeTo); - } - else - printf("GPU [0x%x] is missing its direct IO LINK\n", - temp_nodes[i].gpu_id); - } - } - /* Create the reverse io_link for all the CPU nodes */ for (i = 0; i < sys_props.NumNodes; i++) { if (temp_nodes[i].gpu_id == 0) { - if (temp_nodes[i].link) { - printf("Node [%d] has unexpected io_link. Skipping.\n", i); - continue; - } - temp_nodes[i].link = calloc(temp_nodes[i].node.NumIOLinks, - sizeof(HsaIoLinkProperties)); if (!temp_nodes[i].link) { + printf("Unexpected NULL pointer. Node [%d].link\n", i); ret = HSAKMT_STATUS_NO_MEMORY; free_nodes(temp_nodes, i + 1); goto err; From 7042292c6031dcd84022f3b5579d040fafc333b8 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 8 Mar 2016 15:32:50 -0500 Subject: [PATCH 0158/1247] Add indirect io_links Connect (Peer-to-Peer) GPUs that belong to same NUMA node. Connect all [GPU] <--> [Non Parent NUMA] node Change-Id: Ib4b08a6545d28b7dce4c9b1a90378bfc51bed07e --- src/topology.c | 103 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/src/topology.c b/src/topology.c index 0f21fc7e7e..5dfb8ca8f4 100644 --- a/src/topology.c +++ b/src/topology.c @@ -872,6 +872,29 @@ static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node(uint32_t no return &props[temp_nodes[node_id].node.NumIOLinks]; } +/* topology_add_io_link_for_node - If a free slot is available, + * add io_link for the given Node. + * TODO: Add other members of HsaIoLinkProperties + */ +static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, + const HsaSystemProperties *sys_props, node_t *temp_nodes, + HSA_IOLINKTYPE IoLinkType, uint32_t NodeTo, + uint32_t Weight) +{ + HsaIoLinkProperties *props; + props = topology_get_free_io_link_slot_for_node(node_id, + sys_props, temp_nodes); + if (!props) + return HSAKMT_STATUS_NO_MEMORY; + + props->IoLinkType = IoLinkType; + props->NodeFrom = node_id; + props->NodeTo = NodeTo; + props->Weight = Weight; + temp_nodes[node_id].node.NumIOLinks++; + + return HSAKMT_STATUS_SUCCESS; +} /* topology_create_reverse_io_link - Create io_links from the given CPU * NUMA node to all the GPUs attached to that node @@ -880,26 +903,80 @@ static void topology_create_reverse_io_link(uint32_t cpu_node, const HsaSystemProperties *sys_props, node_t *temp_nodes) { unsigned int gpu_node; - HsaIoLinkProperties *props; + HSAKMT_STATUS ret; for (gpu_node = 0; gpu_node < sys_props->NumNodes; gpu_node++) { if (temp_nodes[gpu_node].gpu_id != 0) { /* Check if this GPU is connected to the give cpu_node, * if so create an io_link */ if (temp_nodes[gpu_node].link->NodeTo == cpu_node) { - props = topology_get_free_io_link_slot_for_node(cpu_node, - sys_props, temp_nodes); - if (!props) + ret = topology_add_io_link_for_node(cpu_node, sys_props, + temp_nodes, HSA_IOLINKTYPE_PCIEXPRESS, + gpu_node, temp_nodes[gpu_node].link->Weight); + if (ret != HSAKMT_STATUS_SUCCESS) { + printf("Error [%d]. Failed to create reverse io_links from Node [%d]\n", + ret, cpu_node); return; - props->NodeFrom = cpu_node; - props->NodeTo = gpu_node; - props->Weight = temp_nodes[gpu_node].link->Weight; - temp_nodes[cpu_node].node.NumIOLinks++; + } } } } } +/* topology_create_indirect_gpu_links - For the given cpu_node, + * find all nodes connected to it and create io_links + * among them */ +static void topology_create_indirect_gpu_links(uint32_t cpu_node, + const HsaSystemProperties *sys_props, node_t *temp_nodes) +{ + unsigned int i, j; + HSAKMT_STATUS ret; + HSA_IOLINKTYPE IoLinkType; + HsaIoLinkProperties *props = temp_nodes[cpu_node].link; + + + if (!props || temp_nodes[cpu_node].node.NumIOLinks == 0) { + printf("CPU Node [%d] has no GPU connected\n", cpu_node); + return; + } + + /* props is the list of io_links cpu_node is connected to. + * Make an indirect io_links from props[i].NodeTo --> props[j].NodeTo + * and props[j].NodeTo --> props[i].NodeTo */ + for (i = 0; i < temp_nodes[cpu_node].node.NumIOLinks - 1; i++) + { + for (j = i + 1; j < temp_nodes[cpu_node].node.NumIOLinks; j++) { + /* Ignore CPU <--> CPU node connected as it is handled by QPI + * link function */ + if (temp_nodes[props[i].NodeTo].gpu_id == 0 && + temp_nodes[props[j].NodeTo].gpu_id == 0) + continue; + + /* The link is from GPU to non-parent NUMA node. So set link type + * to HT */ + if (temp_nodes[props[i].NodeTo].gpu_id == 0 || + temp_nodes[props[j].NodeTo].gpu_id == 0) + IoLinkType = HSA_IOLINKTYPE_HYPERTRANSPORT; + else + IoLinkType = HSA_IOLINKTYPE_PCIEXPRESS; + + ret = topology_add_io_link_for_node(props[i].NodeTo, + sys_props, temp_nodes, IoLinkType, + props[j].NodeTo, props[i].Weight + props[j].Weight); + if (ret != HSAKMT_STATUS_SUCCESS) + printf("Error [%d]. Failed to add io_link from Node [%d]->[%d]\n", + ret, i, j); + + ret = topology_add_io_link_for_node(props[j].NodeTo, + sys_props, temp_nodes, IoLinkType, + props[i].NodeTo, props[i].Weight + props[j].Weight); + if (ret != HSAKMT_STATUS_SUCCESS) + printf("Error [%d]. Failed to add io_link from Node [%d]->[%d]\n", + ret, j, i); + } + } +} + HSAKMT_STATUS topology_take_snapshot(void) { @@ -1010,6 +1087,16 @@ retry: } } + /* Create In-direct links for GPUs. Connect all the (Peer-to-Peer) GPUs + * that belong to same NUMA node. + * For each CPU (NUMA) node, interconnect all the GPUs. */ + for (i = 0; i < sys_props.NumNodes; i++) { + if (temp_nodes[i].gpu_id == 0) { + topology_create_indirect_gpu_links(i, &sys_props, temp_nodes); + } + } + + ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, sys_props.NumNodes); From cac0c0849635e2f7eda286b8bd757cc46678c0c1 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 10 Mar 2016 14:37:53 -0500 Subject: [PATCH 0159/1247] Fix lstopo Lstopo doesn't have system memory mappings at low addresses. Make sure we leave enough GPUVM address space for kernel allocations (currently only CWSR) before the start of the user-managed SVM aperture. Change-Id: Ic197f7bd5a3cfb150a0da2bfdbc848664e7869be --- src/fmm.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index a997becb29..71d980696f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1787,6 +1787,9 @@ err: return ret; } +#define DGPU_APERTURE_ADDR_MIN 0x1000000 /* Leave at least 16MB for kernel */ +#define DGPU_APERTURE_ADDR_INC 0x200000 /* Search in huge-page increments */ + static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limit) { bool found; @@ -1815,9 +1818,9 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi found = false; for (len = max_vm_limit+1; !found && len >= min_vm_size; len >>= 1) { - for (addr = (void *)TONGA_PAGE_SIZE, ret_addr = NULL; + for (addr = (void *)DGPU_APERTURE_ADDR_MIN, ret_addr = NULL; (HSAuint64)addr + (len >> 1) < max_vm_limit; - addr = (void *)((HSAuint64)addr + TONGA_PAGE_SIZE)) { + addr = (void *)((HSAuint64)addr + DGPU_APERTURE_ADDR_INC)) { ret_addr = reserve_address(addr, len); if (!ret_addr) break; From 8ff2bcd48d1bbfadb6e749ae9480b64f07752b86 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 11 Mar 2016 11:15:02 -0500 Subject: [PATCH 0160/1247] Fix indirect io_links Connect only (Peer-to-Peer) GPUs that belong to same NUMA node. Without this additional check non direct GPUs would also get connected. Change-Id: I9a5ed19b8f06cd0527854cbbdb51ede99eade28b --- src/topology.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/topology.c b/src/topology.c index 5dfb8ca8f4..2724260473 100644 --- a/src/topology.c +++ b/src/topology.c @@ -952,6 +952,14 @@ static void topology_create_indirect_gpu_links(uint32_t cpu_node, temp_nodes[props[j].NodeTo].gpu_id == 0) continue; + /* For the given cpu_node, connect to or from the GPUs that are + * connected directly to it via PCIEXPRESS */ + if ((temp_nodes[props[i].NodeTo].gpu_id != 0 && + props[i].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS) || + (temp_nodes[props[j].NodeTo].gpu_id != 0 && + props[j].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS)) + continue; + /* The link is from GPU to non-parent NUMA node. So set link type * to HT */ if (temp_nodes[props[i].NodeTo].gpu_id == 0 || From 1c1bc3247738bcbbee5043def44d2d87792f07b1 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 11 Mar 2016 15:24:06 -0500 Subject: [PATCH 0161/1247] Sync IOLINK defines to thunk spec Current thunk spec v1.07 dated Feb 1, 2016 Change-Id: Ie1821f7f1903ac48b76cb68d452a6073d3a3c8d9 --- include/hsakmttypes.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index b49ccf9541..3b9d2cc454 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -365,7 +365,13 @@ typedef enum _HSA_IOLINKTYPE { HSA_IOLINKTYPE_PCIEXPRESS = 2, HSA_IOLINKTYPE_AMBA = 3, HSA_IOLINKTYPE_MIPI = 4, - HSA_IOLINKTYPE_OTHER = 5, + HSA_IOLINK_TYPE_QPI_1_1 = 5, + HSA_IOLINK_TYPE_RESERVED1 = 6, + HSA_IOLINK_TYPE_RESERVED2 = 7, + HSA_IOLINK_TYPE_RAPID_IO = 8, + HSA_IOLINK_TYPE_INFINIBAND = 9, + HSA_IOLINK_TYPE_RESERVED3 = 10, + HSA_IOLINKTYPE_OTHER = 11, HSA_IOLINKTYPE_NUMIOLINKTYPES, HSA_IOLINKTYPE_SIZE = 0xFFFFFFFF } HSA_IOLINKTYPE; @@ -382,7 +388,8 @@ typedef union // memory accesses across must not be set to "host cacheable"! unsigned int NoAtomics32bit : 1; // The link doesn't support 32bit-wide atomic transactions unsigned int NoAtomics64bit : 1; // The link doesn't support 64bit-wide atomic transactions - unsigned int Reserved :28; + unsigned int NoPeerToPeerDMA : 1; // The link doesn't allow device P2P access + unsigned int Reserved :27; } ui32; } HSA_LINKPROPERTY; From 0ed29f5191c4a88aa30b5be0a92a154b0763a92c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 9 Mar 2016 14:29:37 -0500 Subject: [PATCH 0162/1247] Report SVM heap in topology The Runtime requested this information so they can tell easily whether a pointer is part of HSA shared address space or not. Change-Id: If2041ed34031636677d692bc2dc6625634027ed4 --- include/hsakmttypes.h | 1 + src/fmm.c | 10 ++++++++++ src/fmm.h | 1 + src/topology.c | 19 +++++++++++++++---- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 3b9d2cc454..02d8319c6a 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -261,6 +261,7 @@ typedef enum _HSA_HEAPTYPE HSA_HEAPTYPE_GPU_GDS = 3, // GPU internal memory (GDS) HSA_HEAPTYPE_GPU_LDS = 4, // GPU internal memory (LDS) HSA_HEAPTYPE_GPU_SCRATCH = 5, // GPU special memory (scratch) + HSA_HEAPTYPE_DEVICE_SVM = 6, // sys-memory mapped by device page tables HSA_HEAPTYPE_NUMHEAPTYPES, HSA_HEAPTYPE_SIZE = 0xFFFFFFFF diff --git a/src/fmm.c b/src/fmm.c index 71d980696f..0c746f96a9 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1246,6 +1246,16 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA } break; + case FMM_SVM: + /* Report single SVM aperture, starting at base of + * fine-grained, ending at limit of coarse-grained */ + if (aperture_is_valid(svm.dgpu_alt_aperture.base, + svm.dgpu_aperture.limit)) { + *aperture_base = PORT_VPTR_TO_UINT64(svm.dgpu_alt_aperture.base); + *aperture_limit = PORT_VPTR_TO_UINT64(svm.dgpu_aperture.limit); + } + break; + default: err = HSAKMT_STATUS_ERROR; } diff --git a/src/fmm.h b/src/fmm.h index b1c5ec28a3..1601ef0ee4 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -34,6 +34,7 @@ typedef enum { FMM_GPUVM = FMM_FIRST_APERTURE_TYPE, FMM_LDS, FMM_SCRATCH, + FMM_SVM, FMM_LAST_APERTURE_TYPE } aperture_type_e; diff --git a/src/topology.c b/src/topology.c index 2724260473..bb8cbf4e86 100644 --- a/src/topology.c +++ b/src/topology.c @@ -38,7 +38,7 @@ #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) /* Number of memory banks added by thunk on top of topology */ #define NUM_OF_IGPU_HEAPS 3 -#define NUM_OF_DGPU_HEAPS 2 +#define NUM_OF_DGPU_HEAPS 3 /* SYSFS related */ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" @@ -1291,6 +1291,7 @@ hsaKmtGetNodeMemoryProperties( HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; uint32_t i, gpu_id; HSAuint64 aperture_limit; + bool nodeIsDGPU; if (!MemoryProperties) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -1326,6 +1327,8 @@ hsaKmtGetNodeMemoryProperties( if (gpu_id == 0) goto out; + nodeIsDGPU = topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)); + /*Add LDS*/ if (i < NumBanks && fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id, @@ -1337,9 +1340,7 @@ hsaKmtGetNodeMemoryProperties( /* Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE. * For dGPU the topology node contains Local Memory and it is added by the for loop above */ - if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && - i < NumBanks && - node[NodeId].node.LocalMemSize > 0 && + if (!nodeIsDGPU && i < NumBanks && node[NodeId].node.LocalMemSize > 0 && fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; @@ -1356,6 +1357,16 @@ hsaKmtGetNodeMemoryProperties( i++; } + /* On dGPUs add SVM aperture */ + if (nodeIsDGPU && i < NumBanks && + fmm_get_aperture_base_and_limit( + FMM_SVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, + &aperture_limit) == HSAKMT_STATUS_SUCCESS) { + MemoryProperties[i].HeapType = HSA_HEAPTYPE_DEVICE_SVM; + MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1; + i++; + } + out: pthread_mutex_unlock(&hsakmt_mutex); return err; From 79077811f5fa6cbf055aa9b9aae5121eabc7bb94 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Wed, 9 Mar 2016 14:30:12 -0500 Subject: [PATCH 0163/1247] Add Imprecise flag for memory access fault KFD may not be able to provide the precise VM fault address and status. This flag will indicate whether the event data has the fault details Change-Id: I15ffd5c25f555003c6450cc0700efb769418f76b --- include/hsakmttypes.h | 3 ++- include/linux/kfd_ioctl.h | 2 +- src/events.c | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 02d8319c6a..d091c833f0 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -781,7 +781,8 @@ typedef struct _HsaAccessAttributeFailure unsigned int NoExecute : 1; // Execute access to a page marked NX unsigned int GpuAccess : 1; // Host access only unsigned int ECC : 1; // ECC failure (if supported by HW) - unsigned int Reserved : 27; // must be 0 + unsigned int Imprecise : 1; // Can't determine the exact fault address + unsigned int Reserved : 26; // must be 0 } HsaAccessAttributeFailure; // data associated with HSA_EVENTID_MEMORY diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index ffa68a3555..b0d13cc3b7 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -222,7 +222,7 @@ struct kfd_memory_exception_failure { bool NotPresent; /* Page not present or supervisor privilege */ bool ReadOnly; /* Write access to a read-only page */ bool NoExecute; /* Execute access to a page marked NX */ - bool pad; + bool imprecise; /* Can't determine the exact fault address */ }; /* memory exception data*/ diff --git a/src/events.c b/src/events.c index b31353f49c..a45716ad4a 100644 --- a/src/events.c +++ b/src/events.c @@ -278,6 +278,7 @@ hsaKmtWaitOnMultipleEvents( Events[i]->EventData.EventData.MemoryAccessFault.Failure.NotPresent = event_data[i].memory_exception_data.failure.NotPresent; Events[i]->EventData.EventData.MemoryAccessFault.Failure.ReadOnly = event_data[i].memory_exception_data.failure.ReadOnly; Events[i]->EventData.EventData.MemoryAccessFault.Failure.NoExecute = event_data[i].memory_exception_data.failure.NoExecute; + Events[i]->EventData.EventData.MemoryAccessFault.Failure.Imprecise = event_data[i].memory_exception_data.failure.imprecise; Events[i]->EventData.EventData.MemoryAccessFault.Flags = HSA_EVENTID_MEMORY_FATAL_PROCESS; } } From ee1dd5d9c2130ebfa9de37bfce3141ee9497e06f Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 11 Mar 2016 18:24:14 -0500 Subject: [PATCH 0164/1247] Get processor vendor from /proc/cpuinfo Change-Id: I9039385d268ef1693fab121cbf1caf442129a12e --- src/fmm.c | 1 - src/libhsakmt.h | 1 + src/topology.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 0c746f96a9..ae1a5f7aab 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -35,7 +35,6 @@ #include #define NON_VALID_GPU_ID 0 -#define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) #define INIT_MANAGEBLE_APERTURE(base_value, limit_value) { \ .base = (void *) base_value, \ diff --git a/src/libhsakmt.h b/src/libhsakmt.h index e7c3975646..cb28a04a96 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -58,6 +58,7 @@ extern bool is_dgpu; #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) #define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) +#define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); diff --git a/src/topology.c b/src/topology.c index bb8cbf4e86..f3f6e9f135 100644 --- a/src/topology.c +++ b/src/topology.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "libhsakmt.h" #include "fmm.h" @@ -43,6 +44,7 @@ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" #define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" +#define PROC_CPUINFO_PATH "/proc/cpuinfo" #define MAX_CPU_CORES 128 #define MAX_CACHES 256 @@ -57,6 +59,18 @@ typedef struct { static HsaSystemProperties *_system = NULL; static node_t *node = NULL; +static int processor_vendor; +/* Supported System Vendors */ +enum SUPPORTED_PROCESSOR_VENDORS { + GENUINE_INTEL = 0, + AUTHENTIC_AMD +}; +/* Adding newline to make the search easier */ +static const char *supported_processor_vendor_name[] = { + "GenuineIntel\n", + "AuthenticAMD\n" +}; + static HSAKMT_STATUS topology_take_snapshot(void); static HSAKMT_STATUS topology_drop_snapshot(void); //static int get_cpu_stepping(uint16_t* stepping); @@ -312,6 +326,55 @@ bool topology_is_dgpu(uint16_t device_id) return false; } +static int topology_search_processor_vendor(const char *processor_name) +{ + unsigned int i; + for (i = 0; i < ARRAY_LEN(supported_processor_vendor_name); i++) { + if (!strcmp(processor_name, supported_processor_vendor_name[i])) + return i; + } + return -1; +} + +/* topology_set_processor_vendor - Parse /proc/cpuinfo and + * to find processor vendor and set global variable processor_vendor + * + * cat /proc/cpuinfo format is - "token : Value" + * where token = "vendor_id" and + * Value = indicates System Vendor + */ +static void topology_set_processor_vendor(void) +{ + FILE *fd; + char read_buf[256]; + const char *p; + + fd = fopen(PROC_CPUINFO_PATH, "r"); + if (!fd) { + printf("Failed to open [%s]. Setting Processor Vendor to %s", + PROC_CPUINFO_PATH, supported_processor_vendor_name[GENUINE_INTEL]); + processor_vendor = GENUINE_INTEL; + return; + } + + while (fgets(read_buf, sizeof(read_buf), fd) != NULL) { + if (!strncmp("vendor_id", read_buf, sizeof("vendor_id") - 1)) { + p = strrchr(read_buf, ':'); + p++; // remove separor ':' + for (; isspace(*p); p++); /* remove white space */ + processor_vendor = topology_search_processor_vendor(p); + if (processor_vendor != -1) { + fclose(fd); + return; + } + } + } + fclose(fd); + printf("Failed to get Processor Vendor. Setting to %s", + supported_processor_vendor_name[GENUINE_INTEL]); + processor_vendor = GENUINE_INTEL; +} + HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id) { FILE *fd; @@ -993,6 +1056,7 @@ topology_take_snapshot(void) node_t *temp_nodes = 0; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + topology_set_processor_vendor(); retry: ret = topology_sysfs_get_generation(&gen_start); if (ret != HSAKMT_STATUS_SUCCESS) From f1fbacca1551d82c3c4af5b9788edb4a474b4b0e Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 11 Mar 2016 18:54:46 -0500 Subject: [PATCH 0165/1247] Add QPI or HT io_links Create QPI or HT links among all NUMA nodes. For now, assume all the NUMA nodes are interconnected with same Weight (=1). Change-Id: Id48ba95b9d75515a186f7dc5006b19bd92743ae3 --- src/topology.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/src/topology.c b/src/topology.c index f3f6e9f135..65713271e6 100644 --- a/src/topology.c +++ b/src/topology.c @@ -908,6 +908,20 @@ err1: return ret; } +/* topology_get_numa_node_link_tye - Return NUMA node interconnect based + * on processor vendor + */ +static HSA_IOLINKTYPE topology_get_numa_node_link_tye(void) +{ + if (processor_vendor == GENUINE_INTEL) + return HSA_IOLINK_TYPE_QPI_1_1; + else if (processor_vendor == AUTHENTIC_AMD) + return HSA_IOLINKTYPE_HYPERTRANSPORT; + else + return HSA_IOLINKTYPE_UNDEFINED; + +} + /* topology_get_free_io_link_slot_for_node - For the given node_id, find the next * available free slot to add an io_link */ @@ -959,6 +973,39 @@ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, return HSAKMT_STATUS_SUCCESS; } +/* topology_create_qpi_links - Create QPI or HT links among all NUMA nodes + * For now, assume all the nodes are interconnected with same Weight (=1) + */ +static void topology_create_qpi_links(const HsaSystemProperties *sys_props, + node_t *temp_nodes) +{ + unsigned int i, j; + HSAKMT_STATUS ret; + + /* Find all CPU Nodes and connect each other via HT or QPI io_link */ + for (i = 0; i < sys_props->NumNodes - 1; i++) { + for (j = i + 1; j < sys_props->NumNodes; j++) { + if (temp_nodes[i].gpu_id == 0 && + temp_nodes[j].gpu_id == 0) { + ret = topology_add_io_link_for_node(i, + sys_props, temp_nodes, topology_get_numa_node_link_tye(), + j, 1); + if (ret != HSAKMT_STATUS_SUCCESS) + printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n", + ret, i, j); + + ret = topology_add_io_link_for_node(j, + sys_props, temp_nodes, topology_get_numa_node_link_tye(), + i, 1); + if (ret != HSAKMT_STATUS_SUCCESS) + printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n", + ret, j, i); + + } + } + } +} + /* topology_create_reverse_io_link - Create io_links from the given CPU * NUMA node to all the GPUs attached to that node */ @@ -1024,10 +1071,10 @@ static void topology_create_indirect_gpu_links(uint32_t cpu_node, continue; /* The link is from GPU to non-parent NUMA node. So set link type - * to HT */ + * to HT or QPI */ if (temp_nodes[props[i].NodeTo].gpu_id == 0 || temp_nodes[props[j].NodeTo].gpu_id == 0) - IoLinkType = HSA_IOLINKTYPE_HYPERTRANSPORT; + IoLinkType = topology_get_numa_node_link_tye(); else IoLinkType = HSA_IOLINKTYPE_PCIEXPRESS; @@ -1159,6 +1206,10 @@ retry: } } + /* Create QPI or HT links among CPU (NUMA) nodes. For now assume + * all nodes are interconnected with same weight */ + topology_create_qpi_links(&sys_props, temp_nodes); + /* Create In-direct links for GPUs. Connect all the (Peer-to-Peer) GPUs * that belong to same NUMA node. * For each CPU (NUMA) node, interconnect all the GPUs. */ @@ -1168,7 +1219,6 @@ retry: } } - ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, sys_props.NumNodes); From 06d391c6c95e0c420370f4b5c98b5655fcc6c7a4 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 17 Mar 2016 18:45:40 -0400 Subject: [PATCH 0166/1247] Add environment variable to disable GPU caching This option may help debug synchronization or coherency issues involving the GPU caches. It works only on dGPUs, by changing the cache policy of the GPUVM default aperture to "cohrent", which is implemented as non-cached on current dGPU hardware. Change-Id: I544ac9cc5c0cf1fa5c4e30f67aa42b3b5e44ae67 --- src/fmm.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ae1a5f7aab..5d739eaa5c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1052,6 +1052,12 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HsaNodeProperties props; struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + char *disableCache; + + /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ + disableCache = getenv("HSA_DISABLE_CACHE"); + if (disableCache && strcmp(disableCache, "0") == 0) + disableCache = NULL; /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than @@ -1180,9 +1186,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); err = fmm_set_memory_policy(gpu_mem[gpu_mem_id].gpu_id, - KFD_IOC_CACHE_POLICY_NONCOHERENT, - KFD_IOC_CACHE_POLICY_COHERENT, - alt_base, alt_size); + disableCache ? + KFD_IOC_CACHE_POLICY_COHERENT : + KFD_IOC_CACHE_POLICY_NONCOHERENT, + KFD_IOC_CACHE_POLICY_COHERENT, + alt_base, alt_size); if (err != 0) { fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); ret = HSAKMT_STATUS_ERROR; From 682776d89ad9dc34490d01a0d702977c80cd8320 Mon Sep 17 00:00:00 2001 From: David Ogbeide Date: Wed, 16 Mar 2016 11:42:42 -0400 Subject: [PATCH 0167/1247] libhsakmt: get CPU model name from proc/cpuinfo HSA thunk is currently only aware of GPU node model info, CPU names are NULL. Signed-off-by: David Ogbeide Change-Id: I3c2adbb8566a5048b44c39fff4fd8228912468ff --- src/topology.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/src/topology.c b/src/topology.c index 65713271e6..a1f53599bb 100644 --- a/src/topology.c +++ b/src/topology.c @@ -326,6 +326,68 @@ bool topology_is_dgpu(uint16_t device_id) return false; } +static HSAKMT_STATUS +topology_get_cpu_model_name(HsaNodeProperties *props) { + FILE *fd; + char read_buf[256], cpu_model_name[128]; + const char *p; + uint32_t i, apic_id; + + if (!props) + return HSAKMT_STATUS_INVALID_PARAMETER; + + fd = fopen(PROC_CPUINFO_PATH, "r"); + if (!fd) { + printf("Failed to open [%s]. Unable to get CPU Model Name", + PROC_CPUINFO_PATH); + return HSAKMT_STATUS_ERROR; + } + + while (fgets(read_buf, sizeof(read_buf), fd) != NULL) { + /* Get the model name first, in case matching + * apic IDs are also present in the file + */ + if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { + p = strrchr(read_buf, ':'); + if (!p) + goto err; + + p++; // remove separator ':' + for (; isspace(*p); p++); /* remove white space */ + + /* Extract model name from string */ + for (i = 0; i < sizeof(cpu_model_name) - 1 && p[i] != '\n'; i++) + cpu_model_name[i] = p[i]; + cpu_model_name[i] = '\0'; + } + + if (!strncmp("apicid", read_buf, sizeof("apicid") - 1)) { + p = strrchr(read_buf, ':'); + if (!p) + goto err; + + p++; // remove separator ':' + for (; isspace(*p); p++); /* remove white space */ + + /* Extract apic_id from remaining chars */ + apic_id = atoi(p); + + /* Set CPU model name only if corresponding apic id */ + if (props->CComputeIdLo == apic_id) { + /* Convert from UTF8 to UTF16 */ + for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + props->MarketingName[i] = cpu_model_name[i]; + props->MarketingName[i] = '\0'; + } + } + } + fclose(fd); + return HSAKMT_STATUS_SUCCESS; +err: + fclose(fd); + return HSAKMT_STATUS_ERROR; +} + static int topology_search_processor_vendor(const char *processor_name) { unsigned int i; @@ -495,6 +557,19 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 for (i = 0; hsa_gfxip->marketing_name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = hsa_gfxip->marketing_name[i]; props->MarketingName[i] = 0; + } else { + /* Is CPU node */ + if (!props->NumFComputeCores || !props->DeviceId) { + ret = topology_get_cpu_model_name(props); + if (ret != HSAKMT_STATUS_SUCCESS) + { + printf("Failed to get CPU Model Name from %s\n", PROC_CPUINFO_PATH); + ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ + } + } else { + ret = HSAKMT_STATUS_ERROR; + goto err; + } } if (props->NumFComputeCores) assert(props->EngineId.ui32.Major); From 31861c838e652d9625ea982eb3dd21a93e7ed3d0 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Thu, 7 Apr 2016 18:39:35 -0400 Subject: [PATCH 0168/1247] package: change install directory to /opt/rocm Align with the rest of the driver stack on the new installation path /opt/rocm/* This mechanism for generating packages should be changed for something nicer and more standards compliant in the future. Change-Id: Ic31409b0d0b8f6ee4b25296d2580982a76aab564 --- DEBIAN/control | 2 +- DEBIAN/postinst | 7 +++++++ DEBIAN/x86_64-libhsakmt.conf | 1 + RPM/libhsakmt.spec | 8 +++++++- src/Makefile | 29 +++++++++++++++++++++-------- 5 files changed, 37 insertions(+), 10 deletions(-) create mode 100755 DEBIAN/postinst create mode 100644 DEBIAN/x86_64-libhsakmt.conf diff --git a/DEBIAN/control b/DEBIAN/control index 563a85813d..37474f0f19 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -3,6 +3,6 @@ Architecture: amd64 Maintainer: Advanced Micro Devices Inc. Depends: Priority: optional -Version: 1.8.1 +Version: 2.0.0 Description: Thunk library for AMD KFD diff --git a/DEBIAN/postinst b/DEBIAN/postinst new file mode 100755 index 0000000000..b3919c75d4 --- /dev/null +++ b/DEBIAN/postinst @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +echo "Updating ldconfig" +ldconfig +echo "Done" diff --git a/DEBIAN/x86_64-libhsakmt.conf b/DEBIAN/x86_64-libhsakmt.conf new file mode 100644 index 0000000000..a4551d4820 --- /dev/null +++ b/DEBIAN/x86_64-libhsakmt.conf @@ -0,0 +1 @@ +/opt/rocm/lib diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index b9855fcd24..ad24a7eb8c 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -1,5 +1,5 @@ %define name libhsakmt -%define version 1.8.1 +%define version 2.0.0 %define thunkroot %{getenv:OUT_DIR} Name: %{name} @@ -21,6 +21,12 @@ for AMD KFD cp -R %thunkroot/build/thunk/rpm/libhsakmt $RPM_BUILD_ROOT find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> thunk.list +%post +ldconfig + +%postun +ldconfig + %clean rm -rf $RPM_BUILD_ROOT diff --git a/src/Makefile b/src/Makefile index 143385b5f6..c6828e9ae1 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,5 +1,6 @@ # Include directories -INCLUDES += ../include +INCLUDE_DIR := ../include +INCLUDES += $(INCLUDE_DIR) CFLAGS += $(foreach DIR,$(INCLUDES),-I$(DIR)) LIB_NAME = libhsakmt.so @@ -39,6 +40,7 @@ ifdef O else BUILD_ROOT := ../build endif +BUILD_ROOT := $(abspath $(BUILD_ROOT)) BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) TARGET = $(addprefix $(BUILDDIR)/,$(OBJS)) @@ -54,18 +56,29 @@ lnx: LDFLAGS += -m32 lnx: $(BUILDDIR)/$(LIB_NAME) PACKAGE_DIR = $(BUILDDIR)/libhsakmt -INSTALL_DIR_DEB = $(PACKAGE_DIR)/usr/lib/x86_64-linux-gnu -INSTALL_DIR_RPM = $(PACKAGE_DIR)/usr/lib64 +DEBIAN_DIR = ../DEBIAN +LDCONF_FILE = $(DEBIAN_DIR)/x86_64-libhsakmt.conf +LDCONF_INSTALL_DIR = $(PACKAGE_DIR)/etc/ld.so.conf.d +INSTALL_LIB_DIR = $(PACKAGE_DIR)/opt/rocm/lib +INSTALL_INCLUDE_DIR = $(PACKAGE_DIR)/opt/rocm/include/libhsakmt deb: lnx64a - @mkdir -p $(INSTALL_DIR_DEB) - @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_DIR_DEB) - @cp -r ../DEBIAN $(PACKAGE_DIR) + @mkdir -p $(INSTALL_LIB_DIR) + @mkdir -p $(INSTALL_INCLUDE_DIR) + @mkdir -p $(LDCONF_INSTALL_DIR) + @cp -r $(INCLUDE_DIR)/* $(INSTALL_INCLUDE_DIR) + @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_LIB_DIR) + @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) + @cp -r $(DEBIAN_DIR) $(PACKAGE_DIR) @fakeroot dpkg-deb --build $(PACKAGE_DIR) $(BUILDDIR)/libhsakmt.deb rpm: lnx64a - mkdir -p $(INSTALL_DIR_RPM) - @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_DIR_RPM) + @mkdir -p $(INSTALL_LIB_DIR) + @mkdir -p $(INSTALL_INCLUDE_DIR) + @mkdir -p $(LDCONF_INSTALL_DIR) + @cp -r $(INCLUDE_DIR)/* $(INSTALL_INCLUDE_DIR) + @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_LIB_DIR) + @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec lnx64a: $(BUILDDIR)/$(LIB_NAME) From 82b3fad320812e14ce51190c3b27d3d1601bce0d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 8 Apr 2016 19:58:32 -0400 Subject: [PATCH 0169/1247] Fix 4GB and larger system memory allocations Intermediate size was stored in a 32-bit variable. This resulted in 4GB allocations to fail in KFD due to 0 size. Larger allocations would allocate the wrong amount of memory. Change-Id: If19dedf64952f1d2edd813793241e12c0362d220 --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 5d739eaa5c..32972866ba 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -811,7 +811,7 @@ static void* fmm_allocate_host_gpu(uint64_t MemorySizeInBytes, manageble_aperture_t *aperture; uint64_t mmap_offset; uint32_t ioc_flags; - uint32_t size; + uint64_t size; int32_t i; uint32_t gpu_id; From 9f355b78a0f3daa3a5dedd11a470cd5d27fdaf66 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Mon, 11 Apr 2016 18:17:34 -0400 Subject: [PATCH 0170/1247] Adopt new ROCm packaging guidelines All files should go into /opt/rocm/$component For developer convenience, a single include directory is created through symlinks, from the component include directory to /opt/rocm/include. Similarly, a unified linked directory is present in /opt/rocm/lib The component lib directory should not include linker names (library names without version numbers). This commit also fixes 'make rpm' running correctly without the need for sourcing build/envsetup.sh Change-Id: I95a680f6d3e3bd1ae688d0694934a0577dbd007c --- DEBIAN/x86_64-libhsakmt.conf | 2 +- RPM/libhsakmt.spec | 4 ++-- src/Makefile | 45 +++++++++++++++++++++++------------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/DEBIAN/x86_64-libhsakmt.conf b/DEBIAN/x86_64-libhsakmt.conf index a4551d4820..cfa6dc46b8 100644 --- a/DEBIAN/x86_64-libhsakmt.conf +++ b/DEBIAN/x86_64-libhsakmt.conf @@ -1 +1 @@ -/opt/rocm/lib +/opt/rocm/libhsakmt/lib diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index ad24a7eb8c..77d7ed1be1 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -1,6 +1,6 @@ %define name libhsakmt %define version 2.0.0 -%define thunkroot %{getenv:OUT_DIR} +%define packageroot %{getenv:PACKAGE_DIR} Name: %{name} Version: %{version} @@ -18,7 +18,7 @@ for AMD KFD %setup -T -D -c -n %{name} %install -cp -R %thunkroot/build/thunk/rpm/libhsakmt $RPM_BUILD_ROOT +cp -R %packageroot $RPM_BUILD_ROOT find $RPM_BUILD_ROOT \! -type d | sed "s|$RPM_BUILD_ROOT||"> thunk.list %post diff --git a/src/Makefile b/src/Makefile index c6828e9ae1..6d4c89cbe1 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,3 +1,6 @@ +# Our directory +THUNK_ROOT ?= $(abspath ../) + # Include directories INCLUDE_DIR := ../include INCLUDES += $(INCLUDE_DIR) @@ -59,26 +62,36 @@ PACKAGE_DIR = $(BUILDDIR)/libhsakmt DEBIAN_DIR = ../DEBIAN LDCONF_FILE = $(DEBIAN_DIR)/x86_64-libhsakmt.conf LDCONF_INSTALL_DIR = $(PACKAGE_DIR)/etc/ld.so.conf.d -INSTALL_LIB_DIR = $(PACKAGE_DIR)/opt/rocm/lib -INSTALL_INCLUDE_DIR = $(PACKAGE_DIR)/opt/rocm/include/libhsakmt +ROCM_ROOT = $(PACKAGE_DIR)/opt/rocm +ROCM_LIB = $(ROCM_ROOT)/lib +ROCM_INCLUDE = $(ROCM_ROOT)/include +INSTALL_LIB_DIR = libhsakmt/lib +INSTALL_INCLUDE_DIR = libhsakmt/include/libhsakmt +export PACKAGE_DIR -deb: lnx64a - @mkdir -p $(INSTALL_LIB_DIR) - @mkdir -p $(INSTALL_INCLUDE_DIR) - @mkdir -p $(LDCONF_INSTALL_DIR) - @cp -r $(INCLUDE_DIR)/* $(INSTALL_INCLUDE_DIR) - @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_LIB_DIR) - @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) +package-common: lnx64a + @mkdir -p $(PACKAGE_DIR) @cp -r $(DEBIAN_DIR) $(PACKAGE_DIR) + + @mkdir -p $(LDCONF_INSTALL_DIR) + @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) + + @mkdir -p $(ROCM_ROOT)/$(INSTALL_LIB_DIR) + @cp $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_ROOT)/$(INSTALL_LIB_DIR) + + @mkdir -p $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) + @cp -r $(INCLUDE_DIR)/* $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) + + @mkdir -p $(ROCM_LIB) + @ln -fs ../$(INSTALL_LIB_DIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_LIB)/$(LIB_NAME) + + @mkdir -p $(ROCM_INCLUDE) + ln -fs ../$(INSTALL_INCLUDE_DIR) $(ROCM_INCLUDE)/ + +deb: package-common @fakeroot dpkg-deb --build $(PACKAGE_DIR) $(BUILDDIR)/libhsakmt.deb -rpm: lnx64a - @mkdir -p $(INSTALL_LIB_DIR) - @mkdir -p $(INSTALL_INCLUDE_DIR) - @mkdir -p $(LDCONF_INSTALL_DIR) - @cp -r $(INCLUDE_DIR)/* $(INSTALL_INCLUDE_DIR) - @cp $(BUILDDIR)/libhsakmt.so* $(INSTALL_LIB_DIR) - @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) +rpm: package-common @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec lnx64a: $(BUILDDIR)/$(LIB_NAME) From 44572965f683368f290814e47f1b57d3ece18f19 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Wed, 13 Apr 2016 17:41:25 -0400 Subject: [PATCH 0171/1247] package: rename to hsathk-rocm-dev Since we include headers and not just a library anymore, we should be considered a -dev package and not a lib package. Change-Id: I220465ea4ffc8d66d8d76e6716e6c6c50cdacea1 --- DEBIAN/control | 4 ++-- RPM/libhsakmt.spec | 4 ++-- src/Makefile | 12 ++++++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/DEBIAN/control b/DEBIAN/control index 37474f0f19..750ebb4ba1 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -1,8 +1,8 @@ -Package: libhsakmt +Package: hsakmt-rocm-dev Architecture: amd64 Maintainer: Advanced Micro Devices Inc. Depends: Priority: optional -Version: 2.0.0 +Version: $version Description: Thunk library for AMD KFD diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index 77d7ed1be1..9e32e53426 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -1,5 +1,5 @@ -%define name libhsakmt -%define version 2.0.0 +%define name hsakmt-rocm-dev +%define version %{getenv:PACKAGE_VER} %define packageroot %{getenv:PACKAGE_DIR} Name: %{name} diff --git a/src/Makefile b/src/Makefile index 6d4c89cbe1..84a5dcea15 100644 --- a/src/Makefile +++ b/src/Makefile @@ -9,6 +9,12 @@ CFLAGS += $(foreach DIR,$(INCLUDES),-I$(DIR)) LIB_NAME = libhsakmt.so LIB_MAJOR_VER = 1 +PACKAGE_MAJOR_VER = 2 +PACKAGE_MINOR_VER = 0 +PACKAGE_PATCH_VER ?= 0 +PACKAGE_VER = $(PACKAGE_MAJOR_VER).$(PACKAGE_MINOR_VER).$(PACKAGE_PATCH_VER) +export PACKAGE_VER + # Compiler options CFLAGS += -fPIC # Position-independent code required to build shared library CFLAGS += -W -Wall -Wextra -Werror -Wno-unused-parameter @@ -71,7 +77,6 @@ export PACKAGE_DIR package-common: lnx64a @mkdir -p $(PACKAGE_DIR) - @cp -r $(DEBIAN_DIR) $(PACKAGE_DIR) @mkdir -p $(LDCONF_INSTALL_DIR) @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) @@ -89,7 +94,10 @@ package-common: lnx64a ln -fs ../$(INSTALL_INCLUDE_DIR) $(ROCM_INCLUDE)/ deb: package-common - @fakeroot dpkg-deb --build $(PACKAGE_DIR) $(BUILDDIR)/libhsakmt.deb + @mkdir -p $(PACKAGE_DIR)/DEBIAN + @sed 's/\$$version/$(PACKAGE_VER)/g' $(DEBIAN_DIR)/control > $(PACKAGE_DIR)/DEBIAN/control + @fakeroot dpkg-deb --build $(PACKAGE_DIR) \ + $(BUILDDIR)/hsakmt-dev-$(PACKAGE_VER)-amd64.deb rpm: package-common @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec From 97e51ce33d6b841b5b0e34a7529fe2d5bafbfecf Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 25 Apr 2016 14:27:44 -0400 Subject: [PATCH 0172/1247] Add gfx70x support Change-Id: I400adb62b5225ef3a42da279d067fb0a62907089 --- src/queues.c | 24 +++++++++++++++++++++++- src/topology.c | 13 +++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 0867548e7d..505d8dbd0f 100644 --- a/src/queues.c +++ b/src/queues.c @@ -39,13 +39,15 @@ enum asic_family_type { CHIP_KAVERI = 0, + CHIP_HAWAII, CHIP_CARRIZO, CHIP_TONGA, CHIP_FIJI }; #define IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_FIJI) -#define IS_DGPU(chip) ((chip) >= CHIP_TONGA && (chip) <= CHIP_FIJI) +#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_FIJI) || \ + (chip) == CHIP_HAWAII) struct device_info { @@ -62,6 +64,13 @@ struct device_info kaveri_device_info = { .eop_buffer_size = 0, }; +struct device_info hawaii_device_info = { + .asic_family = CHIP_HAWAII, + .ctx_save_restore_size = 0, + .ctl_stack_size = 0, + .eop_buffer_size = 0, +}; + struct device_info carrizo_device_info = { .asic_family = CHIP_CARRIZO, .ctx_save_restore_size = PAGE_ALIGN_UP(2756608 + 4096), @@ -90,6 +99,7 @@ struct device_id struct device_info *dev_info; }; +/* TODO: unify this with the device list in topology.c */ struct device_id supported_devices[] = { { 0x1304, &kaveri_device_info }, /* Kaveri */ { 0x1305, &kaveri_device_info }, /* Kaveri */ @@ -113,6 +123,18 @@ struct device_id supported_devices[] = { { 0x131B, &kaveri_device_info }, /* Kaveri */ { 0x131C, &kaveri_device_info }, /* Kaveri */ { 0x131D, &kaveri_device_info }, /* Kaveri */ + { 0x67A0, &hawaii_device_info }, /* Hawaii */ + { 0x67A1, &hawaii_device_info }, /* Hawaii */ + { 0x67A2, &hawaii_device_info }, /* Hawaii */ + { 0x67A8, &hawaii_device_info }, /* Hawaii */ + { 0x67A9, &hawaii_device_info }, /* Hawaii */ + { 0x67AA, &hawaii_device_info }, /* Hawaii */ + { 0x67B0, &hawaii_device_info }, /* Hawaii */ + { 0x67B1, &hawaii_device_info }, /* Hawaii */ + { 0x67B8, &hawaii_device_info }, /* Hawaii */ + { 0x67B9, &hawaii_device_info }, /* Hawaii */ + { 0x67BA, &hawaii_device_info }, /* Hawaii */ + { 0x67BE, &hawaii_device_info }, /* Hawaii */ { 0x9870, &carrizo_device_info }, /* Carrizo */ { 0x9874, &carrizo_device_info }, /* Carrizo */ { 0x9875, &carrizo_device_info }, /* Carrizo */ diff --git a/src/topology.c b/src/topology.c index a1f53599bb..a9a75869a8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -106,6 +106,19 @@ static struct hsa_gfxip_table { { 0x131B, 7, 0, 0, 0, "Spectre" }, { 0x131C, 7, 0, 0, 0, "Spectre" }, { 0x131D, 7, 0, 0, 0, "Spectre" }, + /* Hawaii Family */ + { 0x67A0, 7, 0, 0, 1, "Hawaii" }, + { 0x67A1, 7, 0, 0, 1, "Hawaii" }, + { 0x67A2, 7, 0, 0, 1, "Hawaii" }, + { 0x67A8, 7, 0, 0, 1, "Hawaii" }, + { 0x67A9, 7, 0, 0, 1, "Hawaii" }, + { 0x67AA, 7, 0, 0, 1, "Hawaii" }, + { 0x67B0, 7, 0, 0, 1, "Hawaii" }, + { 0x67B1, 7, 0, 0, 1, "Hawaii" }, + { 0x67B8, 7, 0, 0, 1, "Hawaii" }, + { 0x67B9, 7, 0, 0, 1, "Hawaii" }, + { 0x67BA, 7, 0, 0, 1, "Hawaii" }, + { 0x67BE, 7, 0, 0, 1, "Hawaii" }, /* Carrizo Family */ { 0x9870, 8, 0, 1, 0, "Carrizo" }, { 0x9874, 8, 0, 1, 0, "Carrizo" }, From 73ad0a1942984d621930228462fd828dad4cb21d Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 15 Apr 2016 15:44:47 -0400 Subject: [PATCH 0173/1247] Correct NumCaches against the CPU node In a NUMA system, topology should report NumCaches as the number of caches within the node but current code reports the total caches in the system. This patch fixes the error. This patch also uses cpuid to get cache information instead of reading from sysfs files. See "Intel Corporation, Intel 64 and IA-32 Architectures Software Developer's Manual Volume 2(2A, 2B & 2C) Instruction Set Reference" 3-179 for cpuid instruction features used in this patch. Change-Id: I8ecece6c2b230741822620b44e66ddc201ff5112 --- src/topology.c | 549 +++++++++++++++++++++++++++++++------------------ 1 file changed, 354 insertions(+), 195 deletions(-) diff --git a/src/topology.c b/src/topology.c index a9a75869a8..2d2d82ff2b 100644 --- a/src/topology.c +++ b/src/topology.c @@ -23,6 +23,7 @@ * DEALINGS IN THE SOFTWARE. */ +#define _GNU_SOURCE #include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include #include "libhsakmt.h" #include "fmm.h" @@ -45,8 +47,6 @@ #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" #define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" #define PROC_CPUINFO_PATH "/proc/cpuinfo" -#define MAX_CPU_CORES 128 -#define MAX_CACHES 256 typedef struct { uint32_t gpu_id; @@ -139,6 +139,53 @@ static struct hsa_gfxip_table { { 0x7300, 8, 0, 3, 1, "Fiji" } }; +enum cache_type { + CACHE_TYPE_NULL = 0, + CACHE_TYPE_DATA = 1, + CACHE_TYPE_INST = 2, + CACHE_TYPE_UNIFIED = 3 +}; + +typedef struct cacheinfo { + HsaCacheProperties hsa_cache_prop; + uint32_t num_threads_sharing; /* how many CPUs share this cache */ +} cacheinfo_t; + +/* CPU cache table for all CPUs on the system. Each entry has the relative CPU + * info and caches connected to that CPU. + */ +typedef struct cpu_cacheinfo { + uint32_t len; /* length of the table -> number of online procs */ + uint32_t num_caches; /* number of caches connected to this cpu */ + uint32_t num_duplicated_caches; /* to count caches being shared */ + uint32_t apicid; /* this cpu's apic id */ + uint32_t max_num_apicid; /* max number of addressable IDs */ + cacheinfo_t *cache_info; /* an array for cache information */ +} cpu_cacheinfo_t; + +/* Deterministic Cache Parameters Leaf in cpuid */ +union _cpuid_leaf_eax { /* Register EAX */ + struct { + enum cache_type type:5; + uint32_t level:3; + uint32_t is_self_initializing:1; + uint32_t is_fully_associative:1; + uint32_t reserved:4; + uint32_t num_threads_sharing:12; + uint32_t num_cores_on_die:6; + } split; + uint32_t full; +}; + +union _cpuid_leaf_ebx { /* Register EBX */ + struct { + uint32_t coherency_line_size:12; + uint32_t physical_line_partition:10; + uint32_t ways_of_associativity:10; + } split; + uint32_t full; +}; + static void free_node(node_t *n) { @@ -195,25 +242,60 @@ static int num_subdirs(char *dirpath, char *prefix) return count; } -/* read_file - Read the content of a file - * @file - file to read - * @buf - [OUT] buffer containing data read from the file - * @buf_sz - buffer size - * Return - data size in the returning buffer +/* cpuid instruction returns processor identification and feature information + * to the EAX, EBX, ECX, and EDX registers, as determined by input entered in + * EAX (in some cases, ECX as well). */ -static size_t read_file(char *file, char *buf, size_t buf_sz) +static inline void +cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) { - int fd; - size_t len = 0; + __asm__ __volatile__( + "cpuid;" + : "=a" (*eax), + "=b" (*ebx), + "=c" (*ecx), + "=d" (*edx) + : "0" (*eax), "2" (*ecx) + : "memory" + ); +} - memset(buf, 0, buf_sz); +/* In cases ECX is also used as an input for cpuid, i.e. cache leaf */ +static void cpuid_count(uint32_t op, int count, uint32_t *eax, uint32_t *ebx, + uint32_t *ecx, uint32_t *edx) +{ + *eax = op; + *ecx = count; + cpuid(eax, ebx, ecx, edx); +} - if ((fd = open(file, O_RDONLY)) < 0) - return 0; - len = read(fd, buf, buf_sz); - close(fd); +/* Lock current process to the specified processor */ +static int lock_to_processor(int processor) +{ + cpu_set_t cpuset; + memset(&cpuset, 0, sizeof(cpu_set_t)); + CPU_SET(processor, &cpuset); + /* 0: this process */ + return sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); +} - return len; +/* Get count's order of 2. In other words, 2^rtn_value = count + * When count is not an order of 2, round it up to the closest. + */ +static int get_count_order(unsigned int count) +{ + int bit; + uint32_t num; + + for (bit = 31; bit >= 0; bit--) { + num = 1 << bit; + if (count >= num) + break; + } + if (count & (count - 1)) + ++bit; + + return bit; } static HSAKMT_STATUS @@ -649,207 +731,282 @@ err1: return ret; } - -/* parse_sysfs_cache - - * @sys_path - cache path in sysfs - * @prop - [OUT] HSA cache property to fill up - * @apicid - an array where contains each processor's apicid - * Return - HSAKMT_STATUS_SUCCESS in success or an error number in failure - */ -static HSAKMT_STATUS -parse_sysfs_cache(char *sys_path, HsaCacheProperties *prop, uint32_t *apicid) +/* cpuid_find_num_cache_leaves - Use cpuid instruction to find out how many + * cache leaves the CPU has. + * @op - cpuid opcode to get cache information + * Return - the number of cache leaves +*/ +static int cpuid_find_num_cache_leaves(uint32_t op) { - char file[256], buf[256]; - int i, j, n, cpu; - int last; /* the last valid entry in array, which is n-1 if n items */ - unsigned long map[32]; - char *token, *str; + union _cpuid_leaf_eax eax; + union _cpuid_leaf_ebx ebx; + unsigned int ecx; + unsigned int edx; + int idx = -1; - /* cache level */ - snprintf(file, 256, "%s/level", sys_path); - read_file(file, buf, sizeof(buf)); - prop->CacheLevel = atoi(buf); + do { + ++idx; + cpuid_count(op, idx, &eax.full, &ebx.full, &ecx, &edx); + } while (eax.split.type != CACHE_TYPE_NULL); + return idx; +} - /* cache size */ - snprintf(file, 256, "%s/size", sys_path); - read_file(file, buf, sizeof(buf)); - prop->CacheSize = (atoi(buf)); +/* cpuid_get_cpu_cache_info - Use cpuid instruction to get cache information + * @op - cpuid opcode to get cache information + * @cpu_ci - this parameter is an input and also an output. + * [IN] cpu_ci->num_caches: the number of caches of this cpu + * [OUT] cpu_ci->cache_info: to store cache info collected + */ +static void cpuid_get_cpu_cache_info(uint32_t op, cpu_cacheinfo_t *cpu_ci) +{ + union _cpuid_leaf_eax eax; + union _cpuid_leaf_ebx ebx; + uint32_t ecx; + uint32_t edx; + uint32_t index; + cacheinfo_t *this_leaf; - /* cache line size in bytes */ - snprintf(file, 256, "%s/coherency_line_size", sys_path); - read_file(file, buf, sizeof(buf)); - prop->CacheLineSize = (atoi(buf)); - - /* cache lines per tag */ - snprintf(file, 256, "%s/physical_line_partition", sys_path); - read_file(file, buf, sizeof(buf)); - prop->CacheLinesPerTag = (atoi(buf)); - - /* cache associativity */ - snprintf(file, 256, "%s/ways_of_associativity", sys_path); - read_file(file, buf, sizeof(buf)); - prop->CacheAssociativity = (atoi(buf)); - - /* cache type */ - prop->CacheType.ui32.CPU = 1; - snprintf(file, 256, "%s/type", sys_path); - read_file(file, buf, sizeof(buf)); - if (buf[0] == 'D') - prop->CacheType.ui32.Data = 1; - else if (buf[0] == 'I') - prop->CacheType.ui32.Instruction = 1; - - /* sibling map */ - snprintf(file, 256, "%s/shared_cpu_map", sys_path); - read_file(file, buf, sizeof(buf)); - /* Data in shared_cpu_map can be XXXXXXXX when the system doesn't have - * more than 32 processors; it also can be XXXXXXXX,XXXXXXXX,XX .... to - * represent more than 32 processors. We'll parse each XXXXXXXX and - * store them into map[]. - * Say shared_cpu_map is "Nn-1,Nn-2,...,N2,N1,N0\n". Because strtok_r() - * parses Nn-1 first, map[] will store data in a reversed order: - * map[0]=Nn-1, map[1]=Nn-2, ... map[n-2]=N1, map[n-1]=N0 - */ - str = (char *)&buf[0]; - for (last = 0; last < 32; last++) { /* declared map[32] */ - token = strtok_r(str, ",", &str); - map[last] = strtol(token, NULL, 16); - if (token[strlen(token)-1] == '\n') /* this is N0 */ - break; - } - if (last >= 32) { - printf("Fail to parse shared_cpu_map. Increase map[].\n"); - return HSAKMT_STATUS_ERROR; + for (index = 0; index < cpu_ci->num_caches; index++) { + cpuid_count(op, index, &eax.full, &ebx.full, &ecx, &edx); + this_leaf = cpu_ci->cache_info + index; + this_leaf->hsa_cache_prop.ProcessorIdLow = cpu_ci->apicid; + this_leaf->num_threads_sharing = + eax.split.num_threads_sharing + 1; + this_leaf->hsa_cache_prop.CacheLevel = eax.split.level; + this_leaf->hsa_cache_prop.CacheType.ui32.CPU = 1; + if (eax.split.type & CACHE_TYPE_DATA ) + this_leaf->hsa_cache_prop.CacheType.ui32.Data = 1; + if (eax.split.type & CACHE_TYPE_INST ) + this_leaf->hsa_cache_prop.CacheType.ui32.Instruction = 1; + this_leaf->hsa_cache_prop.CacheLineSize = + ebx.split.coherency_line_size + 1; + this_leaf->hsa_cache_prop.CacheAssociativity = + ebx.split.ways_of_associativity + 1; + this_leaf->hsa_cache_prop.CacheLinesPerTag = + ebx.split.physical_line_partition + 1; + this_leaf->hsa_cache_prop.CacheSize = (ecx + 1) * + (ebx.split.coherency_line_size + 1) * + (ebx.split.physical_line_partition + 1) * + (ebx.split.ways_of_associativity + 1); } +} - /* Lower processor ID doesn't always have lower apicid. - * Search the lowest apicid for ProcIdLow - */ - prop->ProcessorIdLow = 0xffffffff; - for (i = last; i >= 0; i--) { /* N0 is stored in map[count] */ - if (!map[i]) - continue; - j = 32; - while (j-- > 0) { - if (map[i] & (1<ProcessorIdLow) - prop->ProcessorIdLow = apicid[cpu]; - } - } - } - /* Now fill in SiblingMap using ProcIdLow as the offset */ - for (i = last; i >= 0; i--) { - j = 32; - while (j-- > 0) { - if (map[i] & (1<ProcessorIdLow; - /* Use array instead of bitmask so the software - * is endian-worry free +/* find_cpu_cache_siblings - In the cache list, some caches may be listed more + * than once if they are shared by multiple CPUs. Identify the cache's CPU + * siblings, record it to SiblingMap[], then remove the duplicated cache by + * changing the cache size to 0. + */ +static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) +{ + cacheinfo_t *this_leaf, *leaf2; + uint32_t n, j, idx_msb, apicid1, apicid2; + cpu_cacheinfo_t *this_cpu, *cpu2; + uint32_t index; + + for (n = 0; n < cpu_ci_list->len; n++) { + this_cpu = cpu_ci_list + n; + for (index = 0; index < this_cpu->num_caches; index++) { + this_leaf = this_cpu->cache_info + index; + /* CacheSize 0 means an invalid cache */ + if (!this_leaf->hsa_cache_prop.CacheSize) + continue; + if (this_leaf->num_threads_sharing == 1) // no siblings + continue; + idx_msb = get_count_order(this_leaf->num_threads_sharing); + for (j = n + 1; j < cpu_ci_list->len; j++) { + cpu2 = cpu_ci_list + j; + leaf2 = cpu2->cache_info + index; + apicid1 = this_leaf->hsa_cache_prop.ProcessorIdLow; + apicid2 = leaf2->hsa_cache_prop.ProcessorIdLow; + if ((apicid2 >> idx_msb) != (apicid1 >> idx_msb)) + continue; + /* A sibling leaf is found. Cache properties + * use ProcIdLow as offset to represent siblings + * in SiblingMap, so keep the lower apicid and + * delete the other by changing CacheSize to 0. */ - if (n < HSA_CPU_SIBLINGS) - prop->SiblingMap[n] = 1; + if (apicid1 < apicid2) { + this_leaf->hsa_cache_prop.SiblingMap[0] = 1; + this_leaf->hsa_cache_prop.SiblingMap[apicid2 - apicid1] = 1; + leaf2->hsa_cache_prop.CacheSize = 0; + cpu2->num_duplicated_caches++; + } else { - printf("Increase HSA_CPU_SIBLINGS.\n"); - return HSAKMT_STATUS_ERROR; + leaf2->hsa_cache_prop.SiblingMap[0] = 1; + leaf2->hsa_cache_prop.SiblingMap[apicid1 - apicid2] = 1; + this_leaf->hsa_cache_prop.CacheSize = 0; + this_cpu->num_duplicated_caches++; } } } } - - return HSAKMT_STATUS_SUCCESS; } -/* topology_get_cpu_cache_props - get CPU cache properties and fill in the - * cache entry of the node's table - * @tbl - the node table to fill up +/* topology_destroy_temp_cpu_cache_list - Free the memory allocated in + * topology_create_temp_cpu_cache_list(). + */ +static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) +{ + uint32_t n; + cpu_cacheinfo_t *p_temp_cpu_ci_list = (cpu_cacheinfo_t *)temp_cpu_ci_list; + cpu_cacheinfo_t *this_cpu; + + if (p_temp_cpu_ci_list) { + for (n = 0; n < p_temp_cpu_ci_list->len; n++) { + this_cpu = p_temp_cpu_ci_list + n; + if (this_cpu->cache_info) + free(this_cpu->cache_info); + } + free(p_temp_cpu_ci_list); + } + + p_temp_cpu_ci_list = NULL; +} + +/* topology_create_temp_cpu_cache_list - Create a temporary cpu-cache list to + * store cpu cache information. This list will be used to copy + * cache information to each CPU node. Must call + * topology_destroy_temp_cpu_cache_list to free the memory after + * the information is copied. + * @temp_cpu_ci_list - [OUT] temporary cpu-cache-info list to store data * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure */ static HSAKMT_STATUS -topology_get_cpu_cache_props(node_t *tbl) +topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) { - FILE *fd; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - char *token, *str; - uint32_t apicid[MAX_CPU_CORES]; - int num_cpus = 0, num_caches = 0; - char path[256], buf[256], cache_paths[MAX_CACHES][256]; - int i, j, n; - const char SYSDIR[] = "/sys/devices/system/cpu"; + void *p_temp_cpu_ci_list; + int procs_online; + cpu_set_t orig_cpuset; + int i; + uint32_t cpuid_op_cache; + uint32_t eax, ebx, ecx = 0, edx; /* cpuid registers */ + cpu_cacheinfo_t *cpu_ci_list, *this_cpu; - if (tbl == NULL) - return HSAKMT_STATUS_ERROR; - - /* Get apicid info from /proc/cpuinfo for ProcessorIdLow */ - if (!(fd = fopen("/proc/cpuinfo", "r"))) - return HSAKMT_STATUS_ERROR; - - while (fgets(buf, 256, fd) != NULL) { - /* /proc/cpuinfo lists in format - property : value */ - token = strtok_r(buf, ":", &str); - if (strncmp(token, "apicid", 6) == 0) { - if (num_cpus >= MAX_CPU_CORES) { - printf("MAX_CPU_CORES %d is not enough.", MAX_CPU_CORES); - fclose(fd); - return HSAKMT_STATUS_ERROR; - } - apicid[num_cpus++] = atoi(str); - } + if (!temp_cpu_ci_list) { + ret = HSAKMT_STATUS_ERROR; + goto exit; } - fclose(fd); + *temp_cpu_ci_list = NULL; - /* Get cache data from /sys/devices/system/cpu/cpuX/cache/indexY */ + procs_online = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (procs_online <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto exit; + } - /* 1. Calculate how many caches */ - for (i=0; i= MAX_CACHES) { - printf("MAX_CACHES %d is not enough!\n", MAX_CACHES); - return HSAKMT_STATUS_ERROR; - } - snprintf(cache_paths[num_caches++], 256, - "%s/cpu%d/cache/index%d", SYSDIR, i, j); - } - } - - /* 2. Allocate number of caches for the table */ - tbl->node.NumCaches = num_caches; - tbl->cache = calloc(tbl->node.NumCaches * sizeof(HsaCacheProperties), 1); - if (!tbl->cache) - return HSAKMT_STATUS_NO_MEMORY; - - /* 3. Fill up cache properties */ - for (i=0; icache[i], - &apicid[0]); - if (ret != HSAKMT_STATUS_SUCCESS) { - printf("Failed to parse cache properties.\n"); - free(tbl->cache); - return ret; + p_temp_cpu_ci_list = calloc(sizeof(cpu_cacheinfo_t) * procs_online, 1); + if (!p_temp_cpu_ci_list) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto exit; + } + + cpu_ci_list = (cpu_cacheinfo_t *)p_temp_cpu_ci_list; + cpu_ci_list->len = procs_online; + + if (processor_vendor == AUTHENTIC_AMD) + cpuid_op_cache = 0x8000001d; + else + cpuid_op_cache = 0x4; + + /* lock_to_processor() changes the affinity. Save the current affinity + * so we can restore it after cpuid is done. + */ + CPU_ZERO(&orig_cpuset); + if (sched_getaffinity(0, sizeof(cpu_set_t), &orig_cpuset) != 0) { + printf("Failed to get CPU affinity\n"); + free(p_temp_cpu_ci_list); + ret = HSAKMT_STATUS_ERROR; + goto exit; + } + + for (i = 0; i < procs_online; i++) { + this_cpu = cpu_ci_list + i; + lock_to_processor(i); /* so cpuid is executed in correct cpu */ + + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + this_cpu->apicid = (ebx >> 24) & 0xff; + this_cpu->max_num_apicid = (ebx >> 16) & 0x0FF; + this_cpu->num_caches = cpuid_find_num_cache_leaves(cpuid_op_cache); + this_cpu->num_duplicated_caches = 0; + this_cpu->cache_info = calloc( + sizeof(cacheinfo_t) * this_cpu->num_caches, 1); + if (!this_cpu->cache_info) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err; + } + cpuid_get_cpu_cache_info(cpuid_op_cache, this_cpu); + } + + find_cpu_cache_siblings(cpu_ci_list); + *temp_cpu_ci_list = p_temp_cpu_ci_list; + +err: + /* restore affinity to original */ + sched_setaffinity(0, sizeof(cpu_set_t), &orig_cpuset); +exit: + if (ret != HSAKMT_STATUS_SUCCESS) + topology_destroy_temp_cpu_cache_list(&temp_cpu_ci_list); + return ret; +} + +/* topology_get_cpu_cache_props - Read CPU cache information from the temporary + * cache list and put them to the node's cache properties entry. + * @tbl - the node table to fill up + * @cpu_ci_list - the cpu cache information list to look up cache info + * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure + */ +static HSAKMT_STATUS +topology_get_cpu_cache_props(node_t *tbl, cpu_cacheinfo_t *cpu_ci_list) +{ + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t apicid_low = tbl->node.CComputeIdLo, apicid_max = 0; + uint32_t n, cache_cnt, idx; + cpu_cacheinfo_t *this_cpu; + cacheinfo_t *this_leaf; + + /* CPU cache info list contains all CPUs. Find out CPUs belonging to + * this node and number of caches under, so we can allocate the cache + * properties in the node. + */ + tbl->node.NumCaches = 0; + for (n = 0; n < cpu_ci_list->len; n++) { + this_cpu = cpu_ci_list + n; + if (this_cpu->apicid == apicid_low) + /* found the first cpu in the node */ + apicid_max = apicid_low + this_cpu->max_num_apicid - 1; + + if ((this_cpu->apicid < apicid_low) || + (this_cpu->apicid > apicid_max)) + continue; /* this cpu doesn't belong to the node */ + tbl->node.NumCaches += + this_cpu->num_caches - this_cpu->num_duplicated_caches; + } + + tbl->cache = calloc( + sizeof(HsaCacheProperties) * tbl->node.NumCaches, 1); + if (!tbl->cache) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto exit; + } + + /* Now fill in the information to cache properties. */ + cache_cnt = 0; + for (n = 0; n < cpu_ci_list->len; n++) { + this_cpu = cpu_ci_list + n; + if ((this_cpu->apicid < apicid_low) || this_cpu->apicid > apicid_max) + continue; /* this cpu doesn't belong to the node */ + for (idx = 0; idx < this_cpu->num_caches; idx++) { + this_leaf = this_cpu->cache_info + idx; + if (this_leaf->hsa_cache_prop.CacheSize > 0) + memcpy(&tbl->cache[cache_cnt++], &this_leaf->hsa_cache_prop, sizeof(HsaCacheProperties)); + if (cache_cnt >= tbl->node.NumCaches) + goto exit; } } +exit: return ret; } @@ -1189,6 +1346,7 @@ topology_take_snapshot(void) uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; HsaSystemProperties sys_props; node_t *temp_nodes = 0; + void *cpu_ci_list = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; topology_set_processor_vendor(); @@ -1200,6 +1358,7 @@ retry: if (ret != HSAKMT_STATUS_SUCCESS) return ret; if(sys_props.NumNodes > 0) { + topology_create_temp_cpu_cache_list(&cpu_ci_list); temp_nodes = calloc(sys_props.NumNodes * sizeof(node_t),1); if (!temp_nodes) return HSAKMT_STATUS_NO_MEMORY; @@ -1242,9 +1401,9 @@ retry: } } } - else if (!temp_nodes[i].gpu_id) { /* This is a CPU node */ - /* Get info from /proc/cpuinfo and /sys/devices/system */ - ret = topology_get_cpu_cache_props(&temp_nodes[i]); + else if (!temp_nodes[i].gpu_id) { /* a CPU node */ + ret = topology_get_cpu_cache_props( + &temp_nodes[i], cpu_ci_list); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, i + 1); goto err; @@ -1332,7 +1491,7 @@ retry: free(node); node = temp_nodes; err: - + topology_destroy_temp_cpu_cache_list(cpu_ci_list); return ret; } From fa102f3b8bad55a10f0a0fcc5765fa38f28716c0 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 4 May 2016 13:44:09 -0400 Subject: [PATCH 0174/1247] Report gfx70x engine ID as 7.0.1 Stepping 1 indicates higher double-precision float performance and potentially other runtime workarounds needed for lack of PCIe atomics on gfx70x. Change-Id: I97185c1233e7d24caaf20a1eadea931d5a2bc664 --- src/topology.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/topology.c b/src/topology.c index 2d2d82ff2b..605da25985 100644 --- a/src/topology.c +++ b/src/topology.c @@ -107,18 +107,18 @@ static struct hsa_gfxip_table { { 0x131C, 7, 0, 0, 0, "Spectre" }, { 0x131D, 7, 0, 0, 0, "Spectre" }, /* Hawaii Family */ - { 0x67A0, 7, 0, 0, 1, "Hawaii" }, - { 0x67A1, 7, 0, 0, 1, "Hawaii" }, - { 0x67A2, 7, 0, 0, 1, "Hawaii" }, - { 0x67A8, 7, 0, 0, 1, "Hawaii" }, - { 0x67A9, 7, 0, 0, 1, "Hawaii" }, - { 0x67AA, 7, 0, 0, 1, "Hawaii" }, - { 0x67B0, 7, 0, 0, 1, "Hawaii" }, - { 0x67B1, 7, 0, 0, 1, "Hawaii" }, - { 0x67B8, 7, 0, 0, 1, "Hawaii" }, - { 0x67B9, 7, 0, 0, 1, "Hawaii" }, - { 0x67BA, 7, 0, 0, 1, "Hawaii" }, - { 0x67BE, 7, 0, 0, 1, "Hawaii" }, + { 0x67A0, 7, 0, 1, 1, "Hawaii" }, + { 0x67A1, 7, 0, 1, 1, "Hawaii" }, + { 0x67A2, 7, 0, 1, 1, "Hawaii" }, + { 0x67A8, 7, 0, 1, 1, "Hawaii" }, + { 0x67A9, 7, 0, 1, 1, "Hawaii" }, + { 0x67AA, 7, 0, 1, 1, "Hawaii" }, + { 0x67B0, 7, 0, 1, 1, "Hawaii" }, + { 0x67B1, 7, 0, 1, 1, "Hawaii" }, + { 0x67B8, 7, 0, 1, 1, "Hawaii" }, + { 0x67B9, 7, 0, 1, 1, "Hawaii" }, + { 0x67BA, 7, 0, 1, 1, "Hawaii" }, + { 0x67BE, 7, 0, 1, 1, "Hawaii" }, /* Carrizo Family */ { 0x9870, 8, 0, 1, 0, "Carrizo" }, { 0x9874, 8, 0, 1, 0, "Carrizo" }, From 16d5aa0d8397fd91ce720f5005205c306b542ed8 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Mon, 9 May 2016 16:02:00 -0400 Subject: [PATCH 0175/1247] libhsakmt: Add new device id for virtualized function of gfx803 Signed-off-by: Shaoyun Liu Change-Id: I90b0bdaeaed8e9e80375e5a7a142205f2a542289 --- src/queues.c | 1 + src/topology.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 505d8dbd0f..9c1acdd01d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -150,6 +150,7 @@ struct device_id supported_devices[] = { { 0x6938, &tonga_device_info }, { 0x6939, &tonga_device_info }, { 0x7300, &fiji_device_info }, + { 0x730f, &fiji_device_info }, { 0, NULL } }; diff --git a/src/topology.c b/src/topology.c index 605da25985..409ffdc5e8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -136,7 +136,8 @@ static struct hsa_gfxip_table { { 0x6938, 8, 0, 2, 1, "Tonga" }, { 0x6939, 8, 0, 2, 1, "Tonga" }, /* Fiji */ - { 0x7300, 8, 0, 3, 1, "Fiji" } + { 0x7300, 8, 0, 3, 1, "Fiji" }, + { 0x730f, 8, 0, 3, 1, "Fiji" } }; enum cache_type { From 6d21c4e753f3179d8c6699f61865f6ca6d6c93b2 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 20 May 2016 16:24:36 -0400 Subject: [PATCH 0176/1247] Add performance counters for gfx70x Add performance counters for gfx70x. The reference is the gfx7 register spec. The register being looked at is SQ_PERFCOUNTER0_SELECT. Change-Id: I344bfb7452f6148f4dc268163d12c553c6be8424 --- src/pmc_table.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 38a2013474..b9ae4315b6 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -86,6 +86,25 @@ static uint32_t fiji_sq_counter_ids[] = { 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; +/* Unused counters - 163 - 166, 167 and 251 are *_DUMMY_LAST */ +static uint32_t hawaii_sq_counter_ids[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, + 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, 161, 162, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, + 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, + 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, + 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, + 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -116,6 +135,17 @@ static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = + sizeof(hawaii_sq_counter_ids) / sizeof(*hawaii_sq_counter_ids), + .counter_ids = hawaii_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + HSAKMT_STATUS get_block_properties(uint16_t dev_id, enum perf_block_id block_id, @@ -162,6 +192,22 @@ get_block_properties(uint16_t dev_id, case 0x7300: *block = fiji_blocks[block_id]; break; + + case 0x67A0: + case 0x67A1: + case 0x67A2: + case 0x67A8: + case 0x67A9: + case 0x67AA: + case 0x67B0: + case 0x67B1: + case 0x67B8: + case 0x67B9: + case 0x67BA: + case 0x67BE: + *block = hawaii_blocks[block_id]; + break; + default: rc = HSAKMT_STATUS_INVALID_PARAMETER; } From bf16caa75f5709fd5a8d7b867bbf1cddeb6cb124 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Wed, 8 Jun 2016 17:23:51 -0400 Subject: [PATCH 0177/1247] libhsakmt: Compute context save area size depends on CU num Change-Id: Iaf35ddeee9fe5a6367097483f67c4adaa0796d7d Signed-off-by: shaoyunl --- src/queues.c | 53 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/queues.c b/src/queues.c index 9c1acdd01d..3118d2a12c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -49,47 +49,37 @@ enum asic_family_type { #define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_FIJI) || \ (chip) == CHIP_HAWAII) +#define WG_CONTEXT_DATA_SIZE_PER_CU_VI 344576 +#define WAVES_PER_CU_VI 32 + struct device_info { enum asic_family_type asic_family; - uint32_t ctx_save_restore_size; - uint32_t ctl_stack_size; uint32_t eop_buffer_size; }; struct device_info kaveri_device_info = { .asic_family = CHIP_KAVERI, - .ctx_save_restore_size = 0, - .ctl_stack_size = 0, .eop_buffer_size = 0, }; struct device_info hawaii_device_info = { .asic_family = CHIP_HAWAII, - .ctx_save_restore_size = 0, - .ctl_stack_size = 0, .eop_buffer_size = 0, }; struct device_info carrizo_device_info = { .asic_family = CHIP_CARRIZO, - .ctx_save_restore_size = PAGE_ALIGN_UP(2756608 + 4096), - .ctl_stack_size = 4096, .eop_buffer_size = 4096, }; struct device_info tonga_device_info = { .asic_family = CHIP_TONGA, - .ctx_save_restore_size = PAGE_ALIGN_UP(8269824 + PAGE_ALIGN_UP(6152)), - .ctl_stack_size = PAGE_ALIGN_UP(6152), .eop_buffer_size = TONGA_PAGE_SIZE, }; struct device_info fiji_device_info = { .asic_family = CHIP_FIJI, - /*SR size = work group ctx data + ctl stack size*/ - .ctx_save_restore_size = PAGE_ALIGN_UP(20674560 + PAGE_ALIGN_UP(15368)), - .ctl_stack_size = PAGE_ALIGN_UP(15368), .eop_buffer_size = TONGA_PAGE_SIZE, }; @@ -161,6 +151,8 @@ struct queue uint32_t rptr; void *eop_buffer; void *ctx_save_restore; + uint32_t ctx_save_restore_size; + uint32_t ctl_stack_size; const struct device_info *dev_info; }; @@ -240,6 +232,29 @@ static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) return ptr; } +/* The bool return indicate whether the queue needs a context-save-restore area*/ +static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) +{ + HsaNodeProperties node; + + if (q->dev_info->asic_family < CHIP_CARRIZO) + return false; + if (hsaKmtGetNodeProperties(nodeid, &node)) + return false; + if (node.NumFComputeCores && node.NumSIMDPerCU) { + uint32_t ctl_stack_size, wg_data_size; + uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; + + ctl_stack_size = cu_num * WAVES_PER_CU_VI * 8 + 8; + wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU_VI; + q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size); + q->ctx_save_restore_size = + q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size); + return true; + } + return false; +} + void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId) { @@ -313,7 +328,7 @@ static void free_queue_gpu(struct queue *q) } if (q->ctx_save_restore) { hsaKmtUnmapMemoryToGPU(q->ctx_save_restore); - hsaKmtFreeMemory(q->ctx_save_restore, q->dev_info->ctx_save_restore_size); + hsaKmtFreeMemory(q->ctx_save_restore, q->ctx_save_restore_size); } free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } @@ -344,14 +359,14 @@ static int handle_concrete_asic(struct queue *q, args->eop_buffer_size = dev_info->eop_buffer_size; } if (args->queue_type != KFD_IOC_QUEUE_TYPE_SDMA && - dev_info->ctx_save_restore_size > 0) { - args->ctx_save_restore_size = dev_info->ctx_save_restore_size; - args->ctl_stack_size = dev_info->ctl_stack_size; + update_ctx_save_restore_size(NodeId, q) == true) { + args->ctx_save_restore_size = q->ctx_save_restore_size; + args->ctl_stack_size = q->ctl_stack_size; if (IS_DGPU(dev_info->asic_family)) { void *mem; HsaMemFlags flags; HSAKMT_STATUS ret; - HSAuint64 size = dev_info->ctx_save_restore_size; + HSAuint64 size = q->ctx_save_restore_size; flags.Value = 0; flags.ui32.NonPaged = 1; /* device memory*/ @@ -367,7 +382,7 @@ static int handle_concrete_asic(struct queue *q, } else q->ctx_save_restore = - allocate_exec_aligned_memory(dev_info->ctx_save_restore_size, + allocate_exec_aligned_memory(q->ctx_save_restore_size, PAGE_SIZE, dev_info->asic_family, NodeId); From 6c4d19a9d222f7ed0048df22e3012490647c2b81 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 27 Jul 2016 16:38:04 -0400 Subject: [PATCH 0178/1247] Shorten the device list in PerfCounter get_block_properties uses the complete DID to identify the GPU. This list is getting too long when more devices are added. Reading the 12 most significant digits is good enough to identify the GPU. Change-Id: Ieebb05402bbe08af12eb7289dfeb5bbf1f515b0f --- src/pmc_table.c | 52 +++++++++++-------------------------------------- 1 file changed, 11 insertions(+), 41 deletions(-) diff --git a/src/pmc_table.c b/src/pmc_table.c index b9ae4315b6..117dcd2e69 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -155,56 +155,26 @@ get_block_properties(uint16_t dev_id, if (block_id > PERFCOUNTER_BLOCKID__MAX || block_id < PERFCOUNTER_BLOCKID__FIRST) return HSAKMT_STATUS_INVALID_PARAMETER; - switch(dev_id) { - case 0x1304: - case 0x1305: - case 0x1306: - case 0x1307: - case 0x1309: - case 0x130A: - case 0x130B: - case 0x130C: - case 0x130D: - case 0x130E: - case 0x130F: - case 0x1310: - case 0x1311: - case 0x1312: - case 0x1313: - case 0x1315: - case 0x1316: - case 0x1317: - case 0x1318: - case 0x131B: - case 0x131C: - case 0x131D: + /* To avoid the long list, we read the 12 most significant digits of DID + * to identify the GPU instead of listing the complete 16 bits. If one day + * 12-bits is not good enough to distinguish the GPU, change the code here. + */ + switch(dev_id >> 4) { + case 0x130: + case 0x131: *block = kaveri_blocks[block_id]; break; - case 0x9870: - case 0x9874: - case 0x9875: - case 0x9876: - case 0x9877: + case 0x987: *block = carrizo_blocks[block_id]; break; - case 0x7300: + case 0x730: *block = fiji_blocks[block_id]; break; - case 0x67A0: - case 0x67A1: - case 0x67A2: - case 0x67A8: - case 0x67A9: - case 0x67AA: - case 0x67B0: - case 0x67B1: - case 0x67B8: - case 0x67B9: - case 0x67BA: - case 0x67BE: + case 0x67A: + case 0x67B: *block = hawaii_blocks[block_id]; break; From 876384305b48bb86516d56a4a238b42199d4ebf0 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 8 Aug 2016 11:30:57 -0400 Subject: [PATCH 0179/1247] Add gfx803 support Add gfx803 and gfx80311 device IDs to the support Change-Id: I16220fd811db102c02e5e0c5b82e40ec299877af --- src/queues.c | 24 +++++++++++++++++++++--- src/topology.c | 10 +++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/queues.c b/src/queues.c index 3118d2a12c..e2148a641f 100644 --- a/src/queues.c +++ b/src/queues.c @@ -42,11 +42,13 @@ enum asic_family_type { CHIP_HAWAII, CHIP_CARRIZO, CHIP_TONGA, - CHIP_FIJI + CHIP_FIJI, + CHIP_POLARIS10, + CHIP_POLARIS11 }; -#define IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_FIJI) -#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_FIJI) || \ +#define IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_POLARIS11) +#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_POLARIS11) || \ (chip) == CHIP_HAWAII) #define WG_CONTEXT_DATA_SIZE_PER_CU_VI 344576 @@ -83,6 +85,16 @@ struct device_info fiji_device_info = { .eop_buffer_size = TONGA_PAGE_SIZE, }; +struct device_info polaris10_device_info = { + .asic_family = CHIP_POLARIS10, + .eop_buffer_size = TONGA_PAGE_SIZE, +}; + +struct device_info polaris11_device_info = { + .asic_family = CHIP_POLARIS11, + .eop_buffer_size = TONGA_PAGE_SIZE, +}; + struct device_id { uint16_t dev_id; @@ -141,6 +153,12 @@ struct device_id supported_devices[] = { { 0x6939, &tonga_device_info }, { 0x7300, &fiji_device_info }, { 0x730f, &fiji_device_info }, + { 0x67c4, &polaris10_device_info }, + { 0x67c7, &polaris10_device_info }, + { 0x67df, &polaris10_device_info }, + { 0x67e3, &polaris11_device_info }, + { 0x67ef, &polaris11_device_info }, + { 0x67ff, &polaris11_device_info }, { 0, NULL } }; diff --git a/src/topology.c b/src/topology.c index 409ffdc5e8..cd8317afbd 100644 --- a/src/topology.c +++ b/src/topology.c @@ -137,7 +137,15 @@ static struct hsa_gfxip_table { { 0x6939, 8, 0, 2, 1, "Tonga" }, /* Fiji */ { 0x7300, 8, 0, 3, 1, "Fiji" }, - { 0x730f, 8, 0, 3, 1, "Fiji" } + { 0x730f, 8, 0, 3, 1, "Fiji" }, + /* Polaris10 */ + { 0x67C4, 8, 0, 3, 1, "Polaris10" }, + { 0x67C7, 8, 0, 3, 1, "Polaris10" }, + { 0x67DF, 8, 0, 3, 1, "Polaris10" }, + /* Polaris11 */ + { 0x67E3, 8, 0, 3, 1, "Polaris11" }, + { 0x67EF, 8, 0, 3, 1, "Polaris11" }, + { 0x67FF, 8, 0, 3, 1, "Polaris11" } }; enum cache_type { From 0b5c65a90363d56c5c3dbad3f9e49c820357c014 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 8 Aug 2016 18:10:51 -0400 Subject: [PATCH 0180/1247] Add performance counters for gfx803 Counter IDs in SQ_PERFCOUNTER0_SELECT are identical on gfx803 10 and gfx803 11. Change-Id: I5cfefd44b52989efd1d89311cf8c70c84ea2b230 --- src/pmc_table.c | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 117dcd2e69..3d4b495b11 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -105,6 +105,29 @@ static uint32_t hawaii_sq_counter_ids[] = { 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 }; +/* Polaris 10 and Polaris 11 have the same SQ cpunter IDs */ +/* Unused counters - 167 and 275 are *_DUMMY_LAST */ +static uint32_t polaris_sq_counter_ids[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, + 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, + 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, + 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, 161, 162, 163, 164, 165, 168, 169, 170, 171, 172, 173, 174, + 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, + 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, + 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 276, 277, 278, 279, 280, + 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -146,6 +169,17 @@ static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = sizeof(polaris_sq_counter_ids) / + sizeof(*polaris_sq_counter_ids), + .counter_ids = polaris_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + HSAKMT_STATUS get_block_properties(uint16_t dev_id, enum perf_block_id block_id, @@ -178,6 +212,13 @@ get_block_properties(uint16_t dev_id, *block = hawaii_blocks[block_id]; break; + case 0x67C: + case 0x67D: + case 0x67E: + case 0x67F: + *block = polaris_blocks[block_id]; + break; + default: rc = HSAKMT_STATUS_INVALID_PARAMETER; } From 9c9bfa30c0409f0eed27d2b34708406c66a6d048 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 2 Aug 2016 15:02:11 -0400 Subject: [PATCH 0181/1247] Fix a bug when mmap fails EventId is needed in calling hsaKmtDestroyEvent() when mmap failed, so we should move it ahead of mmap call. Change-Id: I5f4288b953611799a02b0e988d6b2e48104466a0 --- src/events.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/events.c b/src/events.c index a45716ad4a..6de79903c7 100644 --- a/src/events.c +++ b/src/events.c @@ -93,6 +93,8 @@ hsaKmtCreateEvent( return HSAKMT_STATUS_ERROR; } + e->EventId = args.event_id; + if (events_page == NULL && args.event_page_offset > 0) { events_page = mmap(NULL, KFD_SIGNAL_EVENT_LIMIT * 8, PROT_WRITE | PROT_READ, MAP_SHARED, kfd_fd, args.event_page_offset); @@ -109,7 +111,6 @@ hsaKmtCreateEvent( if (args.event_page_offset > 0 && args.event_slot_index < KFD_SIGNAL_EVENT_LIMIT) e->EventData.HWData2 = (HSAuint64)&events_page[args.event_slot_index]; - e->EventId = args.event_id; e->EventData.EventType = EventDesc->EventType; e->EventData.HWData1 = args.event_id; From 70b1b5b17e63d0201d6bd8d99c951a19a06e07f4 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 16 Aug 2016 11:17:19 -0400 Subject: [PATCH 0182/1247] queues.c: Enforce CUMaskCount being a multiple of 32 The thunk spec requires that CUMaskCount be divisible by 32. Check this and return INVALID_PARAMETER if it is not. Change-Id: I4e0c8502d996d3da31224b817a5d4ff2c6054e13 --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index e2148a641f..2f53085d4b 100644 --- a/src/queues.c +++ b/src/queues.c @@ -598,7 +598,7 @@ hsaKmtSetQueueCUMask( CHECK_KFD_OPEN(); - if (CUMaskCount == 0 || QueueCUMask == NULL) + if (CUMaskCount == 0 || QueueCUMask == NULL || ((CUMaskCount % 32) != 0)) return HSAKMT_STATUS_INVALID_PARAMETER; memset(&args, 0, sizeof(args)); From 9cbbf30be718c0faa67e128c8c78adaa14d7a4de Mon Sep 17 00:00:00 2001 From: Lan Xiao Date: Fri, 19 Aug 2016 11:42:05 -0400 Subject: [PATCH 0183/1247] libhsakmt: Add MarketingName and AMDName for all nodes - CPU & GPU HSA thunk API is currently reporting engineering name to MarketingName and returning NULL when querying for AMDName. -Change current name reporting from MarketingName to AMDName. -Use libpci to get MarketingName Change-Id: I819a6de7b067a2e724a6695e7d800274b83a71f8 Signed-off-by: Lan Xiao --- DEBIAN/control | 2 +- include/hsakmttypes.h | 4 +++- src/Makefile | 6 +++--- src/fmm.c | 7 ++++++- src/libhsakmt.h | 4 +++- src/topology.c | 36 +++++++++++++++++++++++++++--------- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/DEBIAN/control b/DEBIAN/control index 750ebb4ba1..0c3484eb5a 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -1,7 +1,7 @@ Package: hsakmt-rocm-dev Architecture: amd64 Maintainer: Advanced Micro Devices Inc. -Depends: +Depends:libpci3 Priority: optional Version: $version Description: Thunk library for AMD KFD diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index d091c833f0..6059f83c6b 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -195,7 +195,7 @@ typedef union // of throughput compute units (= SIMDs) associated with a H-NUMA node. // -#define HSA_PUBLIC_NAME_SIZE 128 +#define HSA_PUBLIC_NAME_SIZE 64 // Marketing name string size typedef struct _HsaNodeProperties { @@ -248,6 +248,8 @@ typedef struct _HsaNodeProperties HSAuint16 MarketingName[HSA_PUBLIC_NAME_SIZE]; // Public name of the "device" on the node (board or APU name). // Unicode string + HSAuint8 AMDName[HSA_PUBLIC_NAME_SIZE]; //CAL Name of the "device", ASCII + HSAuint8 Reserved[64]; } HsaNodeProperties; diff --git a/src/Makefile b/src/Makefile index 84a5dcea15..60d7ea3b23 100644 --- a/src/Makefile +++ b/src/Makefile @@ -30,7 +30,7 @@ ifneq ($(REL),1) CFLAGS += -ggdb endif -LDFLAGS += -lrt -pthread -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) -Wl,-z,nodelete +LDFLAGS += -lrt -pthread -lpci -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) -Wl,-z,nodelete OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ events.o openclose.o queues.o topology.o fmm.o pmc_table.o \ @@ -55,7 +55,7 @@ BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) TARGET = $(addprefix $(BUILDDIR)/,$(OBJS)) $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER): $(TARGET) - gcc -shared $(LDFLAGS) -o $@ $^ + gcc -shared -o $@ $^ $(LDFLAGS) $(BUILDDIR)/$(LIB_NAME): $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) @ln -sf $(LIB_NAME).$(LIB_MAJOR_VER) $(BUILDDIR)/$(LIB_NAME) @@ -73,6 +73,7 @@ ROCM_LIB = $(ROCM_ROOT)/lib ROCM_INCLUDE = $(ROCM_ROOT)/include INSTALL_LIB_DIR = libhsakmt/lib INSTALL_INCLUDE_DIR = libhsakmt/include/libhsakmt + export PACKAGE_DIR package-common: lnx64a @@ -106,7 +107,6 @@ lnx64a: $(BUILDDIR)/$(LIB_NAME) clean: rm -rf $(BUILD_ROOT) - #Rule $(BUILDDIR)/%.o: %.c ../include/hsakmt.h ../include/hsakmttypes.h ../include/linux/kfd_ioctl.h @echo Compiling $^ diff --git a/src/fmm.c b/src/fmm.c index 32972866ba..d05e6944b9 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -33,6 +33,7 @@ #include #include #include +#include #define NON_VALID_GPU_ID 0 @@ -1053,6 +1054,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache; + struct pci_access *pacc; /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ disableCache = getenv("HSA_DISABLE_CACHE"); @@ -1070,8 +1072,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * 0 by calloc. This is necessary because this function * gets called before hsaKmtAcquireSystemProperties() is called.*/ gpu_mem_count = 0; + pacc = pci_alloc(); + pci_init(pacc); while (i < NumNodes) { - ret = topology_sysfs_get_node_props(i, &props, &gpu_id); + ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; @@ -1092,6 +1096,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) } i++; } + pci_cleanup(pacc); /* The ioctl will also return Number of Nodes if args.kfd_process_device_apertures_ptr * is set to NULL. This is not required since Number of nodes is already known. Kernel diff --git a/src/libhsakmt.h b/src/libhsakmt.h index cb28a04a96..b270d458cf 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -30,6 +30,7 @@ #include #include #include +#include extern int kfd_fd; extern unsigned long kfd_open_count; @@ -68,7 +69,8 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id); -HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id); +HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, + uint32_t *gpu_id, struct pci_access* pacc); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); bool topology_is_dgpu(uint16_t device_id); diff --git a/src/topology.c b/src/topology.c index cd8317afbd..f8856f0f34 100644 --- a/src/topology.c +++ b/src/topology.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "libhsakmt.h" #include "fmm.h" @@ -81,7 +82,7 @@ static struct hsa_gfxip_table { unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info unsigned char is_dgpu; // Predicate for dGPU devices - const char* marketing_name; // Marketing Name of the device + const char* amd_name; // CALName of the device } gfxip_lookup_table[] = { /* Kaveri Family */ { 0x1304, 7, 0, 0, 0, "Spectre" }, @@ -433,7 +434,7 @@ bool topology_is_dgpu(uint16_t device_id) static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props) { FILE *fd; - char read_buf[256], cpu_model_name[128]; + char read_buf[256], cpu_model_name[HSA_PUBLIC_NAME_SIZE]; const char *p; uint32_t i, apic_id; @@ -478,6 +479,8 @@ topology_get_cpu_model_name(HsaNodeProperties *props) { /* Set CPU model name only if corresponding apic id */ if (props->CComputeIdLo == apic_id) { + /* Retrieve the CAL name of CPU node */ + strncpy( (char *) props->AMDName, cpu_model_name, sizeof(props->AMDName)); /* Convert from UTF8 to UTF16 */ for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = cpu_model_name[i]; @@ -542,7 +545,9 @@ static void topology_set_processor_vendor(void) } HSAKMT_STATUS -topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id) { +topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, + struct pci_access* pacc ) +{ FILE *fd; char *read_buf, *p; char prop_name[256]; @@ -552,6 +557,8 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 uint16_t fw_version = 0; int read_size; const struct hsa_gfxip_table* hsa_gfxip; + char namebuf[HSA_PUBLIC_NAME_SIZE]; + const char* name; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -652,15 +659,22 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 props->EngineId.ui32.Minor = hsa_gfxip->minor; props->EngineId.ui32.Stepping = hsa_gfxip->stepping; - if (!hsa_gfxip->marketing_name) { + if (!hsa_gfxip->amd_name) { ret = HSAKMT_STATUS_ERROR; goto err; } - /* Retrieve the marketing name of the node, convert UTF8 to UTF16 */ - for (i = 0; hsa_gfxip->marketing_name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) - props->MarketingName[i] = hsa_gfxip->marketing_name[i]; - props->MarketingName[i] = 0; + /* Retrieve the CAL name of the node */ + strncpy( (char *) props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName) ); + /* Retrieve the marketing name of the node using pcilib, + * convert UTF8 to UTF16 + */ + name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, + props->VendorId, props->DeviceId); + + for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + props->MarketingName[i] = name[i]; + props->MarketingName[i] = '\0'; } else { /* Is CPU node */ if (!props->NumFComputeCores || !props->DeviceId) { @@ -1357,6 +1371,7 @@ topology_take_snapshot(void) node_t *temp_nodes = 0; void *cpu_ci_list = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + struct pci_access *pacc; topology_set_processor_vendor(); retry: @@ -1371,10 +1386,12 @@ retry: temp_nodes = calloc(sys_props.NumNodes * sizeof(node_t),1); if (!temp_nodes) return HSAKMT_STATUS_NO_MEMORY; + pacc = pci_alloc(); + pci_init(pacc); for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_nodes[i].node, - &temp_nodes[i].gpu_id); + &temp_nodes[i].gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) { free_nodes(temp_nodes, i); goto err; @@ -1443,6 +1460,7 @@ retry: } } + pci_cleanup(pacc); } /* The Kernel only creates one way direct link - From e0c77a38cbc1e5903c4f2685be879d1c592468a1 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Fri, 19 Aug 2016 11:32:55 -0400 Subject: [PATCH 0184/1247] Makefile: remove 32bit thunk compilation by default Compiling in 32bit mode is broken, and we don't have an intention on restarting compatibility with 32bit apps. Change-Id: I5524b5b63fe62e6026aa04d84c4510e290a86106 --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 60d7ea3b23..db001a63ae 100644 --- a/src/Makefile +++ b/src/Makefile @@ -39,7 +39,7 @@ OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ .PHONY: all lnx lnx64a clean # Default target -all: lnx lnx64a +all: lnx64a # To relocate build output, set "O=target/dir/" # otherwise it defaults to a local directory From b1d2867b60d52b3c2f9c222e3c9e36676e6e7192 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Mon, 22 Aug 2016 11:15:36 -0400 Subject: [PATCH 0185/1247] LICENSE: add X11/MIT license file Change-Id: I2e95af843046896708bb7a116f7b03a0fa30a255 --- LICENSE.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 LICENSE.md diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000000..f9b6bddadc --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,23 @@ +ROCT-Thunk Interface LICENSE + +Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved. + +MIT LICENSE: +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From df593aa0760a058c7b84ade2a0178ec60d614971 Mon Sep 17 00:00:00 2001 From: Lan Xiao Date: Fri, 2 Sep 2016 12:28:17 -0400 Subject: [PATCH 0186/1247] libhsakmt: Marketing Name and AMDName support for APUs For APUs, use /proc/cpuinfo to get Marketing name. Change-Id: I4a17516d26a092683f36631032be00ad44f7e7fe Signed-off-by: Lan Xiao --- src/fmm.c | 1 + src/topology.c | 37 ++++++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index d05e6944b9..bee546b3f7 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1075,6 +1075,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) pacc = pci_alloc(); pci_init(pacc); while (i < NumNodes) { + memset(&props, 0, sizeof(props)); ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; diff --git a/src/topology.c b/src/topology.c index f8856f0f34..2c5d533921 100644 --- a/src/topology.c +++ b/src/topology.c @@ -432,7 +432,7 @@ bool topology_is_dgpu(uint16_t device_id) } static HSAKMT_STATUS -topology_get_cpu_model_name(HsaNodeProperties *props) { +topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { FILE *fd; char read_buf[256], cpu_model_name[HSA_PUBLIC_NAME_SIZE]; const char *p; @@ -480,7 +480,8 @@ topology_get_cpu_model_name(HsaNodeProperties *props) { /* Set CPU model name only if corresponding apic id */ if (props->CComputeIdLo == apic_id) { /* Retrieve the CAL name of CPU node */ - strncpy( (char *) props->AMDName, cpu_model_name, sizeof(props->AMDName)); + if (!is_apu) + strncpy( (char *) props->AMDName, cpu_model_name, sizeof(props->AMDName)); /* Convert from UTF8 to UTF16 */ for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = cpu_model_name[i]; @@ -666,19 +667,29 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 /* Retrieve the CAL name of the node */ strncpy( (char *) props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName) ); - /* Retrieve the marketing name of the node using pcilib, - * convert UTF8 to UTF16 - */ - name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, - props->VendorId, props->DeviceId); - - for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) - props->MarketingName[i] = name[i]; - props->MarketingName[i] = '\0'; + if (props->NumCPUCores) { + /* Is APU node */ + ret = topology_get_cpu_model_name(props, true); + if (ret != HSAKMT_STATUS_SUCCESS) + { + printf("Failed to get APU Model Name from %s\n", PROC_CPUINFO_PATH); + ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ + } + } else { + /* Is dGPU Node + * Retrieve the marketing name of the node using pcilib, + * convert UTF8 to UTF16 + */ + name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, + props->VendorId, props->DeviceId); + for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + props->MarketingName[i] = name[i]; + props->MarketingName[i] = '\0'; + } } else { - /* Is CPU node */ + /* Is CPU Node */ if (!props->NumFComputeCores || !props->DeviceId) { - ret = topology_get_cpu_model_name(props); + ret = topology_get_cpu_model_name(props, false); if (ret != HSAKMT_STATUS_SUCCESS) { printf("Failed to get CPU Model Name from %s\n", PROC_CPUINFO_PATH); From 8351b3d2e844f46db3e9d1690123837b603a88f9 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 18 Aug 2016 16:30:55 -0400 Subject: [PATCH 0187/1247] Implement hsaKmtGetTileConfig in thunk Change-Id: Iba8d8efa46e3c268a03442d3db568e1b19230e94 --- include/hsakmt.h | 10 ++++++++++ include/hsakmttypes.h | 15 +++++++++++++++ include/linux/kfd_ioctl.h | 28 +++++++++++++++++++++++++++- src/libhsakmt.ver | 1 + src/memory.c | 36 ++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 1 deletion(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 9d8b9d5225..ceb939783c 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -651,6 +651,16 @@ hsaKmtSetTrapHandler( HSAuint64 TrapBufferSizeInBytes //IN ); +/** + Gets image tile configuration. + */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetTileConfig( + HSAuint32 NodeId, // IN + HsaGpuTileConfig* config // IN & OUT + ); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 6059f83c6b..4423b4f462 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -967,6 +967,21 @@ typedef struct _HsaPmcTraceRoot HSATraceId TraceId; } HsaPmcTraceRoot; +typedef struct _HsaGpuTileConfig +{ + const HSAuint32 *TileConfig; + const HSAuint32 *MacroTileConfig; + HSAuint32 NumTileConfigs; + HSAuint32 NumMacroTileConfigs; + + HSAuint32 GbAddrConfig; + + HSAuint32 NumBanks; + HSAuint32 NumRanks; + /* 9 dwords on 64-bit system */ + HSAuint32 Reserved[7]; /* Round up to 16 dwords for future extension */ +} HsaGpuTileConfig; + #pragma pack(pop, hsakmttypes_h) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index b0d13cc3b7..a484756ef7 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -364,6 +364,29 @@ struct kfd_ioctl_import_dmabuf_args { uint32_t dmabuf_fd; /* to KFD */ }; +struct kfd_ioctl_get_tile_config_args { + /* to KFD: pointer to tile array */ + uint64_t tile_config_ptr; + /* to KFD: pointer to macro tile array */ + uint64_t macro_tile_config_ptr; + /* to KFD: array size allocated by user mode + * from KFD: array size filled by kernel + */ + uint32_t num_tile_configs; + /* to KFD: array size allocated by user mode + * from KFD: array size filled by kernel + */ + uint32_t num_macro_tile_configs; + + uint32_t gpu_id; /* to KFD */ + uint32_t gb_addr_config; /* from KFD */ + uint32_t num_banks; /* from KFD */ + uint32_t num_ranks; /* from KFD */ + /* struct size can be extended later if needed + * without breaking ABI compatibility + */ +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -466,7 +489,10 @@ struct kfd_ioctl_import_dmabuf_args { #define AMDKFD_IOC_IMPORT_DMABUF \ AMDKFD_IOWR(0x20, struct kfd_ioctl_import_dmabuf_args) +#define AMDKFD_IOC_GET_TILE_CONFIG \ + AMDKFD_IOWR(0x21, struct kfd_ioctl_get_tile_config_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x21 +#define AMDKFD_COMMAND_END 0x22 #endif diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 2d3d5db4f1..ce6ff028c2 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -47,6 +47,7 @@ hsaKmtPmcStopTrace; hsaKmtMapGraphicHandle; hsaKmtUnmapGraphicHandle; hsaKmtSetTrapHandler; +hsaKmtGetTileConfig; local: *; }; diff --git a/src/memory.c b/src/memory.c index 1ef6a83748..35e1c3d427 100644 --- a/src/memory.c +++ b/src/memory.c @@ -386,3 +386,39 @@ hsaKmtUnmapGraphicHandle( return hsaKmtUnmapMemoryToGPU(PORT_UINT64_TO_VPTR(FlatMemoryAddress)); } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetTileConfig( + HSAuint32 NodeId, /* IN */ + HsaGpuTileConfig *config /* IN & OUT */ +) +{ + struct kfd_ioctl_get_tile_config_args args; + uint32_t gpu_id; + HSAKMT_STATUS result; + + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + args.gpu_id = gpu_id; + args.tile_config_ptr = (uint64_t)config->TileConfig; + args.macro_tile_config_ptr = (uint64_t)config->MacroTileConfig; + args.num_tile_configs = config->NumTileConfigs; + args.num_macro_tile_configs = config->NumMacroTileConfigs; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_TILE_CONFIG, &args) != 0) { + return HSAKMT_STATUS_ERROR; + } + + config->NumTileConfigs = args.num_tile_configs; + config->NumMacroTileConfigs = args.num_macro_tile_configs; + + config->GbAddrConfig = args.gb_addr_config; + + config->NumBanks = args.num_banks; + config->NumRanks = args.num_ranks; + + return HSAKMT_STATUS_SUCCESS; +} From 51e4d27c372f54e92f2458bc194c3363a1a4bb72 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 1 Sep 2016 23:25:42 -0400 Subject: [PATCH 0188/1247] Add pointer attributes API Add two pointer attributes APIs: hsaKmtQueryPointerInfo - allow the user to query the memory information using a pointer. This pointer can point to any address inside the range known to HSA. hsaKmtSetMemoryUserData - allow the user to attach data to a pointer to add memory tracking information. This pointer must match the start address of a memory allocation or registration. TODO: This patch implements support on dGPU. Needs to add APU. Change-Id: I4711809274248434901f0794f50ebfa13a7371a8 --- include/hsakmt.h | 20 ++++ include/hsakmttypes.h | 22 ++++ src/fmm.c | 257 ++++++++++++++++++++++++++++++++++++------ src/fmm.h | 5 +- src/libhsakmt.ver | 2 + src/memory.c | 28 ++++- 6 files changed, 294 insertions(+), 40 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index ceb939783c..8604a439be 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -661,6 +661,26 @@ hsaKmtGetTileConfig( HsaGpuTileConfig* config // IN & OUT ); +/** + Returns information about pointers +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueryPointerInfo( + const void * Pointer, //IN + HsaPointerInfo * PointerInfo //OUT + ); + +/** + Associates user data with a memory allocation +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetMemoryUserData( + const void * Pointer, //IN + void * UserData //IN + ); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 4423b4f462..b1b8de205a 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -982,6 +982,28 @@ typedef struct _HsaGpuTileConfig HSAuint32 Reserved[7]; /* Round up to 16 dwords for future extension */ } HsaGpuTileConfig; +typedef enum _HSA_POINTER_TYPE { + HSA_POINTER_UNKNOWN = 0, + HSA_POINTER_ALLOCATED = 1, // Allocated with hsaKmtAllocMemory (except scratch) + HSA_POINTER_REGISTERED_USER = 2, // Registered user pointer + HSA_POINTER_REGISTERED_GRAPHICS = 3 // Registered graphics buffer + // (hsaKmtRegisterGraphicsToNodes) +} HSA_POINTER_TYPE; + +typedef struct _HsaPointerInfo { + HSA_POINTER_TYPE Type; // Pointer type + HSAuint32 Node; // Node where the memory is located + HsaMemFlags MemFlags; // Only valid for HSA_POINTER_ALLOCATED + void *CPUAddress; // Start address for CPU access + HSAuint64 GPUAddress; // Start address for GPU access + HSAuint64 SizeInBytes; // Size in bytes + HSAuint32 NRegisteredNodes; // Number of nodes the memory is registered to + HSAuint32 NMappedNodes; // Number of nodes the memory is mapped to + const HSAuint32 *RegisteredNodes; // Array of registered nodes + const HSAuint32 *MappedNodes; // Array of mapped nodes + void *UserData; // User data associated with the memory +} HsaPointerInfo; + #pragma pack(pop, hsakmttypes_h) diff --git a/src/fmm.c b/src/fmm.c index bee546b3f7..0c2c9e6d0e 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -49,8 +49,13 @@ struct vm_object { void *start; void *userptr; - uint64_t size; + uint64_t userptr_size; + uint64_t size; /* size allocated on GPU. When the user requests a random + * size, Thunk aligns it to page size and allocates this + * aligned size on GPU + */ uint64_t handle; /* opaque */ + uint32_t node_id; struct vm_object *next; struct vm_object *prev; uint32_t flags; /* memory allocation flags */ @@ -59,13 +64,17 @@ struct vm_object { */ uint32_t *registered_device_id_array; uint32_t registered_device_id_array_size; + uint32_t *registered_node_id_array; /* * Nodes that mapped already */ uint32_t *mapped_device_id_array; uint32_t mapped_device_id_array_size; + uint32_t *mapped_node_id_array; /* Metadata of imported graphics buffers */ void *metadata; + /* User data associated with the memory */ + void *user_data; }; typedef struct vm_object vm_object_t; @@ -181,11 +190,14 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, if (object) { object->start = start; object->userptr = NULL; + object->userptr_size = 0; object->size = size; object->handle = handle; object->next = object->prev = NULL; object->registered_device_id_array_size = 0; object->mapped_device_id_array_size = 0; + object->registered_node_id_array = NULL; + object->mapped_node_id_array = NULL; object->flags = flags; object->metadata = NULL; } @@ -276,7 +288,22 @@ static void vm_split_area(manageble_aperture_t *app, vm_area_t *area, } static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, - void *address, uint64_t size) + const void *address) +{ + vm_object_t *cur = app->vm_objects; + + while (cur) { + if (address >= cur->start && + (uint64_t)address < ((uint64_t)cur->start + cur->size)) + break; + cur = cur->next; + } + + return cur; /* NULL if not found */ +} + +static vm_object_t *vm_find_object_by_start_address(manageble_aperture_t *app, + const void *address, uint64_t size) { vm_object_t *cur = app->vm_objects; @@ -287,13 +314,13 @@ static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, if (cur->start == address && (cur->size == size || size == 0)) break; cur = cur->next; - }; + } return cur; /* NULL if not found */ } static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, - void *address) + const void *address) { vm_object_t *cur = app->vm_objects; @@ -466,7 +493,10 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) return -1; } -static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, +/* After allocating the memory, return the vm_object created for this memory. + * Return NULL if any failure. + */ +static vm_object_t *fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture, uint64_t *mmap_offset, @@ -474,9 +504,10 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, { struct kfd_ioctl_alloc_memory_of_gpu_new_args args; struct kfd_ioctl_free_memory_of_gpu_args free_args; + vm_object_t *vm_obj = NULL; if (!mem) - return -1; + return NULL; /* Allocate memory from amdkfd */ args.gpu_id = gpu_id; @@ -490,26 +521,26 @@ static int fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, args.mmap_offset = *mmap_offset; if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW, &args)) - return -1; + return NULL; /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); - if (!aperture_allocate_object(aperture, mem, args.handle, - MemorySizeInBytes, flags)) + if (!(vm_obj = aperture_allocate_object(aperture, mem, args.handle, + MemorySizeInBytes, flags))) goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); if (mmap_offset) *mmap_offset = args.mmap_offset; - return 0; + return vm_obj; err_object_allocation_failed: pthread_mutex_unlock(&aperture->fmm_mutex); free_args.handle = args.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args); - return -1; + return NULL; } bool fmm_is_inside_some_aperture(void *address) @@ -696,9 +727,11 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, manageble_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset, - uint32_t flags) + uint32_t flags, vm_object_t **vm_obj) { void *mem = NULL; + vm_object_t *obj; + /* Check that aperture is properly initialized/supported */ if (!aperture_is_valid(aperture->base, aperture->limit)) return NULL; @@ -713,8 +746,9 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, * Now that we have the area reserved, allocate memory in the device * itself */ - if (fmm_allocate_memory_in_device(gpu_id, mem, - MemorySizeInBytes, aperture, mmap_offset, flags)) { + obj = fmm_allocate_memory_in_device(gpu_id, mem, + MemorySizeInBytes, aperture, mmap_offset, flags); + if (obj == NULL) { /* * allocation of memory in device failed. * Release region in aperture @@ -726,6 +760,8 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Assign NULL to mem to indicate failure to calling function */ mem = NULL; } + if (vm_obj) + *vm_obj = obj; return mem; } @@ -742,6 +778,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla uint32_t ioc_flags, offset; uint64_t size, mmap_offset; void *mem; + vm_object_t *vm_obj = NULL; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -766,7 +803,15 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla mem = __fmm_allocate_device(gpu_id, size, aperture, offset, &mmap_offset, - ioc_flags); + ioc_flags, &vm_obj); + + if (mem && vm_obj) { + pthread_mutex_lock(&aperture->fmm_mutex); + /* Store memory allocation flags, not ioc flags */ + vm_obj->flags = flags.Value; + gpuid_to_nodeid(gpu_id, &vm_obj->node_id); + pthread_mutex_unlock(&aperture->fmm_mutex); + } if (mem && flags.ui32.HostAccess) { void *ret = mmap(mem, MemorySizeInBytes, @@ -805,7 +850,7 @@ static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, return mem; } -static void* fmm_allocate_host_gpu(uint64_t MemorySizeInBytes, +static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { void *mem; @@ -815,6 +860,7 @@ static void* fmm_allocate_host_gpu(uint64_t MemorySizeInBytes, uint64_t size; int32_t i; uint32_t gpu_id; + vm_object_t *vm_obj = NULL; i = find_first_dgpu(&gpu_id); if (i < 0) @@ -833,7 +879,15 @@ static void* fmm_allocate_host_gpu(uint64_t MemorySizeInBytes, mem = __fmm_allocate_device(gpu_id, size, aperture, 0, &mmap_offset, - ioc_flags); + ioc_flags, &vm_obj); + + if (mem && vm_obj) { + /* Store memory allocation flags, not ioc flags */ + pthread_mutex_lock(&aperture->fmm_mutex); + vm_obj->flags = flags.Value; + vm_obj->node_id = node_id; + pthread_mutex_unlock(&aperture->fmm_mutex); + } if (flags.ui32.HostAccess) { void *ret = mmap(mem, MemorySizeInBytes, @@ -856,10 +910,11 @@ static void* fmm_allocate_host_gpu(uint64_t MemorySizeInBytes, return mem; } -void* fmm_allocate_host(uint64_t MemorySizeInBytes, HsaMemFlags flags) +void* fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, + HsaMemFlags flags) { if (is_dgpu) - return fmm_allocate_host_gpu(MemorySizeInBytes, flags); + return fmm_allocate_host_gpu(node_id, MemorySizeInBytes, flags); return fmm_allocate_host_cpu(MemorySizeInBytes, flags); } @@ -933,7 +988,7 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return; @@ -944,6 +999,7 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) object->registered_device_id_array) { object->mapped_device_id_array_size = 0; object->mapped_device_id_array = NULL; + object->mapped_node_id_array = NULL; } free(object->registered_device_id_array); object->registered_device_id_array_size = 0; @@ -959,6 +1015,13 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) if (object->metadata) free(object->metadata); + if (object->registered_node_id_array) + free(object->registered_node_id_array); + object->registered_node_id_array = NULL; + if (object->mapped_node_id_array) + free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; + if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { /* Remove any CPU mapping, but keep the address range reserved */ @@ -1289,7 +1352,7 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, object = obj; if (!object) { /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (!object) goto err_object_not_found; } @@ -1345,6 +1408,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu bool is_debugger = 0; void *mmap_ret = NULL; uint64_t mmap_offset = 0; + /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) @@ -1363,7 +1427,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu if (!ret && !is_debugger) { offset = VOID_PTRS_SUB(address, aperture->base); mem = __fmm_allocate_device(gpu_id, size, aperture, offset, - NULL, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); + NULL, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE, NULL); if (mem == NULL) return -1; @@ -1416,7 +1480,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (!object) goto err_object_not_found; @@ -1583,7 +1647,7 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, /* Find the object to retrieve the handle */ object = obj; if (!object) { - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (!object) { ret = -1; goto err; @@ -1622,6 +1686,9 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, object->mapped_device_id_array = NULL; object->mapped_device_id_array_size = 0; + if (object->mapped_node_id_array) + free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1652,7 +1719,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle and size */ - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (!object) goto err; @@ -1678,6 +1745,9 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, object->mapped_device_id_array = NULL; object->mapped_device_id_array_size = 0; + if (object->mapped_node_id_array) + free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1937,7 +2007,7 @@ bool fmm_get_handle(void *address, uint64_t *handle) pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (object && handle) { *handle = object->handle; found = true; @@ -1974,19 +2044,19 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, - &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR); + &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, &obj); if (svm_addr == NULL) return HSAKMT_STATUS_ERROR; - /* Find the object and set its userptr address */ - pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_address(aperture, svm_addr, aligned_size); - if (obj == NULL) { + if (obj) { + pthread_mutex_lock(&aperture->fmm_mutex); + obj->userptr = addr; + gpuid_to_nodeid(gpu_id, &obj->node_id); + obj->userptr_size = size; pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_ERROR; } - obj->userptr = addr; - pthread_mutex_unlock(&aperture->fmm_mutex); + else + return HSAKMT_STATUS_ERROR; if (obj_ret) *obj_ret = obj; @@ -2026,7 +2096,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (!object) { pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); pthread_mutex_unlock(&aperture->fmm_mutex); } @@ -2117,6 +2187,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, obj->metadata = metadata; obj->registered_device_id_array = gpu_id_array; obj->registered_device_id_array_size = gpu_id_array_size; + gpuid_to_nodeid(infoArgs.gpu_id, &obj->node_id); } pthread_mutex_unlock(&aperture->fmm_mutex); if (!obj) @@ -2205,7 +2276,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; @@ -2230,6 +2301,9 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) free(object->registered_device_id_array); object->registered_device_id_array = NULL; object->registered_device_id_array_size = 0; + if (object->registered_node_id_array) + free(object->registered_node_id_array); + object->registered_node_id_array = NULL; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -2275,7 +2349,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, if (userptr && is_dgpu) object = vm_find_object_by_userptr(aperture, address); else - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_start_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); @@ -2364,3 +2438,112 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return 0; } + +HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) +{ + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t i; + manageble_aperture_t *aperture; + vm_object_t *vm_obj; + + memset(info, 0, sizeof(HsaPointerInfo)); + + /* TODO: APU */ + + if (address >= svm.dgpu_aperture.base && + address <= svm.dgpu_aperture.limit) + aperture = &svm.dgpu_aperture; + else if (address >= svm.dgpu_alt_aperture.base && + address <= svm.dgpu_alt_aperture.limit) + aperture = &svm.dgpu_alt_aperture; + else + /* Not in SVM, it can be system memory registered by userptr */ + aperture = &svm.dgpu_aperture; + + vm_obj = vm_find_object_by_address(aperture, address); + if (!vm_obj) + vm_obj = vm_find_object_by_userptr(aperture, address); + + if (!vm_obj) { + info->Type = HSA_POINTER_UNKNOWN; + ret = HSAKMT_STATUS_ERROR; + goto exit; + } + + if (vm_obj->metadata) + info->Type = HSA_POINTER_REGISTERED_GRAPHICS; + else if (vm_obj->userptr) + info->Type = HSA_POINTER_REGISTERED_USER; + else + info->Type = HSA_POINTER_ALLOCATED; + + info->Node = vm_obj->node_id; + info->GPUAddress = (HSAuint64)vm_obj->start; + info->SizeInBytes = vm_obj->size; + /* registered nodes */ + info->NRegisteredNodes = + vm_obj->registered_device_id_array_size / sizeof(uint32_t); + if (info->NRegisteredNodes && !vm_obj->registered_node_id_array) { + vm_obj->registered_node_id_array = (uint32_t *) + (uint32_t *)malloc(vm_obj->registered_device_id_array_size); + /* vm_obj->registered_node_id_array allocated here will be + * freed whenever the registration is deregistered or the + * memory being freed + */ + for (i=0; iNRegisteredNodes; i++) + gpuid_to_nodeid(vm_obj->registered_device_id_array[i], + &vm_obj->registered_node_id_array[i]); + } + info->RegisteredNodes = vm_obj->registered_node_id_array; + /* mapped nodes */ + info->NMappedNodes = + vm_obj->mapped_device_id_array_size / sizeof(uint32_t); + if (info->NMappedNodes && !vm_obj->mapped_node_id_array) { + vm_obj->mapped_node_id_array = + (uint32_t *)malloc(vm_obj->mapped_device_id_array_size); + /* vm_obj->mapped_node_id_array allocated here will be + * freed whenever the mapping is unmapped or memory being freed + */ + for (i=0; iNMappedNodes; i++) + gpuid_to_nodeid(vm_obj->mapped_device_id_array[i], + &vm_obj->mapped_node_id_array[i]); + } + info->MappedNodes = vm_obj->mapped_node_id_array; + info->UserData = vm_obj->user_data; + + if (info->Type == HSA_POINTER_REGISTERED_USER) { + info->CPUAddress = vm_obj->userptr; + info->SizeInBytes = vm_obj->userptr_size; + info->GPUAddress += ((HSAuint64)info->CPUAddress & (PAGE_SIZE-1)); + } + else if (info->Type == HSA_POINTER_ALLOCATED) { + info->MemFlags.Value = vm_obj->flags; + info->CPUAddress = vm_obj->start; + } + +exit: + return ret; +} + +HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) +{ + manageble_aperture_t *aperture; + vm_object_t *vm_obj; + + /* TODO: APU */ + + if (mem >= svm.dgpu_alt_aperture.base && + mem <= svm.dgpu_alt_aperture.limit) + aperture = &svm.dgpu_alt_aperture; + else + aperture = &svm.dgpu_aperture; + + vm_obj = vm_find_object_by_start_address(aperture, mem, 0); + if (!vm_obj) + vm_obj = vm_find_object_by_userptr(aperture, mem); + if (!vm_obj) + return HSAKMT_STATUS_ERROR; + + vm_obj->user_data = usr_data; + return HSAKMT_STATUS_SUCCESS; +} diff --git a/src/fmm.h b/src/fmm.h index 1601ef0ee4..96c37f2b08 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -52,7 +52,8 @@ void fmm_destroy_process_apertures(void); */ void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); -void* fmm_allocate_host(uint64_t MemorySizeInBytes, HsaMemFlags flags); +void* fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, + HsaMemFlags flags); void* fmm_open_graphic_handle(uint32_t gpu_id, int32_t graphic_device_handle, uint32_t graphic_handle, @@ -63,6 +64,8 @@ void fmm_release(void* address); int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); int fmm_unmap_from_gpu(void *address); bool fmm_get_handle(void *address, uint64_t *handle); +HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info); +HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data); /* Topology interface*/ HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index ce6ff028c2..bf25aa5bd1 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -48,6 +48,8 @@ hsaKmtMapGraphicHandle; hsaKmtUnmapGraphicHandle; hsaKmtSetTrapHandler; hsaKmtGetTileConfig; +hsaKmtQueryPointerInfo; +hsaKmtSetMemoryUserData; local: *; }; diff --git a/src/memory.c b/src/memory.c index 35e1c3d427..bb086d9d6d 100644 --- a/src/memory.c +++ b/src/memory.c @@ -133,7 +133,8 @@ hsaKmtAllocMemory( } if (gpu_id == 0 && !MemFlags.ui32.Scratch) { - *MemoryAddress = fmm_allocate_host(SizeInBytes, MemFlags); + *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, + MemFlags); if (*MemoryAddress == NULL) return HSAKMT_STATUS_ERROR; @@ -161,7 +162,8 @@ hsaKmtAllocMemory( /* Backwards compatibility hack: Allocate system memory if app * asks for paged memory from a GPU node. */ if (gpu_id && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { - *MemoryAddress = fmm_allocate_host(SizeInBytes, MemFlags); + *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, + MemFlags); if (*MemoryAddress == NULL) return HSAKMT_STATUS_ERROR; @@ -422,3 +424,25 @@ hsaKmtGetTileConfig( return HSAKMT_STATUS_SUCCESS; } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueryPointerInfo( + const void *Pointer, /* IN */ + HsaPointerInfo *PointerInfo /* OUT */ +) +{ + if (!PointerInfo) + return HSAKMT_STATUS_INVALID_PARAMETER; + return fmm_get_mem_info(Pointer, PointerInfo); +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetMemoryUserData( + const void *Pointer, /* IN */ + void *UserData /* IN */ +) +{ + return fmm_set_mem_user_data(Pointer, UserData); +} From 19f2676ea70c506e6e3479f9b6907b295d47e1c7 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 8 Sep 2016 15:01:28 -0400 Subject: [PATCH 0189/1247] Pointer attributes on APU Add CPUVM aperture to keep track of memory allocation that is not known to GPU driver. Together with GPUVM, this patch adds the pointer attributes support to APU. Change-Id: If13f9cf01ff8b9f709b99b66661e7505246adf4c --- src/fmm.c | 137 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 49 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 0c2c9e6d0e..411aa8ff1c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -139,6 +139,12 @@ static svm_t svm = { INIT_MANAGEBLE_APERTURE(0, 0) }; +/* On APU, for memory allocated on the system memory that GPU doesn't access + * via GPU driver, they are not managed by GPUVM. cpuvm_aperture keeps track + * of this part of memory. + */ +static manageble_aperture_t cpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0); + /* GPU node array for default mappings */ static uint32_t all_gpu_id_array_size = 0; static uint32_t *all_gpu_id_array = NULL; @@ -230,6 +236,35 @@ static void vm_remove_object(manageble_aperture_t *app, vm_object_t *object) vm_object_t *next; vm_object_t *prev; + /* Free allocations inside the object */ + if (object->registered_device_id_array_size > 0) { + if (object->mapped_device_id_array == + object->registered_device_id_array) { + object->mapped_device_id_array_size = 0; + object->mapped_device_id_array = NULL; + object->mapped_node_id_array = NULL; + } + free(object->registered_device_id_array); + object->registered_device_id_array_size = 0; + } + if (object->mapped_device_id_array != NULL && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + { + free(object->mapped_device_id_array); + object->mapped_device_id_array_size = 0; + } + if (object->metadata) + free(object->metadata); + + if (object->registered_node_id_array) + free(object->registered_node_id_array); + object->registered_node_id_array = NULL; + if (object->mapped_node_id_array) + free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; + next = object->next; prev = object->prev; @@ -242,7 +277,6 @@ static void vm_remove_object(manageble_aperture_t *app, vm_object_t *object) next->prev = prev; free(object); - } static void vm_add_area_after(vm_area_t *after_this, vm_area_t *new_area) @@ -493,6 +527,35 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) return -1; } +static manageble_aperture_t *fmm_find_aperture(const void *address) +{ + manageble_aperture_t *aperture = NULL; + uint32_t i; + + if (is_dgpu) { + if (address >= svm.dgpu_aperture.base && + address <= svm.dgpu_aperture.limit) + aperture = &svm.dgpu_aperture; + else if (address >= svm.dgpu_alt_aperture.base && + address <= svm.dgpu_alt_aperture.limit) + aperture = &svm.dgpu_alt_aperture; + else + /* Not in SVM, it can be system memory registered by userptr */ + aperture = &svm.dgpu_aperture; + } + else { /* APU */ + for (i = 0; i < gpu_mem_count; i++) { + if ((address >= gpu_mem[i].gpuvm_aperture.base) && + (address <= gpu_mem[i].gpuvm_aperture.limit)) + aperture = &gpu_mem[i].gpuvm_aperture; + } + if (!aperture) /* Not in GPUVM */ + aperture = &cpuvm_aperture; + } + + return aperture; +} + /* After allocating the memory, return the vm_object created for this memory. * Return NULL if any failure. */ @@ -832,6 +895,7 @@ static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, int err; HSAuint64 page_size; void *mem = NULL; + vm_object_t *vm_obj; page_size = PageSizeFromFlags(flags.ui32.PageSize); err = posix_memalign(&mem, page_size, MemorySizeInBytes); @@ -847,6 +911,14 @@ static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, return NULL; } } + + pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); + vm_obj = aperture_allocate_object(&cpuvm_aperture, mem, 0, + MemorySizeInBytes, flags.Value); + if (vm_obj) + vm_obj->node_id = 0; /* APU systems only have one CPU node */ + pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); + return mem; } @@ -994,34 +1066,6 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) return; } - if (object->registered_device_id_array_size > 0) { - if (object->mapped_device_id_array == - object->registered_device_id_array) { - object->mapped_device_id_array_size = 0; - object->mapped_device_id_array = NULL; - object->mapped_node_id_array = NULL; - } - free(object->registered_device_id_array); - object->registered_device_id_array_size = 0; - } - if (object->mapped_device_id_array != NULL && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - { - free(object->mapped_device_id_array); - object->mapped_device_id_array_size = 0; - } - if (object->metadata) - free(object->metadata); - - if (object->registered_node_id_array) - free(object->registered_node_id_array); - object->registered_node_id_array = NULL; - if (object->mapped_node_id_array) - free(object->mapped_node_id_array); - object->mapped_node_id_array = NULL; - if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { /* Remove any CPU mapping, but keep the address range reserved */ @@ -1042,6 +1086,7 @@ void fmm_release(void *address) { uint32_t i; bool found = false; + vm_object_t *object; for (i = 0; i < gpu_mem_count && !found; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) @@ -1076,10 +1121,17 @@ void fmm_release(void *address) } /* - * If memory address isn't inside of any defined aperture - it refers - * to the system memory + * If memory address isn't inside of any defined GPU aperture - it + * refers to the system memory */ if (!found) { + /* Release the vm object in CPUVM */ + pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); + object = vm_find_object_by_start_address(&cpuvm_aperture, address, 0); + if (object) + vm_remove_object(&cpuvm_aperture, object); + pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); + /* Free the memory from the system */ free(address); } } @@ -1268,6 +1320,9 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) } } + cpuvm_aperture.align = PAGE_SIZE; + cpuvm_aperture.limit = (void *)0x7FFFFFFFFFFF; /* 2^47 - 1 */ + free(process_apertures); return ret; @@ -2448,17 +2503,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) memset(info, 0, sizeof(HsaPointerInfo)); - /* TODO: APU */ - - if (address >= svm.dgpu_aperture.base && - address <= svm.dgpu_aperture.limit) - aperture = &svm.dgpu_aperture; - else if (address >= svm.dgpu_alt_aperture.base && - address <= svm.dgpu_alt_aperture.limit) - aperture = &svm.dgpu_alt_aperture; - else - /* Not in SVM, it can be system memory registered by userptr */ - aperture = &svm.dgpu_aperture; + aperture = fmm_find_aperture(address); vm_obj = vm_find_object_by_address(aperture, address); if (!vm_obj) @@ -2530,13 +2575,7 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) manageble_aperture_t *aperture; vm_object_t *vm_obj; - /* TODO: APU */ - - if (mem >= svm.dgpu_alt_aperture.base && - mem <= svm.dgpu_alt_aperture.limit) - aperture = &svm.dgpu_alt_aperture; - else - aperture = &svm.dgpu_aperture; + aperture = fmm_find_aperture(mem); vm_obj = vm_find_object_by_start_address(aperture, mem, 0); if (!vm_obj) From 4911c91389bfc6e6e1027cd6d21619da87b78f41 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 13 Sep 2016 12:44:29 -0400 Subject: [PATCH 0190/1247] Search VM object by range Add vm_find_object_by_userptr_range so QueryPointerInfo can find the object as well when the pointer is not the starting address but it's inside the memory range. Also rename vm_find_object_xxx functions to _by_address and _by_address_range to be consistent. Change-Id: I5c2b3a05b41493e32b7fd9154665bf078b043606 --- src/fmm.c | 74 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 411aa8ff1c..87894c21fc 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -322,21 +322,6 @@ static void vm_split_area(manageble_aperture_t *app, vm_area_t *area, } static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, - const void *address) -{ - vm_object_t *cur = app->vm_objects; - - while (cur) { - if (address >= cur->start && - (uint64_t)address < ((uint64_t)cur->start + cur->size)) - break; - cur = cur->next; - } - - return cur; /* NULL if not found */ -} - -static vm_object_t *vm_find_object_by_start_address(manageble_aperture_t *app, const void *address, uint64_t size) { vm_object_t *cur = app->vm_objects; @@ -353,6 +338,21 @@ static vm_object_t *vm_find_object_by_start_address(manageble_aperture_t *app, return cur; /* NULL if not found */ } +static vm_object_t *vm_find_object_by_address_range(manageble_aperture_t *app, + const void *address) +{ + vm_object_t *cur = app->vm_objects; + + while (cur) { + if (address >= cur->start && + (uint64_t)address < ((uint64_t)cur->start + cur->size)) + break; + cur = cur->next; + } + + return cur; /* NULL if not found */ +} + static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, const void *address) { @@ -363,7 +363,23 @@ static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, if (cur->userptr == address) break; cur = cur->next; - }; + } + + return cur; /* NULL if not found */ +} + +static vm_object_t *vm_find_object_by_userptr_range(manageble_aperture_t *app, + const void *address) +{ + vm_object_t *cur = app->vm_objects; + + /* Look up the appropriate address range containing the given address */ + while (cur) { + if (address >= cur->userptr && + (uint64_t)address < (uint64_t)cur->userptr + cur->userptr_size) + break; + cur = cur->next; + } return cur; /* NULL if not found */ } @@ -1060,7 +1076,7 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return; @@ -1127,7 +1143,7 @@ void fmm_release(void *address) if (!found) { /* Release the vm object in CPUVM */ pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); - object = vm_find_object_by_start_address(&cpuvm_aperture, address, 0); + object = vm_find_object_by_address(&cpuvm_aperture, address, 0); if (object) vm_remove_object(&cpuvm_aperture, object); pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); @@ -1407,7 +1423,7 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, object = obj; if (!object) { /* Find the object to retrieve the handle */ - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (!object) goto err_object_not_found; } @@ -1535,7 +1551,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (!object) goto err_object_not_found; @@ -1702,7 +1718,7 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, /* Find the object to retrieve the handle */ object = obj; if (!object) { - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (!object) { ret = -1; goto err; @@ -1774,7 +1790,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle and size */ - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (!object) goto err; @@ -2062,7 +2078,7 @@ bool fmm_get_handle(void *address, uint64_t *handle) pthread_mutex_lock(&aperture->fmm_mutex); /* Find the object to retrieve the handle */ - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (object && handle) { *handle = object->handle; found = true; @@ -2151,7 +2167,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (!object) { pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); pthread_mutex_unlock(&aperture->fmm_mutex); } @@ -2331,7 +2347,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; @@ -2404,7 +2420,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, if (userptr && is_dgpu) object = vm_find_object_by_userptr(aperture, address); else - object = vm_find_object_by_start_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, address, 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); @@ -2505,9 +2521,9 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) aperture = fmm_find_aperture(address); - vm_obj = vm_find_object_by_address(aperture, address); + vm_obj = vm_find_object_by_address_range(aperture, address); if (!vm_obj) - vm_obj = vm_find_object_by_userptr(aperture, address); + vm_obj = vm_find_object_by_userptr_range(aperture, address); if (!vm_obj) { info->Type = HSA_POINTER_UNKNOWN; @@ -2577,7 +2593,7 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) aperture = fmm_find_aperture(mem); - vm_obj = vm_find_object_by_start_address(aperture, mem, 0); + vm_obj = vm_find_object_by_address(aperture, mem, 0); if (!vm_obj) vm_obj = vm_find_object_by_userptr(aperture, mem); if (!vm_obj) From 660a6ebbd44512c96efb0e5b53c8649930ac0e1e Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 14 Sep 2016 14:42:26 -0400 Subject: [PATCH 0191/1247] Disable CPU cache info in non-x86 CPU cache information reported by Thunk topology is obtained from cpuid instruction. This instruction only applies to X86 systems. It can cause compile errors on non-X86 platforms. This patch temporarily disables CPU cache functions in topology for non-X86 platforms in order to compile. Change-Id: If86671817b0d036cb324eebf3f354682bfb75856 --- src/topology.c | 242 ++++++++++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 111 deletions(-) diff --git a/src/topology.c b/src/topology.c index 2c5d533921..e396c7eeb5 100644 --- a/src/topology.c +++ b/src/topology.c @@ -252,6 +252,7 @@ static int num_subdirs(char *dirpath, char *prefix) return count; } +#if defined(__x86_64__) || defined(__i386__) /* cpuid instruction returns processor identification and feature information * to the EAX, EBX, ECX, and EDX registers, as determined by input entered in * EAX (in some cases, ECX as well). @@ -308,6 +309,118 @@ static int get_count_order(unsigned int count) return bit; } +/* cpuid_find_num_cache_leaves - Use cpuid instruction to find out how many + * cache leaves the CPU has. + * @op - cpuid opcode to get cache information + * Return - the number of cache leaves + */ +static int cpuid_find_num_cache_leaves(uint32_t op) +{ + union _cpuid_leaf_eax eax; + union _cpuid_leaf_ebx ebx; + unsigned int ecx; + unsigned int edx; + int idx = -1; + + do { + ++idx; + cpuid_count(op, idx, &eax.full, &ebx.full, &ecx, &edx); + } while (eax.split.type != CACHE_TYPE_NULL); + return idx; +} + +/* cpuid_get_cpu_cache_info - Use cpuid instruction to get cache information + * @op - cpuid opcode to get cache information + * @cpu_ci - this parameter is an input and also an output. + * [IN] cpu_ci->num_caches: the number of caches of this cpu + * [OUT] cpu_ci->cache_info: to store cache info collected + */ +static void cpuid_get_cpu_cache_info(uint32_t op, cpu_cacheinfo_t *cpu_ci) +{ + union _cpuid_leaf_eax eax; + union _cpuid_leaf_ebx ebx; + uint32_t ecx; + uint32_t edx; + uint32_t index; + cacheinfo_t *this_leaf; + + for (index = 0; index < cpu_ci->num_caches; index++) { + cpuid_count(op, index, &eax.full, &ebx.full, &ecx, &edx); + this_leaf = cpu_ci->cache_info + index; + this_leaf->hsa_cache_prop.ProcessorIdLow = cpu_ci->apicid; + this_leaf->num_threads_sharing = + eax.split.num_threads_sharing + 1; + this_leaf->hsa_cache_prop.CacheLevel = eax.split.level; + this_leaf->hsa_cache_prop.CacheType.ui32.CPU = 1; + if (eax.split.type & CACHE_TYPE_DATA ) + this_leaf->hsa_cache_prop.CacheType.ui32.Data = 1; + if (eax.split.type & CACHE_TYPE_INST ) + this_leaf->hsa_cache_prop.CacheType.ui32.Instruction = 1; + this_leaf->hsa_cache_prop.CacheLineSize = + ebx.split.coherency_line_size + 1; + this_leaf->hsa_cache_prop.CacheAssociativity = + ebx.split.ways_of_associativity + 1; + this_leaf->hsa_cache_prop.CacheLinesPerTag = + ebx.split.physical_line_partition + 1; + this_leaf->hsa_cache_prop.CacheSize = (ecx + 1) * + (ebx.split.coherency_line_size + 1) * + (ebx.split.physical_line_partition + 1) * + (ebx.split.ways_of_associativity + 1); + } +} + +/* find_cpu_cache_siblings - In the cache list, some caches may be listed more + * than once if they are shared by multiple CPUs. Identify the cache's CPU + * siblings, record it to SiblingMap[], then remove the duplicated cache by + * changing the cache size to 0. + */ +static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) +{ + cacheinfo_t *this_leaf, *leaf2; + uint32_t n, j, idx_msb, apicid1, apicid2; + cpu_cacheinfo_t *this_cpu, *cpu2; + uint32_t index; + + for (n = 0; n < cpu_ci_list->len; n++) { + this_cpu = cpu_ci_list + n; + for (index = 0; index < this_cpu->num_caches; index++) { + this_leaf = this_cpu->cache_info + index; + /* CacheSize 0 means an invalid cache */ + if (!this_leaf->hsa_cache_prop.CacheSize) + continue; + if (this_leaf->num_threads_sharing == 1) // no siblings + continue; + idx_msb = get_count_order(this_leaf->num_threads_sharing); + for (j = n + 1; j < cpu_ci_list->len; j++) { + cpu2 = cpu_ci_list + j; + leaf2 = cpu2->cache_info + index; + apicid1 = this_leaf->hsa_cache_prop.ProcessorIdLow; + apicid2 = leaf2->hsa_cache_prop.ProcessorIdLow; + if ((apicid2 >> idx_msb) != (apicid1 >> idx_msb)) + continue; + /* A sibling leaf is found. Cache properties + * use ProcIdLow as offset to represent siblings + * in SiblingMap, so keep the lower apicid and + * delete the other by changing CacheSize to 0. + */ + if (apicid1 < apicid2) { + this_leaf->hsa_cache_prop.SiblingMap[0] = 1; + this_leaf->hsa_cache_prop.SiblingMap[apicid2 - apicid1] = 1; + leaf2->hsa_cache_prop.CacheSize = 0; + cpu2->num_duplicated_caches++; + } + else { + leaf2->hsa_cache_prop.SiblingMap[0] = 1; + leaf2->hsa_cache_prop.SiblingMap[apicid1 - apicid2] = 1; + this_leaf->hsa_cache_prop.CacheSize = 0; + this_cpu->num_duplicated_caches++; + } + } + } + } +} +#endif /* X86 platform */ + static HSAKMT_STATUS topology_sysfs_get_generation(uint32_t *gen) { FILE *fd; @@ -765,117 +878,7 @@ err1: return ret; } -/* cpuid_find_num_cache_leaves - Use cpuid instruction to find out how many - * cache leaves the CPU has. - * @op - cpuid opcode to get cache information - * Return - the number of cache leaves -*/ -static int cpuid_find_num_cache_leaves(uint32_t op) -{ - union _cpuid_leaf_eax eax; - union _cpuid_leaf_ebx ebx; - unsigned int ecx; - unsigned int edx; - int idx = -1; - - do { - ++idx; - cpuid_count(op, idx, &eax.full, &ebx.full, &ecx, &edx); - } while (eax.split.type != CACHE_TYPE_NULL); - return idx; -} - -/* cpuid_get_cpu_cache_info - Use cpuid instruction to get cache information - * @op - cpuid opcode to get cache information - * @cpu_ci - this parameter is an input and also an output. - * [IN] cpu_ci->num_caches: the number of caches of this cpu - * [OUT] cpu_ci->cache_info: to store cache info collected - */ -static void cpuid_get_cpu_cache_info(uint32_t op, cpu_cacheinfo_t *cpu_ci) -{ - union _cpuid_leaf_eax eax; - union _cpuid_leaf_ebx ebx; - uint32_t ecx; - uint32_t edx; - uint32_t index; - cacheinfo_t *this_leaf; - - for (index = 0; index < cpu_ci->num_caches; index++) { - cpuid_count(op, index, &eax.full, &ebx.full, &ecx, &edx); - this_leaf = cpu_ci->cache_info + index; - this_leaf->hsa_cache_prop.ProcessorIdLow = cpu_ci->apicid; - this_leaf->num_threads_sharing = - eax.split.num_threads_sharing + 1; - this_leaf->hsa_cache_prop.CacheLevel = eax.split.level; - this_leaf->hsa_cache_prop.CacheType.ui32.CPU = 1; - if (eax.split.type & CACHE_TYPE_DATA ) - this_leaf->hsa_cache_prop.CacheType.ui32.Data = 1; - if (eax.split.type & CACHE_TYPE_INST ) - this_leaf->hsa_cache_prop.CacheType.ui32.Instruction = 1; - this_leaf->hsa_cache_prop.CacheLineSize = - ebx.split.coherency_line_size + 1; - this_leaf->hsa_cache_prop.CacheAssociativity = - ebx.split.ways_of_associativity + 1; - this_leaf->hsa_cache_prop.CacheLinesPerTag = - ebx.split.physical_line_partition + 1; - this_leaf->hsa_cache_prop.CacheSize = (ecx + 1) * - (ebx.split.coherency_line_size + 1) * - (ebx.split.physical_line_partition + 1) * - (ebx.split.ways_of_associativity + 1); - } -} - -/* find_cpu_cache_siblings - In the cache list, some caches may be listed more - * than once if they are shared by multiple CPUs. Identify the cache's CPU - * siblings, record it to SiblingMap[], then remove the duplicated cache by - * changing the cache size to 0. - */ -static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) -{ - cacheinfo_t *this_leaf, *leaf2; - uint32_t n, j, idx_msb, apicid1, apicid2; - cpu_cacheinfo_t *this_cpu, *cpu2; - uint32_t index; - - for (n = 0; n < cpu_ci_list->len; n++) { - this_cpu = cpu_ci_list + n; - for (index = 0; index < this_cpu->num_caches; index++) { - this_leaf = this_cpu->cache_info + index; - /* CacheSize 0 means an invalid cache */ - if (!this_leaf->hsa_cache_prop.CacheSize) - continue; - if (this_leaf->num_threads_sharing == 1) // no siblings - continue; - idx_msb = get_count_order(this_leaf->num_threads_sharing); - for (j = n + 1; j < cpu_ci_list->len; j++) { - cpu2 = cpu_ci_list + j; - leaf2 = cpu2->cache_info + index; - apicid1 = this_leaf->hsa_cache_prop.ProcessorIdLow; - apicid2 = leaf2->hsa_cache_prop.ProcessorIdLow; - if ((apicid2 >> idx_msb) != (apicid1 >> idx_msb)) - continue; - /* A sibling leaf is found. Cache properties - * use ProcIdLow as offset to represent siblings - * in SiblingMap, so keep the lower apicid and - * delete the other by changing CacheSize to 0. - */ - if (apicid1 < apicid2) { - this_leaf->hsa_cache_prop.SiblingMap[0] = 1; - this_leaf->hsa_cache_prop.SiblingMap[apicid2 - apicid1] = 1; - leaf2->hsa_cache_prop.CacheSize = 0; - cpu2->num_duplicated_caches++; - } - else { - leaf2->hsa_cache_prop.SiblingMap[0] = 1; - leaf2->hsa_cache_prop.SiblingMap[apicid1 - apicid2] = 1; - this_leaf->hsa_cache_prop.CacheSize = 0; - this_cpu->num_duplicated_caches++; - } - } - } - } -} - +#if defined(__x86_64__) || defined(__i386__) /* topology_destroy_temp_cpu_cache_list - Free the memory allocated in * topology_create_temp_cpu_cache_list(). */ @@ -1043,6 +1046,23 @@ topology_get_cpu_cache_props(node_t *tbl, cpu_cacheinfo_t *cpu_ci_list) exit: return ret; } +#else /* not X86 */ +static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) +{ +} + +static HSAKMT_STATUS +topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) +{ + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS +topology_get_cpu_cache_props(node_t *tbl, cpu_cacheinfo_t *cpu_ci_list) +{ + return HSAKMT_STATUS_SUCCESS; +} +#endif static HSAKMT_STATUS topology_sysfs_get_cache_props(uint32_t node_id, uint32_t cache_id, HsaCacheProperties *props) { From 2e0a6eb3716b34fec1c32a235a5f786f2bacc29b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 16 Sep 2016 15:56:57 -0400 Subject: [PATCH 0192/1247] Allocate and map doorbells in SVM for discrete GPUs Allocate doorbells for dGPUs in the SVM aperture and map them for GPU access. This is necessary to allow GPU-initiated submissions to user mode queues. Depends on new doorbell BO allocation flag in KFD. Change-Id: I0737bef4a4764bb4a66c43846707ead2108f6601 --- include/linux/kfd_ioctl.h | 2 + src/fmm.c | 51 ++++++++++++++++++ src/fmm.h | 1 + src/queues.c | 110 ++++++++++++++++++++++++++++++-------- 4 files changed, 143 insertions(+), 21 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index a484756ef7..613a2c7b8c 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -324,6 +324,8 @@ enum evict_type { #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 6) +#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 7) + struct kfd_ioctl_alloc_memory_of_gpu_new_args { uint64_t va_addr; /* to KFD */ uint64_t size; /* to KFD */ diff --git a/src/fmm.c b/src/fmm.c index 87894c21fc..8cf6553833 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -905,6 +905,57 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla return mem; } +void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, + uint64_t doorbell_offset) +{ + manageble_aperture_t *aperture; + int32_t gpu_mem_id; + uint32_t ioc_flags; + void *mem; + vm_object_t *vm_obj = NULL; + + /* Retrieve gpu_mem id according to gpu_id */ + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return NULL; + + /* Use fine-grained aperture */ + aperture = &svm.dgpu_alt_aperture; + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL; + + mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, + aperture, 0, NULL, + ioc_flags, &vm_obj); + + if (mem && vm_obj) { + HsaMemFlags flags; + + /* Cook up some flags for storing in the VM object */ + flags.Value = 0; + flags.ui32.NonPaged = 1; + flags.ui32.HostAccess = 1; + flags.ui32.Reserved = 0xBe11; + + pthread_mutex_lock(&aperture->fmm_mutex); + vm_obj->flags = flags.Value; + gpuid_to_nodeid(gpu_id, &vm_obj->node_id); + pthread_mutex_unlock(&aperture->fmm_mutex); + } + + if (mem) { + void *ret = mmap(mem, MemorySizeInBytes, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd, + doorbell_offset); + if (ret == MAP_FAILED) { + __fmm_release(mem, aperture); + return NULL; + } + } + + return mem; +} + static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, HsaMemFlags flags) { diff --git a/src/fmm.h b/src/fmm.h index 96c37f2b08..6fc8c56928 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -52,6 +52,7 @@ void fmm_destroy_process_apertures(void); */ void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); +void* fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset); void* fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); void* fmm_open_graphic_handle(uint32_t gpu_id, diff --git a/src/queues.c b/src/queues.c index 2f53085d4b..3bd9b2827d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -24,6 +24,7 @@ */ #include "libhsakmt.h" +#include "fmm.h" #include "linux/kfd_ioctl.h" #include #include @@ -181,6 +182,7 @@ struct process_doorbells pthread_mutex_t doorbells_mutex; }; +static unsigned int num_doorbells; static struct process_doorbells *doorbells; HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) @@ -199,15 +201,94 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) doorbells[i].doorbells = NULL; pthread_mutex_init(&doorbells[i].doorbells_mutex, NULL); } + + num_doorbells = NumNodes; + return ret; } void destroy_process_doorbells(void) { - if (doorbells) { - free(doorbells); - doorbells = NULL; + unsigned int i; + + if (!doorbells) + return; + + for (i = 0; i < num_doorbells; i++) { + if (doorbells[i].need_mmap) + continue; + + if (topology_is_dgpu(get_device_id_by_node(i))) { + fmm_unmap_from_gpu(doorbells[i].doorbells); + fmm_release(doorbells[i].doorbells); + } else + munmap(doorbells[i].doorbells, DOORBELLS_PAGE_SIZE); } + + free(doorbells); + doorbells = NULL; + num_doorbells = 0; +} + +static HSAKMT_STATUS map_doorbell_apu(HSAuint32 NodeId, HSAuint32 gpu_id, + HSAuint64 doorbell_offset) +{ + void *ptr; + + ptr = mmap(0, DOORBELLS_PAGE_SIZE, PROT_READ|PROT_WRITE, + MAP_SHARED, kfd_fd, doorbell_offset); + + if (ptr == MAP_FAILED) + return HSAKMT_STATUS_ERROR; + + doorbells[NodeId].need_mmap = false; + doorbells[NodeId].doorbells = ptr; + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS map_doorbell_dgpu(HSAuint32 NodeId, HSAuint32 gpu_id, + HSAuint64 doorbell_offset) +{ + void *ptr; + + ptr = fmm_allocate_doorbell(gpu_id, DOORBELLS_PAGE_SIZE, + doorbell_offset); + + if (ptr == NULL) + return HSAKMT_STATUS_ERROR; + + /* map for GPU access */ + if (fmm_map_to_gpu(ptr, DOORBELLS_PAGE_SIZE, NULL)) { + fmm_release(ptr); + return HSAKMT_STATUS_ERROR; + } + + doorbells[NodeId].need_mmap = false; + doorbells[NodeId].doorbells = ptr; + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, + HSAuint64 doorbell_offset) +{ + HSAKMT_STATUS status = HSAKMT_STATUS_SUCCESS; + + pthread_mutex_lock(&doorbells[NodeId].doorbells_mutex); + if (!doorbells[NodeId].need_mmap) { + pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); + return HSAKMT_STATUS_SUCCESS; + } + + if (topology_is_dgpu(get_device_id_by_node(NodeId))) + status = map_doorbell_dgpu(NodeId, gpu_id, doorbell_offset); + else + status = map_doorbell_apu(NodeId, gpu_id, doorbell_offset); + + pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); + + return status; } static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) @@ -432,7 +513,6 @@ hsaKmtCreateQueue( uint16_t dev_id; struct device_info *dev_info; int err; - void* ptr; CHECK_KFD_OPEN(); result = validate_nodeid(NodeId, &gpu_id); @@ -495,25 +575,13 @@ hsaKmtCreateQueue( q->queue_id = args.queue_id; - pthread_mutex_lock(&doorbells[NodeId].doorbells_mutex); - - if (doorbells[NodeId].need_mmap) { - ptr = mmap(0, DOORBELLS_PAGE_SIZE, PROT_READ|PROT_WRITE, - MAP_SHARED, kfd_fd, args.doorbell_offset); - - if (ptr == MAP_FAILED) { - pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); - hsaKmtDestroyQueue(q->queue_id); - free_queue(q); - return HSAKMT_STATUS_ERROR; - } - - doorbells[NodeId].need_mmap = false; - doorbells[NodeId].doorbells = ptr; + err = map_doorbell(NodeId, gpu_id, args.doorbell_offset); + if (err != HSAKMT_STATUS_SUCCESS) { + hsaKmtDestroyQueue(q->queue_id); + free_queue(q); + return HSAKMT_STATUS_ERROR; } - pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); - QueueResource->QueueId = PORT_VPTR_TO_UINT64(q); QueueResource->Queue_DoorBell = VOID_PTR_ADD32(doorbells[NodeId].doorbells, q->queue_id); From 7511631f0870b2ec8dc293706e9e5fb4adb8c602 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Tue, 20 Sep 2016 13:55:35 -0500 Subject: [PATCH 0193/1247] Add libhsakmt cmake build and packaging files. Change-Id: Ic7fa22d5b266480aa0c62628022f39da4e043d23 --- CMakeLists.txt | 136 ++++++++++++++++++++++++++++++++++++++ DEBIAN/postinst | 20 ++++-- DEBIAN/prerm | 19 ++++++ RPM/rpm_post | 1 + RPM/rpm_postun | 1 + cmake_modules/utils.cmake | 71 ++++++++++++++++++++ 6 files changed, 244 insertions(+), 4 deletions(-) create mode 100644 CMakeLists.txt create mode 100755 DEBIAN/prerm create mode 100644 RPM/rpm_post create mode 100644 RPM/rpm_postun create mode 100644 cmake_modules/utils.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..8aff4b410a --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,136 @@ +################################################################################ +## +## Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved. +## +## MIT LICENSE: +## Permission is hereby granted, free of charge, to any person obtaining a copy of +## this software and associated documentation files (the "Software"), to deal in +## the Software without restriction, including without limitation the rights to +## use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +## of the Software, and to permit persons to whom the Software is furnished to do +## so, subject to the following conditions: +## +## The above copyright notice and this permission notice shall be included in all +## copies or substantial portions of the Software. +## +## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +## SOFTWARE. +## +################################################################################ + +cmake_minimum_required(VERSION 2.8.0) + +set ( HSAKMT_NAME "hsakmt" ) +set ( HSAKMT_COMPONENT "lib${HSAKMT_NAME}" ) +set ( HSAKMT_TARGET "${HSAKMT_NAME}" ) + +project ( ${HSAKMT_TARGET} ) + +list ( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules" ) + +include ( utils ) + +if(NOT DEFINED VERSION_STRING) + set ( VERSION_STRING "2.0.1" ) +endif() + +parse_version ( ${VERSION_STRING} ) + +## Verbose output. +set ( CMAKE_VERBOSE_MAKEFILE on ) + +## Compiler flags +set ( CMAKE_CXX_FLAGS "-fPIC -W -Wall -Wextra -Werror -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden" ) + +set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) + +## Linker Flags +set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.$(VERSION_MAJOR) -Wl,-z,nodelete" ) + +## Source files +set ( HSAKMT_SRC ${HSAKMT_SRC} src/events.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/fmm.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/globals.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/libhsakmt.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/memory.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/openclose.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/perfctr.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/pmc_table.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/queues.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/time.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/topology.c ) +set ( HSAKMT_SRC ${HSAKMT_SRC} src/version.c ) + +## Include paths +include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/include ) +include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/src ) + +## Add the library +add_library ( ${HSAKMT_TARGET} SHARED ${HSAKMT_SRC} ) + +## Set the VERSION and SOVERSION values +if ( DEFINED VERSION_STRING ) + set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${VERSION_STRING}" ) +endif () + +if ( DEFINED VERSION_MAJOR ) + set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${VERSION_MAJOR}" ) +else () + set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION 1 ) +endif() + +target_link_libraries ( ${HSAKMT_TARGET} + dl pthread rt +) + +## If the library is a release, strip the target library +if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) + add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} *.so ) +endif () + +## Create temporary directories for packaged symbolic links +set ( LINK_DIRECTORY "links" ) + +add_custom_command ( OUTPUT ${LINK_DIRECTORY} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMAND mkdir -p links ) + +add_custom_command ( OUTPUT ${LINK_DIRECTORY}/${HSAKMT_COMPONENT} COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/include/${HSAKMT_COMPONENT} ${LINK_DIRECTORY}/${HSAKMT_COMPONENT} DEPENDS ${LINK_DIRECTORY} ) + +add_custom_command ( OUTPUT ${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/lib/${HSAKMT_COMPONENT}.so.${VERSION_MAJOR} ${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so DEPENDS ${LINK_DIRECTORY} ) + +add_custom_target ( copy_links ALL DEPENDS ${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so ${LINK_DIRECTORY}/${HSAKMT_COMPONENT} ) + +## Set the install targets +install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux PERMISSIONS OWNER_WRITE OWNER_READ ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${LINK_DIRECTORY}/${HSAKMT_COMPONENT} DESTINATION include PERMISSIONS OWNER_WRITE OWNER_READ ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so DESTINATION lib PERMISSIONS OWNER_WRITE OWNER_READ ) + +## Packaging directives +set ( CPACK_PACKAGE_NAME "hsakmt-roct-dev" ) +set ( CPACK_PACKAGE_VENDOR "AMD" ) +set ( CPACK_PACKAGE_VERSION_MAJOR ${VERSION_MAJOR} ) +set ( CPACK_PACKAGE_VERSION_MINOR ${VERSION_MINOR} ) +set ( CPACK_PACKAGE_VERSION_PATCH ${VERSION_PATCH} ) +set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) +set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "Thunk library for AMD KFD" ) +set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md" ) + +# Debian package specific variables +set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) +set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm" ) +set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libpci3" ) + +# RPM package specific variables +set ( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) +set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) + +include ( CPack ) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index b3919c75d4..b6a16c013a 100755 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -1,7 +1,19 @@ -#!/bin/bash +#/bin/bash set -e -echo "Updating ldconfig" -ldconfig -echo "Done" +do_ldconfig() { + echo /opt/rocm/libhsakmt/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig +} + +case "$1" in + configure) + do_ldconfig + ;; + abort-upgrade|abort-remove|abort-deconfigure) + echo "$1" + ;; + *) + exit 0 + ;; +esac diff --git a/DEBIAN/prerm b/DEBIAN/prerm new file mode 100755 index 0000000000..f8aa5a1993 --- /dev/null +++ b/DEBIAN/prerm @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +rm_ldconfig() { + rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig +} + +case "$1" in + remove) + rm_ldconfig + ;; + purge) + ;; + *) + exit 0 + ;; +esac + diff --git a/RPM/rpm_post b/RPM/rpm_post new file mode 100644 index 0000000000..7585d83dea --- /dev/null +++ b/RPM/rpm_post @@ -0,0 +1 @@ +echo /opt/rocm/libhsakmt/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig diff --git a/RPM/rpm_postun b/RPM/rpm_postun new file mode 100644 index 0000000000..0b130e27e8 --- /dev/null +++ b/RPM/rpm_postun @@ -0,0 +1 @@ +rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig diff --git a/cmake_modules/utils.cmake b/cmake_modules/utils.cmake new file mode 100644 index 0000000000..58067a4d3c --- /dev/null +++ b/cmake_modules/utils.cmake @@ -0,0 +1,71 @@ +################################################################################ +## +## The University of Illinois/NCSA +## Open Source License (NCSA) +## +## Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +## +## Developed by: +## +## AMD Research and AMD HSA Software Development +## +## Advanced Micro Devices, Inc. +## +## www.amd.com +## +## Permission is hereby granted, free of charge, to any person obtaining a copy +## of this software and associated documentation files (the "Software"), to +## deal with the Software without restriction, including without limitation +## the rights to use, copy, modify, merge, publish, distribute, sublicense, +## and#or sell copies of the Software, and to permit persons to whom the +## Software is furnished to do so, subject to the following conditions: +## +## - Redistributions of source code must retain the above copyright notice, +## this list of conditions and the following disclaimers. +## - Redistributions in binary form must reproduce the above copyright +## notice, this list of conditions and the following disclaimers in +## the documentation and#or other materials provided with the distribution. +## - Neither the names of Advanced Micro Devices, Inc, +## nor the names of its contributors may be used to endorse or promote +## products derived from this Software without specific prior written +## permission. +## +## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +## THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +## OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +## ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +## DEALINGS WITH THE SOFTWARE. +## +################################################################################ + +## Parses the VERSION_STRING variable and places +## the first, second and third number values in +## the major, minor and patch variables. +function(parse_version VERSION_STRING) + + string(REGEX MATCHALL "[0123456789]+" VERSIONS ${VERSION_STRING}) + list(LENGTH VERSIONS VERSION_COUNT) + + if (${VERSION_COUNT} GREATER 0) + list(GET VERSIONS 0 MAJOR) + set(VERSION_MAJOR ${MAJOR} PARENT_SCOPE) + set(TEMP_VERSION_STRING "${MAJOR}") + endif () + + if (${VERSION_COUNT} GREATER 1) + list(GET VERSIONS 1 MINOR) + set(VERSION_MINOR ${MINOR} PARENT_SCOPE) + set(TEMP_VERSION_STRING "${TEMP_VERSION_STRING}.${MINOR}") + endif () + + if (${VERSION_COUNT} GREATER 2) + list(GET VERSIONS 2 PATCH) + set(VERSION_PATCH ${PATCH} PARENT_SCOPE) + set(TEMP_VERSION_STRING "${TEMP_VERSION_STRING}.${PATCH}") + endif () + + set(VERSION_STRING "${TEMP_VERSION_STRING}" PARENT_SCOPE) + +endfunction() From b170e0ad8cdeea4ce0b9d3684dfd7850c083c3cd Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 30 Sep 2016 16:36:01 -0500 Subject: [PATCH 0194/1247] Fix CMakeList.txt file to use correct compile options. Fix compilation errors. Change-Id: I6229a83d0823ee7a123cdaa9efd782108aa3a03c --- CMakeLists.txt | 27 ++++++++++++--------------- src/topology.c | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8aff4b410a..8e4ca74a60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,7 @@ parse_version ( ${VERSION_STRING} ) set ( CMAKE_VERBOSE_MAKEFILE on ) ## Compiler flags -set ( CMAKE_CXX_FLAGS "-fPIC -W -Wall -Wextra -Werror -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden" ) +set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Werror -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden" ) set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) @@ -53,6 +53,7 @@ set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.$(VERSION_MAJOR) -Wl,-z,nodelete" ) ## Source files +set ( HSAKMT_SRC ${HSAKMT_SRC} src/debug.c ) set ( HSAKMT_SRC ${HSAKMT_SRC} src/events.c ) set ( HSAKMT_SRC ${HSAKMT_SRC} src/fmm.c ) set ( HSAKMT_SRC ${HSAKMT_SRC} src/globals.c ) @@ -93,26 +94,22 @@ if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} *.so ) endif () -## Create temporary directories for packaged symbolic links -set ( LINK_DIRECTORY "links" ) - -add_custom_command ( OUTPUT ${LINK_DIRECTORY} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMAND mkdir -p links ) - -add_custom_command ( OUTPUT ${LINK_DIRECTORY}/${HSAKMT_COMPONENT} COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/include/${HSAKMT_COMPONENT} ${LINK_DIRECTORY}/${HSAKMT_COMPONENT} DEPENDS ${LINK_DIRECTORY} ) - -add_custom_command ( OUTPUT ${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/lib/${HSAKMT_COMPONENT}.so.${VERSION_MAJOR} ${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so DEPENDS ${LINK_DIRECTORY} ) - -add_custom_target ( copy_links ALL DEPENDS ${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so ${LINK_DIRECTORY}/${HSAKMT_COMPONENT} ) - ## Set the install targets install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux PERMISSIONS OWNER_WRITE OWNER_READ ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${LINK_DIRECTORY}/${HSAKMT_COMPONENT} DESTINATION include PERMISSIONS OWNER_WRITE OWNER_READ ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${LINK_DIRECTORY}/${HSAKMT_COMPONENT}.so DESTINATION lib PERMISSIONS OWNER_WRITE OWNER_READ ) + +#if( CMAKE_HOST_UNIX ) +# install( CODE "EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E create_symlink include/${HSAKMT_COMPONENT} \${CMAKE_INSTALL_PREFIX}/include/${HSAKMT_COMPONENT} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) +# install( CODE "EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E create_symlink lib/${HSAKMT_COMPONENT}.so \${CMAKE_INSTALL_PREFIX}/lib/${HSAKMT_COMPONENT}.so WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) +#endif( ) + +if( CMAKE_HOST_UNIX ) + install( CODE "EXECUTE_PROCESS( COMMAND mkdir -p \${CMAKE_INSTALL_PREFIX}/include COMMAND ln -srf include/${HSAKMT_COMPONENT} \${CMAKE_INSTALL_PREFIX}/include/${HSAKMT_COMPONENT} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) + install( CODE "EXECUTE_PROCESS( COMMAND mkdir -p \${CMAKE_INSTALL_PREFIX}/lib COMMAND ln -srf lib/${HSAKMT_COMPONENT}.so \${CMAKE_INSTALL_PREFIX}/lib/${HSAKMT_COMPONENT}.so WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) +endif( ) ## Packaging directives set ( CPACK_PACKAGE_NAME "hsakmt-roct-dev" ) diff --git a/src/topology.c b/src/topology.c index e396c7eeb5..93de2f2331 100644 --- a/src/topology.c +++ b/src/topology.c @@ -984,7 +984,7 @@ err: sched_setaffinity(0, sizeof(cpu_set_t), &orig_cpuset); exit: if (ret != HSAKMT_STATUS_SUCCESS) - topology_destroy_temp_cpu_cache_list(&temp_cpu_ci_list); + topology_destroy_temp_cpu_cache_list(*temp_cpu_ci_list); return ret; } From 5493ae420bcc207f780d3d2f0c051052d2c03745 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Mon, 10 Oct 2016 18:34:15 -0500 Subject: [PATCH 0195/1247] Disable GPUVM-mapped doorbell on gfx802 gfx802 requires a workaround for a VM TLB bug in which lookups use the ACTIVE bit of the 8th PTE within any aligned group of 8 PTEs. Until this is fixed in amdgpu the GPUVM doorbell logic will fail. Change-Id: I5ec7b1fcd8b7677011a141d27cfc486c45d9a415 Signed-off-by: Jay Cornwall --- src/queues.c | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/queues.c b/src/queues.c index 3bd9b2827d..26b82b4f87 100644 --- a/src/queues.c +++ b/src/queues.c @@ -207,6 +207,33 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) return ret; } +static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) +{ + int i = 0; + while (supported_devices[i].dev_id != 0) { + if (supported_devices[i].dev_id == dev_id) { + return supported_devices[i].dev_info; + } + i++; + } + + return NULL; +} + +static bool use_gpuvm_doorbell(uint16_t dev_id) +{ + struct device_info *dev_info; + + dev_info = get_device_info_by_dev_id(dev_id); + + /* + * GPUVM doorbell on Tonga requires a workaround for VM TLB ACTIVE bit + * lookup bug. Remove ASIC check when this is implemented in amdgpu. + */ + return (topology_is_dgpu(dev_id) && + dev_info->asic_family != CHIP_TONGA); +} + void destroy_process_doorbells(void) { unsigned int i; @@ -218,7 +245,7 @@ void destroy_process_doorbells(void) if (doorbells[i].need_mmap) continue; - if (topology_is_dgpu(get_device_id_by_node(i))) { + if (use_gpuvm_doorbell(get_device_id_by_node(i))) { fmm_unmap_from_gpu(doorbells[i].doorbells); fmm_release(doorbells[i].doorbells); } else @@ -281,7 +308,7 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, return HSAKMT_STATUS_SUCCESS; } - if (topology_is_dgpu(get_device_id_by_node(NodeId))) + if (use_gpuvm_doorbell(get_device_id_by_node(NodeId))) status = map_doorbell_dgpu(NodeId, gpu_id, doorbell_offset); else status = map_doorbell_apu(NodeId, gpu_id, doorbell_offset); @@ -291,19 +318,6 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, return status; } -static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) -{ - int i = 0; - while (supported_devices[i].dev_id != 0) { - if (supported_devices[i].dev_id == dev_id) { - return supported_devices[i].dev_info; - } - i++; - } - - return NULL; -} - static void free_queue_cpu(struct queue *q) { if (q->eop_buffer) From 0de39b6724fbb79d27cf85b015cf8f0edb0567a9 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Tue, 1 Nov 2016 11:59:17 -0400 Subject: [PATCH 0196/1247] Fix hsaKmtOpen incorrectly doing nothing in some fork scenarios Currently, if a process' parent called hsaKmtOpen, the child will be unable to open a connection to KFD, since kfd_open_count will be > 0. When forking, the refcount should be reset, in order to allow the child to re-open /dev/kfd. Change-Id: Ia4b78f6bacc4f82e8ac724e5f488a3eff5084007 --- src/openclose.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/openclose.c b/src/openclose.c index 72baf7c997..5a55a3a50b 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -37,6 +37,19 @@ static const char kfd_device_name[] = "/dev/kfd"; static const char tmp_file[] = "/var/lock/.amd_hsa_thunk_lock"; int amd_hsa_thunk_lock_fd = 0; +static bool is_forked_child(void) +{ + static pid_t open_pid = -1; + pid_t my_pid = getpid(); + + if (open_pid == -1 || open_pid != my_pid) { + open_pid = my_pid; + return true; + } + + return false; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) @@ -47,6 +60,13 @@ hsaKmtOpenKFD(void) pthread_mutex_lock(&hsakmt_mutex); + /* If the process has forked, the child process must re-initialize + * it's connection to KFD. Any references tracked by kfd_open_count + * belong to the parent + */ + if (is_forked_child()) + kfd_open_count = 0; + if (kfd_open_count == 0) { fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); From c54c63fd5602fe8b5dc2699c37b84922f7470b3a Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Wed, 9 Nov 2016 21:15:07 -0600 Subject: [PATCH 0197/1247] Set the default value of userdata in pointer info to NULL. Change-Id: Ie0d94b921bbce880d9548d5a014a2d7c33062f7e --- src/fmm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fmm.c b/src/fmm.c index 8cf6553833..3125ae1f82 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -206,6 +206,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->mapped_node_id_array = NULL; object->flags = flags; object->metadata = NULL; + object->user_data = NULL; } return object; From a1f417715bb4aabc6c06f98769b9b986078b2f0c Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 15 Nov 2016 15:06:05 -0500 Subject: [PATCH 0198/1247] Making the code more robust by checking the NULL pointer Change-Id: I36b9f73eadd7547c71fe3641ac131c7408b14816 Signed-off-by: Yong Zhao --- src/memory.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/memory.c b/src/memory.c index bb086d9d6d..8687977cec 100644 --- a/src/memory.c +++ b/src/memory.c @@ -183,6 +183,11 @@ hsaKmtFreeMemory( { CHECK_KFD_OPEN(); + if (MemoryAddress == NULL) { + fprintf(stderr, "FIXME: freeing NULL pointer\n"); + return HSAKMT_STATUS_ERROR; + } + fmm_release(MemoryAddress); return HSAKMT_STATUS_SUCCESS; } @@ -283,6 +288,11 @@ hsaKmtMapMemoryToGPU( { CHECK_KFD_OPEN(); + if (MemoryAddress == NULL) { + fprintf(stderr, "FIXME: mapping NULL pointer\n"); + return HSAKMT_STATUS_ERROR; + } + if (AlternateVAGPU) *AlternateVAGPU = 0; @@ -306,6 +316,11 @@ hsaKmtMapMemoryToGPUNodes( uint32_t *gpu_id_array; HSAKMT_STATUS ret; + if (MemoryAddress == NULL) { + fprintf(stderr, "FIXME: mapping NULL pointer\n"); + return HSAKMT_STATUS_ERROR; + } + if (!is_dgpu && NumberOfNodes == 1) return hsaKmtMapMemoryToGPU(MemoryAddress, MemorySizeInBytes, From 2a50ebba9883e4bf03b32c2fa2908fed998bb95d Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 9 Nov 2016 14:54:17 -0500 Subject: [PATCH 0199/1247] Allow a memory to be registered multiple times A memory region is allowed to be registered multiple times when the memory is specified by a user pointer. If it's registered with the same user pointer but with different sizes, it's treated as different instances and multiple VM objects are created with different GPU address. Change-Id: I49627111bb5db36d18f1133b252fb62a611f06a4 --- src/fmm.c | 148 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 106 insertions(+), 42 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 3125ae1f82..e5453e33f3 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -65,12 +65,15 @@ struct vm_object { uint32_t *registered_device_id_array; uint32_t registered_device_id_array_size; uint32_t *registered_node_id_array; + uint32_t registration_count; /* the same memory region can be + registered multiple times */ /* * Nodes that mapped already */ uint32_t *mapped_device_id_array; uint32_t mapped_device_id_array_size; uint32_t *mapped_node_id_array; + uint32_t mapping_count; /* Metadata of imported graphics buffers */ void *metadata; /* User data associated with the memory */ @@ -204,6 +207,8 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->mapped_device_id_array_size = 0; object->registered_node_id_array = NULL; object->mapped_node_id_array = NULL; + object->registration_count = 0; + object->mapping_count = 0; object->flags = flags; object->metadata = NULL; object->user_data = NULL; @@ -355,18 +360,36 @@ static vm_object_t *vm_find_object_by_address_range(manageble_aperture_t *app, } static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, - const void *address) + const void *address, HSAuint64 size) { - vm_object_t *cur = app->vm_objects; + vm_object_t *cur = app->vm_objects, *obj; + uint32_t found = 0; - /* Look up the appropriate address range containing the given address */ + /* Look up the userptr that matches the address. If size is specified, + the size needs to match too. */ while (cur) { - if (cur->userptr == address) + if ((cur->userptr == address) && + ((cur->userptr_size == size) || !size)) { + found = 1; break; + } cur = cur->next; } - return cur; /* NULL if not found */ + /* If size is not specified, we need to ensure the vm_obj found is the + only obj having this address. */ + if (found && !size) { + obj = cur->next; + while (obj) { + if (obj->userptr == address) { + cur = NULL; + break; + } + obj = obj->next; + } + } + + return cur; /* NULL if any look-up failure */ } static vm_object_t *vm_find_object_by_userptr_range(manageble_aperture_t *app, @@ -1480,6 +1503,15 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, goto err_object_not_found; } + /* For a memory region that is registered by user pointer, changing + * mapping nodes is not allowed, so we don't need to check the mapping + * nodes or map if it's already mapped. Just increase the reference. + */ + if (object->userptr && object->mapping_count) { + ++object->mapping_count; + goto exit_ok; + } + args.handle = object->handle; if (object->registered_device_id_array_size > 0) { args.device_ids_array = object->registered_device_id_array; @@ -1507,7 +1539,9 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, memcpy(temp_mapped_id_array, args.device_ids_array, args.device_ids_array_size); object->mapped_device_id_array = temp_mapped_id_array; object->mapped_device_id_array_size = args.device_ids_array_size; + object->mapping_count = 1; +exit_ok: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1669,7 +1703,7 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, obj = object; if (!obj) { - obj = vm_find_object_by_userptr(aperture, addr); + obj = vm_find_object_by_userptr(aperture, addr, size); if (obj == NULL) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; @@ -1761,8 +1795,9 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, vm_object_t *obj) { vm_object_t *object; - int ret; + int ret = 0; struct kfd_ioctl_unmap_memory_from_gpu_new_args args; + HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); @@ -1770,13 +1805,19 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, /* Find the object to retrieve the handle */ object = obj; if (!object) { - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, + VOID_PTR_SUB(address, page_offset), 0); if (!object) { ret = -1; - goto err; + goto out; } } + if (object->userptr && object->mapping_count > 1) { + --object->mapping_count; + goto out; + } + args.handle = object->handle; if (device_ids_array && device_ids_array_size > 0) { args.device_ids_array = device_ids_array; @@ -1791,14 +1832,14 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, * need to deploy the change on there side before thunk fails on this case. */ ret = 0; - goto err; + goto out; } print_device_id_array(args.device_ids_array, args.device_ids_array_size); ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); if (ret != 0) - goto err; + goto out; /* Clearing all mapped nodes list */ if (object->mapped_device_id_array != NULL && @@ -1812,12 +1853,9 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; + object->mapping_count = 0; - if (!obj) - pthread_mutex_unlock(&aperture->fmm_mutex); - - return 0; -err: +out: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); return ret; @@ -1894,7 +1932,7 @@ static int _fmm_unmap_from_gpu_userptr(void *addr) /* Find the start address in SVM space for GPU unmapping */ pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_userptr(aperture, addr); + obj = vm_find_object_by_userptr(aperture, addr, 0); if (obj == NULL) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; @@ -2160,10 +2198,14 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Check if this address was already registered */ pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_userptr(aperture, addr); + obj = vm_find_object_by_userptr(aperture, addr, size); + if (obj != NULL) { + ++obj->registration_count; + pthread_mutex_unlock(&aperture->fmm_mutex); + *obj_ret = obj; + return HSAKMT_STATUS_SUCCESS; + } pthread_mutex_unlock(&aperture->fmm_mutex); - if (obj != NULL) - return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, @@ -2176,6 +2218,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj obj->userptr = addr; gpuid_to_nodeid(gpu_id, &obj->node_id); obj->userptr_size = size; + obj->registration_count = 1; pthread_mutex_unlock(&aperture->fmm_mutex); } else @@ -2225,8 +2268,18 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (!object) return HSAKMT_STATUS_NOT_SUPPORTED; - if (object->registered_device_id_array_size > 0) - return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; + if (object->registered_device_id_array_size > 0) { + /* Multiple registration is allowed, but not changing nodes */ + if ((gpu_id_array_size != object->registered_device_id_array_size) + || memcmp(object->registered_device_id_array, + gpu_id_array, gpu_id_array_size)) { + fprintf(stderr, + "Error. Changing nodes in a registered addr.\n"); + return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; + } + else + return HSAKMT_STATUS_SUCCESS; + } if (gpu_id_array_size > 0) { object->registered_device_id_array = gpu_id_array; @@ -2345,10 +2398,10 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) /* Find the size and start address in SVM space */ pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_userptr(aperture, addr); - if (obj == NULL) { + obj = vm_find_object_by_userptr(aperture, addr, 0); + if ((obj == NULL) || (obj->registration_count > 1)) { pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + return HSAKMT_STATUS_ERROR; } svm_addr = obj->start; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -2364,6 +2417,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) manageble_aperture_t *aperture = NULL; vm_object_t *object = NULL; unsigned i; + HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) @@ -2399,18 +2453,24 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_address(aperture, address, 0); + object = vm_find_object_by_address(aperture, + VOID_PTR_SUB(address, page_offset), 0); if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } - if (object->userptr) { + if (object->registration_count > 1) { + --object->registration_count; pthread_mutex_unlock(&aperture->fmm_mutex); - return fmm_deregister_user_memory(object->userptr); - } else if (object->metadata) { + return HSAKMT_STATUS_SUCCESS; + } + + if (object->metadata || object->userptr) { /* An object with metadata is an imported graphics - * buffer. Deregistering it means releasing the buffer. */ + * buffer. Deregistering imported graphics buffers or + * userptrs means releasing the BO. + */ pthread_mutex_unlock(&aperture->fmm_mutex); __fmm_release(address, aperture); return HSAKMT_STATUS_SUCCESS; @@ -2427,6 +2487,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) if (object->registered_node_id_array) free(object->registered_node_id_array); object->registered_node_id_array = NULL; + object->registration_count = 0; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -2470,7 +2531,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, pthread_mutex_lock(&aperture->fmm_mutex); if (userptr && is_dgpu) - object = vm_find_object_by_userptr(aperture, address); + object = vm_find_object_by_userptr(aperture, address, size); else object = vm_find_object_by_address(aperture, address, 0); @@ -2479,6 +2540,17 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_ERROR; } + /* For userptr, we ignore the nodes array and map all registered nodes. + * This is to simply the implementation of allowing the same memory + * region to be registered multiple times. + */ + if (userptr && is_dgpu) { + retcode = _fmm_map_to_gpu_userptr(address, size, + gpuvm_address, object); + pthread_mutex_unlock(&aperture->fmm_mutex); + return retcode; + } + /* Verify that all nodes to map are registered already */ registered_node_id_array = all_gpu_id_array; registered_node_id_array_size = all_gpu_id_array_size; @@ -2539,16 +2611,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, object->registered_device_id_array = nodes_to_map; object->registered_device_id_array_size = nodes_to_map_size; - if (nodes_to_map_size > 0) { - if (userptr && is_dgpu) - retcode = _fmm_map_to_gpu_userptr(address, size, - gpuvm_address, - object); - else - retcode = _fmm_map_to_gpu_gtt(aperture, address, - size, - object); - } + if (nodes_to_map_size > 0) + retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object); /* Restore old registered device id array */ object->registered_device_id_array = temp_node_id_array; @@ -2647,7 +2711,7 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) vm_obj = vm_find_object_by_address(aperture, mem, 0); if (!vm_obj) - vm_obj = vm_find_object_by_userptr(aperture, mem); + vm_obj = vm_find_object_by_userptr(aperture, mem, 0); if (!vm_obj) return HSAKMT_STATUS_ERROR; From 4838f6e740e65ed45f6a092d7f2737b577c08de0 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 22 Nov 2016 09:44:22 -0500 Subject: [PATCH 0200/1247] kfd ioctl. Remove unused definitions Sync the file with Kernel header file Change-Id: I52d52a38fb38bd4b37d8210ce79775b88c8a985d --- include/linux/kfd_ioctl.h | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 613a2c7b8c..74823bd893 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -301,16 +301,6 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { uint64_t dgpu_limit; }; -struct kfd_ioctl_eviction_args { - uint64_t size; - uint64_t type; -}; - -enum evict_type { - EVICT_FIRST_PDD = 0, - EVICT_BIGGEST_PDD -}; - /* * GPU Memory allocation flags */ @@ -482,8 +472,6 @@ struct kfd_ioctl_get_tile_config_args { #define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ AMDKFD_IOWR(0x1d, struct kfd_ioctl_get_process_apertures_new_args) -#define AMDKFD_IOC_EVICT_MEMORY \ - AMDKFD_IOWR(0x1e, struct kfd_ioctl_eviction_args) #define AMDKFD_IOC_GET_DMABUF_INFO \ AMDKFD_IOWR(0x1f, struct kfd_ioctl_get_dmabuf_info_args) From 559e31d6ffc561ad17c0ccd2d01f572ad9bfe668 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 22 Nov 2016 14:35:35 -0500 Subject: [PATCH 0201/1247] Add API entrypoints for IPC functionality Implement three new APIs for IPC buffer sharing: -hsaKmtShareMemory() -hsaKmtRegisterSharedHandle() -hsaKmtRegisterSharedHandleToNodes() Add new ioclts necessary for the above APIs. Change-Id: Ia2b4d0dc91ec64bff959395d11c0536467404792 Signed-off-by: Harish Kasiviswanathan --- include/hsakmt.h | 37 ++++++ include/hsakmttypes.h | 2 + include/linux/kfd_ioctl.h | 24 +++- src/fmm.c | 230 ++++++++++++++++++++++++++++++++++++-- src/fmm.h | 8 ++ src/libhsakmt.h | 1 + src/libhsakmt.ver | 3 + src/memory.c | 73 ++++++++++++ 8 files changed, 367 insertions(+), 11 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 8604a439be..0eea105772 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -406,6 +406,43 @@ hsaKmtRegisterGraphicsHandleToNodes( HSAuint32* NodeArray //IN ); +/** + Export a memory buffer for sharing with other processes + + NOTE: for the current revision of the thunk spec, SizeInBytes + must match whole allocation. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtShareMemory( + void *MemoryAddress, // IN + HSAuint64 SizeInBytes, // IN + HsaSharedMemoryHandle *SharedMemoryHandle // OUT +); + +/** + Register shared memory handle +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterSharedHandle( + const HsaSharedMemoryHandle *SharedMemoryHandle, // IN + void **MemoryAddress, // OUT + HSAuint64 *SizeInBytes // OUT +); + +/** + Register shared memory handle to specific nodes only +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterSharedHandleToNodes( + const HsaSharedMemoryHandle *SharedMemoryHandle, // IN + void **MemoryAddress, // OUT + HSAuint64 *SizeInBytes, // OUT + HSAuint64 NumberOfNodes, // OUT + HSAuint32* NodeArray // OUT +); /** Unregisters with KFD a memory buffer diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index b1b8de205a..2d29a7083c 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1004,6 +1004,8 @@ typedef struct _HsaPointerInfo { void *UserData; // User data associated with the memory } HsaPointerInfo; +typedef HSAuint32 HsaSharedMemoryHandle[8]; + #pragma pack(pop, hsakmttypes_h) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 74823bd893..12090c6419 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -356,6 +356,22 @@ struct kfd_ioctl_import_dmabuf_args { uint32_t dmabuf_fd; /* to KFD */ }; +struct kfd_ioctl_ipc_export_handle_args { + uint64_t handle; /* to KFD */ + uint32_t share_handle[4]; /* from KFD */ + uint32_t gpu_id; /* to KFD */ + uint32_t pad; +}; + +struct kfd_ioctl_ipc_import_handle_args { + uint64_t handle; /* from KFD */ + uint64_t va_addr; /* to KFD */ + uint64_t mmap_offset; /* from KFD */ + uint32_t share_handle[4]; /* to KFD */ + uint32_t gpu_id; /* to KFD */ + uint32_t pad; +}; + struct kfd_ioctl_get_tile_config_args { /* to KFD: pointer to tile array */ uint64_t tile_config_ptr; @@ -482,7 +498,13 @@ struct kfd_ioctl_get_tile_config_args { #define AMDKFD_IOC_GET_TILE_CONFIG \ AMDKFD_IOWR(0x21, struct kfd_ioctl_get_tile_config_args) +#define AMDKFD_IOC_IPC_IMPORT_HANDLE \ + AMDKFD_IOWR(0x22, struct kfd_ioctl_ipc_import_handle_args) + +#define AMDKFD_IOC_IPC_EXPORT_HANDLE \ + AMDKFD_IOWR(0x23, struct kfd_ioctl_ipc_export_handle_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x22 +#define AMDKFD_COMMAND_END 0x24 #endif diff --git a/src/fmm.c b/src/fmm.c index e5453e33f3..67e1f6f8be 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -78,6 +78,8 @@ struct vm_object { void *metadata; /* User data associated with the memory */ void *user_data; + /* Flag to indicate imported KFD buffer */ + bool is_imported_kfd_bo; }; typedef struct vm_object vm_object_t; @@ -152,6 +154,45 @@ static manageble_aperture_t cpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0); static uint32_t all_gpu_id_array_size = 0; static uint32_t *all_gpu_id_array = NULL; +/* IPC structures and helper functions */ +typedef enum _HSA_APERTURE { + HSA_APERTURE_UNSUPPORTED = 0, + HSA_APERTURE_DGPU, + HSA_APERTURE_DGPU_ALT, + HSA_APERTURE_GPUVM, + HSA_APERTURE_CPUVM +} HSA_APERTURE; + +typedef struct _HsaApertureInfo { + HSA_APERTURE type; // Aperture type + HSAuint32 idx; // Aperture index +} HsaApertureInfo; + +typedef struct _HsaSharedMemoryStruct { + HSAuint32 ShareHandle[4]; + HsaApertureInfo ApeInfo; + HSAuint32 SizeInPages; + HSAuint32 ExportGpuId; +} HsaSharedMemoryStruct; + +static inline const HsaSharedMemoryStruct * to_const_hsa_shared_memory_struct( + const HsaSharedMemoryHandle *SharedMemoryHandle) +{ + return (const HsaSharedMemoryStruct *)SharedMemoryHandle; +} + +static inline HsaSharedMemoryStruct * to_hsa_shared_memory_struct( + HsaSharedMemoryHandle *SharedMemoryHandle) +{ + return (HsaSharedMemoryStruct *)SharedMemoryHandle; +} + +static inline HsaSharedMemoryHandle * to_hsa_shared_memory_handle( + HsaSharedMemoryStruct *SharedMemoryStruct) +{ + return (HsaSharedMemoryHandle *)SharedMemoryStruct; +} + extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit); @@ -212,6 +253,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->flags = flags; object->metadata = NULL; object->user_data = NULL; + object->is_imported_kfd_bo = false; } return object; @@ -567,32 +609,64 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) return -1; } -static manageble_aperture_t *fmm_find_aperture(const void *address) +static manageble_aperture_t *fmm_get_aperture(HsaApertureInfo info) +{ + switch (info.type) { + case HSA_APERTURE_DGPU: + return &svm.dgpu_aperture; + case HSA_APERTURE_DGPU_ALT: + return &svm.dgpu_alt_aperture; + case HSA_APERTURE_GPUVM: + return &gpu_mem[info.idx].gpuvm_aperture; + case HSA_APERTURE_CPUVM: + return &cpuvm_aperture; + default: + return NULL; + } +} +static manageble_aperture_t *fmm_find_aperture(const void *address, + HsaApertureInfo *info) { manageble_aperture_t *aperture = NULL; uint32_t i; + HsaApertureInfo _info = { .type = HSA_APERTURE_UNSUPPORTED, .idx = 0}; if (is_dgpu) { if (address >= svm.dgpu_aperture.base && - address <= svm.dgpu_aperture.limit) + address <= svm.dgpu_aperture.limit) { aperture = &svm.dgpu_aperture; + _info.type = HSA_APERTURE_DGPU; + } else if (address >= svm.dgpu_alt_aperture.base && - address <= svm.dgpu_alt_aperture.limit) + address <= svm.dgpu_alt_aperture.limit) { aperture = &svm.dgpu_alt_aperture; - else - /* Not in SVM, it can be system memory registered by userptr */ + _info.type = HSA_APERTURE_DGPU_ALT; + } + else { + /* Not in SVM, it can be system memory registered by userptr */ aperture = &svm.dgpu_aperture; + _info.type = HSA_APERTURE_DGPU; + } } else { /* APU */ for (i = 0; i < gpu_mem_count; i++) { if ((address >= gpu_mem[i].gpuvm_aperture.base) && - (address <= gpu_mem[i].gpuvm_aperture.limit)) + (address <= gpu_mem[i].gpuvm_aperture.limit)) { aperture = &gpu_mem[i].gpuvm_aperture; + _info.type = HSA_APERTURE_GPUVM; + _info.idx = i; + } } - if (!aperture) /* Not in GPUVM */ + if (!aperture) { + /* Not in GPUVM */ aperture = &cpuvm_aperture; + _info.type = HSA_APERTURE_CPUVM; + } } + if (info) + *info = _info; + return aperture; } @@ -2307,6 +2381,9 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, int r; HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; + if (gpu_id_array_size > 0 && gpu_id_array == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + infoArgs.dmabuf_fd = GraphicsResourceHandle; infoArgs.metadata_size = GRAPHICS_METADATA_DEFAULT_SIZE; metadata = calloc(infoArgs.metadata_size, 1); @@ -2388,6 +2465,139 @@ error_free_metadata: return status; } +HSAKMT_STATUS fmm_share_memory(void* MemoryAddress, + HSAuint64 SizeInBytes, + HsaSharedMemoryHandle *SharedMemoryHandle) +{ + int r = 0; + HSAuint32 gpu_id = 0; + vm_object_t *obj = NULL; + manageble_aperture_t * aperture = NULL; + struct kfd_ioctl_ipc_export_handle_args exportArgs; + HsaApertureInfo ApeInfo; + HsaSharedMemoryStruct *SharedMemoryStruct = + to_hsa_shared_memory_struct(SharedMemoryHandle); + + if (SizeInBytes >= (1ULL << ((sizeof(HSAuint32) * 8) + PAGE_SHIFT))) + return HSAKMT_STATUS_INVALID_PARAMETER; + + aperture = fmm_find_aperture(MemoryAddress, &ApeInfo); + if (!aperture) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pthread_mutex_lock(&aperture->fmm_mutex); + obj = vm_find_object_by_address(aperture, MemoryAddress, 0); + pthread_mutex_unlock(&aperture->fmm_mutex); + if (!obj) + return HSAKMT_STATUS_INVALID_PARAMETER; + + r = validate_nodeid(obj->node_id, &gpu_id); + if (r < 0) + return HSAKMT_STATUS_ERROR; + + exportArgs.handle = obj->handle; + exportArgs.gpu_id = gpu_id; + + + r = kmtIoctl(kfd_fd, AMDKFD_IOC_IPC_EXPORT_HANDLE, (void *)&exportArgs); + if (r) + return HSAKMT_STATUS_ERROR; + + memcpy(SharedMemoryStruct->ShareHandle, exportArgs.share_handle, + sizeof(SharedMemoryStruct->ShareHandle)); + SharedMemoryStruct->ApeInfo = ApeInfo; + SharedMemoryStruct->SizeInPages = (HSAuint32) (SizeInBytes >> PAGE_SHIFT); + SharedMemoryStruct->ExportGpuId = gpu_id; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle, + HSAuint64 *SizeInBytes, + void **MemoryAddress, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size) +{ + int r = 0; + HSAKMT_STATUS err = HSAKMT_STATUS_ERROR; + vm_object_t *obj = NULL; + void *reservedMem = NULL; + manageble_aperture_t *aperture; + struct kfd_ioctl_ipc_import_handle_args importArgs; + struct kfd_ioctl_free_memory_of_gpu_args freeArgs; + const HsaSharedMemoryStruct *SharedMemoryStruct = + to_const_hsa_shared_memory_struct(SharedMemoryHandle); + + if (gpu_id_array_size > 0 && gpu_id_array == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + memcpy(importArgs.share_handle, SharedMemoryStruct->ShareHandle, + sizeof(importArgs.share_handle)); + importArgs.gpu_id = SharedMemoryStruct->ExportGpuId; + + aperture = fmm_get_aperture(SharedMemoryStruct->ApeInfo); + + pthread_mutex_lock(&aperture->fmm_mutex); + reservedMem = aperture_allocate_area(aperture, + (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), + 0); + pthread_mutex_unlock(&aperture->fmm_mutex); + if (!reservedMem) { + err = HSAKMT_STATUS_NO_MEMORY; + goto err_free_buffer; + } + + importArgs.va_addr = (uint64_t)reservedMem; + r = kmtIoctl(kfd_fd, AMDKFD_IOC_IPC_IMPORT_HANDLE, (void *)&importArgs); + if (r) { + err = HSAKMT_STATUS_ERROR; + goto err_import; + } + + pthread_mutex_lock(&aperture->fmm_mutex); + obj = aperture_allocate_object(aperture, reservedMem, importArgs.handle, + (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), + 0); + if (!obj) { + err = HSAKMT_STATUS_NO_MEMORY; + goto err_free_mem; + } + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (importArgs.mmap_offset) { + void *ret = mmap(reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd, + importArgs.mmap_offset); + if (ret == MAP_FAILED) { + err = HSAKMT_STATUS_ERROR; + goto err_free_obj; + } + } + + *MemoryAddress = reservedMem; + *SizeInBytes = (SharedMemoryStruct->SizeInPages << PAGE_SHIFT); + + if (gpu_id_array_size > 0) { + obj->registered_device_id_array = gpu_id_array; + obj->registered_device_id_array_size = gpu_id_array_size; + } + obj->is_imported_kfd_bo = true; + + return HSAKMT_STATUS_SUCCESS; +err_free_obj: + pthread_mutex_lock(&aperture->fmm_mutex); + vm_remove_object(aperture, obj); +err_free_mem: + aperture_release_area(aperture, reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT)); + pthread_mutex_unlock(&aperture->fmm_mutex); +err_free_buffer: + freeArgs.handle = importArgs.handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &freeArgs); +err_import: + return err; +} + static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) { manageble_aperture_t *aperture; @@ -2466,7 +2676,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) return HSAKMT_STATUS_SUCCESS; } - if (object->metadata || object->userptr) { + if (object->metadata || object->userptr || object->is_imported_kfd_bo) { /* An object with metadata is an imported graphics * buffer. Deregistering imported graphics buffers or * userptrs means releasing the BO. @@ -2635,7 +2845,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) memset(info, 0, sizeof(HsaPointerInfo)); - aperture = fmm_find_aperture(address); + aperture = fmm_find_aperture(address, NULL); vm_obj = vm_find_object_by_address_range(aperture, address); if (!vm_obj) @@ -2707,7 +2917,7 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) manageble_aperture_t *aperture; vm_object_t *vm_obj; - aperture = fmm_find_aperture(mem); + aperture = fmm_find_aperture(mem, NULL); vm_obj = vm_find_object_by_address(aperture, mem, 0); if (!vm_obj) diff --git a/src/fmm.h b/src/fmm.h index 6fc8c56928..2fb1d60cf9 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -82,6 +82,14 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, uint32_t *gpu_id_array, uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_deregister_memory(void *address); +HSAKMT_STATUS fmm_share_memory(void* MemoryAddress, + HSAuint64 SizeInBytes, + HsaSharedMemoryHandle *SharedMemoryHandle); +HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle, + HSAuint64 *SizeInBytes, + void **MemoryAddress, + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint32_t nodes_to_map_size, uint64_t *gpuvm_address); #endif /* FMM_H_ */ diff --git a/src/libhsakmt.h b/src/libhsakmt.h index b270d458cf..2243521df1 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -50,6 +50,7 @@ extern bool is_dgpu; do { if (kfd_open_count == 0) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) #define PAGE_SIZE 4096 +#define PAGE_SHIFT 12 /* VI HW bug requires this virtual address alignment */ #define TONGA_PAGE_SIZE 0x8000 diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index bf25aa5bd1..e1b30fb87d 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -27,6 +27,9 @@ hsaKmtFreeMemory; hsaKmtRegisterMemory; hsaKmtRegisterMemoryToNodes; hsaKmtRegisterGraphicsHandleToNodes; +hsaKmtShareMemory; +hsaKmtRegisterSharedHandle; +hsaKmtRegisterSharedHandleToNodes; hsaKmtDeregisterMemory; hsaKmtMapMemoryToGPU; hsaKmtMapMemoryToGPUNodes; diff --git a/src/memory.c b/src/memory.c index 8687977cec..24cc5a818c 100644 --- a/src/memory.c +++ b/src/memory.c @@ -267,6 +267,79 @@ hsaKmtRegisterGraphicsHandleToNodes( return ret; } +HSAKMT_STATUS +HSAKMTAPI +hsaKmtShareMemory( + void *MemoryAddress, /* IN */ + HSAuint64 SizeInBytes, /* IN */ + HsaSharedMemoryHandle *SharedMemoryHandle /* OUT */ +) +{ + CHECK_KFD_OPEN(); + + if (!SharedMemoryHandle) + return HSAKMT_STATUS_INVALID_PARAMETER; + + return fmm_share_memory(MemoryAddress, SizeInBytes, SharedMemoryHandle); +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterSharedHandle( + const HsaSharedMemoryHandle *SharedMemoryHandle, /* IN */ + void **MemoryAddress, /* OUT */ + HSAuint64 *SizeInBytes /* OUT */ +) +{ + CHECK_KFD_OPEN(); + + return hsaKmtRegisterSharedHandleToNodes(SharedMemoryHandle, + MemoryAddress, + SizeInBytes, + 0, + NULL); +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterSharedHandleToNodes( + const HsaSharedMemoryHandle *SharedMemoryHandle, /* IN */ + void **MemoryAddress, /* OUT */ + HSAuint64 *SizeInBytes, /* OUT */ + HSAuint64 NumberOfNodes, /* OUT */ + HSAuint32* NodeArray /* OUT */ +) +{ + CHECK_KFD_OPEN(); + + uint32_t *gpu_id_array = NULL; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + if (!SharedMemoryHandle) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (NodeArray) { + ret = validate_nodeid_array(&gpu_id_array, NumberOfNodes, NodeArray); + if (ret != HSAKMT_STATUS_SUCCESS) + goto error; + } + + ret = fmm_register_shared_memory(SharedMemoryHandle, + SizeInBytes, + MemoryAddress, + gpu_id_array, + NumberOfNodes*sizeof(uint32_t)); + if (ret != HSAKMT_STATUS_SUCCESS) + goto error; + + return ret; + +error: + if (gpu_id_array) + free(gpu_id_array); + return ret; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory( From 4181b408fc260b5750769f3466aeee11d757551a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 4 Jan 2017 16:07:56 -0500 Subject: [PATCH 0202/1247] Allocate CWSR buffer in system memory CWSR buffers can be large on dGPUs (~21MB on gfx803). Allocating them in VRAM limits the number of queues that can be created unnecessarily. Also make freeing of per-queue buffers symmetric with allocation. All buffers are now allocated with allocate_exec_aligned_memory on dGPUs and APUs, so use free_exec_aligned_memory to free them. Change-Id: I45e8cb1801857d0268750202cdd422426611e457 --- src/queues.c | 63 ++++++++++++---------------------------------------- 1 file changed, 14 insertions(+), 49 deletions(-) diff --git a/src/queues.c b/src/queues.c index 26b82b4f87..e4e383c08c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -318,15 +318,6 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, return status; } -static void free_queue_cpu(struct queue *q) -{ - if (q->eop_buffer) - free(q->eop_buffer); - if (q->ctx_save_restore) - free(q->ctx_save_restore); - free(q); -} - static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) { void *ptr; @@ -433,24 +424,17 @@ static void free_exec_aligned_memory(void *addr, uint32_t size, uint32_t align, free(addr); } -static void free_queue_gpu(struct queue *q) -{ - if (q->eop_buffer) { - hsaKmtUnmapMemoryToGPU(q->eop_buffer); - hsaKmtFreeMemory(q->eop_buffer, q->dev_info->eop_buffer_size); - } - if (q->ctx_save_restore) { - hsaKmtUnmapMemoryToGPU(q->ctx_save_restore); - hsaKmtFreeMemory(q->ctx_save_restore, q->ctx_save_restore_size); - } - free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); -} - static void free_queue(struct queue *q) { - if (IS_DGPU(q->dev_info->asic_family)) - return free_queue_gpu(q); - return free_queue_cpu(q); + if (q->eop_buffer) + free_exec_aligned_memory(q->eop_buffer, + q->dev_info->eop_buffer_size, + PAGE_SIZE, q->dev_info->asic_family); + if (q->ctx_save_restore) + free_exec_aligned_memory(q->ctx_save_restore, + q->ctx_save_restore_size, + PAGE_SIZE, q->dev_info->asic_family); + free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } static int handle_concrete_asic(struct queue *q, @@ -475,30 +459,11 @@ static int handle_concrete_asic(struct queue *q, update_ctx_save_restore_size(NodeId, q) == true) { args->ctx_save_restore_size = q->ctx_save_restore_size; args->ctl_stack_size = q->ctl_stack_size; - if (IS_DGPU(dev_info->asic_family)) { - void *mem; - HsaMemFlags flags; - HSAKMT_STATUS ret; - HSAuint64 size = q->ctx_save_restore_size; - flags.Value = 0; - flags.ui32.NonPaged = 1; /* device memory*/ - - ret = hsaKmtAllocMemory(NodeId, size, flags, &mem); - if (ret != HSAKMT_STATUS_SUCCESS) - return ret; - ret = hsaKmtMapMemoryToGPU(mem, size, NULL); - if (ret != HSAKMT_STATUS_SUCCESS) { - hsaKmtFreeMemory(mem, size); - return ret; - } - q->ctx_save_restore = mem; - - } else - q->ctx_save_restore = - allocate_exec_aligned_memory(q->ctx_save_restore_size, - PAGE_SIZE, - dev_info->asic_family, - NodeId); + q->ctx_save_restore = + allocate_exec_aligned_memory(q->ctx_save_restore_size, + PAGE_SIZE, + dev_info->asic_family, + NodeId); if (q->ctx_save_restore == NULL) {; return HSAKMT_STATUS_NO_MEMORY; } From f1f62d863c4e3744b3b15859b17d4a787e924dae Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 10 Jan 2017 14:34:47 -0500 Subject: [PATCH 0203/1247] Add fork support If fork() is called, clear all duplicated data that is invalid in the child process. Change-Id: I4e27198060db593c630c6337b7071dfbd0d80b83 Signed-off-by: Harish Kasiviswanathan --- src/events.c | 5 ++++ src/fmm.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ src/libhsakmt.h | 3 +++ src/openclose.c | 31 +++++++++++++++++++----- src/queues.c | 23 ++++++++++++++++++ 5 files changed, 120 insertions(+), 6 deletions(-) diff --git a/src/events.c b/src/events.c index 6de79903c7..392d0b4cf8 100644 --- a/src/events.c +++ b/src/events.c @@ -36,6 +36,11 @@ static HSAuint64 *events_page = NULL; +void clear_events_page(void) +{ + events_page = NULL; +} + static bool IsSystemEventType(HSA_EVENTTYPE type) { // Debug events behave as signal events. diff --git a/src/fmm.c b/src/fmm.c index 67e1f6f8be..0fd0320432 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2928,3 +2928,67 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) vm_obj->user_data = usr_data; return HSAKMT_STATUS_SUCCESS; } + +static void fmm_clear_aperture(manageble_aperture_t *app) +{ + while (app->vm_objects) + vm_remove_object(app, app->vm_objects); + + while (app->vm_ranges) + vm_remove_area(app, app->vm_ranges); + +} + +/* This is a special funcion that should be called only from the child process + * after a fork(). This will clear all vm_objects and mmaps duplicated from + * the parent. + */ +void fmm_clear_all_mem(void) +{ + uint32_t i; + void *map_addr; + + /* Nothing is initialized. */ + if (!gpu_mem) + return; + + fmm_clear_aperture(&cpuvm_aperture); + + for (i = 0; i < gpu_mem_count; i++) { + fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); + fmm_clear_aperture(&gpu_mem[i].scratch_aperture); + fmm_clear_aperture(&gpu_mem[i].scratch_physical); + } + + if (is_dgpu_mem_init) { + fmm_clear_aperture(&svm.dgpu_aperture); + fmm_clear_aperture(&svm.dgpu_alt_aperture); + + /* Use the same dgpu range as the parent. If failed, then set + * is_dgpu_mem_init to false. Later on dgpu_mem_init will try + * to get a new range + */ + map_addr = mmap(dgpu_shared_aperture_base, (HSAuint64)(dgpu_shared_aperture_limit)- + (HSAuint64)(dgpu_shared_aperture_base) + 1, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); + + if (map_addr == MAP_FAILED) { + munmap(dgpu_shared_aperture_base, + (HSAuint64)(dgpu_shared_aperture_limit) - + (HSAuint64)(dgpu_shared_aperture_base) + 1); + + dgpu_shared_aperture_base = NULL; + dgpu_shared_aperture_limit = NULL; + is_dgpu_mem_init = false; + } + } + + if (all_gpu_id_array) + free(all_gpu_id_array); + + all_gpu_id_array_size = 0; + all_gpu_id_array = NULL; + + gpu_mem_count = 0; + free(gpu_mem); +} diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 2243521df1..e3b07dc90a 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -95,4 +95,7 @@ extern int kmtIoctl(int fd, unsigned long request, void *arg); #define VOID_PTR_SUB(ptr,n) (void*)((uint8_t*)(ptr) - n)/*ptr - offset*/ #define VOID_PTRS_SUB(ptr1,ptr2) (uint64_t)((uint8_t*)(ptr1) - (uint8_t*)(ptr2)) /*ptr1 - ptr2*/ +void clear_events_page(void); +void fmm_clear_all_mem(void); +void clear_process_doorbells(void); #endif diff --git a/src/openclose.c b/src/openclose.c index 5a55a3a50b..8901cffcdc 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -36,20 +36,39 @@ static const char kfd_device_name[] = "/dev/kfd"; static const char tmp_file[] = "/var/lock/.amd_hsa_thunk_lock"; int amd_hsa_thunk_lock_fd = 0; +static pid_t parent_pid = -1; + static bool is_forked_child(void) { - static pid_t open_pid = -1; - pid_t my_pid = getpid(); + pid_t cur_pid = getpid(); - if (open_pid == -1 || open_pid != my_pid) { - open_pid = my_pid; - return true; + if (parent_pid == -1) { + parent_pid = cur_pid; + return false; } + if (parent_pid != cur_pid) + return true; + return false; } +/* Call this from the child process after fork. This will clear all + * data that is duplicated from the parent process, that is not valid + * in the child. + * The topology information is duplicated from the parent is valid + * in the child process so it is not cleared + */ +static void clear_after_fork(void) +{ + clear_process_doorbells(); + clear_events_page(); + fmm_clear_all_mem(); + destroy_device_debugging_memory(); + kfd_open_count = 0; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) @@ -65,7 +84,7 @@ hsaKmtOpenKFD(void) * belong to the parent */ if (is_forked_child()) - kfd_open_count = 0; + clear_after_fork(); if (kfd_open_count == 0) { diff --git a/src/queues.c b/src/queues.c index e4e383c08c..a2ddb535a8 100644 --- a/src/queues.c +++ b/src/queues.c @@ -257,6 +257,29 @@ void destroy_process_doorbells(void) num_doorbells = 0; } +/* This is a special funcion that should be called only from the child process + * after a fork(). This will clear doorbells duplicated from the parent. + */ +void clear_process_doorbells(void) +{ + unsigned int i; + + if (!doorbells) + return; + + for (i = 0; i < num_doorbells; i++) { + if (doorbells[i].need_mmap) + continue; + + if (!use_gpuvm_doorbell(get_device_id_by_node(i))) + munmap(doorbells[i].doorbells, DOORBELLS_PAGE_SIZE); + } + + free(doorbells); + doorbells = NULL; + num_doorbells = 0; +} + static HSAKMT_STATUS map_doorbell_apu(HSAuint32 NodeId, HSAuint32 gpu_id, HSAuint64 doorbell_offset) { From 72b842a6dceb6d6b21b7af09d7dfd336e2f1b1dd Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 1 Feb 2017 10:30:22 -0500 Subject: [PATCH 0204/1247] Sync up gfx803 DIDs with KFD Add gfx803 10/11 device IDs that were recently added to KFD. Change-Id: Id40b117ae47bacedefa6e333fdfdf58dea92cd2d Signed-off-by: Amber Lin --- src/queues.c | 32 +++++++++++++++++++++++--------- src/topology.c | 16 +++++++++++++++- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/queues.c b/src/queues.c index a2ddb535a8..46de72d9b8 100644 --- a/src/queues.c +++ b/src/queues.c @@ -147,19 +147,33 @@ struct device_id supported_devices[] = { { 0x6921, &tonga_device_info }, { 0x6928, &tonga_device_info }, { 0x6929, &tonga_device_info }, - { 0x692b, &tonga_device_info }, - { 0x692f, &tonga_device_info }, + { 0x692B, &tonga_device_info }, + { 0x692F, &tonga_device_info }, { 0x6930, &tonga_device_info }, { 0x6938, &tonga_device_info }, { 0x6939, &tonga_device_info }, { 0x7300, &fiji_device_info }, - { 0x730f, &fiji_device_info }, - { 0x67c4, &polaris10_device_info }, - { 0x67c7, &polaris10_device_info }, - { 0x67df, &polaris10_device_info }, - { 0x67e3, &polaris11_device_info }, - { 0x67ef, &polaris11_device_info }, - { 0x67ff, &polaris11_device_info }, + { 0x730F, &fiji_device_info }, + { 0x67C0, &polaris10_device_info }, + { 0x67C1, &polaris10_device_info }, + { 0x67C2, &polaris10_device_info }, + { 0x67C4, &polaris10_device_info }, + { 0x67C7, &polaris10_device_info }, + { 0x67C8, &polaris10_device_info }, + { 0x67C9, &polaris10_device_info }, + { 0x67CA, &polaris10_device_info }, + { 0x67CC, &polaris10_device_info }, + { 0x67CF, &polaris10_device_info }, + { 0x67DF, &polaris10_device_info }, + { 0x67E0, &polaris11_device_info }, + { 0x67E1, &polaris11_device_info }, + { 0x67E3, &polaris11_device_info }, + { 0x67E7, &polaris11_device_info }, + { 0x67E8, &polaris11_device_info }, + { 0x67E9, &polaris11_device_info }, + { 0x67EB, &polaris11_device_info }, + { 0x67EF, &polaris11_device_info }, + { 0x67FF, &polaris11_device_info }, { 0, NULL } }; diff --git a/src/topology.c b/src/topology.c index 93de2f2331..5d74b8a00b 100644 --- a/src/topology.c +++ b/src/topology.c @@ -138,13 +138,27 @@ static struct hsa_gfxip_table { { 0x6939, 8, 0, 2, 1, "Tonga" }, /* Fiji */ { 0x7300, 8, 0, 3, 1, "Fiji" }, - { 0x730f, 8, 0, 3, 1, "Fiji" }, + { 0x730F, 8, 0, 3, 1, "Fiji" }, /* Polaris10 */ + { 0x67C0, 8, 0, 3, 1, "Polaris10" }, + { 0x67C1, 8, 0, 3, 1, "Polaris10" }, + { 0x67C2, 8, 0, 3, 1, "Polaris10" }, { 0x67C4, 8, 0, 3, 1, "Polaris10" }, { 0x67C7, 8, 0, 3, 1, "Polaris10" }, + { 0x67C8, 8, 0, 3, 1, "Polaris10" }, + { 0x67C9, 8, 0, 3, 1, "Polaris10" }, + { 0x67CA, 8, 0, 3, 1, "Polaris10" }, + { 0x67CC, 8, 0, 3, 1, "Polaris10" }, + { 0x67CF, 8, 0, 3, 1, "Polaris10" }, { 0x67DF, 8, 0, 3, 1, "Polaris10" }, /* Polaris11 */ + { 0x67E0, 8, 0, 3, 1, "Polaris11" }, + { 0x67E1, 8, 0, 3, 1, "Polaris11" }, { 0x67E3, 8, 0, 3, 1, "Polaris11" }, + { 0x67E7, 8, 0, 3, 1, "Polaris11" }, + { 0x67E8, 8, 0, 3, 1, "Polaris11" }, + { 0x67E9, 8, 0, 3, 1, "Polaris11" }, + { 0x67EB, 8, 0, 3, 1, "Polaris11" }, { 0x67EF, 8, 0, 3, 1, "Polaris11" }, { 0x67FF, 8, 0, 3, 1, "Polaris11" } }; From a90abcb317bf0d152b8e1e1de921352fccff9c5e Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 3 Feb 2016 18:24:38 -0500 Subject: [PATCH 0205/1247] Allocate paged system memory as userptr Change-Id: I0864e678681788df37eccd9d7ebc70086e1f93bf --- src/events.c | 2 +- src/fmm.c | 98 +++++++++++++++++++++++++++++++++++++------------ src/libhsakmt.h | 2 +- src/queues.c | 6 ++- 4 files changed, 81 insertions(+), 27 deletions(-) diff --git a/src/events.c b/src/events.c index 392d0b4cf8..8908817116 100644 --- a/src/events.c +++ b/src/events.c @@ -83,7 +83,7 @@ hsaKmtCreateEvent( if (is_dgpu && events_page == NULL) { events_page = allocate_exec_aligned_memory_gpu( - KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0); + KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true); if (!events_page) { pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; diff --git a/src/fmm.c b/src/fmm.c index 0fd0320432..1aee7a72c6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -129,6 +129,9 @@ typedef struct { /* used for coherent (fine-grain) system memory on dGPU, * This aperture is shared by all dGPUs */ manageble_aperture_t dgpu_alt_aperture; + + /* whether to use userptr for paged memory */ + bool userptr_for_paged_mem; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -141,7 +144,8 @@ static void *dgpu_shared_aperture_limit = NULL; static svm_t svm = { INIT_MANAGEBLE_APERTURE(0, 0), - INIT_MANAGEBLE_APERTURE(0, 0) + INIT_MANAGEBLE_APERTURE(0, 0), + true }; /* On APU, for memory allocated on the system memory that GPU doesn't access @@ -1104,7 +1108,7 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, return NULL; size = MemorySizeInBytes; - ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST; + ioc_flags = 0; if (flags.ui32.CoarseGrain) aperture = &svm.dgpu_aperture; else @@ -1114,9 +1118,69 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; } - mem = __fmm_allocate_device(gpu_id, size, - aperture, 0, &mmap_offset, - ioc_flags, &vm_obj); + /* Paged memory is allocated as a userptr mapping, non-paged + * memory is allocated from KFD */ + if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { + /* Allocate address space */ + pthread_mutex_lock(&aperture->fmm_mutex); + mem = aperture_allocate_area(aperture, size, 0); + pthread_mutex_unlock(&aperture->fmm_mutex); + if (mem == NULL) + return NULL; + + /* Map anonymous pages */ + if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) + == MAP_FAILED) { + /* Release address space */ + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, size); + pthread_mutex_unlock(&aperture->fmm_mutex); + return NULL; + } + + /* Create userptr BO */ + mmap_offset = (uint64_t)mem; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_USERPTR; + vm_obj = fmm_allocate_memory_in_device(gpu_id, mem, size, + aperture, &mmap_offset, + ioc_flags); + if (!vm_obj) { + /* Release address space */ + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, size); + pthread_mutex_unlock(&aperture->fmm_mutex); + /* Remove any CPU mapping, but keep the + * address range reserved */ + mmap(mem, MemorySizeInBytes, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | + MAP_PRIVATE | MAP_FIXED, -1, 0); + return NULL; + } + } else { + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST; + mem = __fmm_allocate_device(gpu_id, size, + aperture, 0, &mmap_offset, + ioc_flags, &vm_obj); + + if (mem && flags.ui32.HostAccess) { + void *ret = mmap(mem, MemorySizeInBytes, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + if (ret == MAP_FAILED) { + __fmm_release(mem, aperture); + return NULL; + } + + if (flags.ui32.AQLQueueMemory) { + uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; + memset(ret, 0, MemorySizeInBytes); + mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + } + } + } if (mem && vm_obj) { /* Store memory allocation flags, not ioc flags */ @@ -1126,24 +1190,6 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, pthread_mutex_unlock(&aperture->fmm_mutex); } - if (flags.ui32.HostAccess) { - void *ret = mmap(mem, MemorySizeInBytes, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); - if (ret == MAP_FAILED) { - __fmm_release(mem, aperture); - return NULL; - } - if (flags.ui32.AQLQueueMemory) { - uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; - memset(ret, 0, MemorySizeInBytes); - mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); - } - } - - return mem; } @@ -1334,6 +1380,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache; + char *pagedUserptr; struct pci_access *pacc; /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ @@ -1341,6 +1388,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (disableCache && strcmp(disableCache, "0") == 0) disableCache = NULL; + /* If HSA_USERPTR_FOR_PAGED_MEM unset or set to a non-0 value, + * enable userptr for all paged memory allocations */ + pagedUserptr = getenv("HSA_USERPTR_FOR_PAGED_MEM"); + svm.userptr_for_paged_mem = (!pagedUserptr || strcmp(pagedUserptr, "0")); + /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than * necessary*/ diff --git a/src/libhsakmt.h b/src/libhsakmt.h index e3b07dc90a..e31aeef42e 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -78,7 +78,7 @@ bool topology_is_dgpu(uint16_t device_id); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId); + uint32_t NodeId, bool NonPaged); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes); void destroy_process_doorbells(void); diff --git a/src/queues.c b/src/queues.c index 46de72d9b8..a6c9527b01 100644 --- a/src/queues.c +++ b/src/queues.c @@ -397,7 +397,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) } void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId) + uint32_t NodeId, bool nonPaged) { void *mem; HSAuint64 gpu_va; @@ -407,6 +407,7 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, flags.Value = 0; flags.ui32.HostAccess = 1; flags.ui32.ExecuteAccess = 1; + flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; size = ALIGN_UP(size, align); @@ -448,7 +449,8 @@ static void* allocate_exec_aligned_memory(uint32_t size, uint32_t NodeId) { if (IS_DGPU(type)) - return allocate_exec_aligned_memory_gpu(size, align, NodeId); + return allocate_exec_aligned_memory_gpu(size, align, NodeId, + false); return allocate_exec_aligned_memory_cpu(size, align); } From d4dbf562a9eba6c5a05eb0272c76f34f80e7376f Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 3 Feb 2017 09:59:21 -0500 Subject: [PATCH 0206/1247] Replace spaces with tabs Thunk follows Linux kernel coding convention to use tabs instead of spaces. Change-Id: I4eddcfa9a0513f16c869d9cc63f9f1dae0c39f83 Signed-off-by: Amber Lin --- src/perfctr.c | 370 ++++++++++++++++++++++++------------------------ src/pmc_table.c | 297 +++++++++++++++++++------------------- src/pmc_table.h | 20 +-- 3 files changed, 344 insertions(+), 343 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 95102a19ed..2b312affd4 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -29,19 +29,19 @@ #include "linux/kfd_ioctl.h" #include -#define BITS_PER_BYTE CHAR_BIT +#define BITS_PER_BYTE CHAR_BIT -#define HSA_PERF_MAGIC4CC 0x54415348 +#define HSA_PERF_MAGIC4CC 0x54415348 enum perf_trace_state { - PERF_TRACE_STATE__STOPPED = 0, - PERF_TRACE_STATE__STARTED + PERF_TRACE_STATE__STOPPED = 0, + PERF_TRACE_STATE__STARTED }; struct perf_trace { - uint32_t magic4cc; - uint32_t gpu_id; - enum perf_trace_state state; + uint32_t magic4cc; + uint32_t gpu_id; + enum perf_trace_state state; }; extern int amd_hsa_thunk_lock_fd; @@ -77,94 +77,92 @@ void destroy_counter_props(void) static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) { - int rc = 0; - switch (block_id) { - case PERFCOUNTER_BLOCKID__SQ: - *uuid = HSA_PROFILEBLOCK_AMD_SQ; - break; - default: - /* If we reach this point, it's a bug */ - rc = -1; - } + int rc = 0; + switch (block_id) { + case PERFCOUNTER_BLOCKID__SQ: + *uuid = HSA_PROFILEBLOCK_AMD_SQ; + break; + default: + /* If we reach this point, it's a bug */ + rc = -1; + } - return rc; + return rc; } HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties( - HSAuint32 NodeId, //IN - HsaCounterProperties** CounterProperties //OUT - ) + HSAuint32 NodeId, //IN + HsaCounterProperties** CounterProperties //OUT + ) { - HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; - uint32_t gpu_id, i, block_id; - uint16_t dev_id; - uint32_t counter_props_size = 0; - uint32_t total_counters = 0; - uint32_t total_concurrent = 0; - struct perf_counter_block block = {0}; + HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; + uint32_t gpu_id, i, block_id; + uint16_t dev_id; + uint32_t counter_props_size = 0; + uint32_t total_counters = 0; + uint32_t total_concurrent = 0; + struct perf_counter_block block = {0}; if (counter_props == NULL) return HSAKMT_STATUS_NO_MEMORY; - if (CounterProperties == NULL) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (CounterProperties == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; - if (validate_nodeid(NodeId, &gpu_id) != 0) - return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (validate_nodeid(NodeId, &gpu_id) != 0) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (counter_props[NodeId] == NULL) { + dev_id = get_device_id_by_node(NodeId); + for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { + rc = get_block_properties(dev_id, i, &block); + if (rc != HSAKMT_STATUS_SUCCESS) + return rc; + total_concurrent += block.num_of_slots; + total_counters += block.num_of_counters; + } + counter_props_size = sizeof(HsaCounterProperties) + + sizeof(HsaCounterBlockProperties)*(PERFCOUNTER_BLOCKID__MAX-1) + + sizeof(HsaCounter)*(total_counters-1); - if (counter_props[NodeId] == NULL) { - dev_id = get_device_id_by_node(NodeId); - for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { - rc = get_block_properties(dev_id, i, &block); - if (rc != HSAKMT_STATUS_SUCCESS) - return rc; - total_concurrent += block.num_of_slots; - total_counters += block.num_of_counters; - } + counter_props[NodeId] = malloc(counter_props_size); - counter_props_size = sizeof(HsaCounterProperties) + - sizeof(HsaCounterBlockProperties)*(PERFCOUNTER_BLOCKID__MAX-1) + - sizeof(HsaCounter)*(total_counters-1); + if (counter_props[NodeId] == NULL) + return HSAKMT_STATUS_NO_MEMORY; - counter_props[NodeId] = malloc(counter_props_size); + counter_props[NodeId]->NumBlocks = PERFCOUNTER_BLOCKID__MAX; + counter_props[NodeId]->NumConcurrent = total_concurrent; - if (counter_props[NodeId] == NULL) - return HSAKMT_STATUS_NO_MEMORY; + for (block_id = 0; block_id < PERFCOUNTER_BLOCKID__MAX; block_id++) + { + rc = get_block_properties(dev_id, block_id, &block); + if (rc != HSAKMT_STATUS_SUCCESS) { + free(counter_props[NodeId]); + return rc; + } - counter_props[NodeId]->NumBlocks = PERFCOUNTER_BLOCKID__MAX; - counter_props[NodeId]->NumConcurrent = total_concurrent; + /* Filling the SQ block */ + blockid2uuid(block_id, &counter_props[NodeId]->Blocks[block_id].BlockId); + counter_props[NodeId]->Blocks[block_id].NumCounters = block.num_of_counters; + counter_props[NodeId]->Blocks[block_id].NumConcurrent = block.num_of_slots; - for (block_id = 0; block_id < PERFCOUNTER_BLOCKID__MAX; block_id++) - { - rc = get_block_properties(dev_id, block_id, &block); - if (rc != HSAKMT_STATUS_SUCCESS) { - free(counter_props[NodeId]); - return rc; - } + for (i = 0; i < block.num_of_counters; i++) { + counter_props[NodeId]->Blocks[block_id].Counters[i].BlockIndex = block_id; + counter_props[NodeId]->Blocks[block_id].Counters[i].CounterId = block.counter_ids[i]; + counter_props[NodeId]->Blocks[block_id].Counters[i].CounterSizeInBits = block.counter_size_in_bits; + counter_props[NodeId]->Blocks[block_id].Counters[i].CounterMask = block.counter_mask; + counter_props[NodeId]->Blocks[block_id].Counters[i].Flags.ui32.Global = 1; + counter_props[NodeId]->Blocks[block_id].Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; + } + } + } - /* Filling the SQ block */ - blockid2uuid(block_id, &counter_props[NodeId]->Blocks[block_id].BlockId); - counter_props[NodeId]->Blocks[block_id].NumCounters = block.num_of_counters; - counter_props[NodeId]->Blocks[block_id].NumConcurrent = block.num_of_slots; + *CounterProperties = counter_props[NodeId]; - for (i = 0; i < block.num_of_counters; i++) { - counter_props[NodeId]->Blocks[block_id].Counters[i].BlockIndex = block_id; - counter_props[NodeId]->Blocks[block_id].Counters[i].CounterId = block.counter_ids[i]; - counter_props[NodeId]->Blocks[block_id].Counters[i].CounterSizeInBits = block.counter_size_in_bits; - counter_props[NodeId]->Blocks[block_id].Counters[i].CounterMask = block.counter_mask; - counter_props[NodeId]->Blocks[block_id].Counters[i].Flags.ui32.Global = 1; - counter_props[NodeId]->Blocks[block_id].Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; - } - } - } - - *CounterProperties = counter_props[NodeId]; - - return HSAKMT_STATUS_SUCCESS; + return HSAKMT_STATUS_SUCCESS; } /** @@ -174,51 +172,51 @@ hsaKmtPmcGetCounterProperties( HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace( - HSAuint32 NodeId, //IN - HSAuint32 NumberOfCounters, //IN - HsaCounter* Counters, //IN - HsaPmcTraceRoot* TraceRoot //OUT - ) + HSAuint32 NodeId, //IN + HSAuint32 NumberOfCounters, //IN + HsaCounter* Counters, //IN + HsaPmcTraceRoot* TraceRoot //OUT + ) { - uint32_t gpu_id, i; - uint64_t min_buf_size = 0; - uint32_t concurrent_counters[PERFCOUNTER_BLOCKID__MAX] = {0}; - struct perf_trace *trace = NULL; + uint32_t gpu_id, i; + uint64_t min_buf_size = 0; + uint32_t concurrent_counters[PERFCOUNTER_BLOCKID__MAX] = {0}; + struct perf_trace *trace = NULL; if (counter_props == NULL) return HSAKMT_STATUS_NO_MEMORY; - if (Counters == NULL || TraceRoot == NULL || NumberOfCounters == 0) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (Counters == NULL || TraceRoot == NULL || NumberOfCounters == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; - if (validate_nodeid(NodeId, &gpu_id) != 0) - return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (validate_nodeid(NodeId, &gpu_id) != 0) + return HSAKMT_STATUS_INVALID_NODE_UNIT; - /* Calculating the minimum buffer size */ - for (i = 0; i < NumberOfCounters; i++) { - if (Counters[i].BlockIndex >= PERFCOUNTER_BLOCKID__MAX) - return HSAKMT_STATUS_INVALID_PARAMETER; - min_buf_size += Counters[i].CounterSizeInBits/BITS_PER_BYTE; - concurrent_counters[Counters[i].BlockIndex]++; - } + /* Calculating the minimum buffer size */ + for (i = 0; i < NumberOfCounters; i++) { + if (Counters[i].BlockIndex >= PERFCOUNTER_BLOCKID__MAX) + return HSAKMT_STATUS_INVALID_PARAMETER; + min_buf_size += Counters[i].CounterSizeInBits/BITS_PER_BYTE; + concurrent_counters[Counters[i].BlockIndex]++; + } - /* Verifying that the number of counters per block is not larger than the amount of slots */ - if (concurrent_counters[PERFCOUNTER_BLOCKID__SQ] > counter_props[NodeId]->Blocks[PERFCOUNTER_BLOCKID__SQ].NumConcurrent) - return HSAKMT_STATUS_INVALID_PARAMETER; + /* Verifying that the number of counters per block is not larger than the amount of slots */ + if (concurrent_counters[PERFCOUNTER_BLOCKID__SQ] > counter_props[NodeId]->Blocks[PERFCOUNTER_BLOCKID__SQ].NumConcurrent) + return HSAKMT_STATUS_INVALID_PARAMETER; - trace = malloc(sizeof(trace)); - if (trace == NULL) - return HSAKMT_STATUS_NO_MEMORY; + trace = malloc(sizeof(trace)); + if (trace == NULL) + return HSAKMT_STATUS_NO_MEMORY; - trace->magic4cc = HSA_PERF_MAGIC4CC; - trace->gpu_id = gpu_id; - trace->state = PERF_TRACE_STATE__STOPPED; + trace->magic4cc = HSA_PERF_MAGIC4CC; + trace->gpu_id = gpu_id; + trace->state = PERF_TRACE_STATE__STOPPED; - TraceRoot->NumberOfPasses = 1; - TraceRoot->TraceBufferMinSizeBytes = PAGE_ALIGN_UP(min_buf_size); - TraceRoot->TraceId = PORT_VPTR_TO_UINT64(trace); + TraceRoot->NumberOfPasses = 1; + TraceRoot->TraceBufferMinSizeBytes = PAGE_ALIGN_UP(min_buf_size); + TraceRoot->TraceId = PORT_VPTR_TO_UINT64(trace); - return HSAKMT_STATUS_SUCCESS; + return HSAKMT_STATUS_SUCCESS; } /** @@ -228,37 +226,37 @@ hsaKmtPmcRegisterTrace( HSAKMT_STATUS HSAKMTAPI hsaKmtPmcUnregisterTrace( - HSAuint32 NodeId, //IN - HSATraceId TraceId //IN - ) + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ) { - uint32_t gpu_id; - struct perf_trace *trace; + uint32_t gpu_id; + struct perf_trace *trace; - if (TraceId == 0) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; - if (validate_nodeid(NodeId, &gpu_id) != 0) - return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (validate_nodeid(NodeId, &gpu_id) != 0) + return HSAKMT_STATUS_INVALID_NODE_UNIT; - trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - if (trace->magic4cc != HSA_PERF_MAGIC4CC) - return HSAKMT_STATUS_INVALID_HANDLE; + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; - if (trace->gpu_id != gpu_id) - return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (trace->gpu_id != gpu_id) + return HSAKMT_STATUS_INVALID_NODE_UNIT; - /* If the trace is in the running state, stop it */ - if (trace->state == PERF_TRACE_STATE__STARTED) { - HSAKMT_STATUS status = hsaKmtPmcStopTrace(TraceId); - if (status != HSAKMT_STATUS_SUCCESS) - return status; - } + /* If the trace is in the running state, stop it */ + if (trace->state == PERF_TRACE_STATE__STARTED) { + HSAKMT_STATUS status = hsaKmtPmcStopTrace(TraceId); + if (status != HSAKMT_STATUS_SUCCESS) + return status; + } - free(trace); + free(trace); - return HSAKMT_STATUS_SUCCESS; + return HSAKMT_STATUS_SUCCESS; } @@ -270,29 +268,29 @@ hsaKmtPmcUnregisterTrace( HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess( - HSAuint32 NodeId, //IN - HSATraceId TraceId //IN - ) + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ) { - struct perf_trace *trace; + struct perf_trace *trace; - if (TraceId == 0) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; - trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - if (trace->magic4cc != HSA_PERF_MAGIC4CC) - return HSAKMT_STATUS_INVALID_HANDLE; + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; - if (amd_hsa_thunk_lock_fd > 0) { + if (amd_hsa_thunk_lock_fd > 0) { if (lockf( amd_hsa_thunk_lock_fd, F_TLOCK, 0 ) != 0) - return HSAKMT_STATUS_ERROR; + return HSAKMT_STATUS_ERROR; else return HSAKMT_STATUS_SUCCESS; - } - else { - return HSAKMT_STATUS_ERROR; - } + } + else { + return HSAKMT_STATUS_ERROR; + } } @@ -304,29 +302,29 @@ hsaKmtPmcAcquireTraceAccess( HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess( - HSAuint32 NodeId, //IN - HSATraceId TraceId //IN - ) + HSAuint32 NodeId, //IN + HSATraceId TraceId //IN + ) { - struct perf_trace *trace; + struct perf_trace *trace; - if (TraceId == 0) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; - trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - if (trace->magic4cc != HSA_PERF_MAGIC4CC) - return HSAKMT_STATUS_INVALID_HANDLE; + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; - if (amd_hsa_thunk_lock_fd > 0) { + if (amd_hsa_thunk_lock_fd > 0) { if (lockf( amd_hsa_thunk_lock_fd, F_ULOCK, 0 ) != 0) - return HSAKMT_STATUS_ERROR; + return HSAKMT_STATUS_ERROR; else return HSAKMT_STATUS_SUCCESS; - } - else { - return HSAKMT_STATUS_ERROR; - } + } + else { + return HSAKMT_STATUS_ERROR; + } } @@ -338,22 +336,22 @@ hsaKmtPmcReleaseTraceAccess( HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStartTrace( - HSATraceId TraceId, //IN - void* TraceBuffer, //IN (page aligned) - HSAuint64 TraceBufferSizeBytes //IN (page aligned) - ) + HSATraceId TraceId, //IN + void* TraceBuffer, //IN (page aligned) + HSAuint64 TraceBufferSizeBytes //IN (page aligned) + ) { - struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - if (TraceId == 0 || TraceBuffer == NULL || TraceBufferSizeBytes == 0) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (TraceId == 0 || TraceBuffer == NULL || TraceBufferSizeBytes == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; - if (trace->magic4cc != HSA_PERF_MAGIC4CC) - return HSAKMT_STATUS_INVALID_HANDLE; + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; - trace->state = PERF_TRACE_STATE__STARTED; + trace->state = PERF_TRACE_STATE__STARTED; - return HSAKMT_STATUS_SUCCESS; + return HSAKMT_STATUS_SUCCESS; } @@ -364,18 +362,18 @@ hsaKmtPmcStartTrace( HSAKMT_STATUS HSAKMTAPI hsaKmtPmcQueryTrace( - HSATraceId TraceId //IN - ) + HSATraceId TraceId //IN + ) { - struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - if (TraceId == 0) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; - if (trace->magic4cc != HSA_PERF_MAGIC4CC) - return HSAKMT_STATUS_INVALID_HANDLE; + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; - return HSAKMT_STATUS_SUCCESS; + return HSAKMT_STATUS_SUCCESS; } @@ -386,18 +384,18 @@ hsaKmtPmcQueryTrace( HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStopTrace( - HSATraceId TraceId //IN - ) + HSATraceId TraceId //IN + ) { - struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); - if (TraceId == 0) - return HSAKMT_STATUS_INVALID_PARAMETER; + if (TraceId == 0) + return HSAKMT_STATUS_INVALID_PARAMETER; - if (trace->magic4cc != HSA_PERF_MAGIC4CC) - return HSAKMT_STATUS_INVALID_HANDLE; + if (trace->magic4cc != HSA_PERF_MAGIC4CC) + return HSAKMT_STATUS_INVALID_HANDLE; - trace->state = PERF_TRACE_STATE__STOPPED; + trace->state = PERF_TRACE_STATE__STOPPED; - return HSAKMT_STATUS_SUCCESS; + return HSAKMT_STATUS_SUCCESS; } diff --git a/src/pmc_table.c b/src/pmc_table.c index 3d4b495b11..b0db2fea70 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -26,132 +26,134 @@ #include "libhsakmt.h" #include "pmc_table.h" - static uint32_t kaveri_sq_counter_ids[] = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, +23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, +43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, +63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, +83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, +102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, +118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, +134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, +150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 168, 169, 170, +171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, +187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, +203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, +219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, +235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 }; /* Unused counters - 166, 292 - 297 */ static uint32_t carrizo_sq_counter_ids[] = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, - 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, - 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, +23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, +43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, +63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, +83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, +102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, +118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, +134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, +150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, +167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, +183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, +199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, +215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, +231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, +247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, +263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, +279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; /* Unused counters - 166, 292 - 297 */ static uint32_t fiji_sq_counter_ids[] = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, - 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, - 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, - 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, - 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, - 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, - 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, - 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, +23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, +43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, +63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, +83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, +102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, +118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, +134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, +150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, +167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, +183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, +199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, +215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, +231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, +247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, +263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, +279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; /* Unused counters - 163 - 166, 167 and 251 are *_DUMMY_LAST */ static uint32_t hawaii_sq_counter_ids[] = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, - 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, 161, 162, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, - 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, - 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, - 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, - 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, - 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, +41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, +60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, +79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, +98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, +113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, +128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, +158, 159, 160, 161, 162, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, +178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, +193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, +208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, +223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, +238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 }; /* Polaris 10 and Polaris 11 have the same SQ cpunter IDs */ /* Unused counters - 167 and 275 are *_DUMMY_LAST */ static uint32_t polaris_sq_counter_ids[] = { - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, - 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, - 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, - 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, - 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, - 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, - 158, 159, 160, 161, 162, 163, 164, 165, 168, 169, 170, 171, 172, 173, 174, - 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, - 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, - 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 276, 277, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295 +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, +41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, +60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, +79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, +98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, +113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, +128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, +158, 159, 160, 161, 162, 163, 164, 165, 168, 169, 170, 171, 172, 173, 174, +175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, +190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, +205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, +220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, +235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, +250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, +265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 276, 277, 278, 279, 280, +281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295 }; static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { - [PERFCOUNTER_BLOCKID__SQ] = { - .num_of_slots = 8, - .num_of_counters = sizeof(kaveri_sq_counter_ids)/sizeof(*kaveri_sq_counter_ids), - .counter_ids = kaveri_sq_counter_ids, - .counter_size_in_bits = 64, - .counter_mask = BITMASK(64) - }, + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = sizeof(kaveri_sq_counter_ids) / + sizeof(*kaveri_sq_counter_ids), + .counter_ids = kaveri_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { - [PERFCOUNTER_BLOCKID__SQ] = { - .num_of_slots = 8, - .num_of_counters = sizeof(carrizo_sq_counter_ids)/sizeof(*carrizo_sq_counter_ids), - .counter_ids = carrizo_sq_counter_ids, - .counter_size_in_bits = 64, - .counter_mask = BITMASK(64) - }, + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = sizeof(carrizo_sq_counter_ids) / + sizeof(*carrizo_sq_counter_ids), + .counter_ids = carrizo_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, - .num_of_counters = sizeof(fiji_sq_counter_ids) / sizeof(*fiji_sq_counter_ids), + .num_of_counters = sizeof(fiji_sq_counter_ids) / + sizeof(*fiji_sq_counter_ids), .counter_ids = fiji_sq_counter_ids, .counter_size_in_bits = 64, .counter_mask = BITMASK(64) @@ -159,71 +161,72 @@ static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { }; static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { - [PERFCOUNTER_BLOCKID__SQ] = { - .num_of_slots = 8, - .num_of_counters = - sizeof(hawaii_sq_counter_ids) / sizeof(*hawaii_sq_counter_ids), - .counter_ids = hawaii_sq_counter_ids, - .counter_size_in_bits = 64, - .counter_mask = BITMASK(64) - }, + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = sizeof(hawaii_sq_counter_ids) / + sizeof(*hawaii_sq_counter_ids), + .counter_ids = hawaii_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { - [PERFCOUNTER_BLOCKID__SQ] = { - .num_of_slots = 8, - .num_of_counters = sizeof(polaris_sq_counter_ids) / - sizeof(*polaris_sq_counter_ids), - .counter_ids = polaris_sq_counter_ids, - .counter_size_in_bits = 64, - .counter_mask = BITMASK(64) - }, + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = sizeof(polaris_sq_counter_ids) / + sizeof(*polaris_sq_counter_ids), + .counter_ids = polaris_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; HSAKMT_STATUS get_block_properties(uint16_t dev_id, - enum perf_block_id block_id, - struct perf_counter_block *block) + enum perf_block_id block_id, + struct perf_counter_block *block) { - HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; - if (block_id > PERFCOUNTER_BLOCKID__MAX || block_id < PERFCOUNTER_BLOCKID__FIRST) - return HSAKMT_STATUS_INVALID_PARAMETER; + HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; + if (block_id > PERFCOUNTER_BLOCKID__MAX || block_id < PERFCOUNTER_BLOCKID__FIRST) + return HSAKMT_STATUS_INVALID_PARAMETER; - /* To avoid the long list, we read the 12 most significant digits of DID - * to identify the GPU instead of listing the complete 16 bits. If one day - * 12-bits is not good enough to distinguish the GPU, change the code here. - */ - switch(dev_id >> 4) { - case 0x130: - case 0x131: - *block = kaveri_blocks[block_id]; - break; + /* To avoid the long list, we read the 12 most significant digits of DID + * to identify the GPU instead of listing the complete 16 bits. If one + * day 12-bits is not good enough to distinguish the GPU, change the + * code here. + */ + switch(dev_id >> 4) { + case 0x130: + case 0x131: + *block = kaveri_blocks[block_id]; + break; - case 0x987: - *block = carrizo_blocks[block_id]; - break; + case 0x987: + *block = carrizo_blocks[block_id]; + break; - case 0x730: - *block = fiji_blocks[block_id]; - break; + case 0x730: + *block = fiji_blocks[block_id]; + break; - case 0x67A: - case 0x67B: - *block = hawaii_blocks[block_id]; - break; + case 0x67A: + case 0x67B: + *block = hawaii_blocks[block_id]; + break; - case 0x67C: - case 0x67D: - case 0x67E: - case 0x67F: - *block = polaris_blocks[block_id]; - break; + case 0x67C: + case 0x67D: + case 0x67E: + case 0x67F: + *block = polaris_blocks[block_id]; + break; - default: - rc = HSAKMT_STATUS_INVALID_PARAMETER; - } + default: + rc = HSAKMT_STATUS_INVALID_PARAMETER; + } - return rc; + return rc; } diff --git a/src/pmc_table.h b/src/pmc_table.h index 35ed07eb4c..820edae4e0 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -29,22 +29,22 @@ #include "libhsakmt.h" enum perf_block_id { - PERFCOUNTER_BLOCKID__FIRST = 0, - PERFCOUNTER_BLOCKID__SQ = PERFCOUNTER_BLOCKID__FIRST, - PERFCOUNTER_BLOCKID__MAX + PERFCOUNTER_BLOCKID__FIRST = 0, + PERFCOUNTER_BLOCKID__SQ = PERFCOUNTER_BLOCKID__FIRST, + PERFCOUNTER_BLOCKID__MAX }; struct perf_counter_block { - uint32_t num_of_slots; - uint32_t num_of_counters; - uint32_t *counter_ids; - uint32_t counter_size_in_bits; - uint64_t counter_mask; + uint32_t num_of_slots; + uint32_t num_of_counters; + uint32_t *counter_ids; + uint32_t counter_size_in_bits; + uint64_t counter_mask; }; HSAKMT_STATUS get_block_properties(uint16_t dev_id, - enum perf_block_id block_id, - struct perf_counter_block *block); + enum perf_block_id block_id, + struct perf_counter_block *block); #endif // PMC_TABLE_H From 9dadac6dc9ff34fdd7087039c18aa2d33c8dce7c Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 3 Feb 2017 12:07:57 -0500 Subject: [PATCH 0207/1247] Add IOMMU to performance counter table Add IOMMUv2 to blocks returned by hsaKmtPmcGetCounterProperties(). IOMMU information is read from sysfs. Change-Id: I3a1c6f902f947913570a78700fc0ffc444e1dd72 Signed-off-by: Amber Lin --- src/perfctr.c | 103 ++++++++++++++++++++++--------------- src/pmc_table.c | 133 ++++++++++++++++++++++++++++++++++++++++++++++-- src/pmc_table.h | 10 ++-- 3 files changed, 199 insertions(+), 47 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 2b312affd4..0ec59b6ba8 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -56,6 +56,9 @@ HSAKMT_STATUS init_counter_props(unsigned int NumNodes) return HSAKMT_STATUS_NO_MEMORY; counter_props_count = NumNodes; + + alloc_pmc_blocks(); + return HSAKMT_STATUS_SUCCESS; } @@ -73,18 +76,24 @@ void destroy_counter_props(void) } free(counter_props); + free_pmc_blocks(); } static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) { int rc = 0; + switch (block_id) { case PERFCOUNTER_BLOCKID__SQ: *uuid = HSA_PROFILEBLOCK_AMD_SQ; break; + case PERFCOUNTER_BLOCKID__IOMMUV2: + *uuid = HSA_PROFILEBLOCK_AMD_IOMMUV2; + break; default: /* If we reach this point, it's a bug */ rc = -1; + break; } return rc; @@ -99,11 +108,12 @@ hsaKmtPmcGetCounterProperties( { HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; uint32_t gpu_id, i, block_id; - uint16_t dev_id; uint32_t counter_props_size = 0; uint32_t total_counters = 0; uint32_t total_concurrent = 0; struct perf_counter_block block = {0}; + uint32_t total_blocks = 0; + uint32_t entry; if (counter_props == NULL) return HSAKMT_STATUS_NO_MEMORY; @@ -114,50 +124,63 @@ hsaKmtPmcGetCounterProperties( if (validate_nodeid(NodeId, &gpu_id) != 0) return HSAKMT_STATUS_INVALID_NODE_UNIT; - if (counter_props[NodeId] == NULL) { - dev_id = get_device_id_by_node(NodeId); - for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { - rc = get_block_properties(dev_id, i, &block); - if (rc != HSAKMT_STATUS_SUCCESS) - return rc; - total_concurrent += block.num_of_slots; - total_counters += block.num_of_counters; + if (counter_props[NodeId] != NULL) { + *CounterProperties = counter_props[NodeId]; + return HSAKMT_STATUS_SUCCESS; + } + + for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { + rc = get_block_properties(NodeId, i, &block); + if (rc != HSAKMT_STATUS_SUCCESS) + return rc; + total_concurrent += block.num_of_slots; + total_counters += block.num_of_counters; + /* If num_of_slots=0, this block doesn't exist */ + if (block.num_of_slots) + total_blocks++; + } + + counter_props_size = sizeof(HsaCounterProperties) + + sizeof(HsaCounterBlockProperties)*(total_blocks-1) + + sizeof(HsaCounter)*(total_counters-1); + + counter_props[NodeId] = malloc(counter_props_size); + if (counter_props[NodeId] == NULL) + return HSAKMT_STATUS_NO_MEMORY; + + counter_props[NodeId]->NumBlocks = total_blocks; + counter_props[NodeId]->NumConcurrent = total_concurrent; + + entry = 0; + for (block_id = 0; block_id < PERFCOUNTER_BLOCKID__MAX; block_id++) { + rc = get_block_properties(NodeId, block_id, &block); + if (rc != HSAKMT_STATUS_SUCCESS) { + free(counter_props[NodeId]); + return rc; } - counter_props_size = sizeof(HsaCounterProperties) + - sizeof(HsaCounterBlockProperties)*(PERFCOUNTER_BLOCKID__MAX-1) + - sizeof(HsaCounter)*(total_counters-1); + if (!block.num_of_slots) /* not a valid block */ + continue; - counter_props[NodeId] = malloc(counter_props_size); + blockid2uuid(block_id, + &counter_props[NodeId]->Blocks[entry].BlockId); + counter_props[NodeId]->Blocks[entry].NumCounters = + block.num_of_counters; + counter_props[NodeId]->Blocks[entry].NumConcurrent = + block.num_of_slots; - if (counter_props[NodeId] == NULL) - return HSAKMT_STATUS_NO_MEMORY; - - counter_props[NodeId]->NumBlocks = PERFCOUNTER_BLOCKID__MAX; - counter_props[NodeId]->NumConcurrent = total_concurrent; - - for (block_id = 0; block_id < PERFCOUNTER_BLOCKID__MAX; block_id++) - { - rc = get_block_properties(dev_id, block_id, &block); - if (rc != HSAKMT_STATUS_SUCCESS) { - free(counter_props[NodeId]); - return rc; - } - - /* Filling the SQ block */ - blockid2uuid(block_id, &counter_props[NodeId]->Blocks[block_id].BlockId); - counter_props[NodeId]->Blocks[block_id].NumCounters = block.num_of_counters; - counter_props[NodeId]->Blocks[block_id].NumConcurrent = block.num_of_slots; - - for (i = 0; i < block.num_of_counters; i++) { - counter_props[NodeId]->Blocks[block_id].Counters[i].BlockIndex = block_id; - counter_props[NodeId]->Blocks[block_id].Counters[i].CounterId = block.counter_ids[i]; - counter_props[NodeId]->Blocks[block_id].Counters[i].CounterSizeInBits = block.counter_size_in_bits; - counter_props[NodeId]->Blocks[block_id].Counters[i].CounterMask = block.counter_mask; - counter_props[NodeId]->Blocks[block_id].Counters[i].Flags.ui32.Global = 1; - counter_props[NodeId]->Blocks[block_id].Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; - } + for (i = 0; i < block.num_of_counters; i++) { + counter_props[NodeId]->Blocks[entry].Counters[i].BlockIndex = block_id; + counter_props[NodeId]->Blocks[entry].Counters[i].CounterId = block.counter_ids[i]; + counter_props[NodeId]->Blocks[entry].Counters[i].CounterSizeInBits = block.counter_size_in_bits; + counter_props[NodeId]->Blocks[entry].Counters[i].CounterMask = block.counter_mask; + counter_props[NodeId]->Blocks[entry].Counters[i].Flags.ui32.Global = 1; + if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) + counter_props[NodeId]->Blocks[entry].Counters[i].Type = HSA_PROFILE_TYPE_PRIVILEGED_IMMEDIATE; + else + counter_props[NodeId]->Blocks[entry].Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; } + entry++; } *CounterProperties = counter_props[NodeId]; diff --git a/src/pmc_table.c b/src/pmc_table.c index b0db2fea70..6465507c41 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -23,6 +23,11 @@ * DEALINGS IN THE SOFTWARE. */ +#include +#include +#include +#include +#include #include "libhsakmt.h" #include "pmc_table.h" @@ -182,15 +187,135 @@ static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +/* Current APUs only have one IOMMU. If NUMA is introduced to APUs, we'll need + * to expand the struct here to an array. + */ +static struct perf_counter_block iommu_block; + +static HSAKMT_STATUS +alloc_pmc_blocks_iommu(void) +{ + DIR *dir; + struct dirent *dent; + const char sysfs_amdiommu_event_path[] = + "/sys/bus/event_source/devices/amd_iommu/events"; + /* Counter source in IOMMU's Counter Bank Addressing register is 8 bits, + * so the biggest counter number/id possible is 0xff. + */ + const int max_counter_id = 0xff; + char path[256]; + const int len = sizeof(path); + FILE *file; + int num; + char counter_id[max_counter_id + 1]; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t *ptr; + struct perf_counter_block *block = &iommu_block; + + memset(block, 0, sizeof(struct perf_counter_block)); + + dir = opendir(sysfs_amdiommu_event_path); + if (!dir) + goto out; + + memset(counter_id, 0, max_counter_id + 1); + while ((dent = readdir(dir))) { + if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) + continue; + if (snprintf(path, len, "%s/%s", sysfs_amdiommu_event_path, + dent->d_name) >= len) { + fprintf(stderr, "Increase path length.\n"); + ret = HSAKMT_STATUS_NO_MEMORY; + goto out; + } + file = fopen(path, "r"); + if (!file) { + ret = HSAKMT_STATUS_ERROR; + goto out; + } + if (fscanf(file, "csource=0x%x", &num) != 1) { + ret = HSAKMT_STATUS_ERROR; + fclose(file); + goto out; + } + if (num > max_counter_id) + /* This should never happen. If it does, check IOMMU driver. */ + fprintf(stderr, + "Error: max_counter_id %d is set too small.\n", + max_counter_id); + else { + counter_id[num] = 1; + ++block->num_of_counters; + } + fclose(file); + } + + block->counter_ids = malloc(sizeof(uint32_t) * block->num_of_counters); + if (!block->counter_ids) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto out; + } + ptr = block->counter_ids; + for (num = 0; num < (max_counter_id + 1); num++) { + if (counter_id[num]) { + ptr[0] = num; + ++ptr; + } + } + + if (snprintf(path, len, "%s/%d/%s", + "/sys/devices/virtual/kfd/kfd/topology/nodes", + 0, /* IOMMU is in node 0. Change this if NUMA is introduced to APU. */ + "perf/iommu/max_concurrent") >= len) { + fprintf(stderr, "Increase path length\n"); + ret = HSAKMT_STATUS_NO_MEMORY; + goto out; + }; + file = fopen(path, "r"); + if (!file) { + ret = HSAKMT_STATUS_ERROR; + goto out; + } + if (fscanf(file, "%d", &block->num_of_slots) != 1) + ret = HSAKMT_STATUS_ERROR; + fclose(file); + +out: + if (dir) + closedir(dir); + return ret; +} + +HSAKMT_STATUS alloc_pmc_blocks(void) +{ + return alloc_pmc_blocks_iommu(); +} + +void free_pmc_blocks(void) +{ + if (iommu_block.counter_ids) + free(iommu_block.counter_ids); + iommu_block.counter_ids = NULL; + iommu_block.num_of_counters = 0; +} + HSAKMT_STATUS -get_block_properties(uint16_t dev_id, - enum perf_block_id block_id, - struct perf_counter_block *block) +get_block_properties(uint32_t node_id, + enum perf_block_id block_id, + struct perf_counter_block *block) { HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; - if (block_id > PERFCOUNTER_BLOCKID__MAX || block_id < PERFCOUNTER_BLOCKID__FIRST) + uint16_t dev_id = get_device_id_by_node(node_id); + + if (block_id > PERFCOUNTER_BLOCKID__MAX || + block_id < PERFCOUNTER_BLOCKID__FIRST) return HSAKMT_STATUS_INVALID_PARAMETER; + if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) { + *block = iommu_block; + return HSAKMT_STATUS_SUCCESS; + } + /* To avoid the long list, we read the 12 most significant digits of DID * to identify the GPU instead of listing the complete 16 bits. If one * day 12-bits is not good enough to distinguish the GPU, change the diff --git a/src/pmc_table.h b/src/pmc_table.h index 820edae4e0..adc4af2298 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -31,6 +31,7 @@ enum perf_block_id { PERFCOUNTER_BLOCKID__FIRST = 0, PERFCOUNTER_BLOCKID__SQ = PERFCOUNTER_BLOCKID__FIRST, + PERFCOUNTER_BLOCKID__IOMMUV2, PERFCOUNTER_BLOCKID__MAX }; @@ -42,9 +43,12 @@ struct perf_counter_block { uint64_t counter_mask; }; +HSAKMT_STATUS alloc_pmc_blocks(void); +void free_pmc_blocks(void); + HSAKMT_STATUS -get_block_properties(uint16_t dev_id, - enum perf_block_id block_id, - struct perf_counter_block *block); +get_block_properties(uint32_t node_id, + enum perf_block_id block_id, + struct perf_counter_block *block); #endif // PMC_TABLE_H From cb0f851560b0fd1152674295eabcc7a629168262 Mon Sep 17 00:00:00 2001 From: ozeng Date: Tue, 31 Jan 2017 17:40:50 -0600 Subject: [PATCH 0208/1247] libkmt: Misc fixes in thunk 1. Translate thunk queue priority to kfd priority 2. Initialize event SyncVar 3. Added HSAint32 data type Change-Id: I7decc1be7cbe9c84cb670d9a7c99050b62ba98f3 --- include/hsakmttypes.h | 2 ++ src/events.c | 4 ++++ src/queues.c | 18 ++++++++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) mode change 100644 => 100755 src/events.c mode change 100644 => 100755 src/queues.c diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 2d29a7083c..80c1b29fa3 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -48,6 +48,7 @@ extern "C" { typedef unsigned short HSAuint16; typedef signed short HSAint16; typedef unsigned __int32 HSAuint32; + typedef signed __int32 HSAint32; typedef signed __int64 HSAint64; typedef unsigned __int64 HSAuint64; @@ -63,6 +64,7 @@ extern "C" { typedef uint16_t HSAuint16; typedef int16_t HSAint16; typedef uint32_t HSAuint32; + typedef int32_t HSAint32; typedef int64_t HSAint64; typedef uint64_t HSAuint64; diff --git a/src/events.c b/src/events.c old mode 100644 new mode 100755 index 8908817116..b65ea96e71 --- a/src/events.c +++ b/src/events.c @@ -120,6 +120,10 @@ hsaKmtCreateEvent( e->EventData.HWData1 = args.event_id; e->EventData.HWData3 = args.event_trigger_data; + e->EventData.EventData.SyncVar.SyncVar.UserData = + EventDesc->SyncVar.SyncVar.UserData; + e->EventData.EventData.SyncVar.SyncVarSize = + EventDesc->SyncVar.SyncVarSize; if (IsSignaled && !IsSystemEventType(e->EventData.EventType)) { struct kfd_ioctl_set_event_args set_args; diff --git a/src/queues.c b/src/queues.c old mode 100644 new mode 100755 index a6c9527b01..6a4636a2b1 --- a/src/queues.c +++ b/src/queues.c @@ -513,6 +513,12 @@ static int handle_concrete_asic(struct queue *q, return HSAKMT_STATUS_SUCCESS; } +/* A map to translate thunk queue priority (-3 to +3) + * to KFD queue priority (0 to 15) + * Indexed by thunk_queue_priority+3 + */ +static uint32_t priority_map[] = {0,3,5,7,9,11,15}; + HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue( @@ -533,6 +539,10 @@ hsaKmtCreateQueue( int err; CHECK_KFD_OPEN(); + if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || + Priority > HSA_QUEUE_PRIORITY_MAXIMUM) + return HSAKMT_STATUS_INVALID_PARAMETER; + result = validate_nodeid(NodeId, &gpu_id); if (result != HSAKMT_STATUS_SUCCESS) return result; @@ -581,7 +591,7 @@ hsaKmtCreateQueue( args.ring_base_address = (uintptr_t)QueueAddress; args.ring_size = QueueSizeInBytes; args.queue_percentage = QueuePercentage; - args.queue_priority = Priority; + args.queue_priority = priority_map[Priority+3]; err = kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_QUEUE, &args); @@ -623,13 +633,17 @@ hsaKmtUpdateQueue( CHECK_KFD_OPEN(); + if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || + Priority > HSA_QUEUE_PRIORITY_MAXIMUM) + return HSAKMT_STATUS_INVALID_PARAMETER; + if (q == NULL) return (HSAKMT_STATUS_INVALID_PARAMETER); arg.queue_id = (HSAuint32)q->queue_id; arg.ring_base_address = (uintptr_t)QueueAddress; arg.ring_size = QueueSize; arg.queue_percentage = QueuePercentage; - arg.queue_priority = Priority; + arg.queue_priority = priority_map[Priority+3]; int err = kmtIoctl(kfd_fd, AMDKFD_IOC_UPDATE_QUEUE, &arg); if (err == -1) From 74ebfca9f025c94209110602da879b109cd830c8 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 6 Feb 2017 15:49:12 -0500 Subject: [PATCH 0209/1247] Free BOs before munmapping This avoids unnecessary evictions and failed restores due to the munmap of userptr BOs that are just about to be freed. Change-Id: Icf2f0b73991455556a201c54c05ea7e20af80f47 Signed-off-by: Felix Kuehling --- src/fmm.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 1aee7a72c6..bb9b7ca8bf 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1277,6 +1277,13 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) return; } + /* If memory is user memory and it's still GPU mapped, munmap + * would cause an eviction. If the restore happens quickly + * enough, restore would also fail with an error message. So + * free the BO before unmapping the pages. */ + args.handle = object->handle; + kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); + if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { /* Remove any CPU mapping, but keep the address range reserved */ @@ -1284,9 +1291,6 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); } - args.handle = object->handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); - aperture_release_area(aperture, address, object->size); vm_remove_object(aperture, object); From bc44715be2b810da5fe7c3e83c8b43dda4877f44 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Wed, 8 Feb 2017 14:39:15 -0600 Subject: [PATCH 0210/1247] Update libhsakmt CMakeList.txt file for tag based versioning and proper packaging Change-Id: I63e82deefa8377ced810d99b5b2f0457299048a6 --- CMakeLists.txt | 54 ++++++++++++-------------- cmake_modules/utils.cmake | 81 ++++++++++++++++++++++++++++++--------- 2 files changed, 88 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e4ca74a60..82f4aef425 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ ## ################################################################################ -cmake_minimum_required(VERSION 2.8.0) +cmake_minimum_required ( VERSION 3.5.0 ) set ( HSAKMT_NAME "hsakmt" ) set ( HSAKMT_COMPONENT "lib${HSAKMT_NAME}" ) @@ -35,11 +35,18 @@ list ( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules" ) include ( utils ) -if(NOT DEFINED VERSION_STRING) - set ( VERSION_STRING "2.0.1" ) -endif() +## Get the package version. The defaults to 1.0.0. +get_version ( "1.0.0" ) -parse_version ( ${VERSION_STRING} ) +set ( BUILD_VERSION_MAJOR ${VERSION_MAJOR} ) +set ( BUILD_VERSION_MINOR ${VERSION_MINOR} ) +set ( BUILD_VERSION_PATCH ${VERSION_PATCH} ) +set ( LIB_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) +if ( DEFINED VERSION_BUILD AND NOT ${VERSION_BUILD} STREQUAL "" ) + message ( "VERSION BUILD DEFINED ${VERSION_BUILD}" ) + set ( BUILD_VERSION_PATCH "${BUILD_VERSION_PATCH}-${VERSION_BUILD}" ) +endif () +set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) ## Verbose output. set ( CMAKE_VERBOSE_MAKEFILE on ) @@ -75,15 +82,8 @@ include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/src ) add_library ( ${HSAKMT_TARGET} SHARED ${HSAKMT_SRC} ) ## Set the VERSION and SOVERSION values -if ( DEFINED VERSION_STRING ) - set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${VERSION_STRING}" ) -endif () - -if ( DEFINED VERSION_MAJOR ) - set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${VERSION_MAJOR}" ) -else () - set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION 1 ) -endif() +set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" ) +set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${BUILD_VERSION_MAJOR}" ) target_link_libraries ( ${HSAKMT_TARGET} dl pthread rt @@ -91,34 +91,30 @@ target_link_libraries ( ${HSAKMT_TARGET} ## If the library is a release, strip the target library if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) - add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} *.so ) + add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} *.so* ) endif () +## Create symlinks for packaging and install +add_custom_target ( ${HSAKMT_COMPONENT} ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../${HSAKMT_COMPONENT}/include/${HSAKMT_COMPONENT} ${HSAKMT_COMPONENT}-link ) +add_custom_target ( ${HSAKMT_COMPONENT}.so-link ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../${HSAKMT_COMPONENT}/lib/${HSAKMT_COMPONENT}.so ${HSAKMT_COMPONENT}.so-link ) + ## Set the install targets install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux PERMISSIONS OWNER_WRITE OWNER_READ ) - -#if( CMAKE_HOST_UNIX ) -# install( CODE "EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E create_symlink include/${HSAKMT_COMPONENT} \${CMAKE_INSTALL_PREFIX}/include/${HSAKMT_COMPONENT} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) -# install( CODE "EXECUTE_PROCESS( COMMAND ${CMAKE_COMMAND} -E create_symlink lib/${HSAKMT_COMPONENT}.so \${CMAKE_INSTALL_PREFIX}/lib/${HSAKMT_COMPONENT}.so WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) -#endif( ) - -if( CMAKE_HOST_UNIX ) - install( CODE "EXECUTE_PROCESS( COMMAND mkdir -p \${CMAKE_INSTALL_PREFIX}/include COMMAND ln -srf include/${HSAKMT_COMPONENT} \${CMAKE_INSTALL_PREFIX}/include/${HSAKMT_COMPONENT} WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) - install( CODE "EXECUTE_PROCESS( COMMAND mkdir -p \${CMAKE_INSTALL_PREFIX}/lib COMMAND ln -srf lib/${HSAKMT_COMPONENT}.so \${CMAKE_INSTALL_PREFIX}/lib/${HSAKMT_COMPONENT}.so WORKING_DIRECTORY \${CMAKE_INSTALL_PREFIX}/libhsakmt )" ) -endif( ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}-link DESTINATION include PERMISSIONS OWNER_WRITE OWNER_READ RENAME ${HSAKMT_COMPONENT} ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}.so-link DESTINATION lib PERMISSIONS OWNER_WRITE OWNER_READ RENAME ${HSAKMT_COMPONENT}.so ) ## Packaging directives set ( CPACK_PACKAGE_NAME "hsakmt-roct-dev" ) set ( CPACK_PACKAGE_VENDOR "AMD" ) -set ( CPACK_PACKAGE_VERSION_MAJOR ${VERSION_MAJOR} ) -set ( CPACK_PACKAGE_VERSION_MINOR ${VERSION_MINOR} ) -set ( CPACK_PACKAGE_VERSION_PATCH ${VERSION_PATCH} ) +set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) +set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) +set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) -set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "Thunk library for AMD KFD" ) +set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT library for AMD KFD support" ) set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md" ) # Debian package specific variables diff --git a/cmake_modules/utils.cmake b/cmake_modules/utils.cmake index 58067a4d3c..21fcf25137 100644 --- a/cmake_modules/utils.cmake +++ b/cmake_modules/utils.cmake @@ -3,7 +3,7 @@ ## The University of Illinois/NCSA ## Open Source License (NCSA) ## -## Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +## Copyright (c) 2014-2017, Advanced Micro Devices, Inc. All rights reserved. ## ## Developed by: ## @@ -41,31 +41,76 @@ ################################################################################ ## Parses the VERSION_STRING variable and places -## the first, second and third number values in +## the first, second and third number values in ## the major, minor and patch variables. -function(parse_version VERSION_STRING) +function( parse_version VERSION_STRING ) - string(REGEX MATCHALL "[0123456789]+" VERSIONS ${VERSION_STRING}) - list(LENGTH VERSIONS VERSION_COUNT) + string ( FIND ${VERSION_STRING} "-" STRING_INDEX ) - if (${VERSION_COUNT} GREATER 0) - list(GET VERSIONS 0 MAJOR) - set(VERSION_MAJOR ${MAJOR} PARENT_SCOPE) - set(TEMP_VERSION_STRING "${MAJOR}") + if ( ${STRING_INDEX} GREATER -1 ) + math ( EXPR STRING_INDEX "${STRING_INDEX} + 1" ) + string ( SUBSTRING ${VERSION_STRING} ${STRING_INDEX} -1 VERSION_BUILD ) endif () - if (${VERSION_COUNT} GREATER 1) - list(GET VERSIONS 1 MINOR) - set(VERSION_MINOR ${MINOR} PARENT_SCOPE) - set(TEMP_VERSION_STRING "${TEMP_VERSION_STRING}.${MINOR}") + string ( REGEX MATCHALL "[0123456789]+" VERSIONS ${VERSION_STRING} ) + list (LENGTH VERSIONS VERSION_COUNT ) + + if ( ${VERSION_COUNT} GREATER 0) + list ( GET VERSIONS 0 MAJOR ) + set ( VERSION_MAJOR ${MAJOR} PARENT_SCOPE ) + set ( TEMP_VERSION_STRING "${MAJOR}" ) endif () - if (${VERSION_COUNT} GREATER 2) - list(GET VERSIONS 2 PATCH) - set(VERSION_PATCH ${PATCH} PARENT_SCOPE) - set(TEMP_VERSION_STRING "${TEMP_VERSION_STRING}.${PATCH}") + if ( ${VERSION_COUNT} GREATER 1 ) + list ( GET VERSIONS 1 MINOR ) + set ( VERSION_MINOR ${MINOR} PARENT_SCOPE ) + set ( TEMP_VERSION_STRING "${TEMP_VERSION_STRING}.${MINOR}" ) endif () - set(VERSION_STRING "${TEMP_VERSION_STRING}" PARENT_SCOPE) + if ( ${VERSION_COUNT} GREATER 2 ) + list ( GET VERSIONS 2 PATCH ) + set ( VERSION_PATCH ${PATCH} PARENT_SCOPE ) + set ( TEMP_VERSION_STRING "${TEMP_VERSION_STRING}.${PATCH}" ) + endif () + + if ( DEFINED VERSION_BUILD ) + set ( VERSION_BUILD "${VERSION_BUILD}" PARENT_SCOPE ) + endif () + + set ( VERSION_STRING "${TEMP_VERSION_STRING}" PARENT_SCOPE ) + +endfunction () + +## Gets the current version of the repository +## using versioning tags and git describe. +## Passes back a packaging version string +## and a library version string. +function ( get_version DEFAULT_VERSION_STRING ) + + parse_version ( ${DEFAULT_VERSION_STRING} ) + + find_program ( GIT NAMES git ) + + if ( GIT ) + + execute_process ( COMMAND git describe --long --match [0-9]* + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GIT_TAG_STRING + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE RESULT ) + + if ( ${RESULT} EQUAL 0 ) + + parse_version ( ${GIT_TAG_STRING} ) + + endif () + + endif () + + set( VERSION_STRING "${VERSION_STRING}" PARENT_SCOPE ) + set( VERSION_MAJOR "${VERSION_MAJOR}" PARENT_SCOPE ) + set( VERSION_MINOR "${VERSION_MINOR}" PARENT_SCOPE ) + set( VERSION_PATCH "${VERSION_PATCH}" PARENT_SCOPE ) + set( VERSION_BUILD "${VERSION_BUILD}" PARENT_SCOPE ) endfunction() From a5a30e81990e42d32218df044e4bce05451e410b Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 9 Feb 2017 14:42:14 -0600 Subject: [PATCH 0211/1247] Fix file stripping for release builds. Change-Id: I538c366f0992980ffff1ef337807035b9378845c --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 82f4aef425..1adbe8f102 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,7 @@ target_link_libraries ( ${HSAKMT_TARGET} ## If the library is a release, strip the target library if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) - add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} *.so* ) + add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} ${HSAKMT_COMPONENT}.so ) endif () ## Create symlinks for packaging and install From 9e81f0f5e249c8d41a24d8a0f0f631ae98ff6fcd Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sat, 11 Feb 2017 16:24:54 -0600 Subject: [PATCH 0212/1247] Add libpci to CMakeLists.txt for thunk. Change-Id: I0228035ce769feaf54cbca75f076f73614cbb9cc --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1adbe8f102..dc8285ef53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,7 +52,7 @@ set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUIL set ( CMAKE_VERBOSE_MAKEFILE on ) ## Compiler flags -set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Werror -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden" ) +set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Werror -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden -O2" ) set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) @@ -86,7 +86,7 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${BUILD_VERSION_MAJOR}" ) target_link_libraries ( ${HSAKMT_TARGET} - dl pthread rt + pthread rt pci ) ## If the library is a release, strip the target library From acb1f0b689f7fec0092cccc69fe13f353fed25f4 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Tue, 14 Feb 2017 11:17:00 -0600 Subject: [PATCH 0213/1247] Add --dirty tag to utils.cmake Change-Id: Ib2487eade8d88530df34dfb8e9b442e547e9f52d --- cmake_modules/utils.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake_modules/utils.cmake b/cmake_modules/utils.cmake index 21fcf25137..5b4e73ce23 100644 --- a/cmake_modules/utils.cmake +++ b/cmake_modules/utils.cmake @@ -53,7 +53,7 @@ function( parse_version VERSION_STRING ) endif () string ( REGEX MATCHALL "[0123456789]+" VERSIONS ${VERSION_STRING} ) - list (LENGTH VERSIONS VERSION_COUNT ) + list ( LENGTH VERSIONS VERSION_COUNT ) if ( ${VERSION_COUNT} GREATER 0) list ( GET VERSIONS 0 MAJOR ) @@ -93,7 +93,7 @@ function ( get_version DEFAULT_VERSION_STRING ) if ( GIT ) - execute_process ( COMMAND git describe --long --match [0-9]* + execute_process ( COMMAND git describe --dirty --long --match [0-9]* WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_TAG_STRING OUTPUT_STRIP_TRAILING_WHITESPACE From b3c3f7bae163d670c374b347282582553a0c3fe2 Mon Sep 17 00:00:00 2001 From: ozeng Date: Mon, 13 Feb 2017 14:46:40 -0600 Subject: [PATCH 0214/1247] libkmt: Change files mode back to 644 events.c and queues.c were accidently changed to 755 by change fc70f0c30976f4021f7d763bfc10d76a76029553. Change them back. Change-Id: If51c0b91139afc23e9051cf94c83d61fc20297e6 Signed-off-by: Oak Zeng --- src/events.c | 0 src/queues.c | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/events.c mode change 100755 => 100644 src/queues.c diff --git a/src/events.c b/src/events.c old mode 100755 new mode 100644 diff --git a/src/queues.c b/src/queues.c old mode 100755 new mode 100644 From e79521b556df3aa526b7ea82c2790120900af7af Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 27 Jan 2017 18:10:24 -0500 Subject: [PATCH 0215/1247] Add API entrypoints for Cross Memory Attach Implement two new API for cross memory read and write operation. - hsaKmtProcessVMRead - hsaKmtProcessVMWrite Add new ioclts necessary for the above APIs. Change-Id: I0c153e3b4e1f32b7a8b102ad5c774d9ae9bfc2fa Signed-off-by: Harish Kasiviswanathan --- include/hsakmt.h | 31 ++++++++++++++ include/hsakmttypes.h | 5 +++ include/linux/kfd_ioctl.h | 36 ++++++++++++++++- src/libhsakmt.ver | 2 + src/memory.c | 85 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 1 deletion(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 0eea105772..ed5aff7144 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -444,6 +444,37 @@ hsaKmtRegisterSharedHandleToNodes( HSAuint32* NodeArray // OUT ); +/** + Copy data from the GPU address space of the process identified + by Pid. Size Copied will return actual amount of data copied. + If return is not SUCCESS, partial copies could have happened. + */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtProcessVMRead( + HSAuint32 Pid, // IN + HsaMemoryRange *LocalMemoryArray, // IN + HSAuint64 LocalMemoryArrayCount, // IN + HsaMemoryRange *RemoteMemoryArray, // IN + HSAuint64 RemoteMemoryArrayCount, // IN + HSAuint64 *SizeCopied // OUT +); + +/** + Write data to the GPU address space of the process identified + by Pid. See also hsaKmtProcessVMRead. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtProcessVMWrite( + HSAuint32 Pid, // IN + HsaMemoryRange *LocalMemoryArray, // IN + HSAuint64 LocalMemoryArrayCount, // IN + HsaMemoryRange *RemoteMemoryArray, // IN + HSAuint64 RemoteMemoryArrayCount, // IN + HSAuint64 *SizeCopied // OUT +); + /** Unregisters with KFD a memory buffer */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 80c1b29fa3..fe3d0b220e 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1008,6 +1008,11 @@ typedef struct _HsaPointerInfo { typedef HSAuint32 HsaSharedMemoryHandle[8]; +typedef struct _HsaMemoryRange { + void *MemoryAddress; // Pointer to GPU memory + HSAuint64 SizeInBytes; // Size of above memory +} HsaMemoryRange; + #pragma pack(pop, hsakmttypes_h) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 12090c6419..6805834941 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -395,6 +395,37 @@ struct kfd_ioctl_get_tile_config_args { */ }; +struct kfd_memory_range { + uint64_t va_addr; + uint64_t size; +}; + +/* flags definitions + * BIT0: 0: read operation, 1: write operation. + * This also identifies if the src or dst array belongs to remote process + */ +#define KFD_CROSS_MEMORY_RW_BIT (1 << 0) +#define KFD_SET_CROSS_MEMORY_READ(flags) (flags &= ~KFD_CROSS_MEMORY_RW_BIT) +#define KFD_SET_CROSS_MEMORY_WRITE(flags) (flags |= KFD_CROSS_MEMORY_RW_BIT) +#define KFD_IS_CROSS_MEMORY_WRITE(flags) (flags & KFD_CROSS_MEMORY_RW_BIT) + +struct kfd_ioctl_cross_memory_copy_args { + /* to KFD: Process ID of the remote process */ + uint32_t pid; + /* to KFD: See above definition */ + uint32_t flags; + /* to KFD: Source GPU VM range */ + uint64_t src_mem_range_array; + /* to KFD: Size of above array */ + uint64_t src_mem_array_size; + /* to KFD: Destination GPU VM range */ + uint64_t dst_mem_range_array; + /* to KFD: Size of above array */ + uint64_t dst_mem_array_size; + /* from KFD: Total amount of bytes copied */ + uint64_t bytes_copied; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -504,7 +535,10 @@ struct kfd_ioctl_get_tile_config_args { #define AMDKFD_IOC_IPC_EXPORT_HANDLE \ AMDKFD_IOWR(0x23, struct kfd_ioctl_ipc_export_handle_args) +#define AMDKFD_IOC_CROSS_MEMORY_COPY \ + AMDKFD_IOWR(0x24, struct kfd_ioctl_cross_memory_copy_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x24 +#define AMDKFD_COMMAND_END 0x25 #endif diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index e1b30fb87d..e955a31c97 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -30,6 +30,8 @@ hsaKmtRegisterGraphicsHandleToNodes; hsaKmtShareMemory; hsaKmtRegisterSharedHandle; hsaKmtRegisterSharedHandleToNodes; +hsaKmtProcessVMRead; +hsaKmtProcessVMWrite; hsaKmtDeregisterMemory; hsaKmtMapMemoryToGPU; hsaKmtMapMemoryToGPUNodes; diff --git a/src/memory.c b/src/memory.c index 24cc5a818c..b3883c1477 100644 --- a/src/memory.c +++ b/src/memory.c @@ -340,6 +340,91 @@ error: return ret; } +static uint64_t convertHsaToKfdRange( + HsaMemoryRange *HsaRange) +{ + if (sizeof(struct kfd_memory_range) != + sizeof(HsaMemoryRange)) { + fprintf(stderr, "Struct size mismatch in thunk." + "Cannot cast Hsa Range to KFD IOCTL range\n"); + return 0; + } + return (uint64_t) HsaRange; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtProcessVMRead( + HSAuint32 Pid, /* IN */ + HsaMemoryRange *LocalMemoryArray, /* IN */ + HSAuint64 LocalMemoryArrayCount, /* IN */ + HsaMemoryRange *RemoteMemoryArray, /* IN */ + HSAuint64 RemoteMemoryArrayCount,/* IN */ + HSAuint64 *SizeCopied /* OUT */ +) +{ + struct kfd_ioctl_cross_memory_copy_args args; + + if (LocalMemoryArray == NULL || RemoteMemoryArray == NULL || + LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) + return HSAKMT_STATUS_ERROR; + + args.flags = 0; + KFD_SET_CROSS_MEMORY_READ(args.flags); + args.pid = Pid; + args.src_mem_range_array = convertHsaToKfdRange(RemoteMemoryArray); + args.src_mem_array_size = RemoteMemoryArrayCount; + args.dst_mem_range_array = convertHsaToKfdRange(LocalMemoryArray); + args.dst_mem_array_size = LocalMemoryArrayCount; + args.bytes_copied = 0; + + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args); + if (err) + return HSAKMT_STATUS_ERROR; + + if (SizeCopied) + *SizeCopied = args.bytes_copied; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtProcessVMWrite( + HSAuint32 Pid, /* IN */ + HsaMemoryRange *LocalMemoryArray, /* IN */ + HSAuint64 LocalMemoryArrayCount, /* IN */ + HsaMemoryRange *RemoteMemoryArray, /* IN */ + HSAuint64 RemoteMemoryArrayCount, /* IN */ + HSAuint64 *SizeCopied /* OUT */ +) +{ + struct kfd_ioctl_cross_memory_copy_args args; + + if (LocalMemoryArray == NULL || RemoteMemoryArray == NULL || + LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) + return HSAKMT_STATUS_ERROR; + + args.flags = 0; + KFD_SET_CROSS_MEMORY_WRITE(args.flags); + args.pid = Pid; + args.src_mem_range_array = convertHsaToKfdRange(LocalMemoryArray); + args.src_mem_array_size = LocalMemoryArrayCount; + args.dst_mem_range_array = convertHsaToKfdRange(RemoteMemoryArray); + args.dst_mem_array_size = RemoteMemoryArrayCount; + args.bytes_copied = 0; + + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args); + if (err) + return HSAKMT_STATUS_ERROR; + + if (SizeCopied) + *SizeCopied = args.bytes_copied; + + return HSAKMT_STATUS_SUCCESS; +} + + HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory( From cb60c5f18a23d3d2c7f538c9de09d46e62176fd0 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 6 Feb 2017 15:06:32 -0500 Subject: [PATCH 0216/1247] Support multiple blocks in RegisterTrace Existing code assumes all counters sent to hsaKmtPmcRegisterTrace belong to one PMC block and this block is SQ. This patch considers cases when counters are in different blocks, and removes the hard-coded SQ. As a matter of fact, SQ is non-privileged so the user even shouldn't use SQ counters to register/release trace. This patch also ignores non-privileged blocks as what HSA Thunk spec describes. This patch also records counters information in trace structure so AcquireTrace can get counters information using that TraceId. Change-Id: Ifa5741050553d4615baab01f7485a9e09435b019 Signed-off-by: Amber Lin --- src/perfctr.c | 105 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 0ec59b6ba8..1dbcd856e4 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -24,6 +24,7 @@ */ #include +#include #include "libhsakmt.h" #include "pmc_table.h" #include "linux/kfd_ioctl.h" @@ -38,10 +39,18 @@ enum perf_trace_state { PERF_TRACE_STATE__STARTED }; +struct perf_trace_block { + enum perf_block_id block_id; + uint32_t num_counters; + uint64_t *counter_id; +}; + struct perf_trace { uint32_t magic4cc; - uint32_t gpu_id; + uint32_t gpu_id; enum perf_trace_state state; + uint32_t num_blocks; + struct perf_trace_block blocks[0]; }; extern int amd_hsa_thunk_lock_fd; @@ -99,6 +108,19 @@ static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) return rc; } +static HSAuint32 get_block_concurrent_limit(uint32_t node_id, + HSAuint32 block_id) +{ + uint32_t i; + + for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) + if (counter_props[node_id]->Blocks[i].Counters[0].BlockIndex == + block_id) + return counter_props[node_id]->Blocks[i].NumConcurrent; + + return 0; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties( @@ -201,10 +223,15 @@ hsaKmtPmcRegisterTrace( HsaPmcTraceRoot* TraceRoot //OUT ) { - uint32_t gpu_id, i; + uint32_t gpu_id, i, j; uint64_t min_buf_size = 0; - uint32_t concurrent_counters[PERFCOUNTER_BLOCKID__MAX] = {0}; struct perf_trace *trace = NULL; + uint32_t concurrent_limit; + const uint32_t MAX_COUNTERS = 512; + uint64_t counter_id[PERFCOUNTER_BLOCKID__MAX][MAX_COUNTERS]; + uint32_t num_counters[PERFCOUNTER_BLOCKID__MAX] = {0}; + uint32_t block, num_blocks = 0, total_counters = 0; + uint64_t *counter_id_ptr; if (counter_props == NULL) return HSAKMT_STATUS_NO_MEMORY; @@ -212,28 +239,90 @@ hsaKmtPmcRegisterTrace( if (Counters == NULL || TraceRoot == NULL || NumberOfCounters == 0) return HSAKMT_STATUS_INVALID_PARAMETER; - if (validate_nodeid(NodeId, &gpu_id) != 0) + if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (NumberOfCounters > MAX_COUNTERS) { + fprintf(stderr, "Error: MAX_COUNTERS is too small for %d.\n", + NumberOfCounters); + return HSAKMT_STATUS_NO_MEMORY; + } + /* Calculating the minimum buffer size */ for (i = 0; i < NumberOfCounters; i++) { if (Counters[i].BlockIndex >= PERFCOUNTER_BLOCKID__MAX) return HSAKMT_STATUS_INVALID_PARAMETER; + /* Only privileged counters need to register */ + if (Counters[i].Type > HSA_PROFILE_TYPE_PRIVILEGED_STREAMING) + continue; min_buf_size += Counters[i].CounterSizeInBits/BITS_PER_BYTE; - concurrent_counters[Counters[i].BlockIndex]++; + /* j: the first blank entry in the block to record counter_id */ + j = num_counters[Counters[i].BlockIndex]; + counter_id[Counters[i].BlockIndex][j] = Counters[i].CounterId; + num_counters[Counters[i].BlockIndex]++; + total_counters++; } - /* Verifying that the number of counters per block is not larger than the amount of slots */ - if (concurrent_counters[PERFCOUNTER_BLOCKID__SQ] > counter_props[NodeId]->Blocks[PERFCOUNTER_BLOCKID__SQ].NumConcurrent) + /* Verify that the number of counters per block is not larger than the + * number of slots. + */ + for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { + if (!num_counters[i]) + continue; + concurrent_limit = get_block_concurrent_limit(NodeId, i); + if (!concurrent_limit) { + fprintf(stderr, "Invalid block ID: %d\n", i); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + if (num_counters[i] > concurrent_limit) { + fprintf(stderr, "Counters exceed the limit.\n"); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + num_blocks++; + } + + if (!num_blocks) return HSAKMT_STATUS_INVALID_PARAMETER; - trace = malloc(sizeof(trace)); + /* Now we have sorted blocks/counters information in + * num_counters[block_id] and counter_id[block_id][]. Allocate trace + * and record the information. + */ + trace = (struct perf_trace *)calloc(sizeof(struct perf_trace) + + sizeof(struct perf_trace_block) * num_blocks + + sizeof(uint64_t) * total_counters, + 1); if (trace == NULL) return HSAKMT_STATUS_NO_MEMORY; + block = 0; + counter_id_ptr = (uint64_t *)((char *) + trace + sizeof(struct perf_trace) + + sizeof(struct perf_trace_block) * num_blocks); + /* Fill in each block's information to the TraceId */ + for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { + if (!num_counters[i]) /* not a block to trace */ + continue; + /* Following perf_trace + perf_trace_block x N are those + * counter_id arrays. Assign the counter_id array belonging to + * this block. + */ + trace->blocks[block].counter_id = counter_id_ptr; + /* Fill in counter IDs to the counter_id array. */ + for (j = 0; j < num_counters[i]; j++) + trace->blocks[block].counter_id[j] = counter_id[i][j]; + /* how many counters to trace */ + trace->blocks[block].num_counters = num_counters[i]; + /* block index in "enum perf_block_id" */ + trace->blocks[block].block_id = i; + block++; /* move to next */ + counter_id_ptr += num_counters[i]; + } + trace->magic4cc = HSA_PERF_MAGIC4CC; trace->gpu_id = gpu_id; trace->state = PERF_TRACE_STATE__STOPPED; + trace->num_blocks = num_blocks; TraceRoot->NumberOfPasses = 1; TraceRoot->TraceBufferMinSizeBytes = PAGE_ALIGN_UP(min_buf_size); From 64104fc8d966b354f0f3885c22f65517388e86f8 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 21 Feb 2017 14:13:05 -0500 Subject: [PATCH 0217/1247] Avoid COW after fork for API-allocated system memory Change-Id: I5c7175114c4e6411d3beb5557e16cb71ddb01189 Signed-off-by: Felix Kuehling --- src/fmm.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index bb9b7ca8bf..3dc51ab6e8 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1139,6 +1139,11 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, return NULL; } + /* Mappings in the DGPU aperture don't need to be copied on + * fork. This avoids MMU notifiers and evictions due to user + * memory mappings on fork. */ + madvise(mem, MemorySizeInBytes, MADV_DONTFORK); + /* Create userptr BO */ mmap_offset = (uint64_t)mem; ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_USERPTR; From 1025579c0bb171cbd109830e65955f6d6aa92989 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 15 Feb 2017 11:32:49 -0500 Subject: [PATCH 0218/1247] Implement PMC AcquireTrace Existing code uses lockf to ensure exclusive PMC access of one process and one TraceId. However Thunk spec allows hsaKmtPmcAcquireTraceAccess to get exclusive access to the defined set of counters, not exclusive to one process or one TraceId. Multiple counter sets of multiple TraceIds is allowed if they meet the concurrent access limit evaluated by the hardware /driver. Change-Id: I59cacb855a707fe326a4070452fcbbd3c95ac223 --- src/openclose.c | 13 +- src/perfctr.c | 309 +++++++++++++++++++++++++++++++++++++++++++----- src/pmc_table.c | 10 ++ src/pmc_table.h | 1 + 4 files changed, 300 insertions(+), 33 deletions(-) diff --git a/src/openclose.c b/src/openclose.c index 8901cffcdc..9983ab098b 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -113,13 +113,16 @@ hsaKmtOpenKFD(void) if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) printf("Insufficient Memory. Debugging unavailable\n"); - if (init_counter_props(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) - printf("Insufficient Memory. Performance Counter information unavailable\n"); - amd_hsa_thunk_lock_fd = open(tmp_file, O_CREAT | //create the file if it's not present. - O_RDWR, //only need write access for the internal locking semantics. - S_IRUSR | S_IWUSR); //permissions on the file, 600 here. + O_RDWR, + S_IROTH | S_IWOTH); //allow others to read/write + if (amd_hsa_thunk_lock_fd < 0) + fprintf(stderr, + "Profiling of privileged counters is not available\n"); + if (init_counter_props(sys_props.NumNodes) != + HSAKMT_STATUS_SUCCESS) + fprintf(stderr, "Profiling is not available\n"); } else { diff --git a/src/perfctr.c b/src/perfctr.c index 1dbcd856e4..669f801d7b 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -25,10 +25,15 @@ #include #include +#include +#include +#include #include "libhsakmt.h" #include "pmc_table.h" #include "linux/kfd_ioctl.h" #include +#include +#include #define BITS_PER_BYTE CHAR_BIT @@ -43,6 +48,7 @@ struct perf_trace_block { enum perf_block_id block_id; uint32_t num_counters; uint64_t *counter_id; + int *perf_event_fd; }; struct perf_trace { @@ -53,11 +59,84 @@ struct perf_trace { struct perf_trace_block blocks[0]; }; +enum perf_trace_action { + PERF_TRACE_ACTION__ACQUIRE = 0, + PERF_TRACE_ACTION__RELEASE +}; + +struct perf_lockf_tbl { + uint32_t magic4cc; + uint32_t iommu_slots_left; +}; + +struct perf_counts_values { + union { + struct { + u64 val; + u64 ena; + u64 run; + }; + u64 values[3]; + }; +}; + extern int amd_hsa_thunk_lock_fd; static HsaCounterProperties **counter_props; static unsigned int counter_props_count; +static ssize_t readn(int fd, void *buf, size_t n) +{ + size_t left = n; + ssize_t bytes; + + while (left) { + bytes = read(fd, buf, left); + if (bytes < 0 && errno == EINTR) /* read got interrupted */ + continue; + if (bytes <= 0) + return (n - left); + left -= bytes; + buf = VOID_PTR_ADD(buf, bytes); + } + return n; +} + +static HSAKMT_STATUS init_lockf_perf_section(void) +{ + struct perf_lockf_tbl lockf_tbl; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + if (amd_hsa_thunk_lock_fd <= 0) + return HSAKMT_STATUS_UNAVAILABLE; + + if (lockf(amd_hsa_thunk_lock_fd, F_TLOCK, sizeof(lockf_tbl))) + return HSAKMT_STATUS_ERROR; + + memset(&lockf_tbl, 0, sizeof(lockf_tbl)); + if (readn(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) { + ret = HSAKMT_STATUS_ERROR; + goto out; + } + /* If the magic number exists, the lock file table has been + * initialized by another process and is in use. Don't overwrite it. + */ + if (lockf_tbl.magic4cc == HSA_PERF_MAGIC4CC) + goto out; + /* write the perf content */ + lockf_tbl.magic4cc = HSA_PERF_MAGIC4CC; + lockf_tbl.iommu_slots_left = + pmc_table_get_max_concurrent(PERFCOUNTER_BLOCKID__IOMMUV2); + if (write(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) + ret = HSAKMT_STATUS_ERROR; +out: + /* unlock the perf section */ + if (lockf(amd_hsa_thunk_lock_fd, F_ULOCK, sizeof(lockf_tbl))) + ret = HSAKMT_STATUS_ERROR; + + return ret; +} + HSAKMT_STATUS init_counter_props(unsigned int NumNodes) { counter_props = calloc(NumNodes, sizeof(struct HsaCounterProperties*)); @@ -67,6 +146,7 @@ HSAKMT_STATUS init_counter_props(unsigned int NumNodes) counter_props_count = NumNodes; alloc_pmc_blocks(); + init_lockf_perf_section(); return HSAKMT_STATUS_SUCCESS; } @@ -121,6 +201,149 @@ static HSAuint32 get_block_concurrent_limit(uint32_t node_id, return 0; } +static HSAKMT_STATUS update_block_slots(enum perf_trace_action action, + uint32_t block_id, uint32_t num_slots) +{ + struct perf_lockf_tbl lockf_tbl; + uint32_t *slots_left; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + if (amd_hsa_thunk_lock_fd <= 0) + return HSAKMT_STATUS_UNAVAILABLE; + + if (lockf(amd_hsa_thunk_lock_fd, F_TLOCK, sizeof(lockf_tbl)) < 0) + return HSAKMT_STATUS_ERROR; + + if (readn(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) { + ret = HSAKMT_STATUS_ERROR; + goto out; + } + + if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) + slots_left = &lockf_tbl.iommu_slots_left; + else { + ret = HSAKMT_STATUS_UNAVAILABLE; + goto out; + } + + switch (action) { + case PERF_TRACE_ACTION__ACQUIRE: + if (*slots_left >= num_slots) + *slots_left -= num_slots; + else + ret = HSAKMT_STATUS_UNAVAILABLE; + break; + case PERF_TRACE_ACTION__RELEASE: + if ((*slots_left + num_slots) <= + pmc_table_get_max_concurrent(block_id)) + *slots_left += num_slots; + else + ret = HSAKMT_STATUS_ERROR; + break; + default: + ret = HSAKMT_STATUS_INVALID_PARAMETER; + break; + } + + if (ret == HSAKMT_STATUS_SUCCESS) { + if (write(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) + ret = HSAKMT_STATUS_ERROR; + } +out: + /* unlock the perf section */ + if (lockf(amd_hsa_thunk_lock_fd, F_ULOCK, sizeof(lockf_tbl))) + ret = HSAKMT_STATUS_ERROR; + + return ret; +} + +static unsigned int get_perf_event_type(enum perf_block_id block_id) +{ + FILE *file = NULL; + unsigned int type = 0; + + if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) + file = fopen("/sys/bus/event_source/devices/amd_iommu/type", + "r"); + if (!file) + return 0; + + if (fscanf(file, "%d", &type) != 1) + type = 0; + fclose(file); + + return type; +} + +/* close_perf_event_fd - Close all FDs opened for this block. + * When RT acquires the trace access, RT has no ideas about each + * individual FD opened for this block. We should treat the whole + * block as one and close all of them. + */ +static void close_perf_event_fd(struct perf_trace_block *block) +{ + uint32_t i; + + if (!block || !block->perf_event_fd) + return; + + for (i = 0; i < block->num_counters; i++) + if (block->perf_event_fd[i] > 0) { + close(block->perf_event_fd[i]); + block->perf_event_fd[i] = 0; + } +} + +/* open_perf_event_fd - Open FDs required for this block. + * If one of them fails, we should close all FDs that have been + * opened because RT has no ideas about those FDs successfully + * opened and it won't send anything to close them. + */ +static HSAKMT_STATUS open_perf_event_fd(struct perf_trace_block *block) +{ + struct perf_event_attr attr; + uint32_t i; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + if (!block || !block->perf_event_fd) + return HSAKMT_STATUS_INVALID_HANDLE; + + if (getuid()) { + fprintf(stderr, + "Error. Must be root to open perf_event.\n"); + return HSAKMT_STATUS_ERROR; + } + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.type = get_perf_event_type(block->block_id); + if (!attr.type) + return HSAKMT_STATUS_ERROR; + + for (i = 0; i < block->num_counters; i++) { + attr.size = sizeof(struct perf_event_attr); + attr.config = block->counter_id[i]; + attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | + PERF_FORMAT_TOTAL_TIME_RUNNING; + attr.disabled = 1; + attr.inherit = 1; + + /* We are profiling system wide, not per cpu, so no threads, + * no groups -> pid=-1 and group_fd=-1. cpu = 0 + * flags=PERF_FLAG_FD_NO_GROUP + */ + block->perf_event_fd[i] = syscall(__NR_perf_event_open, &attr, + -1, 0, -1, PERF_FLAG_FD_NO_GROUP); + + if (block->perf_event_fd[i] < 0) { + ret = HSAKMT_STATUS_ERROR; + close_perf_event_fd(block); + break; + } + } + + return ret; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties( @@ -232,6 +455,7 @@ hsaKmtPmcRegisterTrace( uint32_t num_counters[PERFCOUNTER_BLOCKID__MAX] = {0}; uint32_t block, num_blocks = 0, total_counters = 0; uint64_t *counter_id_ptr; + int *fd_ptr; if (counter_props == NULL) return HSAKMT_STATUS_NO_MEMORY; @@ -290,15 +514,34 @@ hsaKmtPmcRegisterTrace( */ trace = (struct perf_trace *)calloc(sizeof(struct perf_trace) + sizeof(struct perf_trace_block) * num_blocks - + sizeof(uint64_t) * total_counters, + + sizeof(uint64_t) * total_counters + + sizeof(int) * total_counters, 1); if (trace == NULL) return HSAKMT_STATUS_NO_MEMORY; + /* Allocated area is partitioned as: + * +---------------------------------+ trace + * | perf_trace | + * |---------------------------------| trace->blocks[0] + * | perf_trace_block 0 | + * | .... | + * | perf_trace_block N-1 | trace->blocks[N-1] + * |---------------------------------| <-- counter_id_ptr starts here + * | block 0's counter IDs(uint64_t) | + * | ...... | + * | block N-1's counter IDs | + * |---------------------------------| <-- perf_event_fd starts here + * | block 0's perf_event_fds(int) | + * | ...... | + * | block N-1's perf_event_fds | + * +---------------------------------+ + */ block = 0; counter_id_ptr = (uint64_t *)((char *) trace + sizeof(struct perf_trace) + sizeof(struct perf_trace_block) * num_blocks); + fd_ptr = (int *)(counter_id_ptr + total_counters); /* Fill in each block's information to the TraceId */ for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { if (!num_counters[i]) /* not a block to trace */ @@ -311,12 +554,14 @@ hsaKmtPmcRegisterTrace( /* Fill in counter IDs to the counter_id array. */ for (j = 0; j < num_counters[i]; j++) trace->blocks[block].counter_id[j] = counter_id[i][j]; + trace->blocks[block].perf_event_fd = fd_ptr; /* how many counters to trace */ trace->blocks[block].num_counters = num_counters[i]; /* block index in "enum perf_block_id" */ trace->blocks[block].block_id = i; block++; /* move to next */ counter_id_ptr += num_counters[i]; + fd_ptr += num_counters[i]; } trace->magic4cc = HSA_PERF_MAGIC4CC; @@ -371,12 +616,6 @@ hsaKmtPmcUnregisterTrace( return HSAKMT_STATUS_SUCCESS; } - -/** - Allows a user mode process to get exclusive access to the defined set of (HW) counters - used for tracing/profiling -*/ - HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess( @@ -385,6 +624,9 @@ hsaKmtPmcAcquireTraceAccess( ) { struct perf_trace *trace; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t gpu_id, i; + int j; if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -394,23 +636,35 @@ hsaKmtPmcAcquireTraceAccess( if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; - if (amd_hsa_thunk_lock_fd > 0) { - if (lockf( amd_hsa_thunk_lock_fd, F_TLOCK, 0 ) != 0) - return HSAKMT_STATUS_ERROR; - else - return HSAKMT_STATUS_SUCCESS; + if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + for (i = 0; i < trace->num_blocks; i++) { + ret = update_block_slots(PERF_TRACE_ACTION__ACQUIRE, + trace->blocks[i].block_id, + trace->blocks[i].num_counters); + if (ret != HSAKMT_STATUS_SUCCESS) + goto out; + ret = open_perf_event_fd(&trace->blocks[i]); + if (ret != HSAKMT_STATUS_SUCCESS) { + i++; /* to release slots just reserved */ + goto out; + } } - else { - return HSAKMT_STATUS_ERROR; + +out: + if (ret != HSAKMT_STATUS_SUCCESS) { + for (j = i-1; j >= 0; j--) { + update_block_slots(PERF_TRACE_ACTION__RELEASE, + trace->blocks[j].block_id, + trace->blocks[j].num_counters); + close_perf_event_fd(&trace->blocks[j]); + } } + + return ret; } - -/** - Allows a user mode process to release exclusive access to the defined set of (HW) counters - used for tracing/profiling -*/ - HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess( @@ -419,6 +673,7 @@ hsaKmtPmcReleaseTraceAccess( ) { struct perf_trace *trace; + uint32_t i; if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -428,16 +683,14 @@ hsaKmtPmcReleaseTraceAccess( if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; - if (amd_hsa_thunk_lock_fd > 0) { - if (lockf( amd_hsa_thunk_lock_fd, F_ULOCK, 0 ) != 0) - return HSAKMT_STATUS_ERROR; - else - return HSAKMT_STATUS_SUCCESS; - } - else { - return HSAKMT_STATUS_ERROR; + for (i = 0; i < trace->num_blocks; i++) { + update_block_slots(PERF_TRACE_ACTION__RELEASE, + trace->blocks[i].block_id, + trace->blocks[i].num_counters); + close_perf_event_fd(&trace->blocks[i]); } + return HSAKMT_STATUS_SUCCESS; } diff --git a/src/pmc_table.c b/src/pmc_table.c index 6465507c41..5bd16c44cf 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -192,6 +192,16 @@ static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { */ static struct perf_counter_block iommu_block; +uint32_t pmc_table_get_max_concurrent(int block_id) +{ + switch (block_id) { + case PERFCOUNTER_BLOCKID__IOMMUV2: + return iommu_block.num_of_slots; + default: + return 0; + } +} + static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) { diff --git a/src/pmc_table.h b/src/pmc_table.h index adc4af2298..2aa1002bdb 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -45,6 +45,7 @@ struct perf_counter_block { HSAKMT_STATUS alloc_pmc_blocks(void); void free_pmc_blocks(void); +uint32_t pmc_table_get_max_concurrent(int block_id); HSAKMT_STATUS get_block_properties(uint32_t node_id, From 9ba2b68fdb34fe2deaa59325128ae1976019bf64 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 22 Sep 2016 14:59:20 -0400 Subject: [PATCH 0219/1247] Add gfx900 support Add gfx900 device to the support Change-Id: I71f30ef43e5e0ef0e7b5f18205b6cc4767d9d861 --- src/queues.c | 12 ++++++++++-- src/topology.c | 5 ++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/queues.c b/src/queues.c index 6a4636a2b1..5a17dfc950 100644 --- a/src/queues.c +++ b/src/queues.c @@ -45,11 +45,12 @@ enum asic_family_type { CHIP_TONGA, CHIP_FIJI, CHIP_POLARIS10, - CHIP_POLARIS11 + CHIP_POLARIS11, + CHIP_VEGA10 }; #define IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_POLARIS11) -#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_POLARIS11) || \ +#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_VEGA10) || \ (chip) == CHIP_HAWAII) #define WG_CONTEXT_DATA_SIZE_PER_CU_VI 344576 @@ -96,6 +97,11 @@ struct device_info polaris11_device_info = { .eop_buffer_size = TONGA_PAGE_SIZE, }; +struct device_info vega10_device_info = { + .asic_family = CHIP_VEGA10, + .eop_buffer_size = 4096, +}; + struct device_id { uint16_t dev_id; @@ -174,6 +180,8 @@ struct device_id supported_devices[] = { { 0x67EB, &polaris11_device_info }, { 0x67EF, &polaris11_device_info }, { 0x67FF, &polaris11_device_info }, + { 0x6860, &vega10_device_info }, + { 0x687F, &vega10_device_info }, { 0, NULL } }; diff --git a/src/topology.c b/src/topology.c index 5d74b8a00b..2a4f5a61fa 100644 --- a/src/topology.c +++ b/src/topology.c @@ -160,7 +160,10 @@ static struct hsa_gfxip_table { { 0x67E9, 8, 0, 3, 1, "Polaris11" }, { 0x67EB, 8, 0, 3, 1, "Polaris11" }, { 0x67EF, 8, 0, 3, 1, "Polaris11" }, - { 0x67FF, 8, 0, 3, 1, "Polaris11" } + { 0x67FF, 8, 0, 3, 1, "Polaris11" }, + /* Vega10 */ + { 0x6860, 9, 0, 0, 1, "Vega10" }, + { 0x687F, 9, 0, 0, 1, "Vega10" } }; enum cache_type { From 48207af92a119b50ff4bc6f4596f7573465b51da Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 28 Oct 2016 18:48:44 -0400 Subject: [PATCH 0220/1247] Make doorbell-size ASIC specific This is in preparation for gfx900, which uses 64-bit doorbells. We maintain the same number of doorbells per process by making the doorbell page size bigger. KFD will need to implement the same rule. Change-Id: I3c4110869b191b83943b5a390a48edfc94d941d8 --- src/queues.c | 64 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/queues.c b/src/queues.c index 5a17dfc950..186c6cad28 100644 --- a/src/queues.c +++ b/src/queues.c @@ -35,8 +35,11 @@ #include #include -/* 1024 doorbells, 4 bytes each doorbell */ -#define DOORBELLS_PAGE_SIZE 1024 * 4 +/* 1024 doorbells, 4 or 8 bytes each doorbell depending on ASIC generation */ +#define DOORBELL_SIZE_GFX7 4 +#define DOORBELL_SIZE_GFX8 4 +#define DOORBELL_SIZE_GFX9 8 +#define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) enum asic_family_type { CHIP_KAVERI = 0, @@ -60,46 +63,55 @@ struct device_info { enum asic_family_type asic_family; uint32_t eop_buffer_size; + uint32_t doorbell_size; }; struct device_info kaveri_device_info = { .asic_family = CHIP_KAVERI, .eop_buffer_size = 0, + .doorbell_size = DOORBELL_SIZE_GFX7, }; struct device_info hawaii_device_info = { .asic_family = CHIP_HAWAII, .eop_buffer_size = 0, + .doorbell_size = DOORBELL_SIZE_GFX7, }; struct device_info carrizo_device_info = { .asic_family = CHIP_CARRIZO, .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX8, }; struct device_info tonga_device_info = { .asic_family = CHIP_TONGA, .eop_buffer_size = TONGA_PAGE_SIZE, + .doorbell_size = DOORBELL_SIZE_GFX8, }; struct device_info fiji_device_info = { .asic_family = CHIP_FIJI, .eop_buffer_size = TONGA_PAGE_SIZE, + .doorbell_size = DOORBELL_SIZE_GFX8, }; struct device_info polaris10_device_info = { .asic_family = CHIP_POLARIS10, .eop_buffer_size = TONGA_PAGE_SIZE, + .doorbell_size = DOORBELL_SIZE_GFX8, }; struct device_info polaris11_device_info = { .asic_family = CHIP_POLARIS11, .eop_buffer_size = TONGA_PAGE_SIZE, + .doorbell_size = DOORBELL_SIZE_GFX8, }; struct device_info vega10_device_info = { .asic_family = CHIP_VEGA10, .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, }; struct device_id @@ -199,7 +211,8 @@ struct queue struct process_doorbells { - bool need_mmap; + bool use_gpuvm; + uint32_t size; void* doorbells; pthread_mutex_t doorbells_mutex; }; @@ -219,7 +232,8 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) return HSAKMT_STATUS_NO_MEMORY; for (i = 0; i < NumNodes; i++) { - doorbells[i].need_mmap = true; + doorbells[i].use_gpuvm = false; + doorbells[i].size = 0; doorbells[i].doorbells = NULL; pthread_mutex_init(&doorbells[i].doorbells_mutex, NULL); } @@ -242,7 +256,8 @@ static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) return NULL; } -static bool use_gpuvm_doorbell(uint16_t dev_id) +static void get_doorbell_map_info(uint16_t dev_id, + struct process_doorbells *doorbell) { struct device_info *dev_info; @@ -252,8 +267,9 @@ static bool use_gpuvm_doorbell(uint16_t dev_id) * GPUVM doorbell on Tonga requires a workaround for VM TLB ACTIVE bit * lookup bug. Remove ASIC check when this is implemented in amdgpu. */ - return (topology_is_dgpu(dev_id) && - dev_info->asic_family != CHIP_TONGA); + doorbell->use_gpuvm = (topology_is_dgpu(dev_id) && + dev_info->asic_family != CHIP_TONGA); + doorbell->size = DOORBELLS_PAGE_SIZE(dev_info->doorbell_size); } void destroy_process_doorbells(void) @@ -264,14 +280,14 @@ void destroy_process_doorbells(void) return; for (i = 0; i < num_doorbells; i++) { - if (doorbells[i].need_mmap) + if (!doorbells[i].size) continue; - if (use_gpuvm_doorbell(get_device_id_by_node(i))) { + if (doorbells[i].use_gpuvm) { fmm_unmap_from_gpu(doorbells[i].doorbells); fmm_release(doorbells[i].doorbells); } else - munmap(doorbells[i].doorbells, DOORBELLS_PAGE_SIZE); + munmap(doorbells[i].doorbells, doorbells[i].size); } free(doorbells); @@ -290,11 +306,11 @@ void clear_process_doorbells(void) return; for (i = 0; i < num_doorbells; i++) { - if (doorbells[i].need_mmap) + if (!doorbells[i].size) continue; - if (!use_gpuvm_doorbell(get_device_id_by_node(i))) - munmap(doorbells[i].doorbells, DOORBELLS_PAGE_SIZE); + if (!doorbells[i].use_gpuvm) + munmap(doorbells[i].doorbells, doorbells[i].size); } free(doorbells); @@ -307,13 +323,12 @@ static HSAKMT_STATUS map_doorbell_apu(HSAuint32 NodeId, HSAuint32 gpu_id, { void *ptr; - ptr = mmap(0, DOORBELLS_PAGE_SIZE, PROT_READ|PROT_WRITE, + ptr = mmap(0, doorbells[NodeId].size, PROT_READ|PROT_WRITE, MAP_SHARED, kfd_fd, doorbell_offset); if (ptr == MAP_FAILED) return HSAKMT_STATUS_ERROR; - doorbells[NodeId].need_mmap = false; doorbells[NodeId].doorbells = ptr; return HSAKMT_STATUS_SUCCESS; @@ -324,19 +339,18 @@ static HSAKMT_STATUS map_doorbell_dgpu(HSAuint32 NodeId, HSAuint32 gpu_id, { void *ptr; - ptr = fmm_allocate_doorbell(gpu_id, DOORBELLS_PAGE_SIZE, + ptr = fmm_allocate_doorbell(gpu_id, doorbells[NodeId].size, doorbell_offset); if (ptr == NULL) return HSAKMT_STATUS_ERROR; /* map for GPU access */ - if (fmm_map_to_gpu(ptr, DOORBELLS_PAGE_SIZE, NULL)) { + if (fmm_map_to_gpu(ptr, doorbells[NodeId].size, NULL)) { fmm_release(ptr); return HSAKMT_STATUS_ERROR; } - doorbells[NodeId].need_mmap = false; doorbells[NodeId].doorbells = ptr; return HSAKMT_STATUS_SUCCESS; @@ -348,16 +362,22 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, HSAKMT_STATUS status = HSAKMT_STATUS_SUCCESS; pthread_mutex_lock(&doorbells[NodeId].doorbells_mutex); - if (!doorbells[NodeId].need_mmap) { + if (doorbells[NodeId].size) { pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); return HSAKMT_STATUS_SUCCESS; } - if (use_gpuvm_doorbell(get_device_id_by_node(NodeId))) + get_doorbell_map_info(get_device_id_by_node(NodeId), + &doorbells[NodeId]); + + if (doorbells[NodeId].use_gpuvm) status = map_doorbell_dgpu(NodeId, gpu_id, doorbell_offset); else status = map_doorbell_apu(NodeId, gpu_id, doorbell_offset); + if (status != HSAKMT_STATUS_SUCCESS) + doorbells[NodeId].size = 0; + pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); return status; @@ -619,7 +639,9 @@ hsaKmtCreateQueue( } QueueResource->QueueId = PORT_VPTR_TO_UINT64(q); - QueueResource->Queue_DoorBell = VOID_PTR_ADD32(doorbells[NodeId].doorbells, q->queue_id); + QueueResource->Queue_DoorBell = VOID_PTR_ADD(doorbells[NodeId].doorbells, + (q->queue_id * + dev_info->doorbell_size)); return HSAKMT_STATUS_SUCCESS; } From e5dd2f88c6a123b8b1b69bf6281670af6448e29a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 15 Nov 2016 14:36:54 -0500 Subject: [PATCH 0221/1247] Update kfd_ioctl.h Copied from kernel repository. Change-Id: I9ed021cfb5b297d9a91dce93ed6355c95fb1127b Signed-off-by: Felix Kuehling Reviewed-by: Shaoyun Liu --- include/linux/kfd_ioctl.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 6805834941..6473ae9040 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -311,10 +311,9 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { #define KFD_IOC_ALLOC_MEM_FLAGS_APU_SCRATCH (1 << 4) #define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM (1 << 5) - #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 6) - #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 7) +#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 8) struct kfd_ioctl_alloc_memory_of_gpu_new_args { uint64_t va_addr; /* to KFD */ From 8cb89b692649621fdc18fb54c5f68212e7661a42 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 15 Nov 2016 14:37:31 -0500 Subject: [PATCH 0222/1247] Use KFD_IOC_ALLOC_MEM_FLAGS_COHERENT for fine-grained memory Use KFD_IOC_ALLOC_MEM_FLAGS_COHERENT when allocating fine-grained memory and doorbell BOs so that they will be mapped with MTYPE_UC on GFX9 hardware. Change-Id: I51adf45b13105f479e6bcdaf54955b467920ee9a Signed-off-by: Felix Kuehling Reviewed-by: Shaoyun Liu --- src/fmm.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 3dc51ab6e8..067b9c28c1 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1023,7 +1023,8 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Use fine-grained aperture */ aperture = &svm.dgpu_alt_aperture; - ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL; + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | + KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, 0, NULL, @@ -1109,10 +1110,12 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, size = MemorySizeInBytes; ioc_flags = 0; - if (flags.ui32.CoarseGrain) + if (flags.ui32.CoarseGrain) { aperture = &svm.dgpu_aperture; - else + } else { aperture = &svm.dgpu_alt_aperture; /* coherent */ + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + } if (flags.ui32.AQLQueueMemory) { size = MemorySizeInBytes * 2; ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; From d7063dd102919ffc061a75e487c87d10d4b864a9 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 21 Nov 2016 20:05:27 -0500 Subject: [PATCH 0223/1247] Allocate 64-bit for doorbells and write pointers On gfx900 we need 64-bit for all doorbells and SDMA WPTRs. Change-Id: I9b922e16442e967599ae3c928308451d5cc470b3 Signed-off-by: Felix Kuehling --- src/queues.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/queues.c b/src/queues.c index 186c6cad28..faa943c0c7 100644 --- a/src/queues.c +++ b/src/queues.c @@ -200,8 +200,8 @@ struct device_id supported_devices[] = { struct queue { uint32_t queue_id; - uint32_t wptr; - uint32_t rptr; + uint64_t wptr; + uint64_t rptr; void *eop_buffer; void *ctx_save_restore; uint32_t ctx_save_restore_size; From 7de66d149b97093f0eb58ff0fb32bf82b06f8a3a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 25 Nov 2016 19:29:33 -0500 Subject: [PATCH 0224/1247] gfx900: Allow doorbell allocation independent of queue ID On SOC15 chips, the ABI for the create_queue ioctl is changed to allow doorbell allocation independent of the queue ID. This is necessary to accommodate doorbell routing to specific engines in the BIF. Change-Id: Ie98d0a758758149dd5fc09ae088afccc29904124 Signed-off-by: Felix Kuehling --- src/queues.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/queues.c b/src/queues.c index faa943c0c7..c9eab609b8 100644 --- a/src/queues.c +++ b/src/queues.c @@ -55,6 +55,7 @@ enum asic_family_type { #define IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_POLARIS11) #define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_VEGA10) || \ (chip) == CHIP_HAWAII) +#define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) #define WG_CONTEXT_DATA_SIZE_PER_CU_VI 344576 #define WAVES_PER_CU_VI 32 @@ -563,6 +564,8 @@ hsaKmtCreateQueue( HSAKMT_STATUS result; uint32_t gpu_id; uint16_t dev_id; + uint64_t doorbell_mmap_offset; + unsigned doorbell_offset; struct device_info *dev_info; int err; CHECK_KFD_OPEN(); @@ -631,7 +634,24 @@ hsaKmtCreateQueue( q->queue_id = args.queue_id; - err = map_doorbell(NodeId, gpu_id, args.doorbell_offset); + if (IS_SOC15(dev_info->asic_family)) { + /* On SOC15 chips, the doorbell offset within the + * doorbell page is included in the doorbell offset + * returned by KFD. This allows doorbells to be + * allocated per-device, independent of the + * per-process queue ID. */ + doorbell_mmap_offset = args.doorbell_offset & + ~(HSAuint64)(doorbells[NodeId].size - 1); + doorbell_offset = args.doorbell_offset & + (doorbells[NodeId].size - 1); + } else { + /* On older chips, the doorbell offset within the + * doorbell page is based on the queue ID. */ + doorbell_mmap_offset = args.doorbell_offset; + doorbell_offset = q->queue_id * dev_info->doorbell_size; + } + + err = map_doorbell(NodeId, gpu_id, doorbell_mmap_offset); if (err != HSAKMT_STATUS_SUCCESS) { hsaKmtDestroyQueue(q->queue_id); free_queue(q); @@ -640,8 +660,7 @@ hsaKmtCreateQueue( QueueResource->QueueId = PORT_VPTR_TO_UINT64(q); QueueResource->Queue_DoorBell = VOID_PTR_ADD(doorbells[NodeId].doorbells, - (q->queue_id * - dev_info->doorbell_size)); + doorbell_offset); return HSAKMT_STATUS_SUCCESS; } From 116e5c5e8bc0f32cdbe9a11cb366c33b96b375ff Mon Sep 17 00:00:00 2001 From: "shaoyun.liu" Date: Fri, 20 Jan 2017 13:33:38 -0500 Subject: [PATCH 0225/1247] Thunk: Don't allocate extra control stack memory for gfx900 The control stack memory for CWSR is allocate in kernel together with MQD allocation. Change-Id: Ib1c0ab9402df3431e9555649394320380d6c6dd8 Signed-off-by: shaoyun.liu --- src/queues.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/queues.c b/src/queues.c index c9eab609b8..2aeda1bda6 100644 --- a/src/queues.c +++ b/src/queues.c @@ -418,8 +418,12 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) ctl_stack_size = cu_num * WAVES_PER_CU_VI * 8 + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU_VI; q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size); - q->ctx_save_restore_size = - q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size); + q->ctx_save_restore_size = PAGE_ALIGN_UP(wg_data_size); + + if (q->dev_info->asic_family < CHIP_VEGA10) + /* GFX8 chips store ctl-stack with WG data */ + q->ctx_save_restore_size += q->ctl_stack_size; + return true; } return false; From c99195128809d53ba05e17ebfaec426ef47135eb Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 28 Feb 2017 12:36:30 -0500 Subject: [PATCH 0226/1247] fmm.c: Disable userptr for paged mem by default Unless HSA_USERPTR_FOR_PAGED_MEM is explicitly set, don't use userptr for all paged memory. This will also allow us to work around some 4.9 issues, and then we can explicitly set HSA_USERPTR_FOR_PAGED_MEM for all usage once those issues are resolved. Change-Id: I25ce22b73ae6e93f1567f2318d9d2b47d4a44e69 --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 067b9c28c1..1a10aa82d2 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1400,10 +1400,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (disableCache && strcmp(disableCache, "0") == 0) disableCache = NULL; - /* If HSA_USERPTR_FOR_PAGED_MEM unset or set to a non-0 value, + /* If HSA_USERPTR_FOR_PAGED_MEM is set to a non-0 value, * enable userptr for all paged memory allocations */ pagedUserptr = getenv("HSA_USERPTR_FOR_PAGED_MEM"); - svm.userptr_for_paged_mem = (!pagedUserptr || strcmp(pagedUserptr, "0")); + svm.userptr_for_paged_mem = (pagedUserptr && strcmp(pagedUserptr, "0")); /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than From ec84fbe264b76f9c96b8fa4d9c6694446c3f4f26 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 2 Mar 2017 12:00:09 -0600 Subject: [PATCH 0227/1247] Fix permissions on hsakmt include files. Change-Id: I1d428e60268e6d2de6776ff5f16d03503d00ddcc --- CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dc8285ef53..bc8ce505bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,12 +100,12 @@ add_custom_target ( ${HSAKMT_COMPONENT}.so-link ALL WORKING_DIRECTORY ${CMAKE_CU ## Set the install targets install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt PERMISSIONS OWNER_WRITE OWNER_READ ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux PERMISSIONS OWNER_WRITE OWNER_READ ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}-link DESTINATION include PERMISSIONS OWNER_WRITE OWNER_READ RENAME ${HSAKMT_COMPONENT} ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}.so-link DESTINATION lib PERMISSIONS OWNER_WRITE OWNER_READ RENAME ${HSAKMT_COMPONENT}.so ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}-link DESTINATION include RENAME ${HSAKMT_COMPONENT} ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}.so-link DESTINATION lib RENAME ${HSAKMT_COMPONENT}.so ) ## Packaging directives set ( CPACK_PACKAGE_NAME "hsakmt-roct-dev" ) From e17c67f049e9dbd8519652e7859aa2215ed36755 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 23 Feb 2017 18:26:23 -0500 Subject: [PATCH 0228/1247] Implement Start/Stop/Query Trace StartTrace and StopTrace send ioctl requests to enable/disable performance counters. QueryTrace reads the counter from the perf_event fd. Change-Id: Ibf79675bc23fcf129371bfd100f8e262121bc684 --- src/perfctr.c | 109 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 11 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 669f801d7b..ce991dd228 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -56,6 +56,8 @@ struct perf_trace { uint32_t gpu_id; enum perf_trace_state state; uint32_t num_blocks; + void *buf; + uint64_t buf_size; struct perf_trace_block blocks[0]; }; @@ -92,10 +94,14 @@ static ssize_t readn(int fd, void *buf, size_t n) while (left) { bytes = read(fd, buf, left); - if (bytes < 0 && errno == EINTR) /* read got interrupted */ - continue; - if (bytes <= 0) + if (!bytes) /* reach EOF */ return (n - left); + if (bytes < 0 ) { + if (errno == EINTR) /* read got interrupted */ + continue; + else + return -errno; + } left -= bytes; buf = VOID_PTR_ADD(buf, bytes); } @@ -144,11 +150,9 @@ HSAKMT_STATUS init_counter_props(unsigned int NumNodes) return HSAKMT_STATUS_NO_MEMORY; counter_props_count = NumNodes; - alloc_pmc_blocks(); - init_lockf_perf_section(); - return HSAKMT_STATUS_SUCCESS; + return init_lockf_perf_section(); } void destroy_counter_props(void) @@ -214,7 +218,13 @@ static HSAKMT_STATUS update_block_slots(enum perf_trace_action action, if (lockf(amd_hsa_thunk_lock_fd, F_TLOCK, sizeof(lockf_tbl)) < 0) return HSAKMT_STATUS_ERROR; - if (readn(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) { + if (lseek(amd_hsa_thunk_lock_fd, 0, SEEK_SET)) { + ret = HSAKMT_STATUS_ERROR; + goto out; + } + + if (readn(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) + != sizeof(lockf_tbl)) { ret = HSAKMT_STATUS_ERROR; goto out; } @@ -344,6 +354,34 @@ static HSAKMT_STATUS open_perf_event_fd(struct perf_trace_block *block) return ret; } +static HSAKMT_STATUS perf_trace_ioctl(struct perf_trace_block *block, + uint32_t cmd) +{ + uint32_t i; + + for (i = 0; i < block->num_counters; i++) { + if (block->perf_event_fd[i] < 0) + return HSAKMT_STATUS_UNAVAILABLE; + if (ioctl(block->perf_event_fd[i], cmd, NULL)) + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS query_trace(int fd, uint64_t *buf) +{ + struct perf_counts_values content; + + if (fd < 0) + return HSAKMT_STATUS_ERROR; + if(readn(fd, &content, sizeof(content)) != sizeof(content)) + return HSAKMT_STATUS_ERROR; + + *buf = content.val; + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties( @@ -706,7 +744,11 @@ hsaKmtPmcStartTrace( HSAuint64 TraceBufferSizeBytes //IN (page aligned) ) { - struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + struct perf_trace *trace = + (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + uint32_t i; + int32_t j; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; if (TraceId == 0 || TraceBuffer == NULL || TraceBufferSizeBytes == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -714,7 +756,24 @@ hsaKmtPmcStartTrace( if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; + for (i = 0; i < trace->num_blocks; i++) { + ret = perf_trace_ioctl(&trace->blocks[i], + PERF_EVENT_IOC_ENABLE); + if (ret != HSAKMT_STATUS_SUCCESS) + break; + } + if (ret != HSAKMT_STATUS_SUCCESS) { + /* Disable enabled blocks before returning the failure. */ + j = (int32_t)i; + while (--j >= 0) + perf_trace_ioctl(&trace->blocks[j], + PERF_EVENT_IOC_DISABLE); + return ret; + } + trace->state = PERF_TRACE_STATE__STARTED; + trace->buf = TraceBuffer; + trace->buf_size = TraceBufferSizeBytes; return HSAKMT_STATUS_SUCCESS; } @@ -730,7 +789,12 @@ hsaKmtPmcQueryTrace( HSATraceId TraceId //IN ) { - struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + struct perf_trace *trace = + (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + uint32_t i, j; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint64_t *buf; + uint64_t buf_filled = 0; if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -738,6 +802,19 @@ hsaKmtPmcQueryTrace( if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; + buf = (uint64_t *)trace->buf; + for (i = 0; i < trace->num_blocks; i++) + for (j = 0; j < trace->blocks[i].num_counters; j++) { + buf_filled += sizeof(uint64_t); + if (buf_filled > trace->buf_size) + return HSAKMT_STATUS_NO_MEMORY; + ret = query_trace(trace->blocks[i].perf_event_fd[j], + buf); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + buf++; + } + return HSAKMT_STATUS_SUCCESS; } @@ -752,7 +829,10 @@ hsaKmtPmcStopTrace( HSATraceId TraceId //IN ) { - struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + struct perf_trace *trace = + (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); + uint32_t i; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -760,7 +840,14 @@ hsaKmtPmcStopTrace( if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; + for (i = 0; i < trace->num_blocks; i++) { + ret = perf_trace_ioctl(&trace->blocks[i], + PERF_EVENT_IOC_DISABLE); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + } + trace->state = PERF_TRACE_STATE__STOPPED; - return HSAKMT_STATUS_SUCCESS; + return ret; } From 1a8a9cb57b6cc1d8acd11b734c08e032c205189e Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 2 Mar 2017 16:59:35 -0500 Subject: [PATCH 0229/1247] Make the lock file writable by others Though S_IWOTH flag is set in the open() call, the lock file is not created as accessable by others if others try to open the file with O_RDWR permission. It's because the default umask masks off S_IWOTH. This patch changes the umask to S_IXOTH since others don't need that permission but it'll open up S_IWOTH. Restore the umask to original after the file is opened. Change-Id: I8a239e1566ce0b0b18821913385f239db7c3588e --- src/openclose.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/openclose.c b/src/openclose.c index 9983ab098b..3e0f5a8267 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -76,6 +76,7 @@ hsaKmtOpenKFD(void) HSAKMT_STATUS result; int fd; HsaSystemProperties sys_props; + mode_t mask; pthread_mutex_lock(&hsakmt_mutex); @@ -113,10 +114,13 @@ hsaKmtOpenKFD(void) if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) printf("Insufficient Memory. Debugging unavailable\n"); + mask = umask(0); /* save the current umask */ + /* We don't want the existing umask to mask out S_IWOTH */ + umask(S_IXOTH); amd_hsa_thunk_lock_fd = open(tmp_file, - O_CREAT | //create the file if it's not present. - O_RDWR, - S_IROTH | S_IWOTH); //allow others to read/write + O_CREAT | O_RDWR, + 0666); + umask(mask); /* restore the original umask */ if (amd_hsa_thunk_lock_fd < 0) fprintf(stderr, "Profiling of privileged counters is not available\n"); From 4827b0911944b61660e4e316e19e90b5fc7d20de Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 1 Mar 2017 16:25:43 -0500 Subject: [PATCH 0230/1247] Unify the device ID list Integrate the supported device ID list distributed in topology, queue, and pmc into one place: topology. Change-Id: If035cf8e4a6fc6caff6c94ec627647cfb11c3d79 --- src/libhsakmt.h | 16 +++++ src/pmc_table.c | 45 ++++++-------- src/queues.c | 117 +++++------------------------------ src/topology.c | 158 ++++++++++++++++++++++++++---------------------- 4 files changed, 135 insertions(+), 201 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index e31aeef42e..437cf14582 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -62,6 +62,20 @@ extern bool is_dgpu; #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) +enum asic_family_type { + CHIP_KAVERI = 0, + CHIP_HAWAII, + CHIP_CARRIZO, + CHIP_TONGA, + CHIP_FIJI, + CHIP_POLARIS10, + CHIP_POLARIS11, + CHIP_VEGA10 +}; +#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_VEGA10) || \ + (chip) == CHIP_HAWAII) +#define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) + HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint16_t get_device_id_by_node(HSAuint32 node_id); @@ -74,6 +88,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties uint32_t *gpu_id, struct pci_access* pacc); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); bool topology_is_dgpu(uint16_t device_id); +HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, + enum asic_family_type *asic); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); diff --git a/src/pmc_table.c b/src/pmc_table.c index 5bd16c44cf..de04230bd0 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -314,8 +314,8 @@ get_block_properties(uint32_t node_id, enum perf_block_id block_id, struct perf_counter_block *block) { - HSAKMT_STATUS rc = HSAKMT_STATUS_SUCCESS; uint16_t dev_id = get_device_id_by_node(node_id); + enum asic_family_type asic; if (block_id > PERFCOUNTER_BLOCKID__MAX || block_id < PERFCOUNTER_BLOCKID__FIRST) @@ -326,42 +326,31 @@ get_block_properties(uint32_t node_id, return HSAKMT_STATUS_SUCCESS; } - /* To avoid the long list, we read the 12 most significant digits of DID - * to identify the GPU instead of listing the complete 16 bits. If one - * day 12-bits is not good enough to distinguish the GPU, change the - * code here. - */ - switch(dev_id >> 4) { - case 0x130: - case 0x131: + if (topology_get_asic_family(dev_id, &asic) != HSAKMT_STATUS_SUCCESS) + return HSAKMT_STATUS_INVALID_PARAMETER; + + switch (asic) { + case CHIP_KAVERI: *block = kaveri_blocks[block_id]; break; - - case 0x987: - *block = carrizo_blocks[block_id]; - break; - - case 0x730: - *block = fiji_blocks[block_id]; - break; - - case 0x67A: - case 0x67B: + case CHIP_HAWAII: *block = hawaii_blocks[block_id]; break; - - case 0x67C: - case 0x67D: - case 0x67E: - case 0x67F: + case CHIP_CARRIZO: + *block = carrizo_blocks[block_id]; + break; + case CHIP_FIJI: + *block = fiji_blocks[block_id]; + break; + case CHIP_POLARIS10: + case CHIP_POLARIS11: *block = polaris_blocks[block_id]; break; - default: - rc = HSAKMT_STATUS_INVALID_PARAMETER; + return HSAKMT_STATUS_INVALID_PARAMETER; } - return rc; + return HSAKMT_STATUS_SUCCESS; } diff --git a/src/queues.c b/src/queues.c index 2aeda1bda6..6840a5df1a 100644 --- a/src/queues.c +++ b/src/queues.c @@ -41,22 +41,6 @@ #define DOORBELL_SIZE_GFX9 8 #define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) -enum asic_family_type { - CHIP_KAVERI = 0, - CHIP_HAWAII, - CHIP_CARRIZO, - CHIP_TONGA, - CHIP_FIJI, - CHIP_POLARIS10, - CHIP_POLARIS11, - CHIP_VEGA10 -}; - -#define IS_VI(chip) ((chip) >= CHIP_CARRIZO && (chip) <= CHIP_POLARIS11) -#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_VEGA10) || \ - (chip) == CHIP_HAWAII) -#define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) - #define WG_CONTEXT_DATA_SIZE_PER_CU_VI 344576 #define WAVES_PER_CU_VI 32 @@ -115,89 +99,23 @@ struct device_info vega10_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +static struct device_info *dev_lookup_table[] = { + [CHIP_KAVERI] = &kaveri_device_info, + [CHIP_HAWAII] = &hawaii_device_info, + [CHIP_CARRIZO] = &carrizo_device_info, + [CHIP_TONGA] = &tonga_device_info, + [CHIP_FIJI] = &fiji_device_info, + [CHIP_POLARIS10] = &polaris10_device_info, + [CHIP_POLARIS11] = &polaris11_device_info, + [CHIP_VEGA10] = &vega10_device_info +}; + struct device_id { uint16_t dev_id; struct device_info *dev_info; }; -/* TODO: unify this with the device list in topology.c */ -struct device_id supported_devices[] = { - { 0x1304, &kaveri_device_info }, /* Kaveri */ - { 0x1305, &kaveri_device_info }, /* Kaveri */ - { 0x1306, &kaveri_device_info }, /* Kaveri */ - { 0x1307, &kaveri_device_info }, /* Kaveri */ - { 0x1309, &kaveri_device_info }, /* Kaveri */ - { 0x130A, &kaveri_device_info }, /* Kaveri */ - { 0x130B, &kaveri_device_info }, /* Kaveri */ - { 0x130C, &kaveri_device_info }, /* Kaveri */ - { 0x130D, &kaveri_device_info }, /* Kaveri */ - { 0x130E, &kaveri_device_info }, /* Kaveri */ - { 0x130F, &kaveri_device_info }, /* Kaveri */ - { 0x1310, &kaveri_device_info }, /* Kaveri */ - { 0x1311, &kaveri_device_info }, /* Kaveri */ - { 0x1312, &kaveri_device_info }, /* Kaveri */ - { 0x1313, &kaveri_device_info }, /* Kaveri */ - { 0x1315, &kaveri_device_info }, /* Kaveri */ - { 0x1316, &kaveri_device_info }, /* Kaveri */ - { 0x1317, &kaveri_device_info }, /* Kaveri */ - { 0x1318, &kaveri_device_info }, /* Kaveri */ - { 0x131B, &kaveri_device_info }, /* Kaveri */ - { 0x131C, &kaveri_device_info }, /* Kaveri */ - { 0x131D, &kaveri_device_info }, /* Kaveri */ - { 0x67A0, &hawaii_device_info }, /* Hawaii */ - { 0x67A1, &hawaii_device_info }, /* Hawaii */ - { 0x67A2, &hawaii_device_info }, /* Hawaii */ - { 0x67A8, &hawaii_device_info }, /* Hawaii */ - { 0x67A9, &hawaii_device_info }, /* Hawaii */ - { 0x67AA, &hawaii_device_info }, /* Hawaii */ - { 0x67B0, &hawaii_device_info }, /* Hawaii */ - { 0x67B1, &hawaii_device_info }, /* Hawaii */ - { 0x67B8, &hawaii_device_info }, /* Hawaii */ - { 0x67B9, &hawaii_device_info }, /* Hawaii */ - { 0x67BA, &hawaii_device_info }, /* Hawaii */ - { 0x67BE, &hawaii_device_info }, /* Hawaii */ - { 0x9870, &carrizo_device_info }, /* Carrizo */ - { 0x9874, &carrizo_device_info }, /* Carrizo */ - { 0x9875, &carrizo_device_info }, /* Carrizo */ - { 0x9876, &carrizo_device_info }, /* Carrizo */ - { 0x9877, &carrizo_device_info }, /* Carrizo */ - { 0x6920, &tonga_device_info }, - { 0x6921, &tonga_device_info }, - { 0x6928, &tonga_device_info }, - { 0x6929, &tonga_device_info }, - { 0x692B, &tonga_device_info }, - { 0x692F, &tonga_device_info }, - { 0x6930, &tonga_device_info }, - { 0x6938, &tonga_device_info }, - { 0x6939, &tonga_device_info }, - { 0x7300, &fiji_device_info }, - { 0x730F, &fiji_device_info }, - { 0x67C0, &polaris10_device_info }, - { 0x67C1, &polaris10_device_info }, - { 0x67C2, &polaris10_device_info }, - { 0x67C4, &polaris10_device_info }, - { 0x67C7, &polaris10_device_info }, - { 0x67C8, &polaris10_device_info }, - { 0x67C9, &polaris10_device_info }, - { 0x67CA, &polaris10_device_info }, - { 0x67CC, &polaris10_device_info }, - { 0x67CF, &polaris10_device_info }, - { 0x67DF, &polaris10_device_info }, - { 0x67E0, &polaris11_device_info }, - { 0x67E1, &polaris11_device_info }, - { 0x67E3, &polaris11_device_info }, - { 0x67E7, &polaris11_device_info }, - { 0x67E8, &polaris11_device_info }, - { 0x67E9, &polaris11_device_info }, - { 0x67EB, &polaris11_device_info }, - { 0x67EF, &polaris11_device_info }, - { 0x67FF, &polaris11_device_info }, - { 0x6860, &vega10_device_info }, - { 0x687F, &vega10_device_info }, - { 0, NULL } -}; - struct queue { uint32_t queue_id; @@ -246,15 +164,12 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) { - int i = 0; - while (supported_devices[i].dev_id != 0) { - if (supported_devices[i].dev_id == dev_id) { - return supported_devices[i].dev_info; - } - i++; - } + enum asic_family_type asic; - return NULL; + if (topology_get_asic_family(dev_id, &asic) != HSAKMT_STATUS_SUCCESS) + return NULL; + + return dev_lookup_table[asic]; } static void get_doorbell_map_info(uint16_t dev_id, diff --git a/src/topology.c b/src/topology.c index 2a4f5a61fa..acdee5d673 100644 --- a/src/topology.c +++ b/src/topology.c @@ -83,87 +83,88 @@ static struct hsa_gfxip_table { unsigned char stepping; // GFXIP Stepping info unsigned char is_dgpu; // Predicate for dGPU devices const char* amd_name; // CALName of the device + enum asic_family_type asic_family; } gfxip_lookup_table[] = { /* Kaveri Family */ - { 0x1304, 7, 0, 0, 0, "Spectre" }, - { 0x1305, 7, 0, 0, 0, "Spectre" }, - { 0x1306, 7, 0, 0, 0, "Spectre" }, - { 0x1307, 7, 0, 0, 0, "Spectre" }, - { 0x1309, 7, 0, 0, 0, "Spectre" }, - { 0x130A, 7, 0, 0, 0, "Spectre" }, - { 0x130B, 7, 0, 0, 0, "Spectre" }, - { 0x130C, 7, 0, 0, 0, "Spectre" }, - { 0x130D, 7, 0, 0, 0, "Spectre" }, - { 0x130E, 7, 0, 0, 0, "Spectre" }, - { 0x130F, 7, 0, 0, 0, "Spectre" }, - { 0x1310, 7, 0, 0, 0, "Spectre" }, - { 0x1311, 7, 0, 0, 0, "Spectre" }, - { 0x1312, 7, 0, 0, 0, "Spooky" }, - { 0x1313, 7, 0, 0, 0, "Spectre" }, - { 0x1315, 7, 0, 0, 0, "Spectre" }, - { 0x1316, 7, 0, 0, 0, "Spooky" }, - { 0x1317, 7, 0, 0, 0, "Spooky" }, - { 0x1318, 7, 0, 0, 0, "Spectre" }, - { 0x131B, 7, 0, 0, 0, "Spectre" }, - { 0x131C, 7, 0, 0, 0, "Spectre" }, - { 0x131D, 7, 0, 0, 0, "Spectre" }, + { 0x1304, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1305, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1306, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1307, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1309, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130A, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130B, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130C, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130D, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130E, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130F, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1310, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1311, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1312, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, + { 0x1313, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1315, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1316, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, + { 0x1317, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, + { 0x1318, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x131B, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x131C, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x131D, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, /* Hawaii Family */ - { 0x67A0, 7, 0, 1, 1, "Hawaii" }, - { 0x67A1, 7, 0, 1, 1, "Hawaii" }, - { 0x67A2, 7, 0, 1, 1, "Hawaii" }, - { 0x67A8, 7, 0, 1, 1, "Hawaii" }, - { 0x67A9, 7, 0, 1, 1, "Hawaii" }, - { 0x67AA, 7, 0, 1, 1, "Hawaii" }, - { 0x67B0, 7, 0, 1, 1, "Hawaii" }, - { 0x67B1, 7, 0, 1, 1, "Hawaii" }, - { 0x67B8, 7, 0, 1, 1, "Hawaii" }, - { 0x67B9, 7, 0, 1, 1, "Hawaii" }, - { 0x67BA, 7, 0, 1, 1, "Hawaii" }, - { 0x67BE, 7, 0, 1, 1, "Hawaii" }, + { 0x67A0, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A1, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A2, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A8, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A9, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67AA, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B0, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B1, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B8, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B9, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67BA, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67BE, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, /* Carrizo Family */ - { 0x9870, 8, 0, 1, 0, "Carrizo" }, - { 0x9874, 8, 0, 1, 0, "Carrizo" }, - { 0x9875, 8, 0, 1, 0, "Carrizo" }, - { 0x9876, 8, 0, 1, 0, "Carrizo" }, - { 0x9877, 8, 0, 1, 0, "Carrizo" }, + { 0x9870, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, + { 0x9874, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, + { 0x9875, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, + { 0x9876, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, + { 0x9877, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, /* Tonga Family */ - { 0x6920, 8, 0, 2, 1, "Tonga" }, - { 0x6921, 8, 0, 2, 1, "Tonga" }, - { 0x6928, 8, 0, 2, 1, "Tonga" }, - { 0x6929, 8, 0, 2, 1, "Tonga" }, - { 0x692B, 8, 0, 2, 1, "Tonga" }, - { 0x692F, 8, 0, 2, 1, "Tonga" }, - { 0x6930, 8, 0, 2, 1, "Tonga" }, - { 0x6938, 8, 0, 2, 1, "Tonga" }, - { 0x6939, 8, 0, 2, 1, "Tonga" }, + { 0x6920, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x6921, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x6928, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x6929, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x692B, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x692F, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x6930, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x6938, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x6939, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, /* Fiji */ - { 0x7300, 8, 0, 3, 1, "Fiji" }, - { 0x730F, 8, 0, 3, 1, "Fiji" }, + { 0x7300, 8, 0, 3, 1, "Fiji", CHIP_FIJI }, + { 0x730F, 8, 0, 3, 1, "Fiji", CHIP_FIJI }, /* Polaris10 */ - { 0x67C0, 8, 0, 3, 1, "Polaris10" }, - { 0x67C1, 8, 0, 3, 1, "Polaris10" }, - { 0x67C2, 8, 0, 3, 1, "Polaris10" }, - { 0x67C4, 8, 0, 3, 1, "Polaris10" }, - { 0x67C7, 8, 0, 3, 1, "Polaris10" }, - { 0x67C8, 8, 0, 3, 1, "Polaris10" }, - { 0x67C9, 8, 0, 3, 1, "Polaris10" }, - { 0x67CA, 8, 0, 3, 1, "Polaris10" }, - { 0x67CC, 8, 0, 3, 1, "Polaris10" }, - { 0x67CF, 8, 0, 3, 1, "Polaris10" }, - { 0x67DF, 8, 0, 3, 1, "Polaris10" }, + { 0x67C0, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67C1, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67C2, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67C4, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67C7, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67C8, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67C9, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67CA, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67CC, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67CF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67DF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, /* Polaris11 */ - { 0x67E0, 8, 0, 3, 1, "Polaris11" }, - { 0x67E1, 8, 0, 3, 1, "Polaris11" }, - { 0x67E3, 8, 0, 3, 1, "Polaris11" }, - { 0x67E7, 8, 0, 3, 1, "Polaris11" }, - { 0x67E8, 8, 0, 3, 1, "Polaris11" }, - { 0x67E9, 8, 0, 3, 1, "Polaris11" }, - { 0x67EB, 8, 0, 3, 1, "Polaris11" }, - { 0x67EF, 8, 0, 3, 1, "Polaris11" }, - { 0x67FF, 8, 0, 3, 1, "Polaris11" }, + { 0x67E0, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67E1, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67E3, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67E7, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67E8, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67E9, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67EB, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67EF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67FF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, /* Vega10 */ - { 0x6860, 9, 0, 0, 1, "Vega10" }, - { 0x687F, 9, 0, 0, 1, "Vega10" } + { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 } }; enum cache_type { @@ -549,6 +550,19 @@ static const struct hsa_gfxip_table* find_hsa_gfxip_device(uint16_t device_id) return NULL; } +HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, + enum asic_family_type *asic) +{ + const struct hsa_gfxip_table *hsa_gfxip = + find_hsa_gfxip_device(device_id); + + if (!hsa_gfxip) + return HSAKMT_STATUS_INVALID_PARAMETER; + + *asic = hsa_gfxip->asic_family; + return HSAKMT_STATUS_SUCCESS; +} + bool topology_is_dgpu(uint16_t device_id) { const struct hsa_gfxip_table* hsa_gfxip = From b3b6367cb8a4917fe3c8f625fb66bd90f46e53ec Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 7 Mar 2017 07:25:49 -0500 Subject: [PATCH 0231/1247] Add gfx803 DID Add 0x67D0 to gfx803 support list. Change-Id: Ifdb1fad4a3c42bea54856f6d5248c00ed546ad85 --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index acdee5d673..95860ae639 100644 --- a/src/topology.c +++ b/src/topology.c @@ -151,6 +151,7 @@ static struct hsa_gfxip_table { { 0x67CA, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67CC, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67CF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67D0, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67DF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, /* Polaris11 */ { 0x67E0, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, From 9e32cdb113965f5e27738ea322c07dafb63412b4 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 3 Mar 2017 11:52:28 -0500 Subject: [PATCH 0232/1247] Don't duplicate PMC tables Many devices have the same counter IDs for the hardware block. Devices in the same GFX generation usually have the same block counters. No need to list each device individually. Instead, have a table to share with all devices that have the same counter IDs, and have separated tables for devices that don't have the same counter IDs. Change-Id: I857056edc6f491f61af6e9598580e5dc7d372f94 --- src/pmc_table.c | 81 ++++++++++++------------------------------------- 1 file changed, 20 insertions(+), 61 deletions(-) diff --git a/src/pmc_table.c b/src/pmc_table.c index de04230bd0..3036189131 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -31,7 +31,8 @@ #include "libhsakmt.h" #include "pmc_table.h" -static uint32_t kaveri_sq_counter_ids[] = { +/* Unused counters - 163-167 */ +static uint32_t gfx7_sq_counter_ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, @@ -49,7 +50,7 @@ static uint32_t kaveri_sq_counter_ids[] = { }; /* Unused counters - 166, 292 - 297 */ -static uint32_t carrizo_sq_counter_ids[] = { +static uint32_t gfx8_sq_counter_ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, @@ -69,49 +70,9 @@ static uint32_t carrizo_sq_counter_ids[] = { 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; -/* Unused counters - 166, 292 - 297 */ -static uint32_t fiji_sq_counter_ids[] = { -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, -23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, -43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, -83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, -102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, -134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, -150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, -167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, -183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, -199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, -215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, -231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, -247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, -263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, -279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 -}; - -/* Unused counters - 163 - 166, 167 and 251 are *_DUMMY_LAST */ -static uint32_t hawaii_sq_counter_ids[] = { -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, -22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, -41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, -60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, -79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, -98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, -113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, -128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, -143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, -158, 159, 160, 161, 162, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, -178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, -193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, -208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, -223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, -238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 -}; - -/* Polaris 10 and Polaris 11 have the same SQ cpunter IDs */ +/* Polaris 10/11 have the same SQ cpunter IDs but different from other gfx8's. */ /* Unused counters - 167 and 275 are *_DUMMY_LAST */ -static uint32_t polaris_sq_counter_ids[] = { +static uint32_t gfx8_pl_sq_counter_ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, @@ -135,9 +96,9 @@ static uint32_t polaris_sq_counter_ids[] = { static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, - .num_of_counters = sizeof(kaveri_sq_counter_ids) / - sizeof(*kaveri_sq_counter_ids), - .counter_ids = kaveri_sq_counter_ids, + .num_of_counters = sizeof(gfx7_sq_counter_ids) / + sizeof(*gfx7_sq_counter_ids), + .counter_ids = gfx7_sq_counter_ids, .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, @@ -146,9 +107,9 @@ static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, - .num_of_counters = sizeof(carrizo_sq_counter_ids) / - sizeof(*carrizo_sq_counter_ids), - .counter_ids = carrizo_sq_counter_ids, + .num_of_counters = sizeof(gfx8_sq_counter_ids) / + sizeof(*gfx8_sq_counter_ids), + .counter_ids = gfx8_sq_counter_ids, .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, @@ -157,9 +118,9 @@ static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, - .num_of_counters = sizeof(fiji_sq_counter_ids) / - sizeof(*fiji_sq_counter_ids), - .counter_ids = fiji_sq_counter_ids, + .num_of_counters = sizeof(gfx8_sq_counter_ids) / + sizeof(*gfx8_sq_counter_ids), + .counter_ids = gfx8_sq_counter_ids, .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, @@ -168,9 +129,9 @@ static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, - .num_of_counters = sizeof(hawaii_sq_counter_ids) / - sizeof(*hawaii_sq_counter_ids), - .counter_ids = hawaii_sq_counter_ids, + .num_of_counters = sizeof(gfx7_sq_counter_ids) / + sizeof(*gfx7_sq_counter_ids), + .counter_ids = gfx7_sq_counter_ids, .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, @@ -179,9 +140,9 @@ static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, - .num_of_counters = sizeof(polaris_sq_counter_ids) / - sizeof(*polaris_sq_counter_ids), - .counter_ids = polaris_sq_counter_ids, + .num_of_counters = sizeof(gfx8_pl_sq_counter_ids) / + sizeof(*gfx8_pl_sq_counter_ids), + .counter_ids = gfx8_pl_sq_counter_ids, .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, @@ -352,5 +313,3 @@ get_block_properties(uint32_t node_id, return HSAKMT_STATUS_SUCCESS; } - - From 2c2b1e0db25ceddbf5041a65602e012d10ab21ef Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 3 Mar 2017 12:08:38 -0500 Subject: [PATCH 0233/1247] Support profiling on gfx900 Add gfx900 to PMC support. This patch lists SQ counters. Change-Id: Ia1e60e76ff71ab2e38d9d5de12ac9d527b3e8c6a --- src/pmc_table.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 3036189131..1e184cb4cb 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -93,6 +93,28 @@ static uint32_t gfx8_pl_sq_counter_ids[] = { 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295 }; +static uint32_t gfx9_sq_counter_ids[] = { +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, +41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, +60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, +79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, +98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, +113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, +128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, +158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, +173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 255, 256, 257, 258, +259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, +274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, +289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, +304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, +319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, +334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, +349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, +364, 365, 366, 367, 368, 369, 370, 371, 372 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -148,6 +170,17 @@ static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 16, + .num_of_counters = sizeof(gfx9_sq_counter_ids) / + sizeof(*gfx9_sq_counter_ids), + .counter_ids = gfx9_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + /* Current APUs only have one IOMMU. If NUMA is introduced to APUs, we'll need * to expand the struct here to an array. */ @@ -307,6 +340,9 @@ get_block_properties(uint32_t node_id, case CHIP_POLARIS11: *block = polaris_blocks[block_id]; break; + case CHIP_VEGA10: + *block = vega_blocks[block_id]; + break; default: return HSAKMT_STATUS_INVALID_PARAMETER; } From 3738a1b5f2ae809657636248a78c813bbef907fd Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 9 Mar 2017 11:08:22 -0500 Subject: [PATCH 0234/1247] Re-formatting IO link code - Typo fix: *_link_tye to *_link_type and a missing word in comments - Replace printf with fprintf(stderr - Shorten lines to fit in 80 characters Change-Id: Ibeb0b98d5c59d617ae06d9854a9dde16251ded52 Signed-off-by: Amber Lin --- src/topology.c | 155 ++++++++++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 73 deletions(-) diff --git a/src/topology.c b/src/topology.c index 95860ae639..29cec2883f 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1239,10 +1239,10 @@ err1: return ret; } -/* topology_get_numa_node_link_tye - Return NUMA node interconnect based +/* topology_get_numa_node_link_type - Return NUMA node interconnect based * on processor vendor */ -static HSA_IOLINKTYPE topology_get_numa_node_link_tye(void) +static HSA_IOLINKTYPE topology_get_numa_node_link_type(void) { if (processor_vendor == GENUINE_INTEL) return HSA_IOLINK_TYPE_QPI_1_1; @@ -1253,31 +1253,33 @@ static HSA_IOLINKTYPE topology_get_numa_node_link_tye(void) } -/* topology_get_free_io_link_slot_for_node - For the given node_id, find the next - * available free slot to add an io_link +/* topology_get_free_io_link_slot_for_node - For the given node_id, find the + * next available free slot to add an io_link */ -static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node(uint32_t node_id, - const HsaSystemProperties *sys_props, node_t *temp_nodes) +static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node( + uint32_t node_id, const HsaSystemProperties *sys_props, + node_t *nodes) { HsaIoLinkProperties *props; if (node_id >= sys_props->NumNodes) { - printf("Invalid node [%d]\n", node_id); + fprintf(stderr, "Invalid node [%d]\n", node_id); return NULL; } - props = temp_nodes[node_id].link; + props = nodes[node_id].link; if (!props) { - printf("Error. No io_link reported for Node [%d]\n", node_id); + fprintf(stderr, "No io_link reported for Node [%d]\n", node_id); return NULL; } - if (temp_nodes[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { - printf("Error. No more space for io_link for Node [%d]\n", node_id); + if (nodes[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { + fprintf(stderr, "No more space for io_link for Node [%d]\n", + node_id); return NULL; } - return &props[temp_nodes[node_id].node.NumIOLinks]; + return &props[nodes[node_id].node.NumIOLinks]; } /* topology_add_io_link_for_node - If a free slot is available, @@ -1308,7 +1310,7 @@ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, * For now, assume all the nodes are interconnected with same Weight (=1) */ static void topology_create_qpi_links(const HsaSystemProperties *sys_props, - node_t *temp_nodes) + node_t *nodes) { unsigned int i, j; HSAKMT_STATUS ret; @@ -1316,23 +1318,23 @@ static void topology_create_qpi_links(const HsaSystemProperties *sys_props, /* Find all CPU Nodes and connect each other via HT or QPI io_link */ for (i = 0; i < sys_props->NumNodes - 1; i++) { for (j = i + 1; j < sys_props->NumNodes; j++) { - if (temp_nodes[i].gpu_id == 0 && - temp_nodes[j].gpu_id == 0) { - ret = topology_add_io_link_for_node(i, - sys_props, temp_nodes, topology_get_numa_node_link_tye(), + if (nodes[i].gpu_id || nodes[j].gpu_id) + continue; + ret = topology_add_io_link_for_node(i, sys_props, nodes, + topology_get_numa_node_link_type(), j, 1); - if (ret != HSAKMT_STATUS_SUCCESS) - printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n", - ret, i, j); + if (ret != HSAKMT_STATUS_SUCCESS) + fprintf(stderr, + "Error %d: Fail to add QPI link [%d]->[%d]\n", + ret, i, j); - ret = topology_add_io_link_for_node(j, - sys_props, temp_nodes, topology_get_numa_node_link_tye(), + ret = topology_add_io_link_for_node(j, sys_props, nodes, + topology_get_numa_node_link_type(), i, 1); - if (ret != HSAKMT_STATUS_SUCCESS) - printf("Error [%d]. Failed to add QPI link from Node [%d]->[%d]\n", - ret, j, i); - - } + if (ret != HSAKMT_STATUS_SUCCESS) + fprintf(stderr, + "Error %d: failed to add QPI link [%d]->[%d]\n", + ret, j, i); } } } @@ -1341,24 +1343,25 @@ static void topology_create_qpi_links(const HsaSystemProperties *sys_props, * NUMA node to all the GPUs attached to that node */ static void topology_create_reverse_io_link(uint32_t cpu_node, - const HsaSystemProperties *sys_props, node_t *temp_nodes) + const HsaSystemProperties *sys_props, node_t *nodes) { unsigned int gpu_node; HSAKMT_STATUS ret; for (gpu_node = 0; gpu_node < sys_props->NumNodes; gpu_node++) { - if (temp_nodes[gpu_node].gpu_id != 0) { - /* Check if this GPU is connected to the give cpu_node, - * if so create an io_link */ - if (temp_nodes[gpu_node].link->NodeTo == cpu_node) { - ret = topology_add_io_link_for_node(cpu_node, sys_props, - temp_nodes, HSA_IOLINKTYPE_PCIEXPRESS, - gpu_node, temp_nodes[gpu_node].link->Weight); - if (ret != HSAKMT_STATUS_SUCCESS) { - printf("Error [%d]. Failed to create reverse io_links from Node [%d]\n", - ret, cpu_node); - return; - } + if (!nodes[gpu_node].gpu_id) + continue; + /* Check if this GPU is connected to the give cpu_node, + * if so create an io_link */ + if (nodes[gpu_node].link->NodeTo == cpu_node) { + ret = topology_add_io_link_for_node(cpu_node, sys_props, + nodes, HSA_IOLINKTYPE_PCIEXPRESS, + gpu_node, nodes[gpu_node].link->Weight); + if (ret != HSAKMT_STATUS_SUCCESS) { + fprintf(stderr, + "Error %d: Fail to create reverse io_links from Node [%d]\n", + ret, cpu_node); + return; } } } @@ -1368,59 +1371,64 @@ static void topology_create_reverse_io_link(uint32_t cpu_node, * find all nodes connected to it and create io_links * among them */ static void topology_create_indirect_gpu_links(uint32_t cpu_node, - const HsaSystemProperties *sys_props, node_t *temp_nodes) + const HsaSystemProperties *sys_props, node_t *nodes) { unsigned int i, j; HSAKMT_STATUS ret; HSA_IOLINKTYPE IoLinkType; - HsaIoLinkProperties *props = temp_nodes[cpu_node].link; + HsaIoLinkProperties *props = nodes[cpu_node].link; + HSAuint32 num_iolinks = nodes[cpu_node].node.NumIOLinks; - - if (!props || temp_nodes[cpu_node].node.NumIOLinks == 0) { - printf("CPU Node [%d] has no GPU connected\n", cpu_node); + if (!props || !num_iolinks) { + fprintf(stderr, "CPU Node [%d] has no GPU connected\n", + cpu_node); return; } /* props is the list of io_links cpu_node is connected to. * Make an indirect io_links from props[i].NodeTo --> props[j].NodeTo * and props[j].NodeTo --> props[i].NodeTo */ - for (i = 0; i < temp_nodes[cpu_node].node.NumIOLinks - 1; i++) + for (i = 0; i < num_iolinks - 1; i++) { - for (j = i + 1; j < temp_nodes[cpu_node].node.NumIOLinks; j++) { - /* Ignore CPU <--> CPU node connected as it is handled by QPI - * link function */ - if (temp_nodes[props[i].NodeTo].gpu_id == 0 && - temp_nodes[props[j].NodeTo].gpu_id == 0) + for (j = i + 1; j < num_iolinks; j++) { + /* Ignore CPU <--> CPU node connected as it is handled + * by QPI link function */ + if (!nodes[props[i].NodeTo].gpu_id && + !nodes[props[j].NodeTo].gpu_id) continue; - /* For the given cpu_node, connect to or from the GPUs that are - * connected directly to it via PCIEXPRESS */ - if ((temp_nodes[props[i].NodeTo].gpu_id != 0 && + /* For the given cpu_node, connect to or from the GPUs + * that are not connected directly to it via PCIEXPRESS + */ + if ((nodes[props[i].NodeTo].gpu_id && props[i].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS) || - (temp_nodes[props[j].NodeTo].gpu_id != 0 && + (nodes[props[j].NodeTo].gpu_id && props[j].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS)) continue; - /* The link is from GPU to non-parent NUMA node. So set link type - * to HT or QPI */ - if (temp_nodes[props[i].NodeTo].gpu_id == 0 || - temp_nodes[props[j].NodeTo].gpu_id == 0) - IoLinkType = topology_get_numa_node_link_tye(); + /* The link is from GPU to non-parent NUMA node. So set + * link type to HT or QPI */ + if (!nodes[props[i].NodeTo].gpu_id || + !nodes[props[j].NodeTo].gpu_id) + IoLinkType = topology_get_numa_node_link_type(); else IoLinkType = HSA_IOLINKTYPE_PCIEXPRESS; ret = topology_add_io_link_for_node(props[i].NodeTo, - sys_props, temp_nodes, IoLinkType, - props[j].NodeTo, props[i].Weight + props[j].Weight); + sys_props, nodes, IoLinkType, + props[j].NodeTo, + props[i].Weight + props[j].Weight); if (ret != HSAKMT_STATUS_SUCCESS) - printf("Error [%d]. Failed to add io_link from Node [%d]->[%d]\n", + fprintf(stderr, + "Error %d: Fail to add io_link [%d]->[%d]\n", ret, i, j); ret = topology_add_io_link_for_node(props[j].NodeTo, - sys_props, temp_nodes, IoLinkType, - props[i].NodeTo, props[i].Weight + props[j].Weight); + sys_props, nodes, IoLinkType, props[i].NodeTo, + props[i].Weight + props[j].Weight); if (ret != HSAKMT_STATUS_SUCCESS) - printf("Error [%d]. Failed to add io_link from Node [%d]->[%d]\n", + fprintf(stderr, + "Error %d: Failed to add io_link [%d]->[%d]\n", ret, j, i); } } @@ -1534,12 +1542,14 @@ retry: for (i = 0; i < sys_props.NumNodes; i++) { if (temp_nodes[i].gpu_id == 0) { if (!temp_nodes[i].link) { - printf("Unexpected NULL pointer. Node [%d].link\n", i); + fprintf(stderr, + "Unexpected NULL pointer. Node [%d].link\n", i); ret = HSAKMT_STATUS_NO_MEMORY; free_nodes(temp_nodes, i + 1); goto err; } - topology_create_reverse_io_link(i, &sys_props, temp_nodes); + topology_create_reverse_io_link(i, &sys_props, + temp_nodes); } } @@ -1550,11 +1560,10 @@ retry: /* Create In-direct links for GPUs. Connect all the (Peer-to-Peer) GPUs * that belong to same NUMA node. * For each CPU (NUMA) node, interconnect all the GPUs. */ - for (i = 0; i < sys_props.NumNodes; i++) { - if (temp_nodes[i].gpu_id == 0) { - topology_create_indirect_gpu_links(i, &sys_props, temp_nodes); - } - } + for (i = 0; i < sys_props.NumNodes; i++) + if (!temp_nodes[i].gpu_id) + topology_create_indirect_gpu_links(i, &sys_props, + temp_nodes); ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { From 73eff30d7deeb1723f03aaf368fc99b8b9625f69 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 3 Mar 2017 14:36:51 -0500 Subject: [PATCH 0235/1247] Add TCA block to PMC support Add TCA to PMC tables. Change-Id: Ia4164ab4581ea3f539706f534f672e5c24f5362f --- include/hsakmttypes.h | 5 ++++ src/perfctr.c | 3 +++ src/pmc_table.c | 54 +++++++++++++++++++++++++++++++++++++++++++ src/pmc_table.h | 1 + 4 files changed, 63 insertions(+) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index fe3d0b220e..656e85d707 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -876,6 +876,11 @@ typedef struct _HSA_UUID HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_SQ, 0xb5c396b6, 0xd310, 0x47e4, 0x86, 0xfc, 0x5c, 0xc3, 0x4, 0x3a, 0xf5, 0x8); +// HSA_UUID that identifies the GPU TextureCache (TCA) block +// {333e393f-e147-4f49-a6d1-60914c7086b0} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_TCA, +0x333e393f, 0xe147, 0x4f49, 0xa6, 0xd1,0x60, 0x91, 0x4c, 0x70, 0x86, 0xb0); + // GUID that identifies the GPU Memory Controller (MC) block // {13900B57-4956-4D98-81D0-68521937F59C} HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_MC, diff --git a/src/perfctr.c b/src/perfctr.c index ce991dd228..dd82e2114c 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -180,6 +180,9 @@ static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) case PERFCOUNTER_BLOCKID__SQ: *uuid = HSA_PROFILEBLOCK_AMD_SQ; break; + case PERFCOUNTER_BLOCKID__TCA: + *uuid = HSA_PROFILEBLOCK_AMD_TCA; + break; case PERFCOUNTER_BLOCKID__IOMMUV2: *uuid = HSA_PROFILEBLOCK_AMD_IOMMUV2; break; diff --git a/src/pmc_table.c b/src/pmc_table.c index 1e184cb4cb..92c363bfa1 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -49,6 +49,11 @@ static uint32_t gfx7_sq_counter_ids[] = { 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 }; +static uint32_t gfx7_tca_counter_ids[] = { +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, +23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38 +}; + /* Unused counters - 166, 292 - 297 */ static uint32_t gfx8_sq_counter_ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, @@ -70,6 +75,11 @@ static uint32_t gfx8_sq_counter_ids[] = { 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; +static uint32_t gfx8_tca_counter_ids[] = { +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, +23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 +}; + /* Polaris 10/11 have the same SQ cpunter IDs but different from other gfx8's. */ /* Unused counters - 167 and 275 are *_DUMMY_LAST */ static uint32_t gfx8_pl_sq_counter_ids[] = { @@ -135,6 +145,17 @@ static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCA] = { + /* PMC0: PERF_SEL~PERF_SEL3, PMC1: PERF_SEL~PERF_SEL3, PMC2: PERF_SEL + * PMC3: PERF_SEL. So 10 PERF_SELs in total + */ + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_tca_counter_ids) / + sizeof(*gfx8_tca_counter_ids), + .counter_ids = gfx8_tca_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { @@ -146,6 +167,14 @@ static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCA] = { + .num_of_slots = 10, /* same as CZ */ + .num_of_counters = sizeof(gfx8_tca_counter_ids) / + sizeof(*gfx8_tca_counter_ids), + .counter_ids = gfx8_tca_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { @@ -157,6 +186,14 @@ static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCA] = { + .num_of_slots = 10, /* same as CZ */ + .num_of_counters = sizeof(gfx7_tca_counter_ids) / + sizeof(*gfx7_tca_counter_ids), + .counter_ids = gfx7_tca_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { @@ -168,6 +205,14 @@ static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCA] = { + .num_of_slots = 10, /* same as CZ */ + .num_of_counters = sizeof(gfx8_tca_counter_ids) / + sizeof(*gfx8_tca_counter_ids), + .counter_ids = gfx8_tca_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { @@ -179,6 +224,15 @@ static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCA] = { + .num_of_slots = 10, /* same as Fiji */ + /* Greenland has the same TCA counter IDs with Fiji */ + .num_of_counters = sizeof(gfx8_tca_counter_ids) / + sizeof(*gfx8_tca_counter_ids), + .counter_ids = gfx8_tca_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; /* Current APUs only have one IOMMU. If NUMA is introduced to APUs, we'll need diff --git a/src/pmc_table.h b/src/pmc_table.h index 2aa1002bdb..c6d16ad47e 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -31,6 +31,7 @@ enum perf_block_id { PERFCOUNTER_BLOCKID__FIRST = 0, PERFCOUNTER_BLOCKID__SQ = PERFCOUNTER_BLOCKID__FIRST, + PERFCOUNTER_BLOCKID__TCA, PERFCOUNTER_BLOCKID__IOMMUV2, PERFCOUNTER_BLOCKID__MAX }; From 53838c981827feb58106ca19aa15388a4394ece2 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 24 Mar 2017 16:03:46 -0400 Subject: [PATCH 0236/1247] Add missing gfx900 device IDs Change-Id: Ica5deb000279a508106125461af64a3851294b0a Signed-off-by: Felix Kuehling --- src/topology.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/topology.c b/src/topology.c index 29cec2883f..e0ad458d20 100644 --- a/src/topology.c +++ b/src/topology.c @@ -165,6 +165,11 @@ static struct hsa_gfxip_table { { 0x67FF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, /* Vega10 */ { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6861, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6862, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6863, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6867, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 } }; From 11862b9f61db45701885e11aa32e7e7d81f005db Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 30 Mar 2017 11:26:22 -0400 Subject: [PATCH 0237/1247] Add guard page after each address space reservation Guard pages help catch out-of-bounds memory accesses by applications by generating VM faults (GPU) and segfaults (CPU). Remove address space reservation from scratch aperture. That address space is managed by the Thunk client. Guard pages would cause Thunk's address space management to get out of sync with the client's. Change-Id: I2e5aee2923a90186358cc7b0e131baf547996df6 Signed-off-by: Felix Kuehling --- src/fmm.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 1a10aa82d2..f8bd6f6db3 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -475,6 +475,16 @@ static bool aperture_is_valid(void *app_base, void *app_limit) return false; } +/* Align size of a VM area + * + * Leave at least one guard page after every object to catch + * out-of-bounds accesses with VM faults. + */ +static uint64_t vm_align_area_size(manageble_aperture_t *app, uint64_t size) +{ + return ALIGN_UP(size + PAGE_SIZE, app->align); +} + /* * Assumes that fmm_mutex is locked on entry. */ @@ -484,7 +494,7 @@ static void aperture_release_area(manageble_aperture_t *app, void *address, vm_area_t *area; uint64_t SizeOfRegion; - MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); + MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); area = vm_find(app, address); if (!area) @@ -522,7 +532,7 @@ static void *aperture_allocate_area_aligned(manageble_aperture_t *app, vm_area_t *cur, *next; void *start; - MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); + MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); if (align < app->align) align = app->align; @@ -1697,7 +1707,6 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu void *address, uint64_t size) { int32_t gpu_mem_id; - uint64_t offset; void *mem = NULL; int ret; bool is_debugger = 0; @@ -1720,20 +1729,11 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); /* allocate object within the scratch backing aperture */ if (!ret && !is_debugger) { - offset = VOID_PTRS_SUB(address, aperture->base); - mem = __fmm_allocate_device(gpu_id, size, aperture, offset, - NULL, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE, NULL); - if (mem == NULL) + vm_object_t *obj = fmm_allocate_memory_in_device( + gpu_id, address, size, aperture, + NULL, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); + if (obj == NULL) return -1; - - if (mem != address) { - fprintf(stderr, - "Got unexpected address for scratch mapping.\n" - " expected: %p\n" - " got: %p\n", address, mem); - __fmm_release(mem, aperture); - return -1; - } } else { fmm_allocate_memory_in_device(gpu_id, address, From c119653add14d543e8602dbd5cb27a3b86fb6cb3 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 22 Mar 2017 11:08:50 -0400 Subject: [PATCH 0238/1247] Create indirect IO links KFD added all direct IO links to sysfs, so this patch removes all direct links related code and modify the indirect links function to reflect the change. Change-Id: Iaec7b5f6c59f9034f8f960ca1fe1145d51dab367 Signed-off-by: Amber Lin --- src/topology.c | 313 +++++++++++++++++++++++-------------------------- 1 file changed, 149 insertions(+), 164 deletions(-) diff --git a/src/topology.c b/src/topology.c index e0ad458d20..99460f49ef 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1244,20 +1244,6 @@ err1: return ret; } -/* topology_get_numa_node_link_type - Return NUMA node interconnect based - * on processor vendor - */ -static HSA_IOLINKTYPE topology_get_numa_node_link_type(void) -{ - if (processor_vendor == GENUINE_INTEL) - return HSA_IOLINK_TYPE_QPI_1_1; - else if (processor_vendor == AUTHENTIC_AMD) - return HSA_IOLINKTYPE_HYPERTRANSPORT; - else - return HSA_IOLINKTYPE_UNDEFINED; - -} - /* topology_get_free_io_link_slot_for_node - For the given node_id, find the * next available free slot to add an io_link */ @@ -1288,153 +1274,182 @@ static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node( } /* topology_add_io_link_for_node - If a free slot is available, - * add io_link for the given Node. + * add io_link for the given Node. If bi_directional is true, set up two + * links for both directions. * TODO: Add other members of HsaIoLinkProperties */ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, - const HsaSystemProperties *sys_props, node_t *temp_nodes, + const HsaSystemProperties *sys_props, node_t *nodes, HSA_IOLINKTYPE IoLinkType, uint32_t NodeTo, - uint32_t Weight) + uint32_t Weight, bool bi_dir) { HsaIoLinkProperties *props; - props = topology_get_free_io_link_slot_for_node(node_id, - sys_props, temp_nodes); - if (!props) - return HSAKMT_STATUS_NO_MEMORY; + /* If bi-directional is set true, it's two links to add. */ + uint32_t i, num_links = (bi_dir == true) ? 2 : 1; + uint32_t node_from = node_id, node_to = NodeTo; - props->IoLinkType = IoLinkType; - props->NodeFrom = node_id; - props->NodeTo = NodeTo; - props->Weight = Weight; - temp_nodes[node_id].node.NumIOLinks++; + for (i = 0; i < num_links; i++) { + props = topology_get_free_io_link_slot_for_node(node_from, + sys_props, nodes); + if (!props) + return HSAKMT_STATUS_NO_MEMORY; + + props->IoLinkType = IoLinkType; + props->NodeFrom = node_from; + props->NodeTo = node_to; + props->Weight = Weight; + nodes[node_from].node.NumIOLinks++; + /* switch direction on the 2nd link when num_links=2 */ + node_from = NodeTo; + node_to = node_id; + } return HSAKMT_STATUS_SUCCESS; } -/* topology_create_qpi_links - Create QPI or HT links among all NUMA nodes - * For now, assume all the nodes are interconnected with same Weight (=1) - */ -static void topology_create_qpi_links(const HsaSystemProperties *sys_props, - node_t *nodes) +/* Find the CPU that this GPU (gpu_node) directly connects to */ +static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, node_t *nodes) { - unsigned int i, j; - HSAKMT_STATUS ret; + HsaIoLinkProperties *props = nodes[gpu_node].link; + uint32_t i; - /* Find all CPU Nodes and connect each other via HT or QPI io_link */ - for (i = 0; i < sys_props->NumNodes - 1; i++) { - for (j = i + 1; j < sys_props->NumNodes; j++) { - if (nodes[i].gpu_id || nodes[j].gpu_id) - continue; - ret = topology_add_io_link_for_node(i, sys_props, nodes, - topology_get_numa_node_link_type(), - j, 1); - if (ret != HSAKMT_STATUS_SUCCESS) - fprintf(stderr, - "Error %d: Fail to add QPI link [%d]->[%d]\n", - ret, i, j); + if (!nodes[gpu_node].gpu_id || !props || + nodes[gpu_node].node.NumIOLinks == 0) + return -1; - ret = topology_add_io_link_for_node(j, sys_props, nodes, - topology_get_numa_node_link_type(), - i, 1); - if (ret != HSAKMT_STATUS_SUCCESS) - fprintf(stderr, - "Error %d: failed to add QPI link [%d]->[%d]\n", - ret, j, i); - } - } + for (i = 0; i < nodes[gpu_node].node.NumIOLinks; i++) + if (props[i].IoLinkType == HSA_IOLINKTYPE_PCIEXPRESS && + props[i].Weight <= 20) /* >20 is GPU->CPU->GPU */ + return props[i].NodeTo; + + return -1; } -/* topology_create_reverse_io_link - Create io_links from the given CPU - * NUMA node to all the GPUs attached to that node +/* Get node1->node2 IO link information. This should be a direct link that has + * been created in the kernel. */ -static void topology_create_reverse_io_link(uint32_t cpu_node, - const HsaSystemProperties *sys_props, node_t *nodes) +static HSAKMT_STATUS get_direct_iolink_info(uint32_t node1, uint32_t node2, + node_t *nodes, HSAuint32 *weight, HSA_IOLINKTYPE *type) { - unsigned int gpu_node; + HsaIoLinkProperties *props = nodes[node1].link; + uint32_t i; + + if (!props) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + for (i = 0; i < nodes[node1].node.NumIOLinks; i++) + if (props[i].NodeTo == node2) { + if (weight) + *weight = props[i].Weight; + if (type) + *type = props[i].IoLinkType; + return HSAKMT_STATUS_SUCCESS; + } + + return HSAKMT_STATUS_INVALID_PARAMETER; +} + +static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, + node_t *nodes, HSAuint32 *weight, HSA_IOLINKTYPE *type) +{ + int32_t dir_cpu1 = -1, dir_cpu2 = -1; + HSAuint32 weight1 = 0, weight2 = 0, weight3 = 0; HSAKMT_STATUS ret; - for (gpu_node = 0; gpu_node < sys_props->NumNodes; gpu_node++) { - if (!nodes[gpu_node].gpu_id) - continue; - /* Check if this GPU is connected to the give cpu_node, - * if so create an io_link */ - if (nodes[gpu_node].link->NodeTo == cpu_node) { - ret = topology_add_io_link_for_node(cpu_node, sys_props, - nodes, HSA_IOLINKTYPE_PCIEXPRESS, - gpu_node, nodes[gpu_node].link->Weight); - if (ret != HSAKMT_STATUS_SUCCESS) { - fprintf(stderr, - "Error %d: Fail to create reverse io_links from Node [%d]\n", - ret, cpu_node); - return; + *weight = 0; + *type = HSA_IOLINKTYPE_UNDEFINED; + + if (node1 == node2) + return HSAKMT_STATUS_INVALID_PARAMETER; + + /* CPU->CPU is not an indirect link */ + if (!nodes[node1].gpu_id && !nodes[node2].gpu_id) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + if (nodes[node1].gpu_id) + dir_cpu1 = gpu_get_direct_link_cpu(node1, nodes); + if (nodes[node2].gpu_id) + dir_cpu2 = gpu_get_direct_link_cpu(node2, nodes); + + if (dir_cpu1 < 0 && dir_cpu2 < 0) + return HSAKMT_STATUS_ERROR; + + /* Possible topology: + * GPU --(weight1) -- CPU -- (weight2) -- GPU + * GPU --(weight1) -- CPU -- (weight2) -- CPU -- (weight3) -- GPU + * GPU --(weight1) -- CPU -- (weight2) -- CPU + * CPU -- (weight2) -- CPU -- (weight3) -- GPU + */ + if (dir_cpu1 >= 0) { /* GPU->CPU ... */ + if (dir_cpu2 >= 0) { + if (dir_cpu1 == dir_cpu2) /* GPU->CPU->GPU*/ { + ret = get_direct_iolink_info(node1, dir_cpu1, + nodes, &weight1, NULL); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = get_direct_iolink_info(dir_cpu1, node2, + nodes, &weight2, type); + } else /* GPU->CPU->CPU->GPU*/ { + ret = get_direct_iolink_info(node1, dir_cpu1, + nodes, &weight1, NULL); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = get_direct_iolink_info(dir_cpu1, dir_cpu2, + nodes, &weight2, type); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + /* On QPI interconnection, GPUs can't access + * each other if they are attached to different + * CPU sockets. CPU<->CPU weight larger than 20 + * means the two CPUs are in different sockets. + */ + if (*type == HSA_IOLINK_TYPE_QPI_1_1 + && weight2 > 20) + return HSAKMT_STATUS_NOT_SUPPORTED; + ret = get_direct_iolink_info(dir_cpu2, node2, + nodes, &weight3, NULL); } + } else /* GPU->CPU->CPU */ { + ret = get_direct_iolink_info(node1, dir_cpu1, nodes, + &weight1, NULL); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = get_direct_iolink_info(dir_cpu1, node2, nodes, + &weight2, type); } + } else { /* CPU->CPU->GPU */ + ret = get_direct_iolink_info(node1, dir_cpu2, nodes, &weight2, + type); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + ret = get_direct_iolink_info(dir_cpu2, node2, nodes, &weight3, + NULL); } + + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + *weight = weight1 + weight2 + weight3; + return HSAKMT_STATUS_SUCCESS; } -/* topology_create_indirect_gpu_links - For the given cpu_node, - * find all nodes connected to it and create io_links - * among them */ -static void topology_create_indirect_gpu_links(uint32_t cpu_node, +static void topology_create_indirect_gpu_links( const HsaSystemProperties *sys_props, node_t *nodes) { - unsigned int i, j; - HSAKMT_STATUS ret; - HSA_IOLINKTYPE IoLinkType; - HsaIoLinkProperties *props = nodes[cpu_node].link; - HSAuint32 num_iolinks = nodes[cpu_node].node.NumIOLinks; - if (!props || !num_iolinks) { - fprintf(stderr, "CPU Node [%d] has no GPU connected\n", - cpu_node); - return; - } + uint32_t i, j; + HSAuint32 weight; + HSA_IOLINKTYPE type; - /* props is the list of io_links cpu_node is connected to. - * Make an indirect io_links from props[i].NodeTo --> props[j].NodeTo - * and props[j].NodeTo --> props[i].NodeTo */ - for (i = 0; i < num_iolinks - 1; i++) - { - for (j = i + 1; j < num_iolinks; j++) { - /* Ignore CPU <--> CPU node connected as it is handled - * by QPI link function */ - if (!nodes[props[i].NodeTo].gpu_id && - !nodes[props[j].NodeTo].gpu_id) + for (i = 0; i < sys_props->NumNodes - 1; i++) { + for (j = i + 1; j < sys_props->NumNodes; j++) { + get_indirect_iolink_info(i, j, nodes, &weight, &type); + if (!weight) continue; - - /* For the given cpu_node, connect to or from the GPUs - * that are not connected directly to it via PCIEXPRESS - */ - if ((nodes[props[i].NodeTo].gpu_id && - props[i].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS) || - (nodes[props[j].NodeTo].gpu_id && - props[j].IoLinkType != HSA_IOLINKTYPE_PCIEXPRESS)) - continue; - - /* The link is from GPU to non-parent NUMA node. So set - * link type to HT or QPI */ - if (!nodes[props[i].NodeTo].gpu_id || - !nodes[props[j].NodeTo].gpu_id) - IoLinkType = topology_get_numa_node_link_type(); - else - IoLinkType = HSA_IOLINKTYPE_PCIEXPRESS; - - ret = topology_add_io_link_for_node(props[i].NodeTo, - sys_props, nodes, IoLinkType, - props[j].NodeTo, - props[i].Weight + props[j].Weight); - if (ret != HSAKMT_STATUS_SUCCESS) + if (topology_add_io_link_for_node(i, sys_props, nodes, + type, j, weight, true) != HSAKMT_STATUS_SUCCESS) fprintf(stderr, - "Error %d: Fail to add io_link [%d]->[%d]\n", - ret, i, j); - - ret = topology_add_io_link_for_node(props[j].NodeTo, - sys_props, nodes, IoLinkType, props[i].NodeTo, - props[i].Weight + props[j].Weight); - if (ret != HSAKMT_STATUS_SUCCESS) - fprintf(stderr, - "Error %d: Failed to add io_link [%d]->[%d]\n", - ret, j, i); + "Fail to add IO link %d->%d\n", i, j); } } } @@ -1522,10 +1537,6 @@ retry: } if (temp_nodes[i].node.NumIOLinks) { - if (temp_nodes[i].gpu_id == 0) { - printf("Warning. Not expecting CPU Node [%d] to have [%d] io_links.\n", - i, temp_nodes[i].node.NumIOLinks); - } for (link_id = 0; link_id < temp_nodes[i].node.NumIOLinks; link_id++) { ret = topology_sysfs_get_iolink_props(i, link_id, &temp_nodes[i].link[link_id]); if (ret != HSAKMT_STATUS_SUCCESS) { @@ -1539,36 +1550,10 @@ retry: pci_cleanup(pacc); } - /* The Kernel only creates one way direct link - - * GPU(PCI_BUS) --> Parent NUMA Node. Create the reverse direct - * io_link here. [NUMA node] --> GPU */ - - /* Create the reverse io_link for all the CPU nodes */ - for (i = 0; i < sys_props.NumNodes; i++) { - if (temp_nodes[i].gpu_id == 0) { - if (!temp_nodes[i].link) { - fprintf(stderr, - "Unexpected NULL pointer. Node [%d].link\n", i); - ret = HSAKMT_STATUS_NO_MEMORY; - free_nodes(temp_nodes, i + 1); - goto err; - } - topology_create_reverse_io_link(i, &sys_props, - temp_nodes); - } - } - - /* Create QPI or HT links among CPU (NUMA) nodes. For now assume - * all nodes are interconnected with same weight */ - topology_create_qpi_links(&sys_props, temp_nodes); - - /* Create In-direct links for GPUs. Connect all the (Peer-to-Peer) GPUs - * that belong to same NUMA node. - * For each CPU (NUMA) node, interconnect all the GPUs. */ - for (i = 0; i < sys_props.NumNodes; i++) - if (!temp_nodes[i].gpu_id) - topology_create_indirect_gpu_links(i, &sys_props, - temp_nodes); + /* All direct IO links are created in the kernel. Here we need to + * connect GPU<->GPU or GPU<->CPU indirect IO links. + */ + topology_create_indirect_gpu_links(&sys_props, temp_nodes); ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { From 34ddde0c50781c379931551b8b540b848e9ce116 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 5 Apr 2017 17:22:36 -0400 Subject: [PATCH 0239/1247] Add debug option to check userptrs on registration export HSA_CHECK_USERPTR=1 to check user pointers on registration. If the pointer doesn't point to a valid mapping, there will be a segfault. Change-Id: I459c0902cbc90338517fbf79678871ebfbe5183b Signed-off-by: Felix Kuehling --- src/fmm.c | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index f8bd6f6db3..a3d65321c5 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -132,6 +132,9 @@ typedef struct { /* whether to use userptr for paged memory */ bool userptr_for_paged_mem; + + /* whether to check userptrs on registration */ + bool check_userptr; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -145,7 +148,8 @@ static void *dgpu_shared_aperture_limit = NULL; static svm_t svm = { INIT_MANAGEBLE_APERTURE(0, 0), INIT_MANAGEBLE_APERTURE(0, 0), - true + true, + false }; /* On APU, for memory allocated on the system memory that GPU doesn't access @@ -1401,8 +1405,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HsaNodeProperties props; struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - char *disableCache; - char *pagedUserptr; + char *disableCache, *pagedUserptr, *checkUserptr; struct pci_access *pacc; /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ @@ -1415,6 +1418,12 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) pagedUserptr = getenv("HSA_USERPTR_FOR_PAGED_MEM"); svm.userptr_for_paged_mem = (pagedUserptr && strcmp(pagedUserptr, "0")); + /* If HSA_CHECK_USERPTR is set to a non-0 value, check all userptrs + * when they are registered + */ + checkUserptr = getenv("HSA_CHECK_USERPTR"); + svm.check_userptr = (checkUserptr && strcmp(checkUserptr, "0")); + /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than * necessary*/ @@ -2317,6 +2326,22 @@ bool fmm_get_handle(void *address, uint64_t *handle) return found; } +static HSAuint8 fmm_check_user_memory(const void *addr, HSAuint64 size) +{ + volatile const HSAuint8 *ptr = addr; + volatile const HSAuint8 *end = ptr + size; + HSAuint8 sum = 0; + + /* Access every page in the buffer to make sure the mapping is + * valid. If it's not, it will die with a segfault that's easy + * to debug. + */ + for (; ptr < end; ptr = (void *)PAGE_ALIGN_UP(ptr + 1)) + sum += *ptr; + + return sum; +} + static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_object_t **obj_ret) { int32_t i; @@ -2345,6 +2370,10 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj } pthread_mutex_unlock(&aperture->fmm_mutex); + /* Optionally check that the CPU mapping is valid */ + if (svm.check_userptr) + fmm_check_user_memory(addr, size); + /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, &obj); From 3f7e7933e3f0b2e7161a6e19ab97261b26a18548 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 5 Apr 2017 20:54:41 -0400 Subject: [PATCH 0240/1247] Add debug option to control the number of guard pages Change-Id: I18b10bcbb4d74a92f17330e44b2dbb4cea61da00 Signed-off-by: Felix Kuehling --- src/fmm.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index a3d65321c5..e89290bcd6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -41,6 +41,7 @@ .base = (void *) base_value, \ .limit = (void *) limit_value, \ .align = PAGE_SIZE, \ + .guard_pages = 1, \ .vm_ranges = NULL, \ .vm_objects = NULL, \ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER \ @@ -96,6 +97,7 @@ typedef struct { void *base; void *limit; uint64_t align; + uint32_t guard_pages; vm_area_t *vm_ranges; vm_object_t *vm_objects; pthread_mutex_t fmm_mutex; @@ -486,7 +488,8 @@ static bool aperture_is_valid(void *app_base, void *app_limit) */ static uint64_t vm_align_area_size(manageble_aperture_t *app, uint64_t size) { - return ALIGN_UP(size + PAGE_SIZE, app->align); + return ALIGN_UP(size + (uint64_t)app->guard_pages * PAGE_SIZE, + app->align); } /* @@ -1405,7 +1408,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HsaNodeProperties props; struct kfd_process_device_apertures * process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - char *disableCache, *pagedUserptr, *checkUserptr; + char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; + unsigned guardPages = 1; struct pci_access *pacc; /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ @@ -1424,6 +1428,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) checkUserptr = getenv("HSA_CHECK_USERPTR"); svm.check_userptr = (checkUserptr && strcmp(checkUserptr, "0")); + /* Specify number of guard pages for SVM apertures, default is 1 */ + guardPagesStr = getenv("HSA_SVM_GUARD_PAGES"); + if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1) + guardPages = 1; + /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than * necessary*/ @@ -1455,6 +1464,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_aperture.fmm_mutex, NULL); gpu_mem[gpu_mem_count].gpuvm_aperture.align = get_vm_alignment(props.DeviceId); + gpu_mem[gpu_mem_count].gpuvm_aperture.guard_pages = guardPages; pthread_mutex_init(&gpu_mem[gpu_mem_count].gpuvm_aperture.fmm_mutex, NULL); gpu_mem_count++; } @@ -1537,6 +1547,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) (uint64_t)svm.dgpu_aperture.base, (uint64_t)svm.dgpu_aperture.limit); svm.dgpu_aperture.align = vm_alignment; + svm.dgpu_aperture.guard_pages = guardPages; /* Non-canonical per-ASIC GPUVM aperture does * not exist on dGPUs in GPUVM64 address mode */ @@ -1565,6 +1576,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) ret = HSAKMT_STATUS_ERROR; } svm.dgpu_alt_aperture.align = vm_alignment; + svm.dgpu_alt_aperture.guard_pages = guardPages; } } From 9f19acbdb7578b6bc2c496231de4d5c0ecf2d2ac Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 24 Apr 2017 16:26:50 -0400 Subject: [PATCH 0241/1247] Add more non-priv PMC blocks to gfx70x/GFX7 HSA Thunk Spec was updated to include more non-privileged blocks for profiling. This patch adds those newly added non-privileged blocks for gfx70x. Signed-off-by: Amber Lin Change-Id: Id745ac236c871e8e61a128a2460784f9c9c354b6 --- include/hsakmttypes.h | 107 +++++++++- src/perfctr.c | 60 ++++++ src/pmc_table.c | 463 ++++++++++++++++++++++++++++++++++++++---- src/pmc_table.h | 26 ++- 4 files changed, 606 insertions(+), 50 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 656e85d707..df83817010 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -870,21 +870,120 @@ typedef struct _HSA_UUID #define HSA_DEFINE_UUID DEFINE_GUID #endif +// HSA_UUID that identifies the GPU ColorBuffer (CB) block +// {9ba429c6-af2d-4b38-b349-157271beac6a} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_CB, +0x9ba429c6, 0xaf2d, 0x4b38, 0xb3, 0x49, 0x15, 0x72, 0x71, 0xbe, 0xac, 0x6a); + +// HSA_UUID that identifies the GPU (CPF) block +// {2b0ad2b5-1c43-4f46-a7bc-e119411ea6c9} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_CPF, +0x2b0ad2b5, 0x1c43, 0x4f46, 0xa7, 0xbc, 0xe1, 0x19, 0x41, 0x1e, 0xa6, 0xc9); + +// HSA_UUID that identifies the GPU (CPG) block +// {590ec94d-20f0-448f-8dff-316c679de7ff +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_CPG, +0x590ec94d, 0x20f0, 0x448f, 0x8d, 0xff, 0x31, 0x6c, 0x67, 0x9d, 0xe7, 0xff); + +// HSA_UUID that identifies the GPU (DB) block +// {3d1a47fc-0013-4ed4-8306-822ca0b7a6c2 +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_DB, +0x3d1a47fc, 0x0013, 0x4ed4, 0x83, 0x06, 0x82, 0x2c, 0xa0, 0xb7, 0xa6, 0xc2); + +// HSA_UUID that identifies the GPU (GDS) block +// {f59276ec-2526-4bf8-8ec0-118f77700dc9 +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_GDS, +0xf59276ec, 0x2526, 0x4bf8, 0x8e, 0xc0, 0x11, 0x8f, 0x77, 0x70, 0x0d, 0xc9); + +// HSA_UUID that identifies the GPU (GRBM) block +// {8f00933c-c33d-4801-97b7-7007f78573ad +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_GRBM, +0x8f00933c, 0xc33d, 0x4801, 0x97, 0xb7, 0x70, 0x07, 0xf7, 0x85, 0x73, 0xad); + +// HSA_UUID that identifies the GPU (GRBMSE) block +// {34ebd8d7-7c8b-4d15-88fa-0e4e4af59ac1 +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_GRBMSE, +0x34ebd8d7, 0x7c8b, 0x4d15, 0x88, 0xfa, 0x0e, 0x4e, 0x4a, 0xf5, 0x9a, 0xc1); + +// HSA_UUID that identifies the GPU (IA) block +// {34276944-4264-4fcd-9d6e-ae264582ec51 +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_IA, +0x34276944, 0x4264, 0x4fcd, 0x9d, 0x6e, 0xae, 0x26, 0x45, 0x82, 0xec, 0x51); + +// HSA_UUID that identifies the GPU Memory Controller (MC) block +// {13900B57-4956-4D98-81D0-68521937F59C +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_MC, +0x13900b57, 0x4956, 0x4d98, 0x81, 0xd0, 0x68, 0x52, 0x19, 0x37, 0xf5, 0x9c); + +// HSA_UUID that identifies the GPU (PASC) block +// {b0e7fb5d-0efc-4744-b516-5d23dc1fd56c +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_PASC, +0xb0e7fb5d, 0x0efc, 0x4744, 0xb5, 0x16, 0x5d, 0x23, 0xdc, 0x1f, 0xd5, 0x6c); + +// HSA_UUID that identifies the GPU (PASU) block +// {9a152b6a-1fad-45f2-a5bf-f163826bd0cd +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_PASU, +0x9a152b6a, 0x1fad, 0x45f2, 0xa5, 0xbf, 0xf1, 0x63, 0x82, 0x6b, 0xd0, 0xcd); + +// HSA_UUID that identifies the GPU (SPI) block +// {eda81044-d62c-47eb-af89-4f6fbf3b38e0 +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_SPI, +0xeda81044, 0xd62c, 0x47eb, 0xaf, 0x89, 0x4f, 0x6f, 0xbf, 0x3b, 0x38, 0xe0); + +// HSA_UUID that identifies the GPU (SRBM) block +// {9f8040e0-6830-4019-acc8-463c9e445b89 +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_SRBM, +0x9f8040e0, 0x6830, 0x4019, 0xac, 0xc8, 0x46, 0x3c, 0x9e, 0x44, 0x5b, 0x89); // GUID that identifies the GPU Shader Sequencer (SQ) block // {B5C396B6-D310-47E4-86FC-5CC3043AF508} HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_SQ, 0xb5c396b6, 0xd310, 0x47e4, 0x86, 0xfc, 0x5c, 0xc3, 0x4, 0x3a, 0xf5, 0x8); +// HSA_UUID that identifies the GPU (SX) block +// {bdb8d737-43cc-4162-be52-51cfb847beaf} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_SX, +0xbdb8d737, 0x43cc, 0x4162, 0xbe, 0x52, 0x51, 0xcf, 0xb8, 0x47, 0xbe, 0xaf); + +// HSA_UUID that identifies the GPU (TA) block +// {c01ee43d-ad92-44b1-8ab9-be5e696ceea7} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_TA, +0xc01ee43d, 0xad92, 0x44b1, 0x8a, 0xb9, 0xbe, 0x5e, 0x69, 0x6c, 0xee, 0xa7); + // HSA_UUID that identifies the GPU TextureCache (TCA) block // {333e393f-e147-4f49-a6d1-60914c7086b0} HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_TCA, 0x333e393f, 0xe147, 0x4f49, 0xa6, 0xd1,0x60, 0x91, 0x4c, 0x70, 0x86, 0xb0); -// GUID that identifies the GPU Memory Controller (MC) block -// {13900B57-4956-4D98-81D0-68521937F59C} -HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_MC, -0x13900b57, 0x4956, 0x4d98, 0x81, 0xd0, 0x68, 0x52, 0x19, 0x37, 0xf5, 0x9c); +// HSA_UUID that identifies the GPU TextureCache (TCC) block +// {848ce855-d805-4566-a8ab-73e884cc6bff} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_TCC, +0x848ce855, 0xd805, 0x4566, 0xa8, 0xab, 0x73, 0xe8, 0x84, 0xcc, 0x6b, 0xff); + +// HSA_UUID that identifies the GPU (TCP) block +// {e10a013b-17d4-4bf5-b089-429591059b60} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_TCP, +0xe10a013b, 0x17d4, 0x4bf5, 0xb0, 0x89, 0x42, 0x95, 0x91, 0x05, 0x9b, 0x60); + +// HSA_UUID that identifies the GPU (TCS) block +// {4126245c-4d96-4d1a-8aed-a939d4cc8ec9} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_TCS, +0x4126245c, 0x4d96, 0x4d1a, 0x8a, 0xed, 0xa9, 0x39, 0xd4, 0xcc, 0x8e, 0xc9); + +// HSA_UUID that identifies the GPU (TD) block +// {7d7c0fe4-fe41-4fea-92c9-4544d7706dc6} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_TD, +0x7d7c0fe4, 0xfe41, 0x4fea, 0x92, 0xc9, 0x45, 0x44, 0xd7, 0x70, 0x6d, 0xc6); + +// HSA_UUID that identifies the GPU (VGT) block +// {0b6a8cb7-7a01-409f-a22c-3014854f1359} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_VGT, +0x0b6a8cb7, 0x7a01, 0x409f, 0xa2, 0x2c, 0x30, 0x14, 0x85, 0x4f, 0x13, 0x59); + +// HSA_UUID that identifies the GPU (WD) block +// {0e176789-46ed-4b02-972a-916d2fac244a} +HSA_DEFINE_UUID(HSA_PROFILEBLOCK_AMD_WD, +0x0e176789, 0x46ed, 0x4b02, 0x97, 0x2a, 0x91, 0x6d, 0x2f, 0xac, 0x24, 0x4a); // GUID that identifies the IMOMMUv2 HW device // {80969879-B0F6-4BE6-97F6-6A6300F5101D} diff --git a/src/perfctr.c b/src/perfctr.c index dd82e2114c..5267ac1dbe 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -177,12 +177,72 @@ static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) int rc = 0; switch (block_id) { + case PERFCOUNTER_BLOCKID__CB: + *uuid = HSA_PROFILEBLOCK_AMD_CB; + break; + case PERFCOUNTER_BLOCKID__CPF: + *uuid = HSA_PROFILEBLOCK_AMD_CPF; + break; + case PERFCOUNTER_BLOCKID__CPG: + *uuid = HSA_PROFILEBLOCK_AMD_CPG; + break; + case PERFCOUNTER_BLOCKID__GDS: + *uuid = HSA_PROFILEBLOCK_AMD_GDS; + break; + case PERFCOUNTER_BLOCKID__GRBM: + *uuid = HSA_PROFILEBLOCK_AMD_GRBM; + break; + case PERFCOUNTER_BLOCKID__GRBMSE: + *uuid = HSA_PROFILEBLOCK_AMD_GRBMSE; + break; + case PERFCOUNTER_BLOCKID__IA: + *uuid = HSA_PROFILEBLOCK_AMD_IA; + break; + case PERFCOUNTER_BLOCKID__MC: + *uuid = HSA_PROFILEBLOCK_AMD_MC; + break; + case PERFCOUNTER_BLOCKID__PASC: + *uuid = HSA_PROFILEBLOCK_AMD_PASC; + break; + case PERFCOUNTER_BLOCKID__PASU: + *uuid = HSA_PROFILEBLOCK_AMD_PASU; + break; + case PERFCOUNTER_BLOCKID__SPI: + *uuid = HSA_PROFILEBLOCK_AMD_SPI; + break; + case PERFCOUNTER_BLOCKID__SRBM: + *uuid = HSA_PROFILEBLOCK_AMD_SRBM; + break; case PERFCOUNTER_BLOCKID__SQ: *uuid = HSA_PROFILEBLOCK_AMD_SQ; break; + case PERFCOUNTER_BLOCKID__SX: + *uuid = HSA_PROFILEBLOCK_AMD_SX; + break; + case PERFCOUNTER_BLOCKID__TA: + *uuid = HSA_PROFILEBLOCK_AMD_TA; + break; case PERFCOUNTER_BLOCKID__TCA: *uuid = HSA_PROFILEBLOCK_AMD_TCA; break; + case PERFCOUNTER_BLOCKID__TCC: + *uuid = HSA_PROFILEBLOCK_AMD_TCC; + break; + case PERFCOUNTER_BLOCKID__TCP: + *uuid = HSA_PROFILEBLOCK_AMD_TCP; + break; + case PERFCOUNTER_BLOCKID__TCS: + *uuid = HSA_PROFILEBLOCK_AMD_TCS; + break; + case PERFCOUNTER_BLOCKID__TD: + *uuid = HSA_PROFILEBLOCK_AMD_TD; + break; + case PERFCOUNTER_BLOCKID__VGT: + *uuid = HSA_PROFILEBLOCK_AMD_VGT; + break; + case PERFCOUNTER_BLOCKID__WD: + *uuid = HSA_PROFILEBLOCK_AMD_WD; + break; case PERFCOUNTER_BLOCKID__IOMMUV2: *uuid = HSA_PROFILEBLOCK_AMD_IOMMUV2; break; diff --git a/src/pmc_table.c b/src/pmc_table.c index 92c363bfa1..3abb57eb37 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -31,27 +31,155 @@ #include "libhsakmt.h" #include "pmc_table.h" -/* Unused counters - 163-167 */ -static uint32_t gfx7_sq_counter_ids[] = { -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, -23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, -43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, -83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, -102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, -118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, -134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, -150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 168, 169, 170, -171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, -187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, -203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, -219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, -235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250 +/****** CB ******/ +static uint32_t gfx7_cb_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225 }; -static uint32_t gfx7_tca_counter_ids[] = { -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, -23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38 +/****** CPF ******/ +static uint32_t gfx7_cpf_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 +}; + +/****** CPG ******/ +static uint32_t gfx7_cpg_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45 +}; + +/****** DB ******/ +static uint32_t gfx7_db_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256 +}; + +/****** GDS ******/ +static uint32_t gfx7_gds_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120 +}; + +/****** GRBM ******/ +static uint32_t gfx7_grbm_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33 +}; + +/****** GRBMSE ******/ +static uint32_t gfx7_grbmse_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 +}; + +/****** IA ******/ +static uint32_t gfx7_ia_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 +}; + +/****** PASC ******/ +static uint32_t gfx7_pasc_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394 +}; + +/****** PASU ******/ +static uint32_t gfx7_pasu_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152 +}; + +/****** SPI ******/ +static uint32_t gfx7_spi_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185 +}; + +/****** SQ ******/ +/* Unused counters - 163-167 */ +static uint32_t gfx7_sq_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 168, 169, +170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, +186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, +202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, +218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, +234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, +250 }; /* Unused counters - 166, 292 - 297 */ @@ -75,11 +203,6 @@ static uint32_t gfx8_sq_counter_ids[] = { 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; -static uint32_t gfx8_tca_counter_ids[] = { -1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, -23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 -}; - /* Polaris 10/11 have the same SQ cpunter IDs but different from other gfx8's. */ /* Unused counters - 167 and 275 are *_DUMMY_LAST */ static uint32_t gfx8_pl_sq_counter_ids[] = { @@ -125,6 +248,96 @@ static uint32_t gfx9_sq_counter_ids[] = { 364, 365, 366, 367, 368, 369, 370, 371, 372 }; +/****** SRBM ******/ +static uint32_t gfx7_srbm_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 +}; + +/****** SX ******/ +static uint32_t gfx7_sx_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33 +}; + +/****** TA ******/ +static uint32_t gfx7_ta_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110 +}; + +/****** TCA ******/ +static uint32_t gfx7_tca_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38 +}; + +static uint32_t gfx8_tca_counter_ids[] = { +1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, +23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 +}; + +/****** TCC ******/ +static uint32_t gfx7_tcc_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, +81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 159 +}; + +/****** TCP ******/ +static uint32_t gfx7_tcp_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153 +}; + +/****** TCS ******/ +static uint32_t gfx7_tcs_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 64, +65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, +85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, +104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, +120, 121, 122, 123, 124, 125, 126, 127 +}; + +/****** TD ******/ +static uint32_t gfx7_td_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 +}; + +/****** VGT ******/ +static uint32_t gfx7_vgt_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139 +}; + +/****** WD ******/ +static uint32_t gfx7_wd_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -136,6 +349,185 @@ static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__CB] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx7_cb_counter_ids) / + sizeof(*gfx7_cb_counter_ids), + .counter_ids = gfx7_cb_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPF] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx7_cpf_counter_ids) / + sizeof(*gfx7_cpf_counter_ids), + .counter_ids = gfx7_cpf_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPG] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx7_cpg_counter_ids) / + sizeof(*gfx7_cpg_counter_ids), + .counter_ids = gfx7_cpg_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__DB] = { + .num_of_slots = 12, + .num_of_counters = sizeof(gfx7_db_counter_ids) / + sizeof(*gfx7_db_counter_ids), + .counter_ids = gfx7_db_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GDS] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_gds_counter_ids) / + sizeof(*gfx7_gds_counter_ids), + .counter_ids = gfx7_gds_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx7_grbm_counter_ids) / + sizeof(*gfx7_grbm_counter_ids), + .counter_ids = gfx7_grbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBMSE] = { + .num_of_slots = 1, + .num_of_counters = sizeof(gfx7_grbmse_counter_ids) / + sizeof(*gfx7_grbmse_counter_ids), + .counter_ids = gfx7_grbmse_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__IA] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx7_ia_counter_ids) / + sizeof(*gfx7_ia_counter_ids), + .counter_ids = gfx7_ia_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASC] = { + .num_of_slots = 11, + .num_of_counters = sizeof(gfx7_pasc_counter_ids) / + sizeof(*gfx7_pasc_counter_ids), + .counter_ids = gfx7_pasc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASU] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx7_pasu_counter_ids) / + sizeof(*gfx7_pasu_counter_ids), + .counter_ids = gfx7_pasu_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SPI] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_spi_counter_ids) / + sizeof(*gfx7_spi_counter_ids), + .counter_ids = gfx7_spi_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx7_srbm_counter_ids) / + sizeof(*gfx7_srbm_counter_ids), + .counter_ids = gfx7_srbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 8, + .num_of_counters = sizeof(gfx7_sq_counter_ids) / + sizeof(*gfx7_sq_counter_ids), + .counter_ids = gfx7_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SX] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_sx_counter_ids) / + sizeof(*gfx7_sx_counter_ids), + .counter_ids = gfx7_sx_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TA] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx7_ta_counter_ids) / + sizeof(*gfx7_ta_counter_ids), + .counter_ids = gfx7_ta_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCA] = { + .num_of_slots = 10, /* same as CZ */ + .num_of_counters = sizeof(gfx7_tca_counter_ids) / + sizeof(*gfx7_tca_counter_ids), + .counter_ids = gfx7_tca_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCC] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx7_tcc_counter_ids) / + sizeof(*gfx7_tcc_counter_ids), + .counter_ids = gfx7_tcc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCP] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx7_tcp_counter_ids) / + sizeof(*gfx7_tcp_counter_ids), + .counter_ids = gfx7_tcp_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCS] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx7_tcs_counter_ids) / + sizeof(*gfx7_tcs_counter_ids), + .counter_ids = gfx7_tcs_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TD] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx7_td_counter_ids) / + sizeof(*gfx7_td_counter_ids), + .counter_ids = gfx7_td_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__VGT] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx7_vgt_counter_ids) / + sizeof(*gfx7_vgt_counter_ids), + .counter_ids = gfx7_vgt_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__WD] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_wd_counter_ids) / + sizeof(*gfx7_wd_counter_ids), + .counter_ids = gfx7_wd_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -146,8 +538,8 @@ static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_mask = BITMASK(64) }, [PERFCOUNTER_BLOCKID__TCA] = { - /* PMC0: PERF_SEL~PERF_SEL3, PMC1: PERF_SEL~PERF_SEL3, PMC2: PERF_SEL - * PMC3: PERF_SEL. So 10 PERF_SELs in total + /* PMC0: PERF_SEL~PERF_SEL3, PMC1: PERF_SEL~PERF_SEL3, + * PMC2: PERF_SEL, PMC3: PERF_SEL. So 10 PERF_SELs in total */ .num_of_slots = 10, .num_of_counters = sizeof(gfx8_tca_counter_ids) / @@ -177,25 +569,6 @@ static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; -static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { - [PERFCOUNTER_BLOCKID__SQ] = { - .num_of_slots = 8, - .num_of_counters = sizeof(gfx7_sq_counter_ids) / - sizeof(*gfx7_sq_counter_ids), - .counter_ids = gfx7_sq_counter_ids, - .counter_size_in_bits = 64, - .counter_mask = BITMASK(64) - }, - [PERFCOUNTER_BLOCKID__TCA] = { - .num_of_slots = 10, /* same as CZ */ - .num_of_counters = sizeof(gfx7_tca_counter_ids) / - sizeof(*gfx7_tca_counter_ids), - .counter_ids = gfx7_tca_counter_ids, - .counter_size_in_bits = 64, - .counter_mask = BITMASK(64) - }, -}; - static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, diff --git a/src/pmc_table.h b/src/pmc_table.h index c6d16ad47e..aa9d5bd0d1 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -30,8 +30,32 @@ enum perf_block_id { PERFCOUNTER_BLOCKID__FIRST = 0, - PERFCOUNTER_BLOCKID__SQ = PERFCOUNTER_BLOCKID__FIRST, + /* non-privileged */ + PERFCOUNTER_BLOCKID__CB = PERFCOUNTER_BLOCKID__FIRST, + PERFCOUNTER_BLOCKID__CPC, + PERFCOUNTER_BLOCKID__CPF, + PERFCOUNTER_BLOCKID__CPG, + PERFCOUNTER_BLOCKID__DB, + PERFCOUNTER_BLOCKID__GDS, + PERFCOUNTER_BLOCKID__GRBM, + PERFCOUNTER_BLOCKID__GRBMSE, + PERFCOUNTER_BLOCKID__IA, + PERFCOUNTER_BLOCKID__MC, + PERFCOUNTER_BLOCKID__PASC, + PERFCOUNTER_BLOCKID__PASU, + PERFCOUNTER_BLOCKID__SPI, + PERFCOUNTER_BLOCKID__SRBM, + PERFCOUNTER_BLOCKID__SQ, + PERFCOUNTER_BLOCKID__SX, + PERFCOUNTER_BLOCKID__TA, PERFCOUNTER_BLOCKID__TCA, + PERFCOUNTER_BLOCKID__TCC, + PERFCOUNTER_BLOCKID__TCP, + PERFCOUNTER_BLOCKID__TCS, + PERFCOUNTER_BLOCKID__TD, + PERFCOUNTER_BLOCKID__VGT, + PERFCOUNTER_BLOCKID__WD, + /* privileged */ PERFCOUNTER_BLOCKID__IOMMUV2, PERFCOUNTER_BLOCKID__MAX }; From ed4a22e0d3b1f6c2674dfdef0d096d17941469e2 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 25 Apr 2017 11:30:28 -0400 Subject: [PATCH 0242/1247] Add more non-priv PMC blocks to GFX8 This patch adds more non-privileged PMC blocks to GFX8 products: gfx801, gfx803, and gfx803. Most of them have the same counter IDs on the same block. For certain blocks when the product doesn't have the same counter IDs, gfx8_xx_ is used to represent the product. Change-Id: I059913c974bf2eb875fd1cf6f8b0d8c9c9bd7c14 Signed-off-by: Amber Lin --- src/pmc_table.c | 633 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 633 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 3abb57eb37..732f7e47e2 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -48,11 +48,42 @@ static uint32_t gfx7_cb_counter_ids[] = { 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225 }; +static uint32_t gfx8_cb_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394, 395 +}; + /****** CPF ******/ static uint32_t gfx7_cpf_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }; +static uint32_t gfx8_cpf_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 +}; + /****** CPG ******/ static uint32_t gfx7_cpg_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -60,6 +91,12 @@ static uint32_t gfx7_cpg_counter_ids[] = { 42, 43, 44, 45 }; +static uint32_t gfx8_cpg_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48 +}; + /****** DB ******/ static uint32_t gfx7_db_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -78,6 +115,7 @@ static uint32_t gfx7_db_counter_ids[] = { 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256 }; +/* gfx8_db_counter_ids are the same as gfx7_db_counter_ids */ /****** GDS ******/ static uint32_t gfx7_gds_counter_ids[] = { @@ -89,23 +127,31 @@ static uint32_t gfx7_gds_counter_ids[] = { 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120 }; +/* gfx8_gds_counter_ids are the same as gfx7_gds_counter_ids */ /****** GRBM ******/ static uint32_t gfx7_grbm_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33 }; +/* gfx8_grbm_counter_ids are the same as gfx7_grbm_counter_ids */ /****** GRBMSE ******/ static uint32_t gfx7_grbmse_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }; +/* gfx8_grbmse_counter_ids are the same as gfx7_grbmse_counter_ids */ /****** IA ******/ static uint32_t gfx7_ia_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }; +static uint32_t gfx8_ia_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23 +}; + /****** PASC ******/ static uint32_t gfx7_pasc_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -134,6 +180,33 @@ static uint32_t gfx7_pasc_counter_ids[] = { 389, 390, 391, 392, 393, 394 }; +static uint32_t gfx8_pasc_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394, 395, 396 +}; + /****** PASU ******/ static uint32_t gfx7_pasu_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -146,6 +219,7 @@ static uint32_t gfx7_pasu_counter_ids[] = { 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152 }; +/* gfx8_pasu_counter_ids are the same as gfx7_pasu_counter_ids */ /****** SPI ******/ static uint32_t gfx7_spi_counter_ids[] = { @@ -162,6 +236,20 @@ static uint32_t gfx7_spi_counter_ids[] = { 181, 182, 183, 184, 185 }; +static uint32_t gfx8_spi_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196 +}; + /****** SQ ******/ /* Unused counters - 163-167 */ static uint32_t gfx7_sq_counter_ids[] = { @@ -253,11 +341,17 @@ static uint32_t gfx7_srbm_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 }; +static uint32_t gfx8_srbm_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27 +}; + /****** SX ******/ static uint32_t gfx7_sx_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33 }; +/* gfx8_sx_counter_ids are the same as gfx7_sx_counter_ids */ /****** TA ******/ static uint32_t gfx7_ta_counter_ids[] = { @@ -269,6 +363,16 @@ static uint32_t gfx7_ta_counter_ids[] = { 101, 102, 103, 104, 105, 106, 107, 108, 109, 110 }; +static uint32_t gfx8_ta_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118 +}; + /****** TCA ******/ static uint32_t gfx7_tca_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -292,6 +396,36 @@ static uint32_t gfx7_tcc_counter_ids[] = { 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 159 }; +static uint32_t gfx8_tcc_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 128, 129, 130, 131, 132, 133, 134, 135, 136, +137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, +153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, +169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, +185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, +201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, +217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, +233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, +249, 250, 251, 252, 253, 254, 255 +}; + +static uint32_t gfx8_cz_tcc_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 128, 129, 130, 131, 132, 133, 134, 135, 136, +137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, +153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, +169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, +185, 186, 187, 188, 189, 190, 191 +}; + /****** TCP ******/ static uint32_t gfx7_tcp_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -305,6 +439,20 @@ static uint32_t gfx7_tcp_counter_ids[] = { 149, 150, 151, 152, 153 }; +static uint32_t gfx8_tcp_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182 +}; + /****** TCS ******/ static uint32_t gfx7_tcs_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 64, @@ -313,6 +461,7 @@ static uint32_t gfx7_tcs_counter_ids[] = { 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127 }; +/* gfx8 doesn't have TCS */ /****** TD ******/ static uint32_t gfx7_td_counter_ids[] = { @@ -320,6 +469,7 @@ static uint32_t gfx7_td_counter_ids[] = { 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 }; +/* gfx8_td_counter_ids are the same as gfx7_td_counter_ids */ /****** VGT ******/ static uint32_t gfx7_vgt_counter_ids[] = { @@ -333,11 +483,38 @@ static uint32_t gfx7_vgt_counter_ids[] = { 133, 134, 135, 136, 137, 138, 139 }; +static uint32_t gfx8_vgt_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145 +}; + +static uint32_t gfx8_pl_vgt_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146 +}; + /****** WD ******/ static uint32_t gfx7_wd_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; +static uint32_t gfx8_wd_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -529,6 +706,102 @@ static struct perf_counter_block hawaii_blocks[PERFCOUNTER_BLOCKID__MAX] = { }; static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__CB] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx8_cb_counter_ids) / + sizeof(*gfx8_cb_counter_ids), + .counter_ids = gfx8_cb_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPF] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx8_cpf_counter_ids) / + sizeof(*gfx8_cpf_counter_ids), + .counter_ids = gfx8_cpf_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPG] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx8_cpg_counter_ids) / + sizeof(*gfx8_cpg_counter_ids), + .counter_ids = gfx8_cpg_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__DB] = { + .num_of_slots = 12, + .num_of_counters = sizeof(gfx7_db_counter_ids) / + sizeof(*gfx7_db_counter_ids), + .counter_ids = gfx7_db_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GDS] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_gds_counter_ids) / + sizeof(*gfx7_gds_counter_ids), + .counter_ids = gfx7_gds_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx7_grbm_counter_ids) / + sizeof(*gfx7_grbm_counter_ids), + .counter_ids = gfx7_grbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBMSE] = { + .num_of_slots = 1, + .num_of_counters = sizeof(gfx7_grbmse_counter_ids) / + sizeof(*gfx7_grbmse_counter_ids), + .counter_ids = gfx7_grbmse_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__IA] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx8_ia_counter_ids) / + sizeof(*gfx8_ia_counter_ids), + .counter_ids = gfx8_ia_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASC] = { + .num_of_slots = 11, + .num_of_counters = sizeof(gfx8_pasc_counter_ids) / + sizeof(*gfx8_pasc_counter_ids), + .counter_ids = gfx8_pasc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASU] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx7_pasu_counter_ids) / + sizeof(*gfx7_pasu_counter_ids), + .counter_ids = gfx7_pasu_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SPI] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx8_spi_counter_ids) / + sizeof(*gfx8_spi_counter_ids), + .counter_ids = gfx8_spi_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx8_srbm_counter_ids) / + sizeof(*gfx8_srbm_counter_ids), + .counter_ids = gfx8_srbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, .num_of_counters = sizeof(gfx8_sq_counter_ids) / @@ -537,6 +810,22 @@ static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__SX] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_sx_counter_ids) / + sizeof(*gfx7_sx_counter_ids), + .counter_ids = gfx7_sx_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TA] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx8_ta_counter_ids) / + sizeof(*gfx8_ta_counter_ids), + .counter_ids = gfx8_ta_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__TCA] = { /* PMC0: PERF_SEL~PERF_SEL3, PMC1: PERF_SEL~PERF_SEL3, * PMC2: PERF_SEL, PMC3: PERF_SEL. So 10 PERF_SELs in total @@ -548,9 +837,145 @@ static struct perf_counter_block carrizo_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCC] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_cz_tcc_counter_ids) / + sizeof(*gfx8_cz_tcc_counter_ids), + .counter_ids = gfx8_cz_tcc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCP] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_tcp_counter_ids) / + sizeof(*gfx8_tcp_counter_ids), + .counter_ids = gfx8_tcp_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TD] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx7_td_counter_ids) / + sizeof(*gfx7_td_counter_ids), + .counter_ids = gfx7_td_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__VGT] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_vgt_counter_ids) / + sizeof(*gfx8_vgt_counter_ids), + .counter_ids = gfx8_vgt_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__WD] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx8_wd_counter_ids) / + sizeof(*gfx8_wd_counter_ids), + .counter_ids = gfx8_wd_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__CB] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx8_cb_counter_ids) / + sizeof(*gfx8_cb_counter_ids), + .counter_ids = gfx8_cb_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPF] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx8_cpf_counter_ids) / + sizeof(*gfx8_cpf_counter_ids), + .counter_ids = gfx8_cpf_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPG] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx8_cpg_counter_ids) / + sizeof(*gfx8_cpg_counter_ids), + .counter_ids = gfx8_cpg_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__DB] = { + .num_of_slots = 12, + .num_of_counters = sizeof(gfx7_db_counter_ids) / + sizeof(*gfx7_db_counter_ids), + .counter_ids = gfx7_db_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GDS] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_gds_counter_ids) / + sizeof(*gfx7_gds_counter_ids), + .counter_ids = gfx7_gds_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx7_grbm_counter_ids) / + sizeof(*gfx7_grbm_counter_ids), + .counter_ids = gfx7_grbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBMSE] = { + .num_of_slots = 1, + .num_of_counters = sizeof(gfx7_grbmse_counter_ids) / + sizeof(*gfx7_grbmse_counter_ids), + .counter_ids = gfx7_grbmse_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__IA] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx8_ia_counter_ids) / + sizeof(*gfx8_ia_counter_ids), + .counter_ids = gfx8_ia_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASC] = { + .num_of_slots = 11, + .num_of_counters = sizeof(gfx8_pasc_counter_ids) / + sizeof(*gfx8_pasc_counter_ids), + .counter_ids = gfx8_pasc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASU] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx7_pasu_counter_ids) / + sizeof(*gfx7_pasu_counter_ids), + .counter_ids = gfx7_pasu_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SPI] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx8_spi_counter_ids) / + sizeof(*gfx8_spi_counter_ids), + .counter_ids = gfx8_spi_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx8_srbm_counter_ids) / + sizeof(*gfx8_srbm_counter_ids), + .counter_ids = gfx8_srbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, .num_of_counters = sizeof(gfx8_sq_counter_ids) / @@ -559,6 +984,22 @@ static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__SX] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_sx_counter_ids) / + sizeof(*gfx7_sx_counter_ids), + .counter_ids = gfx7_sx_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TA] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx8_ta_counter_ids) / + sizeof(*gfx8_ta_counter_ids), + .counter_ids = gfx8_ta_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__TCA] = { .num_of_slots = 10, /* same as CZ */ .num_of_counters = sizeof(gfx8_tca_counter_ids) / @@ -567,9 +1008,137 @@ static struct perf_counter_block fiji_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCC] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_tcc_counter_ids) / + sizeof(*gfx8_tcc_counter_ids), + .counter_ids = gfx8_tcc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCP] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_tcp_counter_ids) / + sizeof(*gfx8_tcp_counter_ids), + .counter_ids = gfx8_tcp_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TD] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx7_td_counter_ids) / + sizeof(*gfx7_td_counter_ids), + .counter_ids = gfx7_td_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__VGT] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_vgt_counter_ids) / + sizeof(*gfx8_vgt_counter_ids), + .counter_ids = gfx8_vgt_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__WD] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx8_wd_counter_ids) / + sizeof(*gfx8_wd_counter_ids), + .counter_ids = gfx8_wd_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__CB] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx8_cb_counter_ids) / + sizeof(*gfx8_cb_counter_ids), + .counter_ids = gfx8_cb_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPF] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx8_cpf_counter_ids) / + sizeof(*gfx8_cpf_counter_ids), + .counter_ids = gfx8_cpf_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPG] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx8_cpg_counter_ids) / + sizeof(*gfx8_cpg_counter_ids), + .counter_ids = gfx8_cpg_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__DB] = { + .num_of_slots = 12, + .num_of_counters = sizeof(gfx7_db_counter_ids) / + sizeof(*gfx7_db_counter_ids), + .counter_ids = gfx7_db_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GDS] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_gds_counter_ids) / + sizeof(*gfx7_gds_counter_ids), + .counter_ids = gfx7_gds_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx7_grbm_counter_ids) / + sizeof(*gfx7_grbm_counter_ids), + .counter_ids = gfx7_grbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBMSE] = { + .num_of_slots = 1, + .num_of_counters = sizeof(gfx7_grbmse_counter_ids) / + sizeof(*gfx7_grbmse_counter_ids), + .counter_ids = gfx7_grbmse_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__IA] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx8_ia_counter_ids) / + sizeof(*gfx8_ia_counter_ids), + .counter_ids = gfx8_ia_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASC] = { + .num_of_slots = 11, + .num_of_counters = sizeof(gfx8_pasc_counter_ids) / + sizeof(*gfx8_pasc_counter_ids), + .counter_ids = gfx8_pasc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASU] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx7_pasu_counter_ids) / + sizeof(*gfx7_pasu_counter_ids), + .counter_ids = gfx7_pasu_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SPI] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx8_spi_counter_ids) / + sizeof(*gfx8_spi_counter_ids), + .counter_ids = gfx8_spi_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, .num_of_counters = sizeof(gfx8_pl_sq_counter_ids) / @@ -578,6 +1147,30 @@ static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__SRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx8_srbm_counter_ids) / + sizeof(*gfx8_srbm_counter_ids), + .counter_ids = gfx8_srbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SX] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_sx_counter_ids) / + sizeof(*gfx7_sx_counter_ids), + .counter_ids = gfx7_sx_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TA] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx8_ta_counter_ids) / + sizeof(*gfx8_ta_counter_ids), + .counter_ids = gfx8_ta_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__TCA] = { .num_of_slots = 10, /* same as CZ */ .num_of_counters = sizeof(gfx8_tca_counter_ids) / @@ -586,6 +1179,46 @@ static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCC] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_tcc_counter_ids) / + sizeof(*gfx8_tcc_counter_ids), + .counter_ids = gfx8_tcc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCP] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_tcp_counter_ids) / + sizeof(*gfx8_tcp_counter_ids), + .counter_ids = gfx8_tcp_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TD] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx7_td_counter_ids) / + sizeof(*gfx7_td_counter_ids), + .counter_ids = gfx7_td_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__VGT] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_pl_vgt_counter_ids) / + sizeof(*gfx8_pl_vgt_counter_ids), + .counter_ids = gfx8_pl_vgt_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__WD] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx8_wd_counter_ids) / + sizeof(*gfx8_wd_counter_ids), + .counter_ids = gfx8_wd_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { From ca06b0966b9d7b5ada9efbc3cc1713e3db3237c9 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 1 May 2017 09:14:03 -0400 Subject: [PATCH 0243/1247] Add more non-priv PMC blocks to GFX9 This patch adds more non-privileged PMC blocks to GFX9/gfx900 to cover blocks added in HSA Thunk Spec. Change-Id: Ia3d953213a32536b2275231149f11ba060791442 Signed-off-by: Amber Lin --- src/pmc_table.c | 341 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 341 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 732f7e47e2..3ca656a223 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -75,6 +75,36 @@ static uint32_t gfx8_cb_counter_ids[] = { 389, 390, 391, 392, 393, 394, 395 }; +static uint32_t gfx9_cb_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, +405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, +421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, +437 +}; + /****** CPF ******/ static uint32_t gfx7_cpf_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 @@ -84,6 +114,11 @@ static uint32_t gfx8_cpf_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 }; +static uint32_t gfx9_cpf_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, +}; + /****** CPG ******/ static uint32_t gfx7_cpg_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -97,6 +132,12 @@ static uint32_t gfx8_cpg_counter_ids[] = { 42, 43, 44, 45, 46, 47, 48 }; +static uint32_t gfx9_cpg_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58 +}; + /****** DB ******/ static uint32_t gfx7_db_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -117,6 +158,29 @@ static uint32_t gfx7_db_counter_ids[] = { }; /* gfx8_db_counter_ids are the same as gfx7_db_counter_ids */ +static uint32_t gfx9_db_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327 +}; + /****** GDS ******/ static uint32_t gfx7_gds_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -128,6 +192,7 @@ static uint32_t gfx7_gds_counter_ids[] = { 117, 118, 119, 120 }; /* gfx8_gds_counter_ids are the same as gfx7_gds_counter_ids */ +/* gfx9_gds_counter_ids are the same as gfx7_gds_counter_ids */ /****** GRBM ******/ static uint32_t gfx7_grbm_counter_ids[] = { @@ -136,12 +201,21 @@ static uint32_t gfx7_grbm_counter_ids[] = { }; /* gfx8_grbm_counter_ids are the same as gfx7_grbm_counter_ids */ +static uint32_t gfx9_grbm_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37 +}; + /****** GRBMSE ******/ static uint32_t gfx7_grbmse_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }; /* gfx8_grbmse_counter_ids are the same as gfx7_grbmse_counter_ids */ +static uint32_t gfx9_grbmse_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +}; + /****** IA ******/ static uint32_t gfx7_ia_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 @@ -152,6 +226,11 @@ static uint32_t gfx8_ia_counter_ids[] = { 22, 23 }; +static uint32_t gfx9_ia_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +}; + /****** PASC ******/ static uint32_t gfx7_pasc_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -207,6 +286,39 @@ static uint32_t gfx8_pasc_counter_ids[] = { 389, 390, 391, 392, 393, 394, 395, 396 }; +static uint32_t gfx9_pasc_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, +405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, +421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, +437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, +453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, +469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, +485, 486, 487, 488, 489, 490 +}; + /****** PASU ******/ static uint32_t gfx7_pasu_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -221,6 +333,26 @@ static uint32_t gfx7_pasu_counter_ids[] = { }; /* gfx8_pasu_counter_ids are the same as gfx7_pasu_counter_ids */ +static uint32_t gfx9_pasu_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291 +}; + /****** SPI ******/ static uint32_t gfx7_spi_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -250,6 +382,20 @@ static uint32_t gfx8_spi_counter_ids[] = { 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196 }; +static uint32_t gfx9_spi_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195 +}; + /****** SQ ******/ /* Unused counters - 163-167 */ static uint32_t gfx7_sq_counter_ids[] = { @@ -345,6 +491,7 @@ static uint32_t gfx8_srbm_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27 }; +/* gfx9 doesn't have SRBM */ /****** SX ******/ static uint32_t gfx7_sx_counter_ids[] = { @@ -353,6 +500,21 @@ static uint32_t gfx7_sx_counter_ids[] = { }; /* gfx8_sx_counter_ids are the same as gfx7_sx_counter_ids */ +static uint32_t gfx9_sx_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207 +}; + /****** TA ******/ static uint32_t gfx7_ta_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -372,6 +534,7 @@ static uint32_t gfx8_ta_counter_ids[] = { 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118 }; +/* gfx9_ta_counter_ids is same as gfx8_ta_counter_ids */ /****** TCA ******/ static uint32_t gfx7_tca_counter_ids[] = { @@ -383,6 +546,7 @@ static uint32_t gfx8_tca_counter_ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 }; +/* gfx9_tca_counter_ids is same as gfx8_tca_counter_ids */ /****** TCC ******/ static uint32_t gfx7_tcc_counter_ids[] = { @@ -425,6 +589,7 @@ static uint32_t gfx8_cz_tcc_counter_ids[] = { 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191 }; +/* gfx9_tcc_counter_ids is same as gfx8_tcc_counter_ids */ /****** TCP ******/ static uint32_t gfx7_tcp_counter_ids[] = { @@ -453,6 +618,14 @@ static uint32_t gfx8_tcp_counter_ids[] = { 181, 182 }; +static uint32_t gfx9_tcp_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84 +}; + /****** TCS ******/ static uint32_t gfx7_tcs_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 64, @@ -462,6 +635,7 @@ static uint32_t gfx7_tcs_counter_ids[] = { 120, 121, 122, 123, 124, 125, 126, 127 }; /* gfx8 doesn't have TCS */ +/* gfx9 doesn't have TCS */ /****** TD ******/ static uint32_t gfx7_td_counter_ids[] = { @@ -471,6 +645,12 @@ static uint32_t gfx7_td_counter_ids[] = { }; /* gfx8_td_counter_ids are the same as gfx7_td_counter_ids */ +static uint32_t gfx9_td_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56 +}; + /****** VGT ******/ static uint32_t gfx7_vgt_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -505,6 +685,17 @@ static uint32_t gfx8_pl_vgt_counter_ids[] = { 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146 }; +static uint32_t gfx9_vgt_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147 +}; + /****** WD ******/ static uint32_t gfx7_wd_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 @@ -515,6 +706,12 @@ static uint32_t gfx8_wd_counter_ids[] = { 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36 }; +static uint32_t gfx9_wd_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57 +}; + static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 8, @@ -1222,6 +1419,94 @@ static struct perf_counter_block polaris_blocks[PERFCOUNTER_BLOCKID__MAX] = { }; static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__CB] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx9_cb_counter_ids) / + sizeof(*gfx9_cb_counter_ids), + .counter_ids = gfx9_cb_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPF] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx9_cpf_counter_ids) / + sizeof(*gfx9_cpf_counter_ids), + .counter_ids = gfx9_cpf_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPG] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx9_cpg_counter_ids) / + sizeof(*gfx9_cpg_counter_ids), + .counter_ids = gfx9_cpg_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__DB] = { + .num_of_slots = 12, + .num_of_counters = sizeof(gfx9_db_counter_ids) / + sizeof(*gfx9_db_counter_ids), + .counter_ids = gfx9_db_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GDS] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx7_gds_counter_ids) / + sizeof(*gfx7_gds_counter_ids), + .counter_ids = gfx7_gds_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx9_grbm_counter_ids) / + sizeof(*gfx9_grbm_counter_ids), + .counter_ids = gfx9_grbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBMSE] = { + .num_of_slots = 1, + .num_of_counters = sizeof(gfx9_grbmse_counter_ids) / + sizeof(*gfx9_grbmse_counter_ids), + .counter_ids = gfx9_grbmse_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__IA] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx9_ia_counter_ids) / + sizeof(*gfx9_ia_counter_ids), + .counter_ids = gfx9_ia_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASC] = { + .num_of_slots = 11, + .num_of_counters = sizeof(gfx9_pasc_counter_ids) / + sizeof(*gfx9_pasc_counter_ids), + .counter_ids = gfx9_pasc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASU] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx9_pasu_counter_ids) / + sizeof(*gfx9_pasu_counter_ids), + .counter_ids = gfx9_pasu_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SPI] = { + .num_of_slots = 18, + .num_of_counters = sizeof(gfx9_spi_counter_ids) / + sizeof(*gfx9_spi_counter_ids), + .counter_ids = gfx9_spi_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__SQ] = { .num_of_slots = 16, .num_of_counters = sizeof(gfx9_sq_counter_ids) / @@ -1230,6 +1515,22 @@ static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__SX] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx9_sx_counter_ids) / + sizeof(*gfx9_sx_counter_ids), + .counter_ids = gfx9_sx_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TA] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx8_ta_counter_ids) / + sizeof(*gfx8_ta_counter_ids), + .counter_ids = gfx8_ta_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, [PERFCOUNTER_BLOCKID__TCA] = { .num_of_slots = 10, /* same as Fiji */ /* Greenland has the same TCA counter IDs with Fiji */ @@ -1239,6 +1540,46 @@ static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { .counter_size_in_bits = 64, .counter_mask = BITMASK(64) }, + [PERFCOUNTER_BLOCKID__TCC] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx8_tcc_counter_ids) / + sizeof(*gfx8_tcc_counter_ids), + .counter_ids = gfx8_tcc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCP] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx9_tcp_counter_ids) / + sizeof(*gfx9_tcp_counter_ids), + .counter_ids = gfx9_tcp_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TD] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx9_td_counter_ids) / + sizeof(*gfx9_td_counter_ids), + .counter_ids = gfx9_td_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__VGT] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx9_vgt_counter_ids) / + sizeof(*gfx9_vgt_counter_ids), + .counter_ids = gfx9_vgt_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__WD] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx9_wd_counter_ids) / + sizeof(*gfx9_wd_counter_ids), + .counter_ids = gfx9_wd_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, }; /* Current APUs only have one IOMMU. If NUMA is introduced to APUs, we'll need From 3b2f064cbc8b0d37f3af38be9da1637d6b904234 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 28 Apr 2017 16:55:52 -0400 Subject: [PATCH 0244/1247] Get PAGE_SIZE from system configuration Change-Id: I87f383c443b873e13d36e80bfa034665bf493520 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 13 ++++++++----- src/globals.c | 2 ++ src/libhsakmt.h | 13 +++++++++++-- src/openclose.c | 9 +++++++++ src/topology.c | 3 +-- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index e89290bcd6..fcc234ee9c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -40,7 +40,7 @@ #define INIT_MANAGEBLE_APERTURE(base_value, limit_value) { \ .base = (void *) base_value, \ .limit = (void *) limit_value, \ - .align = PAGE_SIZE, \ + .align = 0, \ .guard_pages = 1, \ .vm_ranges = NULL, \ .vm_objects = NULL, \ @@ -1392,11 +1392,14 @@ static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_po static uint32_t get_vm_alignment(uint32_t device_id) { + int page_size = 0; + if (device_id >= 0x6920 && device_id <= 0x6939) /* Tonga */ - return TONGA_PAGE_SIZE; - if (device_id >= 0x9870 && device_id <= 0x9877) /* Carrizo */ - return TONGA_PAGE_SIZE; - return PAGE_SIZE; + page_size = TONGA_PAGE_SIZE; + else if (device_id >= 0x9870 && device_id <= 0x9877) /* Carrizo */ + page_size = TONGA_PAGE_SIZE; + + return MAX(PAGE_SIZE, page_size); } HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) diff --git a/src/globals.c b/src/globals.c index d055cecbde..15d6f11456 100644 --- a/src/globals.c +++ b/src/globals.c @@ -32,3 +32,5 @@ unsigned long kfd_open_count; unsigned long system_properties_count; pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; bool is_dgpu = false; +int PAGE_SIZE; +int PAGE_SHIFT; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 437cf14582..be57fc59f1 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -49,8 +49,9 @@ extern bool is_dgpu; #define CHECK_KFD_OPEN() \ do { if (kfd_open_count == 0) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) -#define PAGE_SIZE 4096 -#define PAGE_SHIFT 12 +extern int PAGE_SIZE; +extern int PAGE_SHIFT; + /* VI HW bug requires this virtual address alignment */ #define TONGA_PAGE_SIZE 0x8000 @@ -111,6 +112,14 @@ extern int kmtIoctl(int fd, unsigned long request, void *arg); #define VOID_PTR_SUB(ptr,n) (void*)((uint8_t*)(ptr) - n)/*ptr - offset*/ #define VOID_PTRS_SUB(ptr1,ptr2) (uint64_t)((uint8_t*)(ptr1) - (uint8_t*)(ptr2)) /*ptr1 - ptr2*/ +#define MIN(a, b) ({ \ + typeof(a) tmp1 = (a), tmp2 = (b); \ + tmp1 < tmp2 ? tmp1 : tmp2; }) + +#define MAX(a, b) ({ \ + typeof(a) tmp1 = (a), tmp2 = (b); \ + tmp1 > tmp2 ? tmp1 : tmp2; }) + void clear_events_page(void); void fmm_clear_all_mem(void); void clear_process_doorbells(void); diff --git a/src/openclose.c b/src/openclose.c index 3e0f5a8267..087fbdc36d 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "fmm.h" static const char kfd_device_name[] = "/dev/kfd"; @@ -69,6 +70,12 @@ static void clear_after_fork(void) kfd_open_count = 0; } +static inline void init_page_size(void) +{ + PAGE_SIZE = sysconf(_SC_PAGESIZE); + PAGE_SHIFT = ffs(PAGE_SIZE) - 1; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) @@ -99,6 +106,8 @@ hsaKmtOpenKFD(void) goto open_failed; } + init_page_size(); + result = topology_sysfs_get_system_props(&sys_props); if (result != HSAKMT_STATUS_SUCCESS) goto topology_sysfs_failed; diff --git a/src/topology.c b/src/topology.c index 99460f49ef..bcd25f91d6 100644 --- a/src/topology.c +++ b/src/topology.c @@ -38,8 +38,7 @@ #include "libhsakmt.h" #include "fmm.h" -#define PAGE_SIZE 4096 -#define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) + /* Number of memory banks added by thunk on top of topology */ #define NUM_OF_IGPU_HEAPS 3 #define NUM_OF_DGPU_HEAPS 3 From 5eb31b2ebede795d9b72b94b79e09a87cc761aeb Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 1 May 2017 19:19:38 -0400 Subject: [PATCH 0245/1247] Switch to cleaned up memory management ioctls Change-Id: Ib8971ef91138f2a051272b9b57f0ebd480e8e738 Signed-off-by: Felix Kuehling --- include/linux/kfd_ioctl.h | 63 ++++++++++++++---------------- src/fmm.c | 81 ++++++++++++++++++++++----------------- 2 files changed, 74 insertions(+), 70 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 6473ae9040..b1bc7a5572 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -156,12 +156,14 @@ struct kfd_ioctl_dbg_unregister_args { uint32_t pad; }; +/* FIXME: This is not 32-on-64 safe, fixed upstream, we should match it */ struct kfd_ioctl_dbg_address_watch_args { uint32_t gpu_id; /* to KFD */ uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ void *content_ptr; /* a pointer to the actual content */ }; +/* FIXME: This is not 32-on-64 safe, fixed upstream, we should match it */ struct kfd_ioctl_dbg_wave_control_args { uint32_t gpu_id; /* to KFD */ uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ @@ -217,7 +219,6 @@ struct kfd_ioctl_reset_event_args { uint32_t pad; }; - struct kfd_memory_exception_failure { bool NotPresent; /* Page not present or supervisor privilege */ bool ReadOnly; /* Write access to a read-only page */ @@ -226,6 +227,7 @@ struct kfd_memory_exception_failure { }; /* memory exception data*/ +/* FIXME: Make this 32-on-64 safe. Will break ABI. */ struct kfd_hsa_memory_exception_data { uint64_t va; uint32_t gpu_id; @@ -252,7 +254,8 @@ struct kfd_ioctl_wait_events_args { uint32_t wait_result; /* from KFD */ }; -struct kfd_ioctl_alloc_memory_of_gpu_args { +/* FIXME: remove unused fields and make it 32-on-64 safe. Will break ABI. */ +struct kfd_ioctl_alloc_memory_of_scratch_args { uint64_t va_addr; /* to KFD */ uint64_t size; /* to KFD */ uint64_t handle; /* from KFD */ @@ -266,23 +269,15 @@ struct kfd_ioctl_free_memory_of_gpu_args { struct kfd_ioctl_map_memory_to_gpu_args { uint64_t handle; /* to KFD */ -}; - -struct kfd_ioctl_map_memory_to_gpu_new_args { - uint64_t handle; /* to KFD */ - uint32_t *device_ids_array; /* to KFD */ + uint64_t device_ids_array_ptr; /* to KFD */ uint32_t device_ids_array_size; /* to KFD */ uint32_t pad; }; struct kfd_ioctl_unmap_memory_from_gpu_args { - uint64_t handle; /* to KFD */ -}; - -struct kfd_ioctl_unmap_memory_from_gpu_new_args { - uint64_t handle; /* to KFD */ - uint32_t *device_ids_array; /* to KFD */ - uint32_t device_ids_array_size; /* to KFD */ + uint64_t handle; /* to KFD */ + uint64_t device_ids_array_ptr; /* to KFD */ + uint32_t device_ids_array_size; /* to KFD */ uint32_t pad; }; @@ -295,6 +290,7 @@ struct kfd_ioctl_open_graphic_handle_args { uint32_t pad; }; +/* FIXME: Make this 32-on-64 safe. Will break ABI. */ struct kfd_ioctl_set_process_dgpu_aperture_args { uint32_t gpu_id; uint64_t dgpu_base; @@ -302,25 +298,30 @@ struct kfd_ioctl_set_process_dgpu_aperture_args { }; /* - * GPU Memory allocation flags + * Memory types */ -#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST (1 << 0) -#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE (1 << 1) -#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_SCRATCH (1 << 2) -#define KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE (1 << 3) -#define KFD_IOC_ALLOC_MEM_FLAGS_APU_SCRATCH (1 << 4) +#define KFD_IOC_ALLOC_MEM_FLAGS_VRAM (1 << 0) +#define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) +#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) +#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) -#define KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM (1 << 5) -#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 6) -#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 7) -#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 8) +/* + * Allocation flags attributes/access options. + */ +#define KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED (1 << 31) +#define KFD_IOC_ALLOC_MEM_FLAGS_READONLY (1 << 30) +#define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29) +#define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28) +#define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) +#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS (1 << 26) +#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 25) -struct kfd_ioctl_alloc_memory_of_gpu_new_args { +struct kfd_ioctl_alloc_memory_of_gpu_args { uint64_t va_addr; /* to KFD */ uint64_t size; /* to KFD */ uint64_t handle; /* from KFD */ - uint32_t gpu_id; /* to KFD */ uint64_t mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ + uint32_t gpu_id; /* to KFD */ uint32_t flags; }; @@ -495,7 +496,7 @@ struct kfd_ioctl_cross_memory_copy_args { AMDKFD_IOWR(0x15, struct kfd_ioctl_open_graphic_handle_args) #define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ - AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) + AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_scratch_args) #define AMDKFD_IOC_SET_CU_MASK \ AMDKFD_IOW(0x17, struct kfd_ioctl_set_cu_mask_args) @@ -504,17 +505,9 @@ struct kfd_ioctl_cross_memory_copy_args { AMDKFD_IOW(0x18, \ struct kfd_ioctl_set_process_dgpu_aperture_args) -#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW \ - AMDKFD_IOWR(0x19, struct kfd_ioctl_alloc_memory_of_gpu_new_args) - #define AMDKFD_IOC_SET_TRAP_HANDLER \ AMDKFD_IOW(0x1a, struct kfd_ioctl_set_trap_handler_args) -#define AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW \ - AMDKFD_IOWR(0x1b, struct kfd_ioctl_map_memory_to_gpu_new_args) -#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW \ - AMDKFD_IOWR(0x1c, struct kfd_ioctl_unmap_memory_from_gpu_new_args) - #define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ AMDKFD_IOWR(0x1d, struct kfd_ioctl_get_process_apertures_new_args) diff --git a/src/fmm.c b/src/fmm.c index fcc234ee9c..94193c0b6f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -700,7 +700,7 @@ static vm_object_t *fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, uint64_t *mmap_offset, uint32_t flags) { - struct kfd_ioctl_alloc_memory_of_gpu_new_args args; + struct kfd_ioctl_alloc_memory_of_gpu_args args; struct kfd_ioctl_free_memory_of_gpu_args free_args; vm_object_t *vm_obj = NULL; @@ -711,14 +711,18 @@ static vm_object_t *fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, args.gpu_id = gpu_id; args.size = ALIGN_UP(MemorySizeInBytes, aperture->align); - args.flags = flags; + args.flags = flags | + KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED | + KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS | + KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; args.va_addr = (uint64_t)mem; - if (flags == KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE) + if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && + (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) args.va_addr = VOID_PTRS_SUB(mem, aperture->base); if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) args.mmap_offset = *mmap_offset; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU_NEW, &args)) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) return NULL; /* Allocate object */ @@ -863,7 +867,7 @@ static void fmm_release_scratch(uint32_t gpu_id) void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { manageble_aperture_t *aperture_phy; - struct kfd_ioctl_alloc_memory_of_gpu_args args; + struct kfd_ioctl_alloc_memory_of_scratch_args args; int32_t gpu_mem_id; void *mem = NULL; uint64_t aligned_size = ALIGN_UP(MemorySizeInBytes, SCRATCH_ALIGN); @@ -973,7 +977,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla { manageble_aperture_t *aperture; int32_t gpu_mem_id; - uint32_t ioc_flags, offset; + uint32_t ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_VRAM, offset; uint64_t size, mmap_offset; void *mem; vm_object_t *vm_obj = NULL; @@ -985,16 +989,17 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla size = MemorySizeInBytes; + if (flags.ui32.HostAccess) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC; + if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { - ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE; aperture = &svm.dgpu_aperture; offset = 0; if (flags.ui32.AQLQueueMemory) { size = MemorySizeInBytes * 2; - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; } } else { - ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_APU_DEVICE; aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; offset = GPUVM_APP_OFFSET; } @@ -1135,7 +1140,7 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, } if (flags.ui32.AQLQueueMemory) { size = MemorySizeInBytes * 2; - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_AQL_QUEUE_MEM; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; } /* Paged memory is allocated as a userptr mapping, non-paged @@ -1183,7 +1188,7 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, return NULL; } } else { - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; mem = __fmm_allocate_device(gpu_id, size, aperture, 0, &mmap_offset, ioc_flags, &vm_obj); @@ -1235,7 +1240,7 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, void *mem = NULL; int32_t i = gpu_mem_find_by_gpu_id(gpu_id); struct kfd_ioctl_open_graphic_handle_args open_graphic_handle_args; - struct kfd_ioctl_unmap_memory_from_gpu_new_args unmap_args; + struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; /* If not found or aperture isn't properly initialized/supported */ if (i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, @@ -1273,9 +1278,9 @@ void *fmm_open_graphic_handle(uint32_t gpu_id, release_mem: unmap_args.handle = open_graphic_handle_args.handle; - unmap_args.device_ids_array = NULL; + unmap_args.device_ids_array_ptr = 0; unmap_args.device_ids_array_size = 0; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &unmap_args); + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); release_area: aperture_release_area(&gpu_mem[i].gpuvm_aperture, mem, MemorySizeInBytes); @@ -1660,7 +1665,7 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj) { - struct kfd_ioctl_map_memory_to_gpu_new_args args; + struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; void *temp_mapped_id_array = NULL; @@ -1686,10 +1691,11 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, args.handle = object->handle; if (object->registered_device_id_array_size > 0) { - args.device_ids_array = object->registered_device_id_array; + args.device_ids_array_ptr = + (uint64_t)object->registered_device_id_array; args.device_ids_array_size = object->registered_device_id_array_size; } else { - args.device_ids_array = all_gpu_id_array; + args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; args.device_ids_array_size = all_gpu_id_array_size; } @@ -1697,10 +1703,11 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, if (!temp_mapped_id_array) goto err_object_not_found; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - print_device_id_array(args.device_ids_array, args.device_ids_array_size); + print_device_id_array((void *)args.device_ids_array_ptr, + args.device_ids_array_size); if (object->mapped_device_id_array != NULL && object->mapped_device_id_array_size > 0 && @@ -1708,7 +1715,8 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, object->mapped_device_id_array != object->registered_device_id_array) free(object->mapped_device_id_array); - memcpy(temp_mapped_id_array, args.device_ids_array, args.device_ids_array_size); + memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, + args.device_ids_array_size); object->mapped_device_id_array = temp_mapped_id_array; object->mapped_device_id_array_size = args.device_ids_array_size; object->mapping_count = 1; @@ -1755,7 +1763,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu if (!ret && !is_debugger) { vm_object_t *obj = fmm_allocate_memory_in_device( gpu_id, address, size, aperture, - NULL, KFD_IOC_ALLOC_MEM_FLAGS_DGPU_DEVICE); + NULL, KFD_IOC_ALLOC_MEM_FLAGS_VRAM); if (obj == NULL) return -1; } else { @@ -1764,7 +1772,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu size, aperture, &mmap_offset, - KFD_IOC_ALLOC_MEM_FLAGS_DGPU_HOST); + KFD_IOC_ALLOC_MEM_FLAGS_GTT); mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, @@ -1788,7 +1796,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) { - struct kfd_ioctl_map_memory_to_gpu_new_args args; + struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; void *temp_mapped_id_array = NULL; @@ -1806,10 +1814,11 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, args.handle = object->handle; if (object->registered_device_id_array_size > 0 && object->registered_device_id_array) { - args.device_ids_array = object->registered_device_id_array; + args.device_ids_array_ptr = + (uint64_t)object->registered_device_id_array; args.device_ids_array_size = object->registered_device_id_array_size; } else { - args.device_ids_array = all_gpu_id_array; + args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; args.device_ids_array_size = all_gpu_id_array_size; } @@ -1817,7 +1826,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, if (!temp_mapped_id_array) goto err_object_not_found; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU_NEW, &args)) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; if (object->mapped_device_id_array != NULL && @@ -1826,7 +1835,8 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, object->mapped_device_id_array != object->registered_device_id_array) free(object->mapped_device_id_array); - memcpy(temp_mapped_id_array, args.device_ids_array, args.device_ids_array_size); + memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, + args.device_ids_array_size); object->mapped_device_id_array = temp_mapped_id_array; object->mapped_device_id_array_size = args.device_ids_array_size; @@ -1958,7 +1968,7 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, { vm_object_t *object; int ret = 0; - struct kfd_ioctl_unmap_memory_from_gpu_new_args args; + struct kfd_ioctl_unmap_memory_from_gpu_args args; HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); if (!obj) @@ -1982,10 +1992,10 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, args.handle = object->handle; if (device_ids_array && device_ids_array_size > 0) { - args.device_ids_array = device_ids_array; + args.device_ids_array_ptr = (uint64_t)device_ids_array; args.device_ids_array_size = device_ids_array_size; } else if (object->mapped_device_id_array_size > 0) { - args.device_ids_array = object->mapped_device_id_array; + args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; args.device_ids_array_size = object->mapped_device_id_array_size; } else { /* @@ -1997,9 +2007,10 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, goto out; } - print_device_id_array(args.device_ids_array, args.device_ids_array_size); + print_device_id_array((void *)args.device_ids_array_ptr, + args.device_ids_array_size); - ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); if (ret != 0) goto out; @@ -2029,7 +2040,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, { int32_t gpu_mem_id; vm_object_t *object; - struct kfd_ioctl_unmap_memory_from_gpu_new_args args; + struct kfd_ioctl_unmap_memory_from_gpu_args args; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -2055,9 +2066,9 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, /* unmap from GPU */ args.handle = object->handle; - args.device_ids_array = object->mapped_device_id_array; + args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; args.device_ids_array_size = object->mapped_device_id_array_size; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU_NEW, &args); + kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); /* Clearing all mapped nodes list */ if (object->mapped_device_id_array != NULL && From 15764e2897df7a15a9296ab665e774c488f41f98 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 9 May 2017 14:59:13 -0400 Subject: [PATCH 0246/1247] Fix KFD ioctl ABI This change breaks the ABI, and aligns it with the upstream ABI. It also fixes some ioctl structures that are not 64-bit safe and consolidates ioctl numbers. Change-Id: Ib79944721534bd55a5299c5baf7bb5b3246cccd2 Signed-off-by: Felix Kuehling --- include/linux/kfd_ioctl.h | 150 +++++++++++++++++++------------------- src/debug.c | 4 +- 2 files changed, 77 insertions(+), 77 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index b1bc7a5572..8f833bdeb1 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -133,6 +133,10 @@ struct kfd_process_device_apertures { uint32_t pad; }; +/* This IOCTL and the limited NUM_OF_SUPPORTED_GPUS is deprecated. Use + * kfd_ioctl_get_process_apertures_new instead, which supports + * arbitrary numbers of GPUs. + */ struct kfd_ioctl_get_process_apertures_args { struct kfd_process_device_apertures process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */ @@ -142,6 +146,19 @@ struct kfd_ioctl_get_process_apertures_args { uint32_t pad; }; +struct kfd_ioctl_get_process_apertures_new_args { + /* User allocated. Pointer to struct kfd_process_device_apertures + * filled in by Kernel + */ + uint64_t kfd_process_device_apertures_ptr; + /* to KFD - indicates amount of memory present in + * kfd_process_device_apertures_ptr + * from KFD - Number of entries filled by KFD. + */ + uint32_t num_of_nodes; + uint32_t pad; +}; + #define MAX_ALLOWED_NUM_POINTS 100 #define MAX_ALLOWED_AW_BUFF_SIZE 4096 #define MAX_ALLOWED_WAC_BUFF_SIZE 128 @@ -156,18 +173,16 @@ struct kfd_ioctl_dbg_unregister_args { uint32_t pad; }; -/* FIXME: This is not 32-on-64 safe, fixed upstream, we should match it */ struct kfd_ioctl_dbg_address_watch_args { + uint64_t content_ptr; /* a pointer to the actual content */ uint32_t gpu_id; /* to KFD */ uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ - void *content_ptr; /* a pointer to the actual content */ }; -/* FIXME: This is not 32-on-64 safe, fixed upstream, we should match it */ struct kfd_ioctl_dbg_wave_control_args { + uint64_t content_ptr; /* a pointer to the actual content */ uint32_t gpu_id; /* to KFD */ uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ - void *content_ptr; /* a pointer to the actual content */ }; /* Matching HSA_EVENTTYPE */ @@ -199,7 +214,8 @@ struct kfd_ioctl_create_event_args { uint32_t event_trigger_data; /* from KFD - signal events only */ uint32_t event_type; /* to KFD */ uint32_t auto_reset; /* to KFD */ - uint32_t node_id; /* to KFD - only valid for certain event types */ + uint32_t node_id; /* to KFD - only valid for certain + event types */ uint32_t event_id; /* from KFD */ uint32_t event_slot_index; /* from KFD */ }; @@ -220,21 +236,21 @@ struct kfd_ioctl_reset_event_args { }; struct kfd_memory_exception_failure { - bool NotPresent; /* Page not present or supervisor privilege */ - bool ReadOnly; /* Write access to a read-only page */ - bool NoExecute; /* Execute access to a page marked NX */ - bool imprecise; /* Can't determine the exact fault address */ + uint32_t NotPresent; /* Page not present or supervisor privilege */ + uint32_t ReadOnly; /* Write access to a read-only page */ + uint32_t NoExecute; /* Execute access to a page marked NX */ + uint32_t imprecise; /* Can't determine the exact fault address */ }; -/* memory exception data*/ -/* FIXME: Make this 32-on-64 safe. Will break ABI. */ +/* memory exception data */ struct kfd_hsa_memory_exception_data { + struct kfd_memory_exception_failure failure; uint64_t va; uint32_t gpu_id; - struct kfd_memory_exception_failure failure; + uint32_t pad; }; -/* Event data*/ +/* Event data */ struct kfd_event_data { union { struct kfd_hsa_memory_exception_data memory_exception_data; @@ -254,13 +270,34 @@ struct kfd_ioctl_wait_events_args { uint32_t wait_result; /* from KFD */ }; -/* FIXME: remove unused fields and make it 32-on-64 safe. Will break ABI. */ struct kfd_ioctl_alloc_memory_of_scratch_args { uint64_t va_addr; /* to KFD */ uint64_t size; /* to KFD */ - uint64_t handle; /* from KFD */ uint32_t gpu_id; /* to KFD */ - uint64_t mmap_offset; /* from KFD */ + uint32_t pad; +}; + +/* Allocation flags: memory types */ +#define KFD_IOC_ALLOC_MEM_FLAGS_VRAM (1 << 0) +#define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) +#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) +#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) +/* Allocation flags: attributes/access options */ +#define KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED (1 << 31) +#define KFD_IOC_ALLOC_MEM_FLAGS_READONLY (1 << 30) +#define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29) +#define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28) +#define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) +#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS (1 << 26) +#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 25) + +struct kfd_ioctl_alloc_memory_of_gpu_args { + uint64_t va_addr; /* to KFD */ + uint64_t size; /* to KFD */ + uint64_t handle; /* from KFD */ + uint64_t mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ + uint32_t gpu_id; /* to KFD */ + uint32_t flags; }; struct kfd_ioctl_free_memory_of_gpu_args { @@ -281,6 +318,11 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { uint32_t pad; }; +/* TODO: remove this. It's only implemented for Kaveri and was never + * upstreamed. There are no open-source users of this interface. It + * has been superseded by the pair of get_dmabuf_info and + * import_dmabuf, which is implemented for all supported GPUs. + */ struct kfd_ioctl_open_graphic_handle_args { uint64_t va_addr; /* to KFD */ uint64_t handle; /* from KFD */ @@ -290,52 +332,10 @@ struct kfd_ioctl_open_graphic_handle_args { uint32_t pad; }; -/* FIXME: Make this 32-on-64 safe. Will break ABI. */ struct kfd_ioctl_set_process_dgpu_aperture_args { - uint32_t gpu_id; uint64_t dgpu_base; uint64_t dgpu_limit; -}; - -/* - * Memory types - */ -#define KFD_IOC_ALLOC_MEM_FLAGS_VRAM (1 << 0) -#define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) -#define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) -#define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) - -/* - * Allocation flags attributes/access options. - */ -#define KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED (1 << 31) -#define KFD_IOC_ALLOC_MEM_FLAGS_READONLY (1 << 30) -#define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29) -#define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28) -#define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) -#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS (1 << 26) -#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 25) - -struct kfd_ioctl_alloc_memory_of_gpu_args { - uint64_t va_addr; /* to KFD */ - uint64_t size; /* to KFD */ - uint64_t handle; /* from KFD */ - uint64_t mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ - uint32_t gpu_id; /* to KFD */ - uint32_t flags; -}; - -struct kfd_ioctl_get_process_apertures_new_args { - /* User allocated. Pointer to struct kfd_process_device_apertures - * filled in by Kernel */ - uint64_t kfd_process_device_apertures_ptr; - - /* to KFD - indicates amount of memory present in - * kfd_process_device_apertures_ptr - * from KFD - Number of entries filled by KFD. - */ - uint32_t num_of_nodes; - + uint32_t gpu_id; uint32_t pad; }; @@ -492,45 +492,45 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ AMDKFD_IOWR(0x14, struct kfd_ioctl_unmap_memory_from_gpu_args) -#define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ - AMDKFD_IOWR(0x15, struct kfd_ioctl_open_graphic_handle_args) - #define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ - AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_scratch_args) + AMDKFD_IOWR(0x15, struct kfd_ioctl_alloc_memory_of_scratch_args) #define AMDKFD_IOC_SET_CU_MASK \ - AMDKFD_IOW(0x17, struct kfd_ioctl_set_cu_mask_args) + AMDKFD_IOW(0x16, struct kfd_ioctl_set_cu_mask_args) #define AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE \ - AMDKFD_IOW(0x18, \ + AMDKFD_IOW(0x17, \ struct kfd_ioctl_set_process_dgpu_aperture_args) #define AMDKFD_IOC_SET_TRAP_HANDLER \ - AMDKFD_IOW(0x1a, struct kfd_ioctl_set_trap_handler_args) + AMDKFD_IOW(0x18, struct kfd_ioctl_set_trap_handler_args) #define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ - AMDKFD_IOWR(0x1d, struct kfd_ioctl_get_process_apertures_new_args) - + AMDKFD_IOWR(0x19, struct kfd_ioctl_get_process_apertures_new_args) #define AMDKFD_IOC_GET_DMABUF_INFO \ - AMDKFD_IOWR(0x1f, struct kfd_ioctl_get_dmabuf_info_args) + AMDKFD_IOWR(0x1A, struct kfd_ioctl_get_dmabuf_info_args) #define AMDKFD_IOC_IMPORT_DMABUF \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_import_dmabuf_args) + AMDKFD_IOWR(0x1B, struct kfd_ioctl_import_dmabuf_args) #define AMDKFD_IOC_GET_TILE_CONFIG \ - AMDKFD_IOWR(0x21, struct kfd_ioctl_get_tile_config_args) + AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_tile_config_args) #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ - AMDKFD_IOWR(0x22, struct kfd_ioctl_ipc_import_handle_args) + AMDKFD_IOWR(0x1D, struct kfd_ioctl_ipc_import_handle_args) #define AMDKFD_IOC_IPC_EXPORT_HANDLE \ - AMDKFD_IOWR(0x23, struct kfd_ioctl_ipc_export_handle_args) + AMDKFD_IOWR(0x1E, struct kfd_ioctl_ipc_export_handle_args) #define AMDKFD_IOC_CROSS_MEMORY_COPY \ - AMDKFD_IOWR(0x24, struct kfd_ioctl_cross_memory_copy_args) + AMDKFD_IOWR(0x1F, struct kfd_ioctl_cross_memory_copy_args) + +/* TODO: remove this */ +#define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ + AMDKFD_IOWR(0x20, struct kfd_ioctl_open_graphic_handle_args) #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x25 +#define AMDKFD_COMMAND_END 0x21 #endif diff --git a/src/debug.c b/src/debug.c index 327fbbcacb..51b6f8b0b9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -159,7 +159,7 @@ hsaKmtDbgWavefrontControl( unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); /* save variable content pointer for kfd */ - args->content_ptr = (void *)run_ptr; + args->content_ptr = (uint64_t)run_ptr; /* insert items, and increment pointer accordingly */ @@ -258,7 +258,7 @@ hsaKmtDbgAddressWatch( unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); /* save variable content pointer for kfd */ - args->content_ptr = (void *)run_ptr; + args->content_ptr = (uint64_t)run_ptr; /* insert items, and increment pointer accordingly */ *((HSAuint32*)run_ptr) = NumWatchPoints; From ea58703ece24a2fe13f4c8d9efcae4b6253b2ad1 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 10 May 2017 18:16:31 -0400 Subject: [PATCH 0247/1247] Fix uninitialized memory bug in hsaKmtWaitOnMultipleEvents Use calloc to allocate event data. Otherwise random data may be filled in for events that haven't actually signalled. This could trigger the VM fault handler in the Runtime when no VM fault actually happened and lead to intermittent HSA conformance test failures. Change-Id: Icf702970e73a485b50633703c1b164f87fbb8606 Signed-off-by: Felix Kuehling --- src/events.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/events.c b/src/events.c index b65ea96e71..373018aa29 100644 --- a/src/events.c +++ b/src/events.c @@ -255,7 +255,7 @@ hsaKmtWaitOnMultipleEvents( if (!Events) return HSAKMT_STATUS_INVALID_HANDLE; - struct kfd_event_data *event_data = malloc(NumEvents * sizeof(struct kfd_event_data)); + struct kfd_event_data *event_data = calloc(NumEvents, sizeof(struct kfd_event_data)); for (HSAuint32 i = 0; i < NumEvents; i++) { event_data[i].event_id = Events[i]->EventId; event_data[i].kfd_event_data_ext = (uint64_t)(uintptr_t)NULL; From 8aeb93342663d688698849376e682ecfce3ec237 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 11 May 2017 16:39:19 -0400 Subject: [PATCH 0248/1247] Add some additional gfx900 PCI IDs Change-Id: I5f00f3b30a27285d75c606c1308abfe032ce1d02 Signed-off-by: Felix Kuehling --- src/topology.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/topology.c b/src/topology.c index bcd25f91d6..4c0ce1a0b0 100644 --- a/src/topology.c +++ b/src/topology.c @@ -167,7 +167,9 @@ static struct hsa_gfxip_table { { 0x6861, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6862, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6863, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6864, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6867, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6868, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 } }; From 59cc20d3cb005e09a2c2ebe0ad1d8b105685fd48 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Tue, 30 May 2017 21:26:55 -0500 Subject: [PATCH 0249/1247] Check mmap return address for allocation, not requested address. Change-Id: Ifeb7b17976fc791e3256c70d57cb4d1324a8b960 --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 94193c0b6f..2f2eb73f8b 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2184,7 +2184,7 @@ static void *reserve_address(void *addr, long long unsigned int len) ret_addr = mmap(addr, len, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); - if (addr == MAP_FAILED) + if (ret_addr == MAP_FAILED) return NULL; return ret_addr; From b78e0e152a9fe3461537c07a97ef4c74b3b05bf0 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 20 Apr 2017 08:25:00 -0400 Subject: [PATCH 0250/1247] Clean up thunk code Use checkpatch.pl to fix the majority of errors. Some that remain and will be excluded: Use of typedefs/externs/volatile/sscanf Lines over 80 characters Remaining errors are due to misunderstanding the * symbol with typedefs Also use this opportunity to spell manageable properly Change-Id: I0b335e9cb3e1eea38bee27eaa1f582b2c9b09b38 --- src/debug.c | 212 ++++++++------------ src/events.c | 92 ++++----- src/fmm.c | 404 +++++++++++++++++++------------------ src/fmm.h | 32 ++- src/globals.c | 2 +- src/libhsakmt.c | 18 +- src/memory.c | 266 +++++++++---------------- src/openclose.c | 32 ++- src/perfctr.c | 134 +++++-------- src/pmc_table.c | 10 +- src/pmc_table.h | 7 +- src/queues.c | 188 ++++++++---------- src/time.c | 8 +- src/topology.c | 515 +++++++++++++++++++++--------------------------- src/version.c | 7 +- 15 files changed, 819 insertions(+), 1108 deletions(-) diff --git a/src/debug.c b/src/debug.c index 51b6f8b0b9..05a21329b9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -29,14 +29,14 @@ #include static bool *is_device_debugged; -int debug_get_reg_status(uint32_t node_id, bool* is_debugged); +int debug_get_reg_status(uint32_t node_id, bool *is_debugged); HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes) { unsigned int i; is_device_debugged = malloc(NumNodes * sizeof(bool)); - if (is_device_debugged == NULL) + if (!is_device_debugged) return HSAKMT_STATUS_NO_MEMORY; for (i = 0; i < NumNodes; i++) @@ -51,17 +51,14 @@ void destroy_device_debugging_memory(void) free(is_device_debugged); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDbgRegister( - HSAuint32 NodeId //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId) { HSAKMT_STATUS result; uint32_t gpu_id; + CHECK_KFD_OPEN(); - if (is_device_debugged == NULL) + if (!is_device_debugged) return HSAKMT_STATUS_NO_MEMORY; result = validate_nodeid(NodeId, &gpu_id); @@ -69,31 +66,28 @@ hsaKmtDbgRegister( return result; struct kfd_ioctl_dbg_register_args args; + memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER, &args); + + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER, &args); if (err == 0) result = HSAKMT_STATUS_SUCCESS; else result = HSAKMT_STATUS_ERROR; - return (result); + return result; } -/* =============================================================================== */ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDbgUnregister( - HSAuint32 NodeId //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgUnregister(HSAuint32 NodeId) { - HSAKMT_STATUS result; uint32_t gpu_id; + HSAKMT_STATUS result; + CHECK_KFD_OPEN(); - if (is_device_debugged == NULL) + if (!is_device_debugged) return HSAKMT_STATUS_NO_MEMORY; result = validate_nodeid(NodeId, &gpu_id); @@ -101,28 +95,22 @@ hsaKmtDbgUnregister( return result; struct kfd_ioctl_dbg_unregister_args args; + memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER, &args); - if (err == 0) - result = HSAKMT_STATUS_SUCCESS; - else - result = HSAKMT_STATUS_ERROR; + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER, &args); - return (result); + if (err) + return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; } -/* =============================================================================== */ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDbgWavefrontControl( - HSAuint32 NodeId, //IN - HSA_DBG_WAVEOP Operand, //IN - HSA_DBG_WAVEMODE Mode, //IN - HSAuint32 TrapId, //IN - HsaDbgWaveMessage* DbgWaveMsgRing //IN (? - see thunk API doc!) - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgWavefrontControl(HSAuint32 NodeId, + HSA_DBG_WAVEOP Operand, + HSA_DBG_WAVEMODE Mode, + HSAuint32 TrapId, + HsaDbgWaveMessage *DbgWaveMsgRing) { HSAKMT_STATUS result; uint32_t gpu_id; @@ -136,18 +124,14 @@ hsaKmtDbgWavefrontControl( return result; -/* Determine Size of the ioctl buffer */ +/* Determine Size of the ioctl buffer */ + uint32_t buff_size = sizeof(Operand) + sizeof(Mode) + sizeof(TrapId) + + sizeof(DbgWaveMsgRing->DbgWaveMsg) + + sizeof(DbgWaveMsgRing->MemoryVA) + sizeof(*args); - uint32_t buff_size = sizeof(Operand)+ - sizeof(Mode) + sizeof(TrapId) + - sizeof(DbgWaveMsgRing->DbgWaveMsg)+ sizeof(DbgWaveMsgRing->MemoryVA) + sizeof(*args); - - - args = (struct kfd_ioctl_dbg_wave_control_args*) malloc(buff_size); - if (args == NULL) - { + args = (struct kfd_ioctl_dbg_wave_control_args *)malloc(buff_size); + if (!args) return HSAKMT_STATUS_ERROR; - } memset(args, 0, buff_size); @@ -155,67 +139,53 @@ hsaKmtDbgWavefrontControl( args->buf_size_in_bytes = buff_size; /* increment pointer to the start of the non fixed part */ - - unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); + unsigned char *run_ptr = (unsigned char *)args + sizeof(*args); /* save variable content pointer for kfd */ args->content_ptr = (uint64_t)run_ptr; /* insert items, and increment pointer accordingly */ - - *((HSA_DBG_WAVEOP*)run_ptr) = Operand; + *((HSA_DBG_WAVEOP *)run_ptr) = Operand; run_ptr += sizeof(Operand); - - *((HSA_DBG_WAVEMODE*)run_ptr) = Mode; + *((HSA_DBG_WAVEMODE *)run_ptr) = Mode; run_ptr += sizeof(Mode); - - *((HSAuint32*)run_ptr) = TrapId; + *((HSAuint32 *)run_ptr) = TrapId; run_ptr += sizeof(TrapId); - *((HsaDbgWaveMessageAMD*)run_ptr) = DbgWaveMsgRing->DbgWaveMsg; - run_ptr += sizeof(DbgWaveMsgRing->DbgWaveMsg); + *((HsaDbgWaveMessageAMD *)run_ptr) = DbgWaveMsgRing->DbgWaveMsg; + run_ptr += sizeof(DbgWaveMsgRing->DbgWaveMsg); - *((void**)run_ptr) = DbgWaveMsgRing->MemoryVA; + *((void **)run_ptr) = DbgWaveMsgRing->MemoryVA; run_ptr += sizeof(DbgWaveMsgRing->MemoryVA); /* send to kernel */ - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_WAVE_CONTROL, args); - free (args); + free(args); - if (err == 0) - return HSAKMT_STATUS_SUCCESS; - else + if (err) return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; } - -/* =============================================================================== */ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDbgAddressWatch( - HSAuint32 NodeId, //IN - HSAuint32 NumWatchPoints, //IN - HSA_DBG_WATCH_MODE WatchMode[], //IN - void* WatchAddress[], //IN - HSAuint64 WatchMask[], //IN, optional - HsaEvent* WatchEvent[] //IN, optional - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(HSAuint32 NodeId, + HSAuint32 NumWatchPoints, + HSA_DBG_WATCH_MODE WatchMode[], + void *WatchAddress[], + HSAuint64 WatchMask[], + HsaEvent *WatchEvent[]) { HSAKMT_STATUS result; uint32_t gpu_id; /* determine the size of the watch mask and event buffers - * the value is NULL if and only if no vector data should be attached - * + * the value is NULL if and only if no vector data should be attached */ - - uint32_t watch_mask_items = WatchMask[0] > 0 ? NumWatchPoints:1; - uint32_t watch_event_items = WatchEvent != NULL ? NumWatchPoints:0; + uint32_t watch_mask_items = WatchMask[0] > 0 ? NumWatchPoints:1; + uint32_t watch_event_items = WatchEvent != NULL ? NumWatchPoints:0; struct kfd_ioctl_dbg_address_watch_args *args; HSAuint32 i = 0; @@ -229,23 +199,18 @@ hsaKmtDbgAddressWatch( if (NumWatchPoints > MAX_ALLOWED_NUM_POINTS) return HSAKMT_STATUS_INVALID_PARAMETER; -/* Size and structure of the ioctl buffer is dynamic in this case - * Here we calculate the buff size. - */ + /* Size and structure of the ioctl buffer is dynamic in this case + * Here we calculate the buff size. + */ + uint32_t buff_size = sizeof(NumWatchPoints) + + (sizeof(WatchMode[0]) + sizeof(WatchAddress[0])) * + NumWatchPoints + + watch_mask_items * sizeof(HSAuint64) + + watch_event_items * sizeof(HsaEvent *) + sizeof(*args); - uint32_t buff_size =sizeof(NumWatchPoints)+ - ( sizeof(WatchMode[0]) + - sizeof(WatchAddress[0]))*NumWatchPoints + - watch_mask_items*sizeof(HSAuint64) + - watch_event_items*sizeof(HsaEvent*)+ - sizeof(*args); - - - args = (struct kfd_ioctl_dbg_address_watch_args*) malloc(buff_size); - if (args == NULL) - { + args = (struct kfd_ioctl_dbg_address_watch_args *) malloc(buff_size); + if (!args) return HSAKMT_STATUS_ERROR; - } memset(args, 0, buff_size); @@ -254,64 +219,51 @@ hsaKmtDbgAddressWatch( /* increment pointer to the start of the non fixed part */ - - unsigned char* run_ptr = (unsigned char*)args + sizeof(*args); + unsigned char *run_ptr = (unsigned char *)args + sizeof(*args); /* save variable content pointer for kfd */ args->content_ptr = (uint64_t)run_ptr; /* insert items, and increment pointer accordingly */ - *((HSAuint32*)run_ptr) = NumWatchPoints; + *((HSAuint32 *)run_ptr) = NumWatchPoints; run_ptr += sizeof(NumWatchPoints); - for (i=0; i < NumWatchPoints; i++) - { - *((HSA_DBG_WATCH_MODE*)run_ptr) = WatchMode[i]; - run_ptr += sizeof(WatchMode[i]); + for (i = 0; i < NumWatchPoints; i++) { + *((HSA_DBG_WATCH_MODE *)run_ptr) = WatchMode[i]; + run_ptr += sizeof(WatchMode[i]); } - for (i=0; i < NumWatchPoints; i++) - { - *((void**)run_ptr) = WatchAddress[i]; - run_ptr += sizeof(WatchAddress[i]); + for (i = 0; i < NumWatchPoints; i++) { + *((void **)run_ptr) = WatchAddress[i]; + run_ptr += sizeof(WatchAddress[i]); } - for (i=0; i < watch_mask_items; i++) - { - *((HSAuint64*)run_ptr) = WatchMask[i]; - run_ptr += sizeof(WatchMask[i]); + for (i = 0; i < watch_mask_items; i++) { + *((HSAuint64 *)run_ptr) = WatchMask[i]; + run_ptr += sizeof(WatchMask[i]); } - for (i=0; i < watch_event_items; i++) - { - *((HsaEvent**)run_ptr) = WatchEvent[i]; - run_ptr += sizeof(WatchEvent[i]); + for (i = 0; i < watch_event_items; i++) { + *((HsaEvent **)run_ptr) = WatchEvent[i]; + run_ptr += sizeof(WatchEvent[i]); } /* send to kernel */ - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_ADDRESS_WATCH, args); - free (args); + free(args); - if (err == 0) - { - return HSAKMT_STATUS_SUCCESS; - } - else - { + if (err) return HSAKMT_STATUS_ERROR; - } + return HSAKMT_STATUS_SUCCESS; } -/* =============================================================================== */ -int debug_get_reg_status(uint32_t node_id, bool* is_debugged) +int debug_get_reg_status(uint32_t node_id, bool *is_debugged) { *is_debugged = NULL; - if (is_device_debugged == NULL) + if (!is_device_debugged) return -1; - else { - *is_debugged = is_device_debugged[node_id]; - return 0; - } + + *is_debugged = is_device_debugged[node_id]; + return 0; } diff --git a/src/events.c b/src/events.c index 373018aa29..d934391c6d 100644 --- a/src/events.c +++ b/src/events.c @@ -47,31 +47,24 @@ static bool IsSystemEventType(HSA_EVENTTYPE type) return (type != HSA_EVENTTYPE_SIGNAL && type != HSA_EVENTTYPE_DEBUG_EVENT); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtCreateEvent( - HsaEventDescriptor* EventDesc, //IN - bool ManualReset, //IN - bool IsSignaled, //IN - HsaEvent** Event //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, + bool ManualReset, bool IsSignaled, + HsaEvent **Event) { CHECK_KFD_OPEN(); if (EventDesc->EventType >= HSA_EVENTTYPE_MAXID) - { return HSAKMT_STATUS_INVALID_PARAMETER; - } - HsaEvent* e = malloc(sizeof(HsaEvent)); - if (e == NULL) - { + HsaEvent *e = malloc(sizeof(HsaEvent)); + + if (!e) return HSAKMT_STATUS_ERROR; - } memset(e, 0, sizeof(*e)); struct kfd_ioctl_create_event_args args; + memset(&args, 0, sizeof(args)); args.event_type = EventDesc->EventType; @@ -81,7 +74,7 @@ hsaKmtCreateEvent( /* dGPU code */ pthread_mutex_lock(&hsakmt_mutex); - if (is_dgpu && events_page == NULL) { + if (is_dgpu && !events_page) { events_page = allocate_exec_aligned_memory_gpu( KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true); if (!events_page) { @@ -100,7 +93,7 @@ hsaKmtCreateEvent( e->EventId = args.event_id; - if (events_page == NULL && args.event_page_offset > 0) { + if (!events_page && args.event_page_offset > 0) { events_page = mmap(NULL, KFD_SIGNAL_EVENT_LIMIT * 8, PROT_WRITE | PROT_READ, MAP_SHARED, kfd_fd, args.event_page_offset); if (events_page == MAP_FAILED) { @@ -127,6 +120,7 @@ hsaKmtCreateEvent( if (IsSignaled && !IsSystemEventType(e->EventData.EventType)) { struct kfd_ioctl_set_event_args set_args; + memset(&set_args, 0, sizeof(set_args)); set_args.event_id = args.event_id; @@ -138,11 +132,7 @@ hsaKmtCreateEvent( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDestroyEvent( - HsaEvent* Event //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyEvent(HsaEvent *Event) { CHECK_KFD_OPEN(); @@ -150,34 +140,33 @@ hsaKmtDestroyEvent( return HSAKMT_STATUS_INVALID_HANDLE; struct kfd_ioctl_destroy_event_args args; + memset(&args, 0, sizeof(args)); args.event_id = Event->EventId; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_EVENT, &args) != 0) { + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_EVENT, &args) != 0) return HSAKMT_STATUS_ERROR; - } free(Event); return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetEvent( - HsaEvent* Event //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtSetEvent(HsaEvent *Event) { CHECK_KFD_OPEN(); if (!Event) return HSAKMT_STATUS_INVALID_HANDLE; - /* Although the spec is doesn't say, don't allow system-defined events to be signaled. */ + /* Although the spec is doesn't say, don't allow system-defined events + * to be signaled. + */ if (IsSystemEventType(Event->EventData.EventType)) return HSAKMT_STATUS_ERROR; struct kfd_ioctl_set_event_args args; + memset(&args, 0, sizeof(args)); args.event_id = Event->EventId; @@ -188,22 +177,21 @@ hsaKmtSetEvent( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtResetEvent( - HsaEvent* Event //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtResetEvent(HsaEvent *Event) { CHECK_KFD_OPEN(); if (!Event) return HSAKMT_STATUS_INVALID_HANDLE; - /* Although the spec is doesn't say, don't allow system-defined events to be signaled. */ + /* Although the spec is doesn't say, don't allow system-defined events + * to be signaled. + */ if (IsSystemEventType(Event->EventData.EventType)) return HSAKMT_STATUS_ERROR; struct kfd_ioctl_reset_event_args args; + memset(&args, 0, sizeof(args)); args.event_id = Event->EventId; @@ -214,11 +202,7 @@ hsaKmtResetEvent( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueryEventState( - HsaEvent* Event //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtQueryEventState(HsaEvent *Event) { CHECK_KFD_OPEN(); @@ -228,12 +212,8 @@ hsaKmtQueryEventState( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtWaitOnEvent( - HsaEvent* Event, //IN - HSAuint32 Milliseconds //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event, + HSAuint32 Milliseconds) { if (!Event) return HSAKMT_STATUS_INVALID_HANDLE; @@ -241,14 +221,10 @@ hsaKmtWaitOnEvent( return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtWaitOnMultipleEvents( - HsaEvent* Events[], //IN - HSAuint32 NumEvents, //IN - bool WaitOnAll, //IN - HSAuint32 Milliseconds //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], + HSAuint32 NumEvents, + bool WaitOnAll, + HSAuint32 Milliseconds) { CHECK_KFD_OPEN(); @@ -256,12 +232,14 @@ hsaKmtWaitOnMultipleEvents( return HSAKMT_STATUS_INVALID_HANDLE; struct kfd_event_data *event_data = calloc(NumEvents, sizeof(struct kfd_event_data)); + for (HSAuint32 i = 0; i < NumEvents; i++) { event_data[i].event_id = Events[i]->EventId; event_data[i].kfd_event_data_ext = (uint64_t)(uintptr_t)NULL; } struct kfd_ioctl_wait_events_args args; + memset(&args, 0, sizeof(args)); args.wait_for_all = WaitOnAll; @@ -271,12 +249,10 @@ hsaKmtWaitOnMultipleEvents( HSAKMT_STATUS result; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_WAIT_EVENTS, &args) == -1) { + if (kmtIoctl(kfd_fd, AMDKFD_IOC_WAIT_EVENTS, &args) == -1) result = HSAKMT_STATUS_ERROR; - } - else if (args.wait_result == KFD_IOC_WAIT_RESULT_TIMEOUT) { + else if (args.wait_result == KFD_IOC_WAIT_RESULT_TIMEOUT) result = HSAKMT_STATUS_WAIT_TIMEOUT; - } else { result = HSAKMT_STATUS_SUCCESS; for (HSAuint32 i = 0; i < NumEvents; i++) { diff --git a/src/fmm.c b/src/fmm.c index 2f2eb73f8b..88e6052851 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -37,7 +37,7 @@ #define NON_VALID_GPU_ID 0 -#define INIT_MANAGEBLE_APERTURE(base_value, limit_value) { \ +#define INIT_MANAGEABLE_APERTURE(base_value, limit_value) { \ .base = (void *) base_value, \ .limit = (void *) limit_value, \ .align = 0, \ @@ -52,25 +52,20 @@ struct vm_object { void *userptr; uint64_t userptr_size; uint64_t size; /* size allocated on GPU. When the user requests a random - * size, Thunk aligns it to page size and allocates this - * aligned size on GPU - */ + * size, Thunk aligns it to page size and allocates this + * aligned size on GPU + */ uint64_t handle; /* opaque */ uint32_t node_id; struct vm_object *next; struct vm_object *prev; uint32_t flags; /* memory allocation flags */ - /* - * Registered nodes to map on SVM mGPU - */ + /* Registered nodes to map on SVM mGPU */ uint32_t *registered_device_id_array; uint32_t registered_device_id_array_size; uint32_t *registered_node_id_array; - uint32_t registration_count; /* the same memory region can be - registered multiple times */ - /* - * Nodes that mapped already - */ + uint32_t registration_count; /* the same memory region can be registered multiple times */ + /* Nodes that mapped already */ uint32_t *mapped_device_id_array; uint32_t mapped_device_id_array_size; uint32_t *mapped_node_id_array; @@ -101,7 +96,7 @@ typedef struct { vm_area_t *vm_ranges; vm_object_t *vm_objects; pthread_mutex_t fmm_mutex; -} manageble_aperture_t; +} manageable_aperture_t; typedef struct { void *base; @@ -114,23 +109,25 @@ typedef struct { uint32_t node_id; uint64_t local_mem_size; aperture_t lds_aperture; - manageble_aperture_t scratch_aperture; - manageble_aperture_t scratch_physical; /* For dGPU, scratch physical - is allocated from dgpu_aperture. When requested by RT, each - GPU will get a differnt range */ - manageble_aperture_t gpuvm_aperture; /* used for GPUVM on APU, outside - * the canonical address range */ + manageable_aperture_t scratch_aperture; + manageable_aperture_t scratch_physical; /* For dGPU, scratch physical is allocated from + * dgpu_aperture. When requested by RT, each + * GPU will get a differnt range + */ + manageable_aperture_t gpuvm_aperture; /* used for GPUVM on APU, outsidethe canonical address range */ } gpu_mem_t; /* The main structure for dGPU Shared Virtual Memory Management */ typedef struct { /* used for non-coherent system and invisible device mem on dGPU. - * This aperture is shared by all dGPUs */ - manageble_aperture_t dgpu_aperture; + * This aperture is shared by all dGPUs + */ + manageable_aperture_t dgpu_aperture; /* used for coherent (fine-grain) system memory on dGPU, - * This aperture is shared by all dGPUs */ - manageble_aperture_t dgpu_alt_aperture; + * This aperture is shared by all dGPUs + */ + manageable_aperture_t dgpu_alt_aperture; /* whether to use userptr for paged memory */ bool userptr_for_paged_mem; @@ -140,16 +137,17 @@ typedef struct { } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU -* specific memory apertures. */ + * specific memory apertures. + */ static gpu_mem_t *gpu_mem; static unsigned int gpu_mem_count; -static void *dgpu_shared_aperture_base = NULL; -static void *dgpu_shared_aperture_limit = NULL; +static void *dgpu_shared_aperture_base; +static void *dgpu_shared_aperture_limit; static svm_t svm = { - INIT_MANAGEBLE_APERTURE(0, 0), - INIT_MANAGEBLE_APERTURE(0, 0), + INIT_MANAGEABLE_APERTURE(0, 0), + INIT_MANAGEABLE_APERTURE(0, 0), true, false }; @@ -158,11 +156,11 @@ static svm_t svm = { * via GPU driver, they are not managed by GPUVM. cpuvm_aperture keeps track * of this part of memory. */ -static manageble_aperture_t cpuvm_aperture = INIT_MANAGEBLE_APERTURE(0, 0); +static manageable_aperture_t cpuvm_aperture = INIT_MANAGEABLE_APERTURE(0, 0); /* GPU node array for default mappings */ -static uint32_t all_gpu_id_array_size = 0; -static uint32_t *all_gpu_id_array = NULL; +static uint32_t all_gpu_id_array_size; +static uint32_t *all_gpu_id_array; /* IPC structures and helper functions */ typedef enum _HSA_APERTURE { @@ -185,34 +183,35 @@ typedef struct _HsaSharedMemoryStruct { HSAuint32 ExportGpuId; } HsaSharedMemoryStruct; -static inline const HsaSharedMemoryStruct * to_const_hsa_shared_memory_struct( +static inline const HsaSharedMemoryStruct *to_const_hsa_shared_memory_struct( const HsaSharedMemoryHandle *SharedMemoryHandle) { return (const HsaSharedMemoryStruct *)SharedMemoryHandle; } -static inline HsaSharedMemoryStruct * to_hsa_shared_memory_struct( +static inline HsaSharedMemoryStruct *to_hsa_shared_memory_struct( HsaSharedMemoryHandle *SharedMemoryHandle) { return (HsaSharedMemoryStruct *)SharedMemoryHandle; } -static inline HsaSharedMemoryHandle * to_hsa_shared_memory_handle( +static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( HsaSharedMemoryStruct *SharedMemoryStruct) { return (HsaSharedMemoryHandle *)SharedMemoryStruct; } -extern int debug_get_reg_status(uint32_t node_id, bool* is_debugged); +extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit); -static void __fmm_release(void *address, manageble_aperture_t *aperture); +static void __fmm_release(void *address, manageable_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, - manageble_aperture_t *aperture, + manageable_aperture_t *aperture, void *address); static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_array_size); -static int32_t find_first_dgpu(HSAuint32 *gpu_id) { +static int32_t find_first_dgpu(HSAuint32 *gpu_id) +{ int32_t i; *gpu_id = NON_VALID_GPU_ID; @@ -270,7 +269,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, } -static void vm_remove_area(manageble_aperture_t *app, vm_area_t *area) +static void vm_remove_area(manageable_aperture_t *app, vm_area_t *area) { vm_area_t *next; vm_area_t *prev; @@ -278,7 +277,7 @@ static void vm_remove_area(manageble_aperture_t *app, vm_area_t *area) next = area->next; prev = area->prev; - if (prev == NULL) /* The first element */ + if (!prev) /* The first element */ app->vm_ranges = next; else prev->next = next; @@ -289,7 +288,7 @@ static void vm_remove_area(manageble_aperture_t *app, vm_area_t *area) free(area); } -static void vm_remove_object(manageble_aperture_t *app, vm_object_t *object) +static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object) { vm_object_t *next; vm_object_t *prev; @@ -305,11 +304,10 @@ static void vm_remove_object(manageble_aperture_t *app, vm_object_t *object) free(object->registered_device_id_array); object->registered_device_id_array_size = 0; } - if (object->mapped_device_id_array != NULL && + if (object->mapped_device_id_array && object->mapped_device_id_array_size > 0 && object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - { + object->mapped_device_id_array != object->registered_device_id_array) { free(object->mapped_device_id_array); object->mapped_device_id_array_size = 0; } @@ -326,7 +324,7 @@ static void vm_remove_object(manageble_aperture_t *app, vm_object_t *object) next = object->next; prev = object->prev; - if (prev == NULL) /* The first element */ + if (!prev) /* The first element */ app->vm_objects = next; else prev->next = next; @@ -362,7 +360,7 @@ static void vm_add_object_before(vm_object_t *before_this, prev->next = new_object; } -static void vm_split_area(manageble_aperture_t *app, vm_area_t *area, +static void vm_split_area(manageable_aperture_t *app, vm_area_t *area, void *address, uint64_t MemorySizeInBytes) { /* @@ -379,7 +377,7 @@ static void vm_split_area(manageble_aperture_t *app, vm_area_t *area, vm_add_area_after(area, new_area); } -static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, +static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app, const void *address, uint64_t size) { vm_object_t *cur = app->vm_objects; @@ -396,8 +394,8 @@ static vm_object_t *vm_find_object_by_address(manageble_aperture_t *app, return cur; /* NULL if not found */ } -static vm_object_t *vm_find_object_by_address_range(manageble_aperture_t *app, - const void *address) +static vm_object_t *vm_find_object_by_address_range(manageable_aperture_t *app, + const void *address) { vm_object_t *cur = app->vm_objects; @@ -411,14 +409,15 @@ static vm_object_t *vm_find_object_by_address_range(manageble_aperture_t *app, return cur; /* NULL if not found */ } -static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, +static vm_object_t *vm_find_object_by_userptr(manageable_aperture_t *app, const void *address, HSAuint64 size) { vm_object_t *cur = app->vm_objects, *obj; uint32_t found = 0; /* Look up the userptr that matches the address. If size is specified, - the size needs to match too. */ + * the size needs to match too. + */ while (cur) { if ((cur->userptr == address) && ((cur->userptr_size == size) || !size)) { @@ -429,7 +428,8 @@ static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, } /* If size is not specified, we need to ensure the vm_obj found is the - only obj having this address. */ + * only obj having this address. + */ if (found && !size) { obj = cur->next; while (obj) { @@ -444,7 +444,7 @@ static vm_object_t *vm_find_object_by_userptr(manageble_aperture_t *app, return cur; /* NULL if any look-up failure */ } -static vm_object_t *vm_find_object_by_userptr_range(manageble_aperture_t *app, +static vm_object_t *vm_find_object_by_userptr_range(manageable_aperture_t *app, const void *address) { vm_object_t *cur = app->vm_objects; @@ -460,7 +460,7 @@ static vm_object_t *vm_find_object_by_userptr_range(manageble_aperture_t *app, return cur; /* NULL if not found */ } -static vm_area_t *vm_find(manageble_aperture_t *app, void *address) +static vm_area_t *vm_find(manageable_aperture_t *app, void *address) { vm_area_t *cur = app->vm_ranges; @@ -486,7 +486,7 @@ static bool aperture_is_valid(void *app_base, void *app_limit) * Leave at least one guard page after every object to catch * out-of-bounds accesses with VM faults. */ -static uint64_t vm_align_area_size(manageble_aperture_t *app, uint64_t size) +static uint64_t vm_align_area_size(manageable_aperture_t *app, uint64_t size) { return ALIGN_UP(size + (uint64_t)app->guard_pages * PAGE_SIZE, app->align); @@ -495,7 +495,7 @@ static uint64_t vm_align_area_size(manageble_aperture_t *app, uint64_t size) /* * Assumes that fmm_mutex is locked on entry. */ -static void aperture_release_area(manageble_aperture_t *app, void *address, +static void aperture_release_area(manageable_aperture_t *app, void *address, uint64_t MemorySizeInBytes) { vm_area_t *area; @@ -531,7 +531,7 @@ static void aperture_release_area(manageble_aperture_t *app, void *address, * returns allocated address or NULL. Assumes, that fmm_mutex is locked * on entry. */ -static void *aperture_allocate_area_aligned(manageble_aperture_t *app, +static void *aperture_allocate_area_aligned(manageable_aperture_t *app, uint64_t MemorySizeInBytes, uint64_t offset, uint64_t align) @@ -584,7 +584,7 @@ static void *aperture_allocate_area_aligned(manageble_aperture_t *app, return start; } -static void *aperture_allocate_area(manageble_aperture_t *app, +static void *aperture_allocate_area(manageable_aperture_t *app, uint64_t MemorySizeInBytes, uint64_t offset) { @@ -592,7 +592,7 @@ static void *aperture_allocate_area(manageble_aperture_t *app, } /* returns 0 on success. Assumes, that fmm_mutex is locked on entry */ -static vm_object_t *aperture_allocate_object(manageble_aperture_t *app, +static vm_object_t *aperture_allocate_object(manageable_aperture_t *app, void *new_address, uint64_t handle, uint64_t MemorySizeInBytes, @@ -610,7 +610,7 @@ static vm_object_t *aperture_allocate_object(manageble_aperture_t *app, return NULL; /* check for non-empty list */ - if (app->vm_objects != NULL) + if (app->vm_objects) /* Add it before the first element */ vm_add_object_before(app->vm_objects, new_object); @@ -630,25 +630,25 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) return -1; } -static manageble_aperture_t *fmm_get_aperture(HsaApertureInfo info) +static manageable_aperture_t *fmm_get_aperture(HsaApertureInfo info) { switch (info.type) { - case HSA_APERTURE_DGPU: - return &svm.dgpu_aperture; - case HSA_APERTURE_DGPU_ALT: - return &svm.dgpu_alt_aperture; - case HSA_APERTURE_GPUVM: - return &gpu_mem[info.idx].gpuvm_aperture; - case HSA_APERTURE_CPUVM: - return &cpuvm_aperture; - default: - return NULL; + case HSA_APERTURE_DGPU: + return &svm.dgpu_aperture; + case HSA_APERTURE_DGPU_ALT: + return &svm.dgpu_alt_aperture; + case HSA_APERTURE_GPUVM: + return &gpu_mem[info.idx].gpuvm_aperture; + case HSA_APERTURE_CPUVM: + return &cpuvm_aperture; + default: + return NULL; } } -static manageble_aperture_t *fmm_find_aperture(const void *address, +static manageable_aperture_t *fmm_find_aperture(const void *address, HsaApertureInfo *info) { - manageble_aperture_t *aperture = NULL; + manageable_aperture_t *aperture = NULL; uint32_t i; HsaApertureInfo _info = { .type = HSA_APERTURE_UNSUPPORTED, .idx = 0}; @@ -657,19 +657,16 @@ static manageble_aperture_t *fmm_find_aperture(const void *address, address <= svm.dgpu_aperture.limit) { aperture = &svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; - } - else if (address >= svm.dgpu_alt_aperture.base && + } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { aperture = &svm.dgpu_alt_aperture; _info.type = HSA_APERTURE_DGPU_ALT; - } - else { + } else { /* Not in SVM, it can be system memory registered by userptr */ aperture = &svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; } - } - else { /* APU */ + } else { /* APU */ for (i = 0; i < gpu_mem_count; i++) { if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) { @@ -696,7 +693,7 @@ static manageble_aperture_t *fmm_find_aperture(const void *address, */ static vm_object_t *fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, uint64_t MemorySizeInBytes, - manageble_aperture_t *aperture, + manageable_aperture_t *aperture, uint64_t *mmap_offset, uint32_t flags) { @@ -727,8 +724,9 @@ static vm_object_t *fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, /* Allocate object */ pthread_mutex_lock(&aperture->fmm_mutex); - if (!(vm_obj = aperture_allocate_object(aperture, mem, args.handle, - MemorySizeInBytes, flags))) + vm_obj = aperture_allocate_object(aperture, mem, args.handle, + MemorySizeInBytes, flags); + if (!vm_obj) goto err_object_allocation_failed; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -773,7 +771,7 @@ static void aperture_print(aperture_t *app) printf("\t Limit: %p\n", app->limit); } -static void manageble_aperture_print(manageble_aperture_t *app) +static void manageable_aperture_print(manageable_aperture_t *app) { vm_area_t *cur = app->vm_ranges; vm_object_t *object = app->vm_objects; @@ -801,17 +799,17 @@ void fmm_print(uint32_t gpu_id) printf("LDS aperture:\n"); aperture_print(&gpu_mem[gpu_mem_id].lds_aperture); printf("GPUVM aperture:\n"); - manageble_aperture_print(&gpu_mem[gpu_mem_id].gpuvm_aperture); + manageable_aperture_print(&gpu_mem[gpu_mem_id].gpuvm_aperture); printf("Scratch aperture:\n"); - manageble_aperture_print(&gpu_mem[gpu_mem_id].scratch_aperture); + manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_aperture); printf("Scratch backing memory:\n"); - manageble_aperture_print(&gpu_mem[gpu_mem_id].scratch_physical); + manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_physical); } printf("dGPU aperture:\n"); - manageble_aperture_print(&svm.dgpu_aperture); + manageable_aperture_print(&svm.dgpu_aperture); printf("dGPU alt aperture:\n"); - manageble_aperture_print(&svm.dgpu_alt_aperture); + manageable_aperture_print(&svm.dgpu_alt_aperture); } #else @@ -825,7 +823,7 @@ static void fmm_release_scratch(uint32_t gpu_id) int32_t gpu_mem_id; uint64_t size; vm_object_t *obj; - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) @@ -840,6 +838,7 @@ static void fmm_release_scratch(uint32_t gpu_id) pthread_mutex_lock(&aperture->fmm_mutex); while ((obj = aperture->vm_objects)) { void *obj_addr = obj->start; + pthread_mutex_unlock(&aperture->fmm_mutex); _fmm_unmap_from_gpu_scratch(gpu_id, aperture, obj_addr); @@ -866,7 +865,7 @@ static void fmm_release_scratch(uint32_t gpu_id) #define SCRATCH_ALIGN 0x10000 void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { - manageble_aperture_t *aperture_phy; + manageable_aperture_t *aperture_phy; struct kfd_ioctl_alloc_memory_of_scratch_args args; int32_t gpu_mem_id; void *mem = NULL; @@ -878,7 +877,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) return NULL; aperture_phy = &gpu_mem[gpu_mem_id].scratch_physical; - if (aperture_phy->base != NULL || aperture_phy->limit != NULL) + if (aperture_phy->base || aperture_phy->limit) /* Scratch was already allocated for this GPU */ return NULL; @@ -893,10 +892,11 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) uint64_t aligned_padded_size = aligned_size + SCRATCH_ALIGN - PAGE_SIZE; void *padded_end, *aligned_start, *aligned_end; + mem = mmap(0, aligned_padded_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == NULL) + if (!mem) return NULL; /* align start and unmap padding */ padded_end = VOID_PTR_ADD(mem, aligned_padded_size); @@ -927,8 +927,8 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) return mem; } -static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, - manageble_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset, +static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, + manageable_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset, uint32_t flags, vm_object_t **vm_obj) { void *mem = NULL; @@ -950,7 +950,7 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, */ obj = fmm_allocate_memory_in_device(gpu_id, mem, MemorySizeInBytes, aperture, mmap_offset, flags); - if (obj == NULL) { + if (!obj) { /* * allocation of memory in device failed. * Release region in aperture @@ -975,7 +975,7 @@ static void* __fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, #define GPUVM_APP_OFFSET 0x10000 void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; int32_t gpu_mem_id; uint32_t ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_VRAM, offset; uint64_t size, mmap_offset; @@ -1019,7 +1019,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla if (mem && flags.ui32.HostAccess) { void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + MAP_SHARED | MAP_FIXED, kfd_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; @@ -1032,7 +1032,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; int32_t gpu_mem_id; uint32_t ioc_flags; void *mem; @@ -1081,7 +1081,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, return mem; } -static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, +static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, HsaMemFlags flags) { int err; @@ -1114,11 +1114,11 @@ static void* fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, return mem; } -static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, +static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { void *mem; - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; uint64_t mmap_offset; uint32_t ioc_flags; uint64_t size; @@ -1144,13 +1144,14 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, } /* Paged memory is allocated as a userptr mapping, non-paged - * memory is allocated from KFD */ + * memory is allocated from KFD + */ if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, size, 0); pthread_mutex_unlock(&aperture->fmm_mutex); - if (mem == NULL) + if (!mem) return NULL; /* Map anonymous pages */ @@ -1166,7 +1167,8 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, /* Mappings in the DGPU aperture don't need to be copied on * fork. This avoids MMU notifiers and evictions due to user - * memory mappings on fork. */ + * memory mappings on fork. + */ madvise(mem, MemorySizeInBytes, MADV_DONTFORK); /* Create userptr BO */ @@ -1181,7 +1183,8 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, aperture_release_area(aperture, mem, size); pthread_mutex_unlock(&aperture->fmm_mutex); /* Remove any CPU mapping, but keep the - * address range reserved */ + * address range reserved + */ mmap(mem, MemorySizeInBytes, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); @@ -1196,7 +1199,7 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, if (mem && flags.ui32.HostAccess) { void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + MAP_SHARED | MAP_FIXED, kfd_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; @@ -1204,10 +1207,11 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, if (flags.ui32.AQLQueueMemory) { uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; + memset(ret, 0, MemorySizeInBytes); mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd , mmap_offset); + MAP_SHARED | MAP_FIXED, kfd_fd, mmap_offset); } } } @@ -1223,7 +1227,7 @@ static void* fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, return mem; } -void* fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, +void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { if (is_dgpu) @@ -1290,7 +1294,7 @@ out: return NULL; } -static void __fmm_release(void *address, manageble_aperture_t *aperture) +static void __fmm_release(void *address, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args; vm_object_t *object; @@ -1310,7 +1314,8 @@ static void __fmm_release(void *address, manageble_aperture_t *aperture) /* If memory is user memory and it's still GPU mapped, munmap * would cause an eviction. If the restore happens quickly * enough, restore would also fail with an error message. So - * free the BO before unmapping the pages. */ + * free the BO before unmapping the pages. + */ args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); @@ -1356,8 +1361,7 @@ void fmm_release(void *address) found = true; __fmm_release(address, &svm.dgpu_aperture); fmm_print(gpu_mem[i].gpu_id); - } - else if (address >= svm.dgpu_alt_aperture.base && + } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { found = true; __fmm_release(address, &svm.dgpu_alt_aperture); @@ -1411,13 +1415,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { struct kfd_ioctl_get_process_apertures_new_args args; uint32_t i = 0; - int32_t gpu_mem_id =0; + int32_t gpu_mem_id = 0; uint32_t gpu_id; HsaNodeProperties props; - struct kfd_process_device_apertures * process_apertures; + struct kfd_process_device_apertures *process_apertures; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; - unsigned guardPages = 1; + unsigned int guardPages = 1; struct pci_access *pacc; /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ @@ -1426,7 +1430,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) disableCache = NULL; /* If HSA_USERPTR_FOR_PAGED_MEM is set to a non-0 value, - * enable userptr for all paged memory allocations */ + * enable userptr for all paged memory allocations + */ pagedUserptr = getenv("HSA_USERPTR_FOR_PAGED_MEM"); svm.userptr_for_paged_mem = (pagedUserptr && strcmp(pagedUserptr, "0")); @@ -1442,15 +1447,17 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) guardPages = 1; /* Trade off - NumNodes includes GPU nodes + CPU Node. So in - * systems with CPU node, slightly more memory is allocated than - * necessary*/ + * systems with CPU node, slightly more memory is allocated than + * necessary + */ gpu_mem = (gpu_mem_t *)calloc(NumNodes, sizeof(gpu_mem_t)); - if (gpu_mem == NULL) + if (!gpu_mem) return HSAKMT_STATUS_NO_MEMORY; - /* Initialize gpu_mem[] from sysfs topology. Rest of the members are set to - * 0 by calloc. This is necessary because this function - * gets called before hsaKmtAcquireSystemProperties() is called.*/ + /* Initialize gpu_mem[] from sysfs topology. Rest of the members are + * set to 0 by calloc. This is necessary because this function + * gets called before hsaKmtAcquireSystemProperties() is called. + */ gpu_mem_count = 0; pacc = pci_alloc(); pci_init(pacc); @@ -1480,11 +1487,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) } pci_cleanup(pacc); - /* The ioctl will also return Number of Nodes if args.kfd_process_device_apertures_ptr - * is set to NULL. This is not required since Number of nodes is already known. Kernel - * will fill in the apertures in kfd_process_device_apertures_ptr */ + /* The ioctl will also return Number of Nodes if + * args.kfd_process_device_apertures_ptr is set to NULL. This is not + * required since Number of nodes is already known. Kernel will fill in + * the apertures in kfd_process_device_apertures_ptr + */ process_apertures = malloc(gpu_mem_count * sizeof(struct kfd_process_device_apertures)); - if (process_apertures == NULL) { + if (!process_apertures) { ret = HSAKMT_STATUS_NO_MEMORY; goto sysfs_parse_failed; } @@ -1501,15 +1510,16 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) all_gpu_id_array = NULL; if (args.num_of_nodes > 0) { all_gpu_id_array = malloc(sizeof(uint32_t) * args.num_of_nodes); - if (all_gpu_id_array == NULL) { + if (!all_gpu_id_array) { ret = HSAKMT_STATUS_NO_MEMORY; goto get_aperture_ioctl_failed; } } for (i = 0 ; i < args.num_of_nodes ; i++) { - /* Map Kernel process device data node i <--> gpu_mem_id which indexes into gpu_mem[] - * based on gpu_id */ + /* Map Kernel process device data node i <--> gpu_mem_id which + * indexes into gpu_mem[] based on gpu_id + */ gpu_mem_id = gpu_mem_find_by_gpu_id(process_apertures[i].gpu_id); if (gpu_mem_id < 0) { ret = HSAKMT_STATUS_ERROR; @@ -1558,13 +1568,15 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) svm.dgpu_aperture.guard_pages = guardPages; /* Non-canonical per-ASIC GPUVM aperture does - * not exist on dGPUs in GPUVM64 address mode */ + * not exist on dGPUs in GPUVM64 address mode + */ gpu_mem[gpu_mem_id].gpuvm_aperture.base = NULL; gpu_mem[gpu_mem_id].gpuvm_aperture.limit = NULL; /* Use the first 1/4 of the dGPU aperture as - * alternate aperture for coherent access. - * Base and size must be 64KB aligned. */ + * alternate aperture for coherent access. + * Base and size must be 64KB aligned. + */ alt_base = (uintptr_t)svm.dgpu_aperture.base; alt_size = (VOID_PTRS_SUB(svm.dgpu_aperture.limit, svm.dgpu_aperture.base) + 1) >> 2; @@ -1580,7 +1592,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) KFD_IOC_CACHE_POLICY_COHERENT, alt_base, alt_size); if (err != 0) { - fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); + fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", + gpu_mem[gpu_mem_id].gpu_id); ret = HSAKMT_STATUS_ERROR; } svm.dgpu_alt_aperture.align = vm_alignment; @@ -1595,7 +1608,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) return ret; get_aperture_ioctl_failed: -invalid_gpu_id : +invalid_gpu_id: free(process_apertures); sysfs_parse_failed: fmm_destroy_process_apertures(); @@ -1647,7 +1660,8 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA case FMM_SVM: /* Report single SVM aperture, starting at base of - * fine-grained, ending at limit of coarse-grained */ + * fine-grained, ending at limit of coarse-grained + */ if (aperture_is_valid(svm.dgpu_alt_aperture.base, svm.dgpu_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(svm.dgpu_alt_aperture.base); @@ -1662,7 +1676,7 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA return err; } -static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, +static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj) { struct kfd_ioctl_map_memory_to_gpu_args args; @@ -1709,7 +1723,7 @@ static int _fmm_map_to_gpu_gtt(manageble_aperture_t *aperture, print_device_id_array((void *)args.device_ids_array_ptr, args.device_ids_array_size); - if (object->mapped_device_id_array != NULL && + if (object->mapped_device_id_array && object->mapped_device_id_array_size > 0 && object->mapped_device_id_array != all_gpu_id_array && object->mapped_device_id_array != object->registered_device_id_array) @@ -1735,7 +1749,7 @@ err_object_not_found: return -1; } -static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *aperture, +static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address, uint64_t size) { int32_t gpu_mem_id; @@ -1750,12 +1764,12 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu if (gpu_mem_id < 0) return -1; - if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) + if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) return 0; /* Nothing to do on APU */ /* sanity check the address */ if (address < aperture->base || - VOID_PTR_ADD(address, size -1) > aperture->limit) + VOID_PTR_ADD(address, size - 1) > aperture->limit) return -1; ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); @@ -1764,7 +1778,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu vm_object_t *obj = fmm_allocate_memory_in_device( gpu_id, address, size, aperture, NULL, KFD_IOC_ALLOC_MEM_FLAGS_VRAM); - if (obj == NULL) + if (!obj) return -1; } else { fmm_allocate_memory_in_device(gpu_id, @@ -1792,7 +1806,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageble_aperture_t *apertu return ret; } -static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, +static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) { @@ -1801,7 +1815,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, void *temp_mapped_id_array = NULL; /* Check that address space was previously reserved */ - if (vm_find(aperture, address) == NULL) + if (!vm_find(aperture, address)) return -1; pthread_mutex_lock(&aperture->fmm_mutex); @@ -1829,7 +1843,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageble_aperture_t *aperture, if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - if (object->mapped_device_id_array != NULL && + if (object->mapped_device_id_array && object->mapped_device_id_array_size > 0 && object->mapped_device_id_array != all_gpu_id_array && object->mapped_device_id_array != object->registered_device_id_array) @@ -1860,7 +1874,7 @@ err_object_not_found: static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, uint64_t *gpuvm_addr, vm_object_t *object) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *obj; void *svm_addr; HSAuint64 svm_size; @@ -1876,7 +1890,7 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, obj = object; if (!obj) { obj = vm_find_object_by_userptr(aperture, addr, size); - if (obj == NULL) { + if (!obj) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } @@ -1885,7 +1899,8 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, svm_size = obj->size; /* Map and return the GPUVM address adjusted by the offset - * from the start of the page */ + * from the start of the page + */ ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size, obj); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -1962,7 +1977,7 @@ static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_ #endif } -static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, +static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, uint32_t *device_ids_array, uint32_t device_ids_array_size, vm_object_t *obj) { @@ -2015,7 +2030,7 @@ static int _fmm_unmap_from_gpu(manageble_aperture_t *aperture, void *address, goto out; /* Clearing all mapped nodes list */ - if (object->mapped_device_id_array != NULL && + if (object->mapped_device_id_array && object->mapped_device_id_array_size > 0 && object->mapped_device_id_array != all_gpu_id_array && object->mapped_device_id_array != object->registered_device_id_array) @@ -2035,7 +2050,7 @@ out: } static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, - manageble_aperture_t *aperture, + manageable_aperture_t *aperture, void *address) { int32_t gpu_mem_id; @@ -2047,7 +2062,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, if (gpu_mem_id < 0) return -1; - if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) + if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) return 0; /* Nothing to do on APU */ pthread_mutex_lock(&aperture->fmm_mutex); @@ -2057,7 +2072,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, if (!object) goto err; - if (object->mapped_device_id_array == NULL || + if (!object->mapped_device_id_array || object->mapped_device_id_array_size == 0) { pthread_mutex_unlock(&aperture->fmm_mutex); return 0; @@ -2071,7 +2086,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); /* Clearing all mapped nodes list */ - if (object->mapped_device_id_array != NULL && + if (object->mapped_device_id_array && object->mapped_device_id_array_size > 0 && object->mapped_device_id_array != all_gpu_id_array && object->mapped_device_id_array != object->registered_device_id_array) @@ -2097,7 +2112,7 @@ err: static int _fmm_unmap_from_gpu_userptr(void *addr) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *obj; void *svm_addr; @@ -2106,7 +2121,7 @@ static int _fmm_unmap_from_gpu_userptr(void *addr) /* Find the start address in SVM space for GPU unmapping */ pthread_mutex_lock(&aperture->fmm_mutex); obj = vm_find_object_by_userptr(aperture, addr, 0); - if (obj == NULL) { + if (!obj) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } @@ -2175,7 +2190,7 @@ static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit) return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE, &args); } -static void *reserve_address(void *addr, long long unsigned int len) +static void *reserve_address(void *addr, unsigned long long int len) { void *ret_addr; @@ -2251,24 +2266,25 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi ret_addr = reserve_address(addr, len); if (!ret_addr) break; - if ((HSAuint64)ret_addr + (len>>1) < max_vm_limit) + if ((HSAuint64)ret_addr + (len >> 1) < max_vm_limit) /* At least half the returned address * space is GPU addressable, we'll - * take it */ + * take it + */ break; - munmap (ret_addr, len); + munmap(ret_addr, len); } if (!ret_addr) { fprintf(stderr, "Failed to reserve %uGB for SVM ...\n", - (unsigned)(len >> 30)); + (unsigned int)(len >> 30)); continue; } if ((HSAuint64)ret_addr + min_vm_size - 1 > max_vm_limit) { /* addressable size is less than the minimum */ fprintf(stderr, "Got %uGB for SVM at %p with only %dGB usable ...\n", - (unsigned)(len >> 30), ret_addr, + (unsigned int)(len >> 30), ret_addr, (int)(((HSAint64)max_vm_limit - (HSAint64)ret_addr) >> 30)); munmap(ret_addr, len); @@ -2306,7 +2322,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi bool fmm_get_handle(void *address, uint64_t *handle) { uint32_t i; - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *object; bool found; @@ -2329,8 +2345,7 @@ bool fmm_get_handle(void *address, uint64_t *handle) if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) { aperture = &svm.dgpu_aperture; - } - else if ((address >= svm.dgpu_alt_aperture.base) && + } else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) { aperture = &svm.dgpu_alt_aperture; } @@ -2372,7 +2387,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj { int32_t i; HSAuint32 gpu_id; - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; void *svm_addr = NULL; vm_object_t *obj; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); @@ -2388,7 +2403,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Check if this address was already registered */ pthread_mutex_lock(&aperture->fmm_mutex); obj = vm_find_object_by_userptr(aperture, addr, size); - if (obj != NULL) { + if (obj) { ++obj->registration_count; pthread_mutex_unlock(&aperture->fmm_mutex); *obj_ret = obj; @@ -2403,7 +2418,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, &obj); - if (svm_addr == NULL) + if (!svm_addr) return HSAKMT_STATUS_ERROR; if (obj) { @@ -2413,8 +2428,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj obj->userptr_size = size; obj->registration_count = 1; pthread_mutex_unlock(&aperture->fmm_mutex); - } - else + } else return HSAKMT_STATUS_ERROR; if (obj_ret) @@ -2426,11 +2440,11 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *object = NULL; HSAKMT_STATUS ret; - if (gpu_id_array_size > 0 && gpu_id_array == NULL) + if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; if ((address >= svm.dgpu_aperture.base) && @@ -2469,8 +2483,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, fprintf(stderr, "Error. Changing nodes in a registered addr.\n"); return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; - } - else + } else return HSAKMT_STATUS_SUCCESS; } @@ -2491,7 +2504,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, struct kfd_ioctl_get_dmabuf_info_args infoArgs; struct kfd_ioctl_import_dmabuf_args importArgs; struct kfd_ioctl_free_memory_of_gpu_args freeArgs; - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *obj; void *metadata; void *mem, *aperture_base; @@ -2500,7 +2513,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, int r; HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; - if (gpu_id_array_size > 0 && gpu_id_array == NULL) + if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; infoArgs.dmabuf_fd = GraphicsResourceHandle; @@ -2541,7 +2554,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, infoArgs.size, offset); pthread_mutex_unlock(&aperture->fmm_mutex); - if (mem == NULL) + if (!mem) goto error_free_metadata; /* Import DMA buffer */ @@ -2569,7 +2582,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, GraphicsResourceInfo->SizeInBytes = infoArgs.size; GraphicsResourceInfo->Metadata = (void *)(unsigned long)infoArgs.metadata_ptr; GraphicsResourceInfo->MetadataSizeInBytes = infoArgs.metadata_size; - GraphicsResourceInfo->Reserved = 0; + GraphicsResourceInfo->Reserved = 0; return HSAKMT_STATUS_SUCCESS; @@ -2584,14 +2597,14 @@ error_free_metadata: return status; } -HSAKMT_STATUS fmm_share_memory(void* MemoryAddress, +HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, HSAuint64 SizeInBytes, HsaSharedMemoryHandle *SharedMemoryHandle) { int r = 0; HSAuint32 gpu_id = 0; vm_object_t *obj = NULL; - manageble_aperture_t * aperture = NULL; + manageable_aperture_t *aperture = NULL; struct kfd_ioctl_ipc_export_handle_args exportArgs; HsaApertureInfo ApeInfo; HsaSharedMemoryStruct *SharedMemoryStruct = @@ -2641,13 +2654,13 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo HSAKMT_STATUS err = HSAKMT_STATUS_ERROR; vm_object_t *obj = NULL; void *reservedMem = NULL; - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; struct kfd_ioctl_ipc_import_handle_args importArgs; struct kfd_ioctl_free_memory_of_gpu_args freeArgs; const HsaSharedMemoryStruct *SharedMemoryStruct = to_const_hsa_shared_memory_struct(SharedMemoryHandle); - if (gpu_id_array_size > 0 && gpu_id_array == NULL) + if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; memcpy(importArgs.share_handle, SharedMemoryStruct->ShareHandle, @@ -2719,7 +2732,7 @@ err_import: static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *obj; void *svm_addr; @@ -2728,7 +2741,7 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) /* Find the size and start address in SVM space */ pthread_mutex_lock(&aperture->fmm_mutex); obj = vm_find_object_by_userptr(aperture, addr, 0); - if ((obj == NULL) || (obj->registration_count > 1)) { + if (!obj || obj->registration_count > 1) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } @@ -2743,9 +2756,9 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) HSAKMT_STATUS fmm_deregister_memory(void *address) { - manageble_aperture_t *aperture = NULL; + manageable_aperture_t *aperture = NULL; vm_object_t *object = NULL; - unsigned i; + unsigned int i; HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); if ((address >= svm.dgpu_aperture.base) && @@ -2832,7 +2845,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint32_t nodes_to_map_size, uint64_t *gpuvm_address) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *object = NULL; uint32_t i, j, temp_node; bool found, userptr = false; @@ -2841,7 +2854,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; - if ((nodes_to_map_size > 0 && nodes_to_map == NULL) || address == NULL) + if ((nodes_to_map_size > 0 && !nodes_to_map) || !address) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -2884,7 +2897,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, registered_node_id_array = all_gpu_id_array; registered_node_id_array_size = all_gpu_id_array_size; if (object->registered_device_id_array_size > 0 && - object->registered_device_id_array != NULL) { + object->registered_device_id_array) { registered_node_id_array = object->registered_device_id_array; registered_node_id_array_size = object->registered_device_id_array_size; } @@ -2959,7 +2972,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; uint32_t i; - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *vm_obj; memset(info, 0, sizeof(HsaPointerInfo)); @@ -2996,7 +3009,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) * freed whenever the registration is deregistered or the * memory being freed */ - for (i=0; iNRegisteredNodes; i++) + for (i = 0; i < info->NRegisteredNodes; i++) gpuid_to_nodeid(vm_obj->registered_device_id_array[i], &vm_obj->registered_node_id_array[i]); } @@ -3010,7 +3023,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) /* vm_obj->mapped_node_id_array allocated here will be * freed whenever the mapping is unmapped or memory being freed */ - for (i=0; iNMappedNodes; i++) + for (i = 0; i < info->NMappedNodes; i++) gpuid_to_nodeid(vm_obj->mapped_device_id_array[i], &vm_obj->mapped_node_id_array[i]); } @@ -3020,9 +3033,8 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) if (info->Type == HSA_POINTER_REGISTERED_USER) { info->CPUAddress = vm_obj->userptr; info->SizeInBytes = vm_obj->userptr_size; - info->GPUAddress += ((HSAuint64)info->CPUAddress & (PAGE_SIZE-1)); - } - else if (info->Type == HSA_POINTER_ALLOCATED) { + info->GPUAddress += ((HSAuint64)info->CPUAddress & (PAGE_SIZE - 1)); + } else if (info->Type == HSA_POINTER_ALLOCATED) { info->MemFlags.Value = vm_obj->flags; info->CPUAddress = vm_obj->start; } @@ -3033,7 +3045,7 @@ exit: HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) { - manageble_aperture_t *aperture; + manageable_aperture_t *aperture; vm_object_t *vm_obj; aperture = fmm_find_aperture(mem, NULL); @@ -3048,7 +3060,7 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) return HSAKMT_STATUS_SUCCESS; } -static void fmm_clear_aperture(manageble_aperture_t *app) +static void fmm_clear_aperture(manageable_aperture_t *app) { while (app->vm_objects) vm_remove_object(app, app->vm_objects); diff --git a/src/fmm.h b/src/fmm.h index 2fb1d60cf9..f5b65e4209 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -41,27 +41,25 @@ typedef enum { typedef struct { aperture_type_e app_type; uint64_t size; - void* start_address; + void *start_address; } aperture_properties_t; HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes); void fmm_destroy_process_apertures(void); -/* - * Memory interface - */ -void* fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); -void* fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); -void* fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset); -void* fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, +/* Memory interface */ +void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); +void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); +void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset); +void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); -void* fmm_open_graphic_handle(uint32_t gpu_id, - int32_t graphic_device_handle, - uint32_t graphic_handle, - uint64_t MemorySizeInBytes); +void *fmm_open_graphic_handle(uint32_t gpu_id, + int32_t graphic_device_handle, + uint32_t graphic_handle, + uint64_t MemorySizeInBytes); void fmm_print(uint32_t node); -bool fmm_is_inside_some_aperture(void* address); -void fmm_release(void* address); +bool fmm_is_inside_some_aperture(void *address); +void fmm_release(void *address); int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); int fmm_unmap_from_gpu(void *address); bool fmm_get_handle(void *address, uint64_t *handle); @@ -75,14 +73,14 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA HSAuint64 *aperture_base, HSAuint64 *aperture_limit); HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, - uint32_t *gpu_id_array, - uint32_t gpu_id_array_size); + uint32_t *gpu_id_array, + uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo *GraphicsResourceInfo, uint32_t *gpu_id_array, uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_deregister_memory(void *address); -HSAKMT_STATUS fmm_share_memory(void* MemoryAddress, +HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, HSAuint64 SizeInBytes, HsaSharedMemoryHandle *SharedMemoryHandle); HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemoryHandle, diff --git a/src/globals.c b/src/globals.c index 15d6f11456..3d75b9f1b7 100644 --- a/src/globals.c +++ b/src/globals.c @@ -31,6 +31,6 @@ int kfd_fd; unsigned long kfd_open_count; unsigned long system_properties_count; pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; -bool is_dgpu = false; +bool is_dgpu; int PAGE_SIZE; int PAGE_SHIFT; diff --git a/src/libhsakmt.c b/src/libhsakmt.c index d7f79d3562..4d0775ccbe 100644 --- a/src/libhsakmt.c +++ b/src/libhsakmt.c @@ -3,16 +3,14 @@ #include "libhsakmt.h" -/** - * Call ioctl, restarting if it is interupted - */ -int -kmtIoctl(int fd, unsigned long request, void *arg) +/* Call ioctl, restarting if it is interrupted */ +int kmtIoctl(int fd, unsigned long request, void *arg) { - int ret; + int ret; - do { - ret = ioctl(fd, request, arg); - } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); - return ret; + do { + ret = ioctl(fd, request, arg); + } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); + + return ret; } diff --git a/src/memory.c b/src/memory.c index b3883c1477..00aff3cfe3 100644 --- a/src/memory.c +++ b/src/memory.c @@ -34,15 +34,11 @@ #include #include "fmm.h" -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetMemoryPolicy( - HSAuint32 Node, - HSAuint32 DefaultPolicy, - HSAuint32 AlternatePolicy, - void *MemoryAddressAlternate, - HSAuint64 MemorySizeInBytes -) +HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, + HSAuint32 DefaultPolicy, + HSAuint32 AlternatePolicy, + void *MemoryAddressAlternate, + HSAuint64 MemorySizeInBytes) { struct kfd_ioctl_set_memory_policy_args args; HSAKMT_STATUS result; @@ -54,7 +50,8 @@ hsaKmtSetMemoryPolicy( /* This is a legacy API useful on Kaveri only. On dGPU * the alternate aperture is setup and used * automatically for coherent allocations. Don't let - * app override it. */ + * app override it. + */ return HSAKMT_STATUS_NOT_IMPLEMENTED; result = validate_nodeid(Node, &gpu_id); @@ -106,14 +103,10 @@ HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags) } } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtAllocMemory( - HSAuint32 PreferredNode, /* IN */ - HSAuint64 SizeInBytes, /* IN (multiple of page size) */ - HsaMemFlags MemFlags, /* IN */ - void **MemoryAddress /* OUT (page-aligned) */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, + HSAuint64 SizeInBytes, + HsaMemFlags MemFlags, + void **MemoryAddress) { HSAKMT_STATUS result; uint32_t gpu_id; @@ -127,16 +120,14 @@ hsaKmtAllocMemory( page_size = PageSizeFromFlags(MemFlags.ui32.PageSize); - if ((!MemoryAddress) || (!SizeInBytes) || - (SizeInBytes & (page_size-1))) { + if (!MemoryAddress || !SizeInBytes || (SizeInBytes & (page_size-1))) return HSAKMT_STATUS_INVALID_PARAMETER; - } if (gpu_id == 0 && !MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, MemFlags); - if (*MemoryAddress == NULL) + if (!(*MemoryAddress)) return HSAKMT_STATUS_ERROR; return HSAKMT_STATUS_SUCCESS; @@ -145,27 +136,28 @@ hsaKmtAllocMemory( if (gpu_id && MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes, MemFlags); - if (*MemoryAddress == NULL) + if (!(*MemoryAddress)) return HSAKMT_STATUS_NO_MEMORY; return HSAKMT_STATUS_SUCCESS; } - if (MemFlags.ui32.Scratch ) { + if (MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_scratch(gpu_id, SizeInBytes); - if (*MemoryAddress == NULL) + if (!(*MemoryAddress)) return HSAKMT_STATUS_NO_MEMORY; return HSAKMT_STATUS_SUCCESS; } /* Backwards compatibility hack: Allocate system memory if app - * asks for paged memory from a GPU node. */ + * asks for paged memory from a GPU node. + */ if (gpu_id && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, MemFlags); - if (*MemoryAddress == NULL) + if (!(*MemoryAddress)) return HSAKMT_STATUS_ERROR; return HSAKMT_STATUS_SUCCESS; @@ -174,16 +166,12 @@ hsaKmtAllocMemory( return HSAKMT_STATUS_INVALID_PARAMETER; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtFreeMemory( - void *MemoryAddress, /* IN (page-aligned) */ - HSAuint64 SizeInBytes /* IN */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, + HSAuint64 SizeInBytes) { CHECK_KFD_OPEN(); - if (MemoryAddress == NULL) { + if (!MemoryAddress) { fprintf(stderr, "FIXME: freeing NULL pointer\n"); return HSAKMT_STATUS_ERROR; } @@ -192,12 +180,8 @@ hsaKmtFreeMemory( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtRegisterMemory( - void *MemoryAddress, /* IN (cache-aligned) */ - HSAuint64 MemorySizeInBytes /* IN (cache-aligned) */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress, + HSAuint64 MemorySizeInBytes) { CHECK_KFD_OPEN(); @@ -206,17 +190,13 @@ hsaKmtRegisterMemory( return HSAKMT_STATUS_SUCCESS; return fmm_register_memory(MemoryAddress, MemorySizeInBytes, - NULL, 0); + NULL, 0); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtRegisterMemoryToNodes( - void *MemoryAddress, /* IN (cache-aligned) */ - HSAuint64 MemorySizeInBytes, /* IN (cache-aligned) */ - HSAuint64 NumberOfNodes, /* IN */ - HSAuint32* NodeArray /* IN */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + HSAuint64 NumberOfNodes, + HSAuint32 *NodeArray) { CHECK_KFD_OPEN(); uint32_t *gpu_id_array; @@ -240,14 +220,10 @@ hsaKmtRegisterMemoryToNodes( return ret; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtRegisterGraphicsHandleToNodes( - HSAuint64 GraphicsResourceHandle, /* IN */ - HsaGraphicsResourceInfo *GraphicsResourceInfo, /* OUT */ - HSAuint64 NumberOfNodes, /* IN */ - HSAuint32* NodeArray /* IN */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes(HSAuint64 GraphicsResourceHandle, + HsaGraphicsResourceInfo *GraphicsResourceInfo, + HSAuint64 NumberOfNodes, + HSAuint32 *NodeArray) { CHECK_KFD_OPEN(); uint32_t *gpu_id_array; @@ -259,7 +235,7 @@ hsaKmtRegisterGraphicsHandleToNodes( if (ret == HSAKMT_STATUS_SUCCESS) { ret = fmm_register_graphics_handle( GraphicsResourceHandle, GraphicsResourceInfo, - gpu_id_array,NumberOfNodes*sizeof(uint32_t)); + gpu_id_array, NumberOfNodes * sizeof(uint32_t)); if (ret != HSAKMT_STATUS_SUCCESS) free(gpu_id_array); } @@ -267,13 +243,9 @@ hsaKmtRegisterGraphicsHandleToNodes( return ret; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtShareMemory( - void *MemoryAddress, /* IN */ - HSAuint64 SizeInBytes, /* IN */ - HsaSharedMemoryHandle *SharedMemoryHandle /* OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtShareMemory(void *MemoryAddress, + HSAuint64 SizeInBytes, + HsaSharedMemoryHandle *SharedMemoryHandle) { CHECK_KFD_OPEN(); @@ -283,13 +255,9 @@ hsaKmtShareMemory( return fmm_share_memory(MemoryAddress, SizeInBytes, SharedMemoryHandle); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtRegisterSharedHandle( - const HsaSharedMemoryHandle *SharedMemoryHandle, /* IN */ - void **MemoryAddress, /* OUT */ - HSAuint64 *SizeInBytes /* OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandle(const HsaSharedMemoryHandle *SharedMemoryHandle, + void **MemoryAddress, + HSAuint64 *SizeInBytes) { CHECK_KFD_OPEN(); @@ -300,15 +268,11 @@ hsaKmtRegisterSharedHandle( NULL); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtRegisterSharedHandleToNodes( - const HsaSharedMemoryHandle *SharedMemoryHandle, /* IN */ - void **MemoryAddress, /* OUT */ - HSAuint64 *SizeInBytes, /* OUT */ - HSAuint64 NumberOfNodes, /* OUT */ - HSAuint32* NodeArray /* OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandleToNodes(const HsaSharedMemoryHandle *SharedMemoryHandle, + void **MemoryAddress, + HSAuint64 *SizeInBytes, + HSAuint64 NumberOfNodes, + HSAuint32 *NodeArray) { CHECK_KFD_OPEN(); @@ -340,32 +304,26 @@ error: return ret; } -static uint64_t convertHsaToKfdRange( - HsaMemoryRange *HsaRange) +static uint64_t convertHsaToKfdRange(HsaMemoryRange *HsaRange) { if (sizeof(struct kfd_memory_range) != sizeof(HsaMemoryRange)) { - fprintf(stderr, "Struct size mismatch in thunk." - "Cannot cast Hsa Range to KFD IOCTL range\n"); + fprintf(stderr, "Struct size mismatch in thunk. Cannot cast Hsa Range to KFD IOCTL range\n"); return 0; } return (uint64_t) HsaRange; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtProcessVMRead( - HSAuint32 Pid, /* IN */ - HsaMemoryRange *LocalMemoryArray, /* IN */ - HSAuint64 LocalMemoryArrayCount, /* IN */ - HsaMemoryRange *RemoteMemoryArray, /* IN */ - HSAuint64 RemoteMemoryArrayCount,/* IN */ - HSAuint64 *SizeCopied /* OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, + HsaMemoryRange *LocalMemoryArray, + HSAuint64 LocalMemoryArrayCount, + HsaMemoryRange *RemoteMemoryArray, + HSAuint64 RemoteMemoryArrayCount, + HSAuint64 *SizeCopied) { struct kfd_ioctl_cross_memory_copy_args args; - if (LocalMemoryArray == NULL || RemoteMemoryArray == NULL || + if (!LocalMemoryArray || !RemoteMemoryArray || LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) return HSAKMT_STATUS_ERROR; @@ -379,6 +337,7 @@ hsaKmtProcessVMRead( args.bytes_copied = 0; int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args); + if (err) return HSAKMT_STATUS_ERROR; @@ -388,20 +347,16 @@ hsaKmtProcessVMRead( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtProcessVMWrite( - HSAuint32 Pid, /* IN */ - HsaMemoryRange *LocalMemoryArray, /* IN */ - HSAuint64 LocalMemoryArrayCount, /* IN */ - HsaMemoryRange *RemoteMemoryArray, /* IN */ - HSAuint64 RemoteMemoryArrayCount, /* IN */ - HSAuint64 *SizeCopied /* OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, + HsaMemoryRange *LocalMemoryArray, + HSAuint64 LocalMemoryArrayCount, + HsaMemoryRange *RemoteMemoryArray, + HSAuint64 RemoteMemoryArrayCount, + HSAuint64 *SizeCopied) { struct kfd_ioctl_cross_memory_copy_args args; - if (LocalMemoryArray == NULL || RemoteMemoryArray == NULL || + if (!LocalMemoryArray || !RemoteMemoryArray || LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) return HSAKMT_STATUS_ERROR; @@ -415,6 +370,7 @@ hsaKmtProcessVMWrite( args.bytes_copied = 0; int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args); + if (err) return HSAKMT_STATUS_ERROR; @@ -425,28 +381,20 @@ hsaKmtProcessVMWrite( } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDeregisterMemory( - void *MemoryAddress /* IN */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory(void *MemoryAddress) { CHECK_KFD_OPEN(); return fmm_deregister_memory(MemoryAddress); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtMapMemoryToGPU( - void *MemoryAddress, /* IN (page-aligned) */ - HSAuint64 MemorySizeInBytes, /* IN (page-aligned) */ - HSAuint64 *AlternateVAGPU /* OUT (page-aligned) */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + HSAuint64 *AlternateVAGPU) { CHECK_KFD_OPEN(); - if (MemoryAddress == NULL) { + if (!MemoryAddress) { fprintf(stderr, "FIXME: mapping NULL pointer\n"); return HSAKMT_STATUS_ERROR; } @@ -460,21 +408,17 @@ hsaKmtMapMemoryToGPU( return HSAKMT_STATUS_ERROR; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtMapMemoryToGPUNodes( - void* MemoryAddress, //IN (page-aligned) - HSAuint64 MemorySizeInBytes, //IN (page-aligned) - HSAuint64* AlternateVAGPU, //OUT (page-aligned) - HsaMemMapFlags MemMapFlags, //IN - HSAuint64 NumberOfNodes, //IN - HSAuint32* NodeArray //IN -) +HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + HSAuint64 *AlternateVAGPU, + HsaMemMapFlags MemMapFlags, + HSAuint64 NumberOfNodes, + HSAuint32 *NodeArray) { uint32_t *gpu_id_array; HSAKMT_STATUS ret; - if (MemoryAddress == NULL) { + if (!MemoryAddress) { fprintf(stderr, "FIXME: mapping NULL pointer\n"); return HSAKMT_STATUS_ERROR; } @@ -493,15 +437,11 @@ hsaKmtMapMemoryToGPUNodes( gpu_id_array, NumberOfNodes * sizeof(uint32_t), AlternateVAGPU); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtUnmapMemoryToGPU( - void *MemoryAddress /* IN (page-aligned) */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) { CHECK_KFD_OPEN(); - if (MemoryAddress == NULL) { + if (!MemoryAddress) { /* Workaround for runtime bug */ fprintf(stderr, "FIXME: Unmapping NULL pointer\n"); return HSAKMT_STATUS_SUCCESS; @@ -513,16 +453,12 @@ hsaKmtUnmapMemoryToGPU( return HSAKMT_STATUS_ERROR; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtMapGraphicHandle( - HSAuint32 NodeId, /* IN */ - HSAuint64 GraphicDeviceHandle, /* IN */ - HSAuint64 GraphicResourceHandle, /* IN */ - HSAuint64 GraphicResourceOffset, /* IN */ - HSAuint64 GraphicResourceSize, /* IN */ - HSAuint64 *FlatMemoryAddress /* OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtMapGraphicHandle(HSAuint32 NodeId, + HSAuint64 GraphicDeviceHandle, + HSAuint64 GraphicResourceHandle, + HSAuint64 GraphicResourceOffset, + HSAuint64 GraphicResourceSize, + HSAuint64 *FlatMemoryAddress) { CHECK_KFD_OPEN(); @@ -550,24 +486,15 @@ hsaKmtMapGraphicHandle( return HSAKMT_STATUS_NO_MEMORY; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtUnmapGraphicHandle( - HSAuint32 NodeId, /* IN */ - HSAuint64 FlatMemoryAddress, /* IN */ - HSAuint64 SizeInBytes /* IN */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapGraphicHandle(HSAuint32 NodeId, + HSAuint64 FlatMemoryAddress, + HSAuint64 SizeInBytes) { return hsaKmtUnmapMemoryToGPU(PORT_UINT64_TO_VPTR(FlatMemoryAddress)); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetTileConfig( - HSAuint32 NodeId, /* IN */ - HsaGpuTileConfig *config /* IN & OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig *config) { struct kfd_ioctl_get_tile_config_args args; uint32_t gpu_id; @@ -583,9 +510,8 @@ hsaKmtGetTileConfig( args.num_tile_configs = config->NumTileConfigs; args.num_macro_tile_configs = config->NumMacroTileConfigs; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_TILE_CONFIG, &args) != 0) { + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_TILE_CONFIG, &args) != 0) return HSAKMT_STATUS_ERROR; - } config->NumTileConfigs = args.num_tile_configs; config->NumMacroTileConfigs = args.num_macro_tile_configs; @@ -598,24 +524,16 @@ hsaKmtGetTileConfig( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueryPointerInfo( - const void *Pointer, /* IN */ - HsaPointerInfo *PointerInfo /* OUT */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtQueryPointerInfo(const void *Pointer, + HsaPointerInfo *PointerInfo) { if (!PointerInfo) return HSAKMT_STATUS_INVALID_PARAMETER; return fmm_get_mem_info(Pointer, PointerInfo); } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetMemoryUserData( - const void *Pointer, /* IN */ - void *UserData /* IN */ -) +HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryUserData(const void *Pointer, + void *UserData) { return fmm_set_mem_user_data(Pointer, UserData); } diff --git a/src/openclose.c b/src/openclose.c index 087fbdc36d..5e8b5e1aa4 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -36,7 +36,7 @@ static const char kfd_device_name[] = "/dev/kfd"; static const char tmp_file[] = "/var/lock/.amd_hsa_thunk_lock"; -int amd_hsa_thunk_lock_fd = 0; +int amd_hsa_thunk_lock_fd; static pid_t parent_pid = -1; @@ -76,9 +76,7 @@ static inline void init_page_size(void) PAGE_SHIFT = ffs(PAGE_SIZE) - 1; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtOpenKFD(void) +HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) { HSAKMT_STATUS result; int fd; @@ -94,8 +92,9 @@ hsaKmtOpenKFD(void) if (is_forked_child()) clear_after_fork(); - if (kfd_open_count == 0) - { + if (kfd_open_count == 0) { + amd_hsa_thunk_lock_fd = 0; + fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); if (fd != -1) { @@ -125,7 +124,7 @@ hsaKmtOpenKFD(void) mask = umask(0); /* save the current umask */ /* We don't want the existing umask to mask out S_IWOTH */ - umask(S_IXOTH); + umask(0001); amd_hsa_thunk_lock_fd = open(tmp_file, O_CREAT | O_RDWR, 0666); @@ -136,9 +135,7 @@ hsaKmtOpenKFD(void) if (init_counter_props(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) fprintf(stderr, "Profiling is not available\n"); - } - else - { + } else { kfd_open_count++; result = HSAKMT_STATUS_SUCCESS; } @@ -157,18 +154,14 @@ open_failed: return result; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtCloseKFD(void) +HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void) { HSAKMT_STATUS result; pthread_mutex_lock(&hsakmt_mutex); - if (kfd_open_count > 0) - { - if (--kfd_open_count == 0) - { + if (kfd_open_count > 0) { + if (--kfd_open_count == 0) { destroy_counter_props(); destroy_device_debugging_memory(); destroy_process_doorbells(); @@ -183,11 +176,8 @@ hsaKmtCloseKFD(void) } result = HSAKMT_STATUS_SUCCESS; - } - else - { + } else result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; - } pthread_mutex_unlock(&hsakmt_mutex); diff --git a/src/perfctr.c b/src/perfctr.c index 5267ac1dbe..962fe56a7e 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -96,7 +96,7 @@ static ssize_t readn(int fd, void *buf, size_t n) bytes = read(fd, buf, left); if (!bytes) /* reach EOF */ return (n - left); - if (bytes < 0 ) { + if (bytes < 0) { if (errno == EINTR) /* read got interrupted */ continue; else @@ -145,8 +145,8 @@ out: HSAKMT_STATUS init_counter_props(unsigned int NumNodes) { - counter_props = calloc(NumNodes, sizeof(struct HsaCounterProperties*)); - if (counter_props == NULL) + counter_props = calloc(NumNodes, sizeof(struct HsaCounterProperties *)); + if (!counter_props) return HSAKMT_STATUS_NO_MEMORY; counter_props_count = NumNodes; @@ -159,11 +159,11 @@ void destroy_counter_props(void) { unsigned int i; - if (counter_props == NULL) + if (!counter_props) return; - for (i = 0; iNumBlocks = total_blocks; @@ -534,18 +530,11 @@ hsaKmtPmcGetCounterProperties( return HSAKMT_STATUS_SUCCESS; } -/** - Registers a set of (HW) counters to be used for tracing/profiling -*/ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtPmcRegisterTrace( - HSAuint32 NodeId, //IN - HSAuint32 NumberOfCounters, //IN - HsaCounter* Counters, //IN - HsaPmcTraceRoot* TraceRoot //OUT - ) +/* Registers a set of (HW) counters to be used for tracing/profiling */ +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace(HSAuint32 NodeId, + HSAuint32 NumberOfCounters, + HsaCounter *Counters, + HsaPmcTraceRoot *TraceRoot) { uint32_t gpu_id, i, j; uint64_t min_buf_size = 0; @@ -558,10 +547,10 @@ hsaKmtPmcRegisterTrace( uint64_t *counter_id_ptr; int *fd_ptr; - if (counter_props == NULL) + if (!counter_props) return HSAKMT_STATUS_NO_MEMORY; - if (Counters == NULL || TraceRoot == NULL || NumberOfCounters == 0) + if (!Counters || !TraceRoot || NumberOfCounters == 0) return HSAKMT_STATUS_INVALID_PARAMETER; if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) @@ -618,7 +607,7 @@ hsaKmtPmcRegisterTrace( + sizeof(uint64_t) * total_counters + sizeof(int) * total_counters, 1); - if (trace == NULL) + if (!trace) return HSAKMT_STATUS_NO_MEMORY; /* Allocated area is partitioned as: @@ -677,16 +666,10 @@ hsaKmtPmcRegisterTrace( return HSAKMT_STATUS_SUCCESS; } -/** - Unregisters a set of (HW) counters used for tracing/profiling -*/ +/* Unregisters a set of (HW) counters used for tracing/profiling */ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtPmcUnregisterTrace( - HSAuint32 NodeId, //IN - HSATraceId TraceId //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcUnregisterTrace(HSAuint32 NodeId, + HSATraceId TraceId) { uint32_t gpu_id; struct perf_trace *trace; @@ -708,6 +691,7 @@ hsaKmtPmcUnregisterTrace( /* If the trace is in the running state, stop it */ if (trace->state == PERF_TRACE_STATE__STARTED) { HSAKMT_STATUS status = hsaKmtPmcStopTrace(TraceId); + if (status != HSAKMT_STATUS_SUCCESS) return status; } @@ -717,12 +701,8 @@ hsaKmtPmcUnregisterTrace( return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtPmcAcquireTraceAccess( - HSAuint32 NodeId, //IN - HSATraceId TraceId //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess(HSAuint32 NodeId, + HSATraceId TraceId) { struct perf_trace *trace; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -766,12 +746,8 @@ out: return ret; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtPmcReleaseTraceAccess( - HSAuint32 NodeId, //IN - HSATraceId TraceId //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess(HSAuint32 NodeId, + HSATraceId TraceId) { struct perf_trace *trace; uint32_t i; @@ -795,17 +771,10 @@ hsaKmtPmcReleaseTraceAccess( } -/** - Starts tracing operation on a previously established set of performance counters -*/ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtPmcStartTrace( - HSATraceId TraceId, //IN - void* TraceBuffer, //IN (page aligned) - HSAuint64 TraceBufferSizeBytes //IN (page aligned) - ) +/* Starts tracing operation on a previously established set of performance counters */ +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStartTrace(HSATraceId TraceId, + void *TraceBuffer, + HSAuint64 TraceBufferSizeBytes) { struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); @@ -813,7 +782,7 @@ hsaKmtPmcStartTrace( int32_t j; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - if (TraceId == 0 || TraceBuffer == NULL || TraceBufferSizeBytes == 0) + if (TraceId == 0 || !TraceBuffer || TraceBufferSizeBytes == 0) return HSAKMT_STATUS_INVALID_PARAMETER; if (trace->magic4cc != HSA_PERF_MAGIC4CC) @@ -842,15 +811,9 @@ hsaKmtPmcStartTrace( } -/** - Forces an update of all the counters that a previously started trace operation has registered -*/ +/*Forces an update of all the counters that a previously started trace operation has registered */ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtPmcQueryTrace( - HSATraceId TraceId //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcQueryTrace(HSATraceId TraceId) { struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); @@ -882,15 +845,8 @@ hsaKmtPmcQueryTrace( } -/** - Stops tracing operation on a previously established set of performance counters -*/ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtPmcStopTrace( - HSATraceId TraceId //IN - ) +/* Stops tracing operation on a previously established set of performance counters */ +HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStopTrace(HSATraceId TraceId) { struct perf_trace *trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); diff --git a/src/pmc_table.c b/src/pmc_table.c index 3ca656a223..50ff81a521 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1597,8 +1597,7 @@ uint32_t pmc_table_get_max_concurrent(int block_id) } } -static HSAKMT_STATUS -alloc_pmc_blocks_iommu(void) +static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) { DIR *dir; struct dirent *dent; @@ -1704,10 +1703,9 @@ void free_pmc_blocks(void) iommu_block.num_of_counters = 0; } -HSAKMT_STATUS -get_block_properties(uint32_t node_id, - enum perf_block_id block_id, - struct perf_counter_block *block) +HSAKMT_STATUS get_block_properties(uint32_t node_id, + enum perf_block_id block_id, + struct perf_counter_block *block) { uint16_t dev_id = get_device_id_by_node(node_id); enum asic_family_type asic; diff --git a/src/pmc_table.h b/src/pmc_table.h index aa9d5bd0d1..7e48cbe801 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -72,9 +72,8 @@ HSAKMT_STATUS alloc_pmc_blocks(void); void free_pmc_blocks(void); uint32_t pmc_table_get_max_concurrent(int block_id); -HSAKMT_STATUS -get_block_properties(uint32_t node_id, - enum perf_block_id block_id, - struct perf_counter_block *block); +HSAKMT_STATUS get_block_properties(uint32_t node_id, + enum perf_block_id block_id, + struct perf_counter_block *block); #endif // PMC_TABLE_H diff --git a/src/queues.c b/src/queues.c index 6840a5df1a..7fbd1083c7 100644 --- a/src/queues.c +++ b/src/queues.c @@ -44,8 +44,7 @@ #define WG_CONTEXT_DATA_SIZE_PER_CU_VI 344576 #define WAVES_PER_CU_VI 32 -struct device_info -{ +struct device_info { enum asic_family_type asic_family; uint32_t eop_buffer_size; uint32_t doorbell_size; @@ -110,14 +109,12 @@ static struct device_info *dev_lookup_table[] = { [CHIP_VEGA10] = &vega10_device_info }; -struct device_id -{ +struct device_id { uint16_t dev_id; struct device_info *dev_info; }; -struct queue -{ +struct queue { uint32_t queue_id; uint64_t wptr; uint64_t rptr; @@ -128,11 +125,10 @@ struct queue const struct device_info *dev_info; }; -struct process_doorbells -{ +struct process_doorbells { bool use_gpuvm; uint32_t size; - void* doorbells; + void *doorbells; pthread_mutex_t doorbells_mutex; }; @@ -145,9 +141,10 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; /* doorbells[] is accessed using Topology NodeId. This means doorbells[0], - * which corresponds to CPU only Node, might not be used */ + * which corresponds to CPU only Node, might not be used + */ doorbells = malloc(NumNodes * sizeof(struct process_doorbells)); - if (doorbells == NULL) + if (!doorbells) return HSAKMT_STATUS_NO_MEMORY; for (i = 0; i < NumNodes; i++) { @@ -258,7 +255,7 @@ static HSAKMT_STATUS map_doorbell_dgpu(HSAuint32 NodeId, HSAuint32 gpu_id, ptr = fmm_allocate_doorbell(gpu_id, doorbells[NodeId].size, doorbell_offset); - if (ptr == NULL) + if (!ptr) return HSAKMT_STATUS_ERROR; /* map for GPU access */ @@ -299,7 +296,7 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, return status; } -static void* allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) +static void *allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) { void *ptr; int retval; @@ -344,7 +341,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) return false; } -void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, +void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId, bool nonPaged) { void *mem; @@ -361,14 +358,13 @@ void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, size = ALIGN_UP(size, align); ret = hsaKmtAllocMemory(0, size, flags, &mem); - if (ret != HSAKMT_STATUS_SUCCESS) { + if (ret != HSAKMT_STATUS_SUCCESS) return NULL; - } if (NodeId != 0) { uint32_t nodes_array[1] = {NodeId}; - if (hsaKmtRegisterMemoryToNodes(mem, size, 1, nodes_array) - != HSAKMT_STATUS_SUCCESS) { + + if (hsaKmtRegisterMemoryToNodes(mem, size, 1, nodes_array) != HSAKMT_STATUS_SUCCESS) { hsaKmtFreeMemory(mem, size); return NULL; } @@ -386,15 +382,14 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) { size = ALIGN_UP(size, align); - if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) { + if (hsaKmtUnmapMemoryToGPU(addr) == HSAKMT_STATUS_SUCCESS) hsaKmtFreeMemory(addr, size); - } } -static void* allocate_exec_aligned_memory(uint32_t size, - uint32_t align, - enum asic_family_type type, - uint32_t NodeId) +static void *allocate_exec_aligned_memory(uint32_t size, + uint32_t align, + enum asic_family_type type, + uint32_t NodeId) { if (IS_DGPU(type)) return allocate_exec_aligned_memory_gpu(size, align, NodeId, @@ -429,6 +424,7 @@ static int handle_concrete_asic(struct queue *q, uint32_t NodeId) { const struct device_info *dev_info = q->dev_info; + if (dev_info) { if (dev_info->eop_buffer_size > 0) { q->eop_buffer = @@ -436,9 +432,9 @@ static int handle_concrete_asic(struct queue *q, PAGE_SIZE, dev_info->asic_family, NodeId); - if (q->eop_buffer == NULL) { + if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; - } + args->eop_buffer_address = (uintptr_t)q->eop_buffer; args->eop_buffer_size = dev_info->eop_buffer_size; } @@ -451,9 +447,9 @@ static int handle_concrete_asic(struct queue *q, PAGE_SIZE, dev_info->asic_family, NodeId); - if (q->ctx_save_restore == NULL) {; + if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; - } + args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; } } @@ -465,31 +461,28 @@ static int handle_concrete_asic(struct queue *q, * to KFD queue priority (0 to 15) * Indexed by thunk_queue_priority+3 */ -static uint32_t priority_map[] = {0,3,5,7,9,11,15}; +static uint32_t priority_map[] = {0, 3, 5, 7, 9, 11, 15}; -HSAKMT_STATUS -HSAKMTAPI -hsaKmtCreateQueue( - HSAuint32 NodeId, //IN - HSA_QUEUE_TYPE Type, //IN - HSAuint32 QueuePercentage, //IN - HSA_QUEUE_PRIORITY Priority, //IN - void* QueueAddress, //IN - HSAuint64 QueueSizeInBytes, //IN - HsaEvent* Event, //IN - HsaQueueResource* QueueResource //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, + HSA_QUEUE_TYPE Type, + HSAuint32 QueuePercentage, + HSA_QUEUE_PRIORITY Priority, + void *QueueAddress, + HSAuint64 QueueSizeInBytes, + HsaEvent *Event, + HsaQueueResource *QueueResource) { HSAKMT_STATUS result; uint32_t gpu_id; uint16_t dev_id; uint64_t doorbell_mmap_offset; - unsigned doorbell_offset; + unsigned int doorbell_offset; struct device_info *dev_info; int err; + CHECK_KFD_OPEN(); - if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || + if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || Priority > HSA_QUEUE_PRIORITY_MAXIMUM) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -500,31 +493,37 @@ hsaKmtCreateQueue( dev_id = get_device_id_by_node(NodeId); dev_info = get_device_info_by_dev_id(dev_id); - struct queue *q = allocate_exec_aligned_memory(sizeof (*q), + struct queue *q = allocate_exec_aligned_memory(sizeof(*q), PAGE_SIZE, dev_info->asic_family, NodeId); - if (q == NULL) + if (!q) return HSAKMT_STATUS_NO_MEMORY; memset(q, 0, sizeof(*q)); struct kfd_ioctl_create_queue_args args; + memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; q->dev_info = dev_info; - switch (Type) - { - case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; break; - case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; - case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; - default: return HSAKMT_STATUS_INVALID_PARAMETER; + switch (Type) { + case HSA_QUEUE_COMPUTE: + args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; + break; + case HSA_QUEUE_SDMA: + args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; + break; + case HSA_QUEUE_COMPUTE_AQL: + args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; + break; + default: + return HSAKMT_STATUS_INVALID_PARAMETER; } - if (Type != HSA_QUEUE_COMPUTE_AQL) - { + if (Type != HSA_QUEUE_COMPUTE_AQL) { QueueResource->QueueRptrValue = (uintptr_t)&q->rptr; QueueResource->QueueWptrValue = (uintptr_t)&q->wptr; } @@ -545,8 +544,7 @@ hsaKmtCreateQueue( err = kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_QUEUE, &args); - if (err == -1) - { + if (err == -1) { free_queue(q); return HSAKMT_STATUS_ERROR; } @@ -558,14 +556,16 @@ hsaKmtCreateQueue( * doorbell page is included in the doorbell offset * returned by KFD. This allows doorbells to be * allocated per-device, independent of the - * per-process queue ID. */ + * per-process queue ID. + */ doorbell_mmap_offset = args.doorbell_offset & ~(HSAuint64)(doorbells[NodeId].size - 1); doorbell_offset = args.doorbell_offset & (doorbells[NodeId].size - 1); } else { /* On older chips, the doorbell offset within the - * doorbell page is based on the queue ID. */ + * doorbell page is based on the queue ID. + */ doorbell_mmap_offset = args.doorbell_offset; doorbell_offset = q->queue_id * dev_info->doorbell_size; } @@ -585,28 +585,24 @@ hsaKmtCreateQueue( } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtUpdateQueue( - HSA_QUEUEID QueueId, //IN - HSAuint32 QueuePercentage,//IN - HSA_QUEUE_PRIORITY Priority, //IN - void* QueueAddress, //IN - HSAuint64 QueueSize, //IN - HsaEvent* Event //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtUpdateQueue(HSA_QUEUEID QueueId, + HSAuint32 QueuePercentage, + HSA_QUEUE_PRIORITY Priority, + void *QueueAddress, + HSAuint64 QueueSize, + HsaEvent *Event) { struct kfd_ioctl_update_queue_args arg; struct queue *q = PORT_UINT64_TO_VPTR(QueueId); CHECK_KFD_OPEN(); - if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || + if (Priority < HSA_QUEUE_PRIORITY_MINIMUM || Priority > HSA_QUEUE_PRIORITY_MAXIMUM) return HSAKMT_STATUS_INVALID_PARAMETER; - if (q == NULL) - return (HSAKMT_STATUS_INVALID_PARAMETER); + if (!q) + return HSAKMT_STATUS_INVALID_PARAMETER; arg.queue_id = (HSAuint32)q->queue_id; arg.ring_base_address = (uintptr_t)QueueAddress; arg.ring_size = QueueSize; @@ -614,27 +610,22 @@ hsaKmtUpdateQueue( arg.queue_priority = priority_map[Priority+3]; int err = kmtIoctl(kfd_fd, AMDKFD_IOC_UPDATE_QUEUE, &arg); + if (err == -1) - { return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDestroyQueue( - HSA_QUEUEID QueueId //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyQueue(HSA_QUEUEID QueueId) { CHECK_KFD_OPEN(); struct queue *q = PORT_UINT64_TO_VPTR(QueueId); struct kfd_ioctl_destroy_queue_args args; - if (q == NULL) - return (HSAKMT_STATUS_INVALID_PARAMETER); + if (!q) + return HSAKMT_STATUS_INVALID_PARAMETER; memset(&args, 0, sizeof(args)); @@ -643,30 +634,22 @@ hsaKmtDestroyQueue( int err = kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_QUEUE, &args); if (err == -1) - { return HSAKMT_STATUS_ERROR; - } - else - { - free_queue(q); - return HSAKMT_STATUS_SUCCESS; - } + + free_queue(q); + return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetQueueCUMask( - HSA_QUEUEID QueueId, //IN - HSAuint32 CUMaskCount, //IN - HSAuint32* QueueCUMask //IN - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtSetQueueCUMask(HSA_QUEUEID QueueId, + HSAuint32 CUMaskCount, + HSAuint32 *QueueCUMask) { struct queue *q = PORT_UINT64_TO_VPTR(QueueId); struct kfd_ioctl_set_cu_mask_args args; CHECK_KFD_OPEN(); - if (CUMaskCount == 0 || QueueCUMask == NULL || ((CUMaskCount % 32) != 0)) + if (CUMaskCount == 0 || !QueueCUMask || ((CUMaskCount % 32) != 0)) return HSAKMT_STATUS_INVALID_PARAMETER; memset(&args, 0, sizeof(args)); @@ -675,23 +658,18 @@ hsaKmtSetQueueCUMask( args.cu_mask_ptr = (uintptr_t)QueueCUMask; int err = kmtIoctl(kfd_fd, AMDKFD_IOC_SET_CU_MASK, &args); + if (err == -1) - { return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetTrapHandler( - HSAuint32 Node, - void *TrapHandlerBaseAddress, - HSAuint64 TrapHandlerSizeInBytes, - void *TrapBufferBaseAddress, - HSAuint64 TrapBufferSizeInBytes -) +HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node, + void *TrapHandlerBaseAddress, + HSAuint64 TrapHandlerSizeInBytes, + void *TrapBufferBaseAddress, + HSAuint64 TrapBufferSizeInBytes) { struct kfd_ioctl_set_trap_handler_args args; HSAKMT_STATUS result; diff --git a/src/time.c b/src/time.c index 45709f9bd6..56e5484850 100644 --- a/src/time.c +++ b/src/time.c @@ -26,12 +26,8 @@ #include "libhsakmt.h" #include "linux/kfd_ioctl.h" -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetClockCounters( - HSAuint32 NodeId, //IN - HsaClockCounters* Counters //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtGetClockCounters(HSAuint32 NodeId, + HsaClockCounters *Counters) { HSAKMT_STATUS result; uint32_t gpu_id; diff --git a/src/topology.c b/src/topology.c index 4c0ce1a0b0..1f36a1cac5 100644 --- a/src/topology.c +++ b/src/topology.c @@ -81,7 +81,7 @@ static struct hsa_gfxip_table { unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info unsigned char is_dgpu; // Predicate for dGPU devices - const char* amd_name; // CALName of the device + const char *amd_name; // CALName of the device enum asic_family_type asic_family; } gfxip_lookup_table[] = { /* Kaveri Family */ @@ -226,7 +226,7 @@ free_node(node_t *n) { assert(n); - if (n == NULL) + if (!n) return; if ((n)->mem) @@ -240,6 +240,7 @@ free_node(node_t *n) static void free_nodes(node_t *temp_nodes, int size) { int i; + if (temp_nodes) { for (i = 0; i < size; i++) free_node(&temp_nodes[i]); @@ -261,7 +262,7 @@ static int num_subdirs(char *dirpath, char *prefix) int prefix_len = strlen(prefix); dirp = opendir(dirpath); - if(dirp) { + if (dirp) { while ((dir = readdir(dirp)) != 0) { if ((strcmp(dir->d_name, ".") == 0) || (strcmp(dir->d_name, "..") == 0)) @@ -282,8 +283,8 @@ static int num_subdirs(char *dirpath, char *prefix) * to the EAX, EBX, ECX, and EDX registers, as determined by input entered in * EAX (in some cases, ECX as well). */ -static inline void -cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { __asm__ __volatile__( "cpuid;" @@ -309,6 +310,7 @@ static void cpuid_count(uint32_t op, int count, uint32_t *eax, uint32_t *ebx, static int lock_to_processor(int processor) { cpu_set_t cpuset; + memset(&cpuset, 0, sizeof(cpu_set_t)); CPU_SET(processor, &cpuset); /* 0: this process */ @@ -377,9 +379,9 @@ static void cpuid_get_cpu_cache_info(uint32_t op, cpu_cacheinfo_t *cpu_ci) eax.split.num_threads_sharing + 1; this_leaf->hsa_cache_prop.CacheLevel = eax.split.level; this_leaf->hsa_cache_prop.CacheType.ui32.CPU = 1; - if (eax.split.type & CACHE_TYPE_DATA ) + if (eax.split.type & CACHE_TYPE_DATA) this_leaf->hsa_cache_prop.CacheType.ui32.Data = 1; - if (eax.split.type & CACHE_TYPE_INST ) + if (eax.split.type & CACHE_TYPE_INST) this_leaf->hsa_cache_prop.CacheType.ui32.Instruction = 1; this_leaf->hsa_cache_prop.CacheLineSize = ebx.split.coherency_line_size + 1; @@ -433,8 +435,7 @@ static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) this_leaf->hsa_cache_prop.SiblingMap[apicid2 - apicid1] = 1; leaf2->hsa_cache_prop.CacheSize = 0; cpu2->num_duplicated_caches++; - } - else { + } else { leaf2->hsa_cache_prop.SiblingMap[0] = 1; leaf2->hsa_cache_prop.SiblingMap[apicid1 - apicid2] = 1; this_leaf->hsa_cache_prop.CacheSize = 0; @@ -446,8 +447,8 @@ static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) } #endif /* X86 platform */ -static HSAKMT_STATUS -topology_sysfs_get_generation(uint32_t *gen) { +static HSAKMT_STATUS topology_sysfs_get_generation(uint32_t *gen) +{ FILE *fd; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -465,12 +466,12 @@ err: return ret; } -HSAKMT_STATUS -topology_sysfs_get_system_props(HsaSystemProperties *props) { +HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) +{ FILE *fd; char *read_buf, *p; char prop_name[256]; - long long unsigned int prop_val; + unsigned long long prop_val; uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -487,28 +488,26 @@ topology_sysfs_get_system_props(HsaSystemProperties *props) { goto err1; } - read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } - /* Since we're using the buffer as a string, we make sure the string terminates */ - if(read_size >= PAGE_SIZE) - read_size = PAGE_SIZE-1; - read_buf[read_size] = 0; + /* Since we're using the buffer as a string, we make sure the string terminates */ + if (read_size >= PAGE_SIZE) + read_size = PAGE_SIZE - 1; + read_buf[read_size] = 0; - /* - * Read the system properties - */ + /* Read the system properties */ prog = 0; p = read_buf; - while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { - if (strcmp(prop_name,"platform_oem") == 0) + while (sscanf(p += prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name, "platform_oem") == 0) props->PlatformOem = (uint32_t)prop_val; - else if (strcmp(prop_name,"platform_id") == 0) + else if (strcmp(prop_name, "platform_id") == 0) props->PlatformId = (uint32_t)prop_val; - else if (strcmp(prop_name,"platform_rev") == 0) + else if (strcmp(prop_name, "platform_rev") == 0) props->PlatformRev = (uint32_t)prop_val; } @@ -526,8 +525,8 @@ err1: return ret; } -HSAKMT_STATUS -topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) { +HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) +{ FILE *fd; char path[256]; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -537,21 +536,20 @@ topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) { fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; - if (fscanf(fd, "%ul", gpu_id) != 1) { + if (fscanf(fd, "%ul", gpu_id) != 1) ret = HSAKMT_STATUS_ERROR; - } fclose(fd); return ret; } -static const struct hsa_gfxip_table* find_hsa_gfxip_device(uint16_t device_id) +static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id) { uint32_t i, table_size; table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); - for (i=0; iis_dgpu) { is_dgpu = true; return true; } + is_dgpu = false; return false; } -static HSAKMT_STATUS -topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { +static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, + bool is_apu) +{ FILE *fd; char read_buf[256], cpu_model_name[HSA_PUBLIC_NAME_SIZE]; const char *p; - uint32_t i, apic_id; + uint32_t i = 0, apic_id = 0; if (!props) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -599,7 +599,7 @@ topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { return HSAKMT_STATUS_ERROR; } - while (fgets(read_buf, sizeof(read_buf), fd) != NULL) { + while (fgets(read_buf, sizeof(read_buf), fd)) { /* Get the model name first, in case matching * apic IDs are also present in the file */ @@ -609,7 +609,8 @@ topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { goto err; p++; // remove separator ':' - for (; isspace(*p); p++); /* remove white space */ + for (; isspace(*p); p++) + ; /* remove white space */ /* Extract model name from string */ for (i = 0; i < sizeof(cpu_model_name) - 1 && p[i] != '\n'; i++) @@ -623,7 +624,8 @@ topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { goto err; p++; // remove separator ':' - for (; isspace(*p); p++); /* remove white space */ + for (; isspace(*p); p++) + ; /* remove white space */ /* Extract apic_id from remaining chars */ apic_id = atoi(p); @@ -632,7 +634,7 @@ topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { if (props->CComputeIdLo == apic_id) { /* Retrieve the CAL name of CPU node */ if (!is_apu) - strncpy( (char *) props->AMDName, cpu_model_name, sizeof(props->AMDName)); + strncpy((char *)props->AMDName, cpu_model_name, sizeof(props->AMDName)); /* Convert from UTF8 to UTF16 */ for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = cpu_model_name[i]; @@ -650,6 +652,7 @@ err: static int topology_search_processor_vendor(const char *processor_name) { unsigned int i; + for (i = 0; i < ARRAY_LEN(supported_processor_vendor_name); i++) { if (!strcmp(processor_name, supported_processor_vendor_name[i])) return i; @@ -678,11 +681,12 @@ static void topology_set_processor_vendor(void) return; } - while (fgets(read_buf, sizeof(read_buf), fd) != NULL) { + while (fgets(read_buf, sizeof(read_buf), fd)) { if (!strncmp("vendor_id", read_buf, sizeof("vendor_id") - 1)) { p = strrchr(read_buf, ':'); - p++; // remove separor ':' - for (; isspace(*p); p++); /* remove white space */ + p++; // remove separator ':' + for (; *p && isspace(*p); p++) + ; /* remove white space */ processor_vendor = topology_search_processor_vendor(p); if (processor_vendor != -1) { fclose(fd); @@ -696,21 +700,22 @@ static void topology_set_processor_vendor(void) processor_vendor = GENUINE_INTEL; } -HSAKMT_STATUS -topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, - struct pci_access* pacc ) +HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, + HsaNodeProperties *props, + uint32_t *gpu_id, + struct pci_access *pacc) { FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; - long long unsigned int prop_val; + unsigned long long prop_val; uint32_t i, prog; uint16_t fw_version = 0; int read_size; - const struct hsa_gfxip_table* hsa_gfxip; + const struct hsa_gfxip_table *hsa_gfxip; char namebuf[HSA_PUBLIC_NAME_SIZE]; - const char* name; + const char *name; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -728,7 +733,7 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 fd = fopen(path, "r"); if (!fd) { free(read_buf); - return HSAKMT_STATUS_ERROR; + return HSAKMT_STATUS_ERROR; } read_size = fread(read_buf, 1, PAGE_SIZE, fd); @@ -738,63 +743,61 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 } /* Since we're using the buffer as a string, we make sure the string terminates */ - if(read_size >= PAGE_SIZE) - read_size = PAGE_SIZE-1; + if (read_size >= PAGE_SIZE) + read_size = PAGE_SIZE - 1; read_buf[read_size] = 0; - /* - * Read the node properties - */ + /* Read the node properties */ prog = 0; p = read_buf; - while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { - if (strcmp(prop_name,"cpu_cores_count") == 0) + while (sscanf(p += prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name, "cpu_cores_count") == 0) props->NumCPUCores = (uint32_t)prop_val; - else if (strcmp(prop_name,"simd_count") == 0) + else if (strcmp(prop_name, "simd_count") == 0) props->NumFComputeCores = (uint32_t)prop_val; - else if (strcmp(prop_name,"mem_banks_count") == 0) + else if (strcmp(prop_name, "mem_banks_count") == 0) props->NumMemoryBanks = (uint32_t)prop_val; - else if (strcmp(prop_name,"caches_count") == 0) + else if (strcmp(prop_name, "caches_count") == 0) props->NumCaches = (uint32_t)prop_val; - else if (strcmp(prop_name,"io_links_count") == 0) + else if (strcmp(prop_name, "io_links_count") == 0) props->NumIOLinks = (uint32_t)prop_val; - else if (strcmp(prop_name,"cpu_core_id_base") == 0) + else if (strcmp(prop_name, "cpu_core_id_base") == 0) props->CComputeIdLo = (uint32_t)prop_val; - else if (strcmp(prop_name,"simd_id_base") == 0) + else if (strcmp(prop_name, "simd_id_base") == 0) props->FComputeIdLo = (uint32_t)prop_val; - else if (strcmp(prop_name,"capability") == 0) + else if (strcmp(prop_name, "capability") == 0) props->Capability.Value = (uint32_t)prop_val; - else if (strcmp(prop_name,"max_waves_per_simd") == 0) + else if (strcmp(prop_name, "max_waves_per_simd") == 0) props->MaxWavesPerSIMD = (uint32_t)prop_val; - else if (strcmp(prop_name,"lds_size_in_kb") == 0) + else if (strcmp(prop_name, "lds_size_in_kb") == 0) props->LDSSizeInKB = (uint32_t)prop_val; - else if (strcmp(prop_name,"gds_size_in_kb") == 0) + else if (strcmp(prop_name, "gds_size_in_kb") == 0) props->GDSSizeInKB = (uint32_t)prop_val; - else if (strcmp(prop_name,"wave_front_size") == 0) + else if (strcmp(prop_name, "wave_front_size") == 0) props->WaveFrontSize = (uint32_t)prop_val; - else if (strcmp(prop_name,"array_count") == 0) + else if (strcmp(prop_name, "array_count") == 0) props->NumShaderBanks = (uint32_t)prop_val; - else if (strcmp(prop_name,"simd_arrays_per_engine") == 0) + else if (strcmp(prop_name, "simd_arrays_per_engine") == 0) props->NumArrays = (uint32_t)prop_val; - else if (strcmp(prop_name,"cu_per_simd_array") == 0) + else if (strcmp(prop_name, "cu_per_simd_array") == 0) props->NumCUPerArray = (uint32_t)prop_val; - else if (strcmp(prop_name,"simd_per_cu") == 0) + else if (strcmp(prop_name, "simd_per_cu") == 0) props->NumSIMDPerCU = (uint32_t)prop_val; - else if (strcmp(prop_name,"max_slots_scratch_cu") == 0) + else if (strcmp(prop_name, "max_slots_scratch_cu") == 0) props->MaxSlotsScratchCU = (uint32_t)prop_val; - else if (strcmp(prop_name,"fw_version") == 0) + else if (strcmp(prop_name, "fw_version") == 0) fw_version = (uint16_t)prop_val; - else if (strcmp(prop_name,"vendor_id") == 0) + else if (strcmp(prop_name, "vendor_id") == 0) props->VendorId = (uint32_t)prop_val; - else if (strcmp(prop_name,"device_id") == 0) + else if (strcmp(prop_name, "device_id") == 0) props->DeviceId = (uint32_t)prop_val; - else if (strcmp(prop_name,"location_id") == 0) + else if (strcmp(prop_name, "location_id") == 0) props->LocationId = (uint32_t)prop_val; - else if (strcmp(prop_name,"max_engine_clk_fcompute") == 0) + else if (strcmp(prop_name, "max_engine_clk_fcompute") == 0) props->MaxEngineClockMhzFCompute = (uint32_t)prop_val; - else if (strcmp(prop_name,"max_engine_clk_ccompute") == 0) + else if (strcmp(prop_name, "max_engine_clk_ccompute") == 0) props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; - else if (strcmp(prop_name,"local_mem_size") == 0) + else if (strcmp(prop_name, "local_mem_size") == 0) props->LocalMemSize = prop_val; } @@ -817,12 +820,11 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 } /* Retrieve the CAL name of the node */ - strncpy( (char *) props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName) ); + strncpy((char *)props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName)); if (props->NumCPUCores) { /* Is APU node */ ret = topology_get_cpu_model_name(props, true); - if (ret != HSAKMT_STATUS_SUCCESS) - { + if (ret != HSAKMT_STATUS_SUCCESS) { printf("Failed to get APU Model Name from %s\n", PROC_CPUINFO_PATH); ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ } @@ -841,8 +843,7 @@ topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32 /* Is CPU Node */ if (!props->NumFComputeCores || !props->DeviceId) { ret = topology_get_cpu_model_name(props, false); - if (ret != HSAKMT_STATUS_SUCCESS) - { + if (ret != HSAKMT_STATUS_SUCCESS) { printf("Failed to get CPU Model Name from %s\n", PROC_CPUINFO_PATH); ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ } @@ -860,52 +861,53 @@ err: return ret; } -static HSAKMT_STATUS -topology_sysfs_get_mem_props(uint32_t node_id, uint32_t mem_id, HsaMemoryProperties *props) { +static HSAKMT_STATUS topology_sysfs_get_mem_props(uint32_t node_id, + uint32_t mem_id, + HsaMemoryProperties *props) +{ FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; - long long unsigned int prop_val; + unsigned long long prop_val; uint32_t prog; - int read_size; + int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); snprintf(path, 256, "%s/%d/mem_banks/%d/properties", KFD_SYSFS_PATH_NODES, node_id, mem_id); fd = fopen(path, "r"); - if (!fd) { + if (!fd) return HSAKMT_STATUS_ERROR; - } read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = HSAKMT_STATUS_NO_MEMORY; goto err1; } - read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } - /* Since we're using the buffer as a string, we make sure the string terminates */ - if(read_size >= PAGE_SIZE) - read_size = PAGE_SIZE-1; - read_buf[read_size] = 0; + /* Since we're using the buffer as a string, we make sure the string terminates */ + if (read_size >= PAGE_SIZE) + read_size = PAGE_SIZE - 1; + read_buf[read_size] = 0; prog = 0; p = read_buf; - while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { - if (strcmp(prop_name,"heap_type") == 0) + while (sscanf(p += prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name, "heap_type") == 0) props->HeapType = (uint32_t)prop_val; - else if (strcmp(prop_name,"size_in_bytes") == 0) + else if (strcmp(prop_name, "size_in_bytes") == 0) props->SizeInBytes = (uint64_t)prop_val; - else if (strcmp(prop_name,"flags") == 0) + else if (strcmp(prop_name, "flags") == 0) props->Flags.MemoryProperty = (uint32_t)prop_val; - else if (strcmp(prop_name,"width") == 0) + else if (strcmp(prop_name, "width") == 0) props->Width = (uint32_t)prop_val; - else if (strcmp(prop_name,"mem_clk_max") == 0) + else if (strcmp(prop_name, "mem_clk_max") == 0) props->MemoryClockMax = (uint32_t)prop_val; } @@ -946,8 +948,7 @@ static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) * @temp_cpu_ci_list - [OUT] temporary cpu-cache-info list to store data * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure */ -static HSAKMT_STATUS -topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) +static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; void *p_temp_cpu_ci_list; @@ -1032,8 +1033,8 @@ exit: * @cpu_ci_list - the cpu cache information list to look up cache info * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure */ -static HSAKMT_STATUS -topology_get_cpu_cache_props(node_t *tbl, cpu_cacheinfo_t *cpu_ci_list) +static HSAKMT_STATUS topology_get_cpu_cache_props(node_t *tbl, + cpu_cacheinfo_t *cpu_ci_list) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; uint32_t apicid_low = tbl->node.CComputeIdLo, apicid_max = 0; @@ -1089,71 +1090,71 @@ static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) { } -static HSAKMT_STATUS -topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) +static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) { return HSAKMT_STATUS_SUCCESS; } -static HSAKMT_STATUS -topology_get_cpu_cache_props(node_t *tbl, cpu_cacheinfo_t *cpu_ci_list) +static HSAKMT_STATUS topology_get_cpu_cache_props(node_t *tbl, + cpu_cacheinfo_t *cpu_ci_list) { return HSAKMT_STATUS_SUCCESS; } #endif -static HSAKMT_STATUS -topology_sysfs_get_cache_props(uint32_t node_id, uint32_t cache_id, HsaCacheProperties *props) { +static HSAKMT_STATUS topology_sysfs_get_cache_props(uint32_t node_id, + uint32_t cache_id, + HsaCacheProperties *props) +{ FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; - long long unsigned int prop_val; + unsigned long long prop_val; uint32_t i, prog; - int read_size; + int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); snprintf(path, 256, "%s/%d/caches/%d/properties", KFD_SYSFS_PATH_NODES, node_id, cache_id); fd = fopen(path, "r"); - if (!fd) { + if (!fd) return HSAKMT_STATUS_ERROR; - } read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = HSAKMT_STATUS_NO_MEMORY; goto err1; } - read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } - /* Since we're using the buffer as a string, we make sure the string terminates */ - if(read_size >= PAGE_SIZE) - read_size = PAGE_SIZE-1; - read_buf[read_size] = 0; + /* Since we're using the buffer as a string, we make sure the string terminates */ + if (read_size >= PAGE_SIZE) + read_size = PAGE_SIZE - 1; + read_buf[read_size] = 0; prog = 0; p = read_buf; - while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { - if (strcmp(prop_name,"processor_id_low") == 0) + while (sscanf(p += prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name, "processor_id_low") == 0) props->ProcessorIdLow = (uint32_t)prop_val; - else if (strcmp(prop_name,"level") == 0) + else if (strcmp(prop_name, "level") == 0) props->CacheLevel = (uint32_t)prop_val; - else if (strcmp(prop_name,"size") == 0) + else if (strcmp(prop_name, "size") == 0) props->CacheSize = (uint32_t)prop_val; - else if (strcmp(prop_name,"cache_line_size") == 0) + else if (strcmp(prop_name, "cache_line_size") == 0) props->CacheLineSize = (uint32_t)prop_val; - else if (strcmp(prop_name,"cache_lines_per_tag") == 0) + else if (strcmp(prop_name, "cache_lines_per_tag") == 0) props->CacheLinesPerTag = (uint32_t)prop_val; - else if (strcmp(prop_name,"association") == 0) + else if (strcmp(prop_name, "association") == 0) props->CacheAssociativity = (uint32_t)prop_val; - else if (strcmp(prop_name,"latency") == 0) + else if (strcmp(prop_name, "latency") == 0) props->CacheLatency = (uint32_t)prop_val; - else if (strcmp(prop_name,"type") == 0) + else if (strcmp(prop_name, "type") == 0) props->CacheType.Value = (uint32_t)prop_val; else if (strcmp(prop_name, "sibling_map") == 0) break; @@ -1163,8 +1164,8 @@ topology_sysfs_get_cache_props(uint32_t node_id, uint32_t cache_id, HsaCacheProp if ((sscanf(p, "sibling_map %n", &prog)) == 0 && prog) { i = 0; while ((i < HSA_CPU_SIBLINGS) && - (sscanf(p+=prog, "%u%*[,\n]%n", &props->SiblingMap[i++], - &prog) == 1)); + (sscanf(p += prog, "%u%*[,\n]%n", &props->SiblingMap[i++], &prog) == 1)) + continue; } err2: @@ -1174,66 +1175,67 @@ err1: return ret; } -static HSAKMT_STATUS -topology_sysfs_get_iolink_props(uint32_t node_id, uint32_t iolink_id, HsaIoLinkProperties *props) { +static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, + uint32_t iolink_id, + HsaIoLinkProperties *props) +{ FILE *fd; char *read_buf, *p; char prop_name[256]; char path[256]; - long long unsigned int prop_val; + unsigned long long prop_val; uint32_t prog; - int read_size; + int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, node_id, iolink_id); fd = fopen(path, "r"); - if (!fd) { + if (!fd) return HSAKMT_STATUS_ERROR; - } read_buf = malloc(PAGE_SIZE); if (!read_buf) { ret = HSAKMT_STATUS_NO_MEMORY; goto err1; } - read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; goto err2; } - /* Since we're using the buffer as a string, we make sure the string terminates */ - if(read_size >= PAGE_SIZE) - read_size = PAGE_SIZE-1; - read_buf[read_size] = 0; + /* Since we're using the buffer as a string, we make sure the string terminates */ + if (read_size >= PAGE_SIZE) + read_size = PAGE_SIZE - 1; + read_buf[read_size] = 0; prog = 0; p = read_buf; - while(sscanf(p+=prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { - if (strcmp(prop_name,"type") == 0) + while (sscanf(p += prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name, "type") == 0) props->IoLinkType = (uint32_t)prop_val; - else if (strcmp(prop_name,"version_major") == 0) + else if (strcmp(prop_name, "version_major") == 0) props->VersionMajor = (uint32_t)prop_val; - else if (strcmp(prop_name,"version_minor") == 0) + else if (strcmp(prop_name, "version_minor") == 0) props->VersionMinor = (uint32_t)prop_val; - else if (strcmp(prop_name,"node_from") == 0) + else if (strcmp(prop_name, "node_from") == 0) props->NodeFrom = (uint32_t)prop_val; - else if (strcmp(prop_name,"node_to") == 0) + else if (strcmp(prop_name, "node_to") == 0) props->NodeTo = (uint32_t)prop_val; - else if (strcmp(prop_name,"weight") == 0) + else if (strcmp(prop_name, "weight") == 0) props->Weight = (uint32_t)prop_val; - else if (strcmp(prop_name,"min_latency") == 0) + else if (strcmp(prop_name, "min_latency") == 0) props->MinimumLatency = (uint32_t)prop_val; - else if (strcmp(prop_name,"max_latency") == 0) + else if (strcmp(prop_name, "max_latency") == 0) props->MaximumLatency = (uint32_t)prop_val; - else if (strcmp(prop_name,"min_bandwidth") == 0) + else if (strcmp(prop_name, "min_bandwidth") == 0) props->MinimumBandwidth = (uint32_t)prop_val; - else if (strcmp(prop_name,"max_bandwidth") == 0) + else if (strcmp(prop_name, "max_bandwidth") == 0) props->MaximumBandwidth = (uint32_t)prop_val; - else if (strcmp(prop_name,"recommended_transfer_size") == 0) + else if (strcmp(prop_name, "recommended_transfer_size") == 0) props->RecTransferSize = (uint32_t)prop_val; - else if (strcmp(prop_name,"flags") == 0) + else if (strcmp(prop_name, "flags") == 0) props->Flags.LinkProperty = (uint32_t)prop_val; } @@ -1246,11 +1248,11 @@ err1: } /* topology_get_free_io_link_slot_for_node - For the given node_id, find the - * next available free slot to add an io_link + * next available free slot to add an io_link */ -static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node( - uint32_t node_id, const HsaSystemProperties *sys_props, - node_t *nodes) +static HsaIoLinkProperties *topology_get_free_io_link_slot_for_node(uint32_t node_id, + const HsaSystemProperties *sys_props, + node_t *nodes) { HsaIoLinkProperties *props; @@ -1275,14 +1277,16 @@ static HsaIoLinkProperties * topology_get_free_io_link_slot_for_node( } /* topology_add_io_link_for_node - If a free slot is available, - * add io_link for the given Node. If bi_directional is true, set up two - * links for both directions. - * TODO: Add other members of HsaIoLinkProperties + * add io_link for the given Node. If bi_directional is true, set up two + * links for both directions. + * TODO: Add other members of HsaIoLinkProperties */ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, - const HsaSystemProperties *sys_props, node_t *nodes, - HSA_IOLINKTYPE IoLinkType, uint32_t NodeTo, - uint32_t Weight, bool bi_dir) + const HsaSystemProperties *sys_props, + node_t *nodes, + HSA_IOLINKTYPE IoLinkType, + uint32_t NodeTo, + uint32_t Weight, bool bi_dir) { HsaIoLinkProperties *props; /* If bi-directional is set true, it's two links to add. */ @@ -1330,7 +1334,8 @@ static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, node_t *nodes) * been created in the kernel. */ static HSAKMT_STATUS get_direct_iolink_info(uint32_t node1, uint32_t node2, - node_t *nodes, HSAuint32 *weight, HSA_IOLINKTYPE *type) + node_t *nodes, HSAuint32 *weight, + HSA_IOLINKTYPE *type) { HsaIoLinkProperties *props = nodes[node1].link; uint32_t i; @@ -1351,7 +1356,8 @@ static HSAKMT_STATUS get_direct_iolink_info(uint32_t node1, uint32_t node2, } static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, - node_t *nodes, HSAuint32 *weight, HSA_IOLINKTYPE *type) + node_t *nodes, HSAuint32 *weight, + HSA_IOLINKTYPE *type) { int32_t dir_cpu1 = -1, dir_cpu2 = -1; HSAuint32 weight1 = 0, weight2 = 0, weight3 = 0; @@ -1389,7 +1395,7 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, if (ret != HSAKMT_STATUS_SUCCESS) return ret; ret = get_direct_iolink_info(dir_cpu1, node2, - nodes, &weight2, type); + nodes, &weight2, type); } else /* GPU->CPU->CPU->GPU*/ { ret = get_direct_iolink_info(node1, dir_cpu1, nodes, &weight1, NULL); @@ -1434,8 +1440,8 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, return HSAKMT_STATUS_SUCCESS; } -static void topology_create_indirect_gpu_links( - const HsaSystemProperties *sys_props, node_t *nodes) +static void topology_create_indirect_gpu_links(const HsaSystemProperties *sys_props, + node_t *nodes) { uint32_t i, j; @@ -1455,8 +1461,7 @@ static void topology_create_indirect_gpu_links( } } -HSAKMT_STATUS -topology_take_snapshot(void) +HSAKMT_STATUS topology_take_snapshot(void) { uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; HsaSystemProperties sys_props; @@ -1473,9 +1478,9 @@ retry: ret = topology_sysfs_get_system_props(&sys_props); if (ret != HSAKMT_STATUS_SUCCESS) return ret; - if(sys_props.NumNodes > 0) { + if (sys_props.NumNodes > 0) { topology_create_temp_cpu_cache_list(&cpu_ci_list); - temp_nodes = calloc(sys_props.NumNodes * sizeof(node_t),1); + temp_nodes = calloc(sys_props.NumNodes * sizeof(node_t), 1); if (!temp_nodes) return HSAKMT_STATUS_NO_MEMORY; pacc = pci_alloc(); @@ -1518,8 +1523,7 @@ retry: goto err; } } - } - else if (!temp_nodes[i].gpu_id) { /* a CPU node */ + } else if (!temp_nodes[i].gpu_id) { /* a CPU node */ ret = topology_get_cpu_cache_props( &temp_nodes[i], cpu_ci_list); if (ret != HSAKMT_STATUS_SUCCESS) { @@ -1529,7 +1533,8 @@ retry: } /* To simplify, allocate maximum needed memory for io_links for each node. This - * removes the need for realloc when indirect and QPI links are added later */ + * removes the need for realloc when indirect and QPI links are added later + */ temp_nodes[i].link = calloc(sys_props.NumNodes - 1, sizeof(HsaIoLinkProperties)); if (!temp_nodes[i].link) { ret = HSAKMT_STATUS_NO_MEMORY; @@ -1585,12 +1590,8 @@ err: return ret; } -/* - * Drop the Snashot of the HSA topology information. - * Assume lock is held. - */ -HSAKMT_STATUS -topology_drop_snapshot(void) +/* Drop the Snashot of the HSA topology information. Assume lock is held. */ +HSAKMT_STATUS topology_drop_snapshot(void) { HSAKMT_STATUS err; @@ -1614,10 +1615,9 @@ out: return err; } -HSAKMT_STATUS -validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) +HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) { - if (!node || !_system || _system->NumNodes <= nodeid) + if (!node || !_system || _system->NumNodes <= nodeid) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (gpu_id) *gpu_id = node[nodeid].gpu_id; @@ -1625,11 +1625,12 @@ validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) return HSAKMT_STATUS_SUCCESS; } -HSAKMT_STATUS -gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id){ +HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id) +{ uint64_t node_idx; - for(node_idx = 0; node_idx < _system->NumNodes; node_idx++){ - if (node[node_idx].gpu_id == gpu_id){ + + for (node_idx = 0; node_idx < _system->NumNodes; node_idx++) { + if (node[node_idx].gpu_id == gpu_id) { *node_id = node_idx; return HSAKMT_STATUS_SUCCESS; } @@ -1639,17 +1640,14 @@ gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id){ } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtAcquireSystemProperties( - HsaSystemProperties* SystemProperties //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties(HsaSystemProperties *SystemProperties) { HSAKMT_STATUS err; + CHECK_KFD_OPEN(); if (!SystemProperties) - return HSAKMT_STATUS_INVALID_PARAMETER; + return HSAKMT_STATUS_INVALID_PARAMETER; pthread_mutex_lock(&hsakmt_mutex); @@ -1667,9 +1665,7 @@ out: return err; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtReleaseSystemProperties(void) +HSAKMT_STATUS HSAKMTAPI hsaKmtReleaseSystemProperties(void) { CHECK_KFD_OPEN(); @@ -1684,12 +1680,8 @@ hsaKmtReleaseSystemProperties(void) return err; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetNodeProperties( - HSAuint32 NodeId, //IN - HsaNodeProperties* NodeProperties //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, + HsaNodeProperties *NodeProperties) { HSAKMT_STATUS err; uint32_t gpu_id; @@ -1701,7 +1693,7 @@ hsaKmtGetNodeProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (_system == NULL) { + if (!_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; @@ -1731,13 +1723,9 @@ out: return err; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetNodeMemoryProperties( - HSAuint32 NodeId, //IN - HSAuint32 NumBanks, //IN - HsaMemoryProperties* MemoryProperties //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, + HSAuint32 NumBanks, + HsaMemoryProperties *MemoryProperties) { HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; uint32_t i, gpu_id; @@ -1751,14 +1739,14 @@ hsaKmtGetNodeMemoryProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (_system == NULL) { + if (!_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; } /* Check still necessary */ - if (NodeId >= _system->NumNodes ) { + if (NodeId >= _system->NumNodes) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -1790,7 +1778,9 @@ hsaKmtGetNodeMemoryProperties( } /* Add Local memory - HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE. - * For dGPU the topology node contains Local Memory and it is added by the for loop above */ + * For dGPU the topology node contains Local Memory and it is added by + * the for loop above + */ if (!nodeIsDGPU && i < NumBanks && node[NodeId].node.LocalMemSize > 0 && fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { @@ -1799,7 +1789,7 @@ hsaKmtGetNodeMemoryProperties( i++; } - /*Add SCRATCH*/ + /* Add SCRATCH */ if (i < NumBanks && fmm_get_aperture_base_and_limit(FMM_SCRATCH, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { @@ -1823,14 +1813,10 @@ out: return err; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetNodeCacheProperties( - HSAuint32 NodeId, //IN - HSAuint32 ProcessorId, //IN - HSAuint32 NumCaches, //IN - HsaCacheProperties* CacheProperties //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeCacheProperties(HSAuint32 NodeId, + HSAuint32 ProcessorId, + HSAuint32 NumCaches, + HsaCacheProperties *CacheProperties) { HSAKMT_STATUS err; uint32_t i; @@ -1842,7 +1828,7 @@ hsaKmtGetNodeCacheProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (_system == NULL) { + if (!_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; @@ -1865,13 +1851,9 @@ out: return err; } -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetNodeIoLinkProperties( - HSAuint32 NodeId, //IN - HSAuint32 NumIoLinks, //IN - HsaIoLinkProperties* IoLinkProperties //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, + HSAuint32 NumIoLinks, + HsaIoLinkProperties *IoLinkProperties) { HSAKMT_STATUS err; uint32_t i; @@ -1884,7 +1866,7 @@ hsaKmtGetNodeIoLinkProperties( pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (_system == NULL) { + if (!_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; assert(_system); goto out; @@ -1909,15 +1891,16 @@ out: uint16_t get_device_id_by_node(HSAuint32 node_id) { - if (!node || !_system || _system->NumNodes <= node_id) - return 0; + if (!node || !_system || _system->NumNodes <= node_id) + return 0; - return node[node_id].node.DeviceId; + return node[node_id].node.DeviceId; } uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; + if (!node || !_system) return 0; @@ -1935,14 +1918,14 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, HSAKMT_STATUS ret; unsigned int i; - if (NumberOfNodes == 0 || NodeArray == NULL || gpu_id_array == NULL) + if (NumberOfNodes == 0 || !NodeArray || !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; /* Translate Node IDs to gpu_ids */ *gpu_id_array = malloc(NumberOfNodes * sizeof(uint32_t)); - if (*gpu_id_array == NULL) + if (!(*gpu_id_array)) return HSAKMT_STATUS_NO_MEMORY; - for (i = 0; i < NumberOfNodes; i++) { + for (i = 0; i < NumberOfNodes; i++) { ret = validate_nodeid(NodeArray[i], *gpu_id_array + i); if (ret != HSAKMT_STATUS_SUCCESS) { free(*gpu_id_array); @@ -1950,45 +1933,5 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, } } - return ret; -} - -#if 0 -static int get_cpu_stepping(uint16_t* stepping) -{ - int ret; - FILE* fd = fopen("/proc/cpuinfo", "r"); - if (!fd) - return -1; - - char* read_buf = malloc(PAGE_SIZE); - if (!read_buf) { - ret = -1; - goto err1; - } - - int read_size = fread(read_buf, 1, PAGE_SIZE, fd); - if (read_size <= 0) { - ret = -2; - goto err2; - } - - /* Since we're using the buffer as a string, we make sure the string terminates */ - if(read_size >= PAGE_SIZE) - read_size = PAGE_SIZE-1; - read_buf[read_size] = 0; - - *stepping = 0; - - char* p = strstr(read_buf, "stepping"); - if (p) - sscanf(p , "stepping\t: %hu\n", stepping); - -err2: - free(read_buf); -err1: - fclose(fd); - return ret; } -#endif diff --git a/src/version.c b/src/version.c index 95bfec6523..0b7c7c86e8 100644 --- a/src/version.c +++ b/src/version.c @@ -28,15 +28,12 @@ #include #include "linux/kfd_ioctl.h" -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetVersion( - HsaVersionInfo* VersionInfo //OUT - ) +HSAKMT_STATUS HSAKMTAPI hsaKmtGetVersion(HsaVersionInfo *VersionInfo) { CHECK_KFD_OPEN(); struct kfd_ioctl_get_version_args args; + memset(&args, 0, sizeof(args)); if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1) From 683fc963258bf9f8dfcc0ac4b06e3eece4c57677 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 25 May 2017 14:51:30 -0400 Subject: [PATCH 0251/1247] Implement hsaKmtGetQueueInfo interface For items in HsaQueueInfo, control stack information comes from KFD, CU mask information is maintained in Thunk, and others (queue detail error and queue type extended) are ignored (value = 0) at this point. Change-Id: Ib21370b0f52b2bb4ebe6a9b4b6ec6139cccb25ca Signed-off-by: Amber Lin --- include/hsakmt.h | 7 +++ include/hsakmttypes.h | 19 ++++++++ include/linux/kfd_ioctl.h | 15 ++++++- src/libhsakmt.h | 1 + src/libhsakmt.ver | 1 + src/queues.c | 91 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 131 insertions(+), 3 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index ed5aff7144..9084079882 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -330,6 +330,13 @@ hsaKmtSetQueueCUMask( HSAuint32* QueueCUMask //IN ); +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueInfo( + HSA_QUEUEID QueueId, //IN + HsaQueueInfo *QueueInfo //IN +); + /** Allows an HSA process to set/change the default and alternate memory coherency, before starting to dispatch. */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index df83817010..eb640d7014 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -582,6 +582,25 @@ typedef enum _HSA_QUEUE_TYPE HSA_QUEUE_TYPE_SIZE = 0xFFFFFFFF //aligns to 32bit enum } HSA_QUEUE_TYPE; +typedef struct +{ + HSAuint32 QueueDetailError; // HW specific queue error state + HSAuint32 QueueTypeExtended; // HW specific queue type info. + // 0 = no information + HSAuint32 NumCUAssigned; // size of *CUMaskInfo bit array, Multiple + // of 32, 0 = no information + HSAuint32* CUMaskInfo; // runtime/system CU assignment for realtime + // queue & reserved CU priority. Ptr to + // bit-array, each bit represents one CU. + // NULL = no information + HSAuint32* UserContextSaveArea; // reference to user space context save area + HSAuint64 SaveAreaSizeInBytes; // Must be 4-Byte aligned + HSAuint32* ControlStackTop; // ptr to the TOS + HSAuint64 ControlStackUsedInBytes; // Must be 4-Byte aligned + HSAuint64 Reserved1; // runtime/system CU assignment + HSAuint64 Reserved2; // runtime/system CU assignment +} HsaQueueInfo; + typedef struct _HsaQueueResource { HSA_QUEUEID QueueId; /** queue ID */ diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 8f833bdeb1..1ff1271e91 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -82,6 +82,14 @@ struct kfd_ioctl_set_cu_mask_args { uint64_t cu_mask_ptr; /* to KFD */ }; +struct kfd_ioctl_get_queue_wave_state_args { + uint64_t ctl_stack_address; /* to KFD */ + uint32_t ctl_stack_used_size; /* from KFD */ + uint32_t save_area_used_size; /* from KFD */ + uint32_t queue_id; /* to KFD */ + uint32_t pad; +}; + /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ #define KFD_IOC_CACHE_POLICY_COHERENT 0 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 @@ -526,11 +534,14 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x1F, struct kfd_ioctl_cross_memory_copy_args) +#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ + AMDKFD_IOWR(0x20, struct kfd_ioctl_get_queue_wave_state_args) + /* TODO: remove this */ #define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_open_graphic_handle_args) + AMDKFD_IOWR(0x21, struct kfd_ioctl_open_graphic_handle_args) #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x21 +#define AMDKFD_COMMAND_END 0x22 #endif diff --git a/src/libhsakmt.h b/src/libhsakmt.h index be57fc59f1..0cc45ea200 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -59,6 +59,7 @@ extern int PAGE_SHIFT; do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1)) #define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index e955a31c97..436e33bf8c 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -55,6 +55,7 @@ hsaKmtSetTrapHandler; hsaKmtGetTileConfig; hsaKmtQueryPointerInfo; hsaKmtSetMemoryUserData; +hsaKmtGetQueueInfo; local: *; }; diff --git a/src/queues.c b/src/queues.c index 7fbd1083c7..9860a6d6ec 100644 --- a/src/queues.c +++ b/src/queues.c @@ -122,7 +122,14 @@ struct queue { void *ctx_save_restore; uint32_t ctx_save_restore_size; uint32_t ctl_stack_size; + void *ctl_stack_copy; const struct device_info *dev_info; + /* This queue structure is allocated from GPU with page aligned size + * but only small bytes are used. We use the extra space in the end for + * cu_mask bits array. + */ + uint32_t cu_mask_count; /* in bits */ + uint32_t cu_mask[0]; }; struct process_doorbells { @@ -416,6 +423,11 @@ static void free_queue(struct queue *q) free_exec_aligned_memory(q->ctx_save_restore, q->ctx_save_restore_size, PAGE_SIZE, q->dev_info->asic_family); + if (q->ctl_stack_copy) + free_exec_aligned_memory(q->ctl_stack_copy, + q->ctl_stack_size, + PAGE_SIZE, q->dev_info->asic_family); + free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } @@ -451,6 +463,11 @@ static int handle_concrete_asic(struct queue *q, return HSAKMT_STATUS_NO_MEMORY; args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; + if (IS_SOC15(q->dev_info->asic_family)) { + q->ctl_stack_copy = malloc(q->ctl_stack_size); + if (q->ctl_stack_copy == NULL) + return HSAKMT_STATUS_NO_MEMORY; + } } } @@ -479,6 +496,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, unsigned int doorbell_offset; struct device_info *dev_info; int err; + HsaNodeProperties props; + uint32_t cu_num, i; CHECK_KFD_OPEN(); @@ -501,6 +520,19 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, memset(q, 0, sizeof(*q)); + /* By default, CUs are all turned on. Initialize cu_mask to '1 + * for all CU bits. + */ + if (hsaKmtGetNodeProperties(NodeId, &props)) + q->cu_mask_count = 0; + else { + cu_num = props.NumFComputeCores / props.NumSIMDPerCU; + /* cu_mask_count counts bits. It must be multiple of 32 */ + q->cu_mask_count = ALIGN_UP_32(cu_num, 32); + for (i = 0; i < cu_num; i++) + q->cu_mask[i/32] |= (1 << (i % 32)); + } + struct kfd_ioctl_create_queue_args args; memset(&args, 0, sizeof(args)); @@ -534,7 +566,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, return err; } - args.read_pointer_address = QueueResource->QueueRptrValue; args.write_pointer_address = QueueResource->QueueWptrValue; args.ring_base_address = (uintptr_t)QueueAddress; @@ -662,6 +693,64 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetQueueCUMask(HSA_QUEUEID QueueId, if (err == -1) return HSAKMT_STATUS_ERROR; + memcpy(q->cu_mask, QueueCUMask, CUMaskCount / 8); + q->cu_mask_count = CUMaskCount; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueInfo( + HSA_QUEUEID QueueId, + HsaQueueInfo *QueueInfo +) +{ + struct queue *q = PORT_UINT64_TO_VPTR(QueueId); + struct kfd_ioctl_get_queue_wave_state_args args; + uintptr_t ctl_stack_base_addr = 0; + + CHECK_KFD_OPEN(); + + if (QueueInfo == NULL || q == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + if (q->ctx_save_restore == NULL) + return HSAKMT_STATUS_ERROR; + + memset(&args, 0, sizeof(args)); + args.queue_id = q->queue_id; + + if (IS_SOC15(q->dev_info->asic_family)) { + /* From SOC15 onwards the control stack is kept in the MQD. + * The save area is at the beginning of the allocated space. + * Request a copy of the control stack from the KFD. + */ + ctl_stack_base_addr = (uintptr_t)q->ctl_stack_copy; + QueueInfo->UserContextSaveArea = q->ctx_save_restore; + args.ctl_stack_address = ctl_stack_base_addr; + } else { + /* Before SOC15 the control stack is already in userspace. + * The save area immediately follows the control stack. + */ + ctl_stack_base_addr = (uintptr_t)q->ctx_save_restore; + QueueInfo->UserContextSaveArea = (void *) + ((uintptr_t)q->ctx_save_restore + q->ctl_stack_size); + args.ctl_stack_address = 0; + } + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_QUEUE_WAVE_STATE, &args) < 0) + return HSAKMT_STATUS_ERROR; + + QueueInfo->ControlStackTop = (void *)(ctl_stack_base_addr + + q->ctl_stack_size - args.ctl_stack_used_size); + QueueInfo->SaveAreaSizeInBytes = args.save_area_used_size; + QueueInfo->ControlStackUsedInBytes = args.ctl_stack_used_size; + QueueInfo->NumCUAssigned = q->cu_mask_count; + QueueInfo->CUMaskInfo = q->cu_mask; + QueueInfo->QueueDetailError = 0; + QueueInfo->QueueTypeExtended = 0; + return HSAKMT_STATUS_SUCCESS; } From 374bd89d8c05e5a2ee850f3577ff8fc5ab763828 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 26 May 2017 15:24:11 -0400 Subject: [PATCH 0252/1247] Remove deprecated implementation of hsaKmtMapGraphicHandle The KFD implementation has been removed and will not be upstreamed. This API has been superseded by hsaKmtRegisterGraphicsHandleToNodes. Change-Id: I5f2d8da3260974618cdb6ea3fdcd77d37b82c9cb Signed-off-by: Felix Kuehling Signed-off-by: Amber Lin --- include/linux/kfd_ioctl.h | 20 +------------ src/fmm.c | 59 --------------------------------------- src/fmm.h | 4 --- src/memory.c | 30 ++++---------------- 4 files changed, 7 insertions(+), 106 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 1ff1271e91..b318a8fd92 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -326,20 +326,6 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { uint32_t pad; }; -/* TODO: remove this. It's only implemented for Kaveri and was never - * upstreamed. There are no open-source users of this interface. It - * has been superseded by the pair of get_dmabuf_info and - * import_dmabuf, which is implemented for all supported GPUs. - */ -struct kfd_ioctl_open_graphic_handle_args { - uint64_t va_addr; /* to KFD */ - uint64_t handle; /* from KFD */ - uint32_t gpu_id; /* to KFD */ - int graphic_device_fd; /* to KFD */ - uint32_t graphic_handle; /* to KFD */ - uint32_t pad; -}; - struct kfd_ioctl_set_process_dgpu_aperture_args { uint64_t dgpu_base; uint64_t dgpu_limit; @@ -537,11 +523,7 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ AMDKFD_IOWR(0x20, struct kfd_ioctl_get_queue_wave_state_args) -/* TODO: remove this */ -#define AMDKFD_IOC_OPEN_GRAPHIC_HANDLE \ - AMDKFD_IOWR(0x21, struct kfd_ioctl_open_graphic_handle_args) - #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x22 +#define AMDKFD_COMMAND_END 0x21 #endif diff --git a/src/fmm.c b/src/fmm.c index 88e6052851..2264e6fa9d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1235,65 +1235,6 @@ void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, return fmm_allocate_host_cpu(MemorySizeInBytes, flags); } -void *fmm_open_graphic_handle(uint32_t gpu_id, - int32_t graphic_device_handle, - uint32_t graphic_handle, - uint64_t MemorySizeInBytes) -{ - - void *mem = NULL; - int32_t i = gpu_mem_find_by_gpu_id(gpu_id); - struct kfd_ioctl_open_graphic_handle_args open_graphic_handle_args; - struct kfd_ioctl_unmap_memory_from_gpu_args unmap_args; - - /* If not found or aperture isn't properly initialized/supported */ - if (i < 0 || !aperture_is_valid(gpu_mem[i].gpuvm_aperture.base, - gpu_mem[i].gpuvm_aperture.limit)) - return NULL; - - pthread_mutex_lock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - /* Allocate address space */ - mem = aperture_allocate_area(&gpu_mem[i].gpuvm_aperture, - MemorySizeInBytes, GPUVM_APP_OFFSET); - if (!mem) - goto out; - - /* Allocate local memory */ - open_graphic_handle_args.gpu_id = gpu_id; - open_graphic_handle_args.graphic_device_fd = graphic_device_handle; - open_graphic_handle_args.graphic_handle = graphic_handle; - open_graphic_handle_args.va_addr = - VOID_PTRS_SUB(mem, gpu_mem[i].gpuvm_aperture.base); - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_OPEN_GRAPHIC_HANDLE, - &open_graphic_handle_args)) - goto release_area; - - /* Allocate object */ - if (!aperture_allocate_object(&gpu_mem[i].gpuvm_aperture, mem, - open_graphic_handle_args.handle, - MemorySizeInBytes, 0)) - goto release_mem; - - pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - - /* That's all. Just return the new address */ - return mem; - -release_mem: - unmap_args.handle = open_graphic_handle_args.handle; - unmap_args.device_ids_array_ptr = 0; - unmap_args.device_ids_array_size = 0; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &unmap_args); -release_area: - aperture_release_area(&gpu_mem[i].gpuvm_aperture, mem, - MemorySizeInBytes); -out: - pthread_mutex_unlock(&gpu_mem[i].gpuvm_aperture.fmm_mutex); - - return NULL; -} - static void __fmm_release(void *address, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args; diff --git a/src/fmm.h b/src/fmm.h index f5b65e4209..f3870842fa 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -53,10 +53,6 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset); void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); -void *fmm_open_graphic_handle(uint32_t gpu_id, - int32_t graphic_device_handle, - uint32_t graphic_handle, - uint64_t MemorySizeInBytes); void fmm_print(uint32_t node); bool fmm_is_inside_some_aperture(void *address); void fmm_release(void *address); diff --git a/src/memory.c b/src/memory.c index 00aff3cfe3..c4785b9ec1 100644 --- a/src/memory.c +++ b/src/memory.c @@ -460,30 +460,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapGraphicHandle(HSAuint32 NodeId, HSAuint64 GraphicResourceSize, HSAuint64 *FlatMemoryAddress) { - - CHECK_KFD_OPEN(); - HSAKMT_STATUS result; - uint32_t gpu_id; - void *graphic_handle; - - if (GraphicResourceOffset != 0) - return HSAKMT_STATUS_NOT_IMPLEMENTED; - - result = validate_nodeid(NodeId, &gpu_id); - if (result != HSAKMT_STATUS_SUCCESS) - return result; - - graphic_handle = fmm_open_graphic_handle(gpu_id, - GraphicDeviceHandle, - GraphicResourceHandle, - GraphicResourceSize); - - *FlatMemoryAddress = PORT_VPTR_TO_UINT64(graphic_handle); - - if (*FlatMemoryAddress) - return HSAKMT_STATUS_SUCCESS; - else - return HSAKMT_STATUS_NO_MEMORY; + /* This API was only ever implemented in KFD for Kaveri and + * was never upstreamed. There are no open-source users of + * this interface. It has been superseded by + * RegisterGraphicsHandleToNodes. + */ + return HSAKMT_STATUS_NOT_IMPLEMENTED; } HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapGraphicHandle(HSAuint32 NodeId, From ceaaa1a57c0be6cbb616551c61bd3ca16c03ecb9 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 5 Jun 2017 12:19:54 -0400 Subject: [PATCH 0253/1247] A missing block in PMC DB block was missing in the UUID look-up. Change-Id: Ife5c25859bab6ec7fd99d0cd4d098ab044a08142 Signed-off-by: Amber Lin --- src/perfctr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/perfctr.c b/src/perfctr.c index 962fe56a7e..f54599cd46 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -186,6 +186,9 @@ static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) case PERFCOUNTER_BLOCKID__CPG: *uuid = HSA_PROFILEBLOCK_AMD_CPG; break; + case PERFCOUNTER_BLOCKID__DB: + *uuid = HSA_PROFILEBLOCK_AMD_DB; + break; case PERFCOUNTER_BLOCKID__GDS: *uuid = HSA_PROFILEBLOCK_AMD_GDS; break; From 5e26827d05c986d3f8a1f9c209e5e807146f2d4e Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 7 Jun 2017 14:48:37 -0400 Subject: [PATCH 0254/1247] Support deb package build for other architectures Use build machine architecture to build debian package. Useful for building on Power8 and ARM64 machines. Change-Id: I97fc80a6723b139e753019a355f11ced0bba0dd4 Signed-off-by: Harish Kasiviswanathan --- DEBIAN/control | 2 +- src/Makefile | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/DEBIAN/control b/DEBIAN/control index 0c3484eb5a..779d712fac 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -1,5 +1,5 @@ Package: hsakmt-rocm-dev -Architecture: amd64 +Architecture: $arch Maintainer: Advanced Micro Devices Inc. Depends:libpci3 Priority: optional diff --git a/src/Makefile b/src/Makefile index db001a63ae..0ecf9d932f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -14,6 +14,7 @@ PACKAGE_MINOR_VER = 0 PACKAGE_PATCH_VER ?= 0 PACKAGE_VER = $(PACKAGE_MAJOR_VER).$(PACKAGE_MINOR_VER).$(PACKAGE_PATCH_VER) export PACKAGE_VER +DEB_BUILD_ARCH = $(shell dpkg --print-architecture | sed 's/ \*$i//g') # Compiler options CFLAGS += -fPIC # Position-independent code required to build shared library @@ -96,9 +97,10 @@ package-common: lnx64a deb: package-common @mkdir -p $(PACKAGE_DIR)/DEBIAN - @sed 's/\$$version/$(PACKAGE_VER)/g' $(DEBIAN_DIR)/control > $(PACKAGE_DIR)/DEBIAN/control + @sed 's/\$$version/$(PACKAGE_VER)/g;s/\$$arch/$(DEB_BUILD_ARCH)/g' $(DEBIAN_DIR)/control > $(PACKAGE_DIR)/DEBIAN/control + @fakeroot dpkg-deb --build $(PACKAGE_DIR) \ - $(BUILDDIR)/hsakmt-dev-$(PACKAGE_VER)-amd64.deb + $(BUILDDIR)/hsakmt-dev-$(PACKAGE_VER)-${DEB_BUILD_ARCH}.deb rpm: package-common @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec From 5114a9368b193c4e893df77587bd73c39b6fcbb8 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 8 Jun 2017 15:22:05 -0400 Subject: [PATCH 0255/1247] Change gfx900 compute capability to 9.0.1 9.0.1 is XNACK enabled gfx900 compile target. Compiler must generate ISA that's XNACK enabled. Change-Id: Ic4987132ef9f8d06d9e2bcdb8f7eeb875cdd2b44 Signed-off-by: Amber Lin --- src/topology.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/topology.c b/src/topology.c index 1f36a1cac5..f15365049e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -163,15 +163,15 @@ static struct hsa_gfxip_table { { 0x67EF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67FF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, /* Vega10 */ - { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6861, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6862, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6863, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6864, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6867, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6868, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 } + { 0x6860, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x6861, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x6862, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x6863, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x6864, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x6867, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x6868, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x686C, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, + { 0x687F, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 } }; enum cache_type { From 6e113e26343d1ac92f7db10e539e9546bd4d0eaf Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 16 Jun 2017 14:30:39 -0400 Subject: [PATCH 0256/1247] Free control stack correctly ctl_stack_copy is allocated from malloc. It should be freed by free. Change-Id: Ib924da20200d91f52f106fe173464d47862759a8 Signed-off-by: Amber Lin --- src/queues.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/queues.c b/src/queues.c index 9860a6d6ec..983ab8705d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -424,9 +424,7 @@ static void free_queue(struct queue *q) q->ctx_save_restore_size, PAGE_SIZE, q->dev_info->asic_family); if (q->ctl_stack_copy) - free_exec_aligned_memory(q->ctl_stack_copy, - q->ctl_stack_size, - PAGE_SIZE, q->dev_info->asic_family); + free(q->ctl_stack_copy); free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } From 5b3c9f0b317bea65cf06e007c1ebef3488706e9a Mon Sep 17 00:00:00 2001 From: Evan Quan Date: Tue, 20 Jun 2017 15:33:37 +0800 Subject: [PATCH 0257/1247] Revert "Change gfx900 compute capability to 9.0.1" This reverts commit 5114a9368b193c4e893df77587bd73c39b6fcbb8. Change-Id: Id9c4f43462820bf09f25674fa30e6eb04098808e Signed-off-by: Evan Quan --- src/topology.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/topology.c b/src/topology.c index f15365049e..1f36a1cac5 100644 --- a/src/topology.c +++ b/src/topology.c @@ -163,15 +163,15 @@ static struct hsa_gfxip_table { { 0x67EF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67FF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, /* Vega10 */ - { 0x6860, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x6861, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x6862, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x6863, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x6864, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x6867, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x6868, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x686C, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 }, - { 0x687F, 9, 0, 1, 1, "Vega10", CHIP_VEGA10 } + { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6861, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6862, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6863, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6864, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6867, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6868, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 } }; enum cache_type { From 13aadde56e22c1e136e20cf00ff3b6eaf18b3b78 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 22 Jun 2017 12:12:17 -0400 Subject: [PATCH 0258/1247] Use environment variable to force gfx version For experimental purpose, we need an option to change compute capability by forcing the GfxIp version. This patch allows to use environment variable HSA_OVERRIDE_GFX_VERSION=major.minor.stepping to replace the default version. For example: export HSA_OVERRIDE_GFX_VERSION=9.0.1 Change-Id: I90cfbd43619d9d3aebf53321d4e058f01bcd7088 Signed-off-by: Amber Lin --- src/topology.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/src/topology.c b/src/topology.c index 1f36a1cac5..2456a9c197 100644 --- a/src/topology.c +++ b/src/topology.c @@ -73,7 +73,6 @@ static const char *supported_processor_vendor_name[] = { static HSAKMT_STATUS topology_take_snapshot(void); static HSAKMT_STATUS topology_drop_snapshot(void); -//static int get_cpu_stepping(uint16_t* stepping); static struct hsa_gfxip_table { uint16_t device_id; // Device ID @@ -706,11 +705,11 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, struct pci_access *pacc) { FILE *fd; - char *read_buf, *p; + char *read_buf, *p, *envvar, dummy; char prop_name[256]; char path[256]; unsigned long long prop_val; - uint32_t i, prog; + uint32_t i, prog, major, minor, step; uint16_t fw_version = 0; int read_size; const struct hsa_gfxip_table *hsa_gfxip; @@ -802,7 +801,6 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, } -// get_cpu_stepping(&stepping); props->EngineId.ui32.uCode = fw_version & 0x3ff; props->EngineId.ui32.Major = 0; props->EngineId.ui32.Minor = 0; @@ -810,9 +808,26 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); if (hsa_gfxip) { - props->EngineId.ui32.Major = hsa_gfxip->major & 0x3f; - props->EngineId.ui32.Minor = hsa_gfxip->minor; - props->EngineId.ui32.Stepping = hsa_gfxip->stepping; + envvar = getenv("HSA_OVERRIDE_GFX_VERSION"); + if (envvar) { + /* HSA_OVERRIDE_GFX_VERSION=major.minor.stepping */ + if ((sscanf(envvar, "%u.%u.%u%c", + &major, &minor, &step, &dummy) != 3) || + (major > 63 || minor > 255 || step > 255)) { + fprintf(stderr, + "HSA_OVERRIDE_GFX_VERSION %s is invalid\n", + envvar); + ret = HSAKMT_STATUS_ERROR; + goto err; + } + props->EngineId.ui32.Major = major & 0x3f; + props->EngineId.ui32.Minor = minor & 0xff; + props->EngineId.ui32.Stepping = step & 0xff; + } else { + props->EngineId.ui32.Major = hsa_gfxip->major & 0x3f; + props->EngineId.ui32.Minor = hsa_gfxip->minor; + props->EngineId.ui32.Stepping = hsa_gfxip->stepping; + } if (!hsa_gfxip->amd_name) { ret = HSAKMT_STATUS_ERROR; From ccfe73992997ad050cfd1f742ad2517f6a1ec7a2 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 26 Jun 2017 15:23:32 -0400 Subject: [PATCH 0259/1247] Introduce debug level to Thunk Existing Thunk has printf/fprintf in the code while normally libraries don't print any message. This patch introduces a print machenism similar to how the Linux kernel prints to console based on the log level. The default is not to print any message, but setting HSAKMT_DEBUG_LEVEL will enable the prints. Change-Id: Ic071e122d35a82260218e9914cde4815e69df742 Signed-off-by: Amber Lin --- src/libhsakmt.h | 18 ++++++++++++++++++ src/openclose.c | 23 ++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 0cc45ea200..7c4cc19f8c 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -64,6 +64,24 @@ extern int PAGE_SHIFT; #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) +/* HSA Thunk logging usage */ +extern int hsakmt_debug_level; +#define hsakmt_print(level, fmt, ...) \ + do { if (level <= hsakmt_debug_level) fprintf(stderr, fmt, ##__VA_ARGS__); } while (0) +#define HSAKMT_DEBUG_LEVEL_DEFAULT -1 +#define HSAKMT_DEBUG_LEVEL_ERR 3 +#define HSAKMT_DEBUG_LEVEL_WARNING 4 +#define HSAKMT_DEBUG_LEVEL_INFO 6 +#define HSAKMT_DEBUG_LEVEL_DEBUG 7 +#define pr_err(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_ERR, fmt, ##__VA_ARGS__) +#define pr_warn(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_WARNING, fmt, ##__VA_ARGS__) +#define pr_info(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__) +#define pr_debug(fmt, ...) \ + hsakmt_print(HSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__) + enum asic_family_type { CHIP_KAVERI = 0, CHIP_HAWAII, diff --git a/src/openclose.c b/src/openclose.c index 5e8b5e1aa4..204b31c360 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -25,6 +25,7 @@ #include "libhsakmt.h" +#include #include #include #include @@ -38,7 +39,7 @@ static const char kfd_device_name[] = "/dev/kfd"; static const char tmp_file[] = "/var/lock/.amd_hsa_thunk_lock"; int amd_hsa_thunk_lock_fd; static pid_t parent_pid = -1; - +int hsakmt_debug_level; static bool is_forked_child(void) { @@ -76,6 +77,25 @@ static inline void init_page_size(void) PAGE_SHIFT = ffs(PAGE_SIZE) - 1; } +/* Normally libraries don't print messages. For debugging purpose, we'll + * print messages if an environment variable, HSAKMT_DEBUG_LEVEL, is set. + */ +static void init_debug_level(void) +{ + char *envvar; + int debug_level; + + hsakmt_debug_level = HSAKMT_DEBUG_LEVEL_DEFAULT; + + envvar = getenv("HSAKMT_DEBUG_LEVEL"); + if (envvar) { + debug_level = atoi(envvar); + if (debug_level >= HSAKMT_DEBUG_LEVEL_ERR && + debug_level <= HSAKMT_DEBUG_LEVEL_DEBUG) + hsakmt_debug_level = debug_level; + } +} + HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) { HSAKMT_STATUS result; @@ -93,6 +113,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) clear_after_fork(); if (kfd_open_count == 0) { + init_debug_level(); amd_hsa_thunk_lock_fd = 0; fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); From 897c4e2fffb32bb0b001c8dde3975264dbc7992f Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 27 Jun 2017 16:42:18 -0400 Subject: [PATCH 0260/1247] Replace printf/fprintf with pr_xxx Libraries normally don't print messages. We use pr_err, pr_warn, pr_info, and pr_debug to print messages to stderr when prints are enabled for debugging. Change-Id: I9caf719343aa618c88e7b500f9737a46702e424a Signed-off-by: Amber Lin --- src/fmm.c | 49 ++++++++++++++++++++++--------------------------- src/memory.c | 10 +++++----- src/openclose.c | 7 +++---- src/perfctr.c | 9 ++++----- src/pmc_table.c | 7 +++---- src/topology.c | 27 ++++++++++++--------------- 6 files changed, 49 insertions(+), 60 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 2264e6fa9d..32562f363e 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -767,8 +767,8 @@ bool fmm_is_inside_some_aperture(void *address) #ifdef DEBUG_PRINT_APERTURE static void aperture_print(aperture_t *app) { - printf("\t Base: %p\n", app->base); - printf("\t Limit: %p\n", app->limit); + pr_info("\t Base: %p\n", app->base); + pr_info("\t Limit: %p\n", app->limit); } static void manageable_aperture_print(manageable_aperture_t *app) @@ -776,16 +776,16 @@ static void manageable_aperture_print(manageable_aperture_t *app) vm_area_t *cur = app->vm_ranges; vm_object_t *object = app->vm_objects; - printf("\t Base: %p\n", app->base); - printf("\t Limit: %p\n", app->limit); - printf("\t Ranges:\n"); + pr_info("\t Base: %p\n", app->base); + pr_info("\t Limit: %p\n", app->limit); + pr_info("\t Ranges:\n"); while (cur) { - printf("\t\t Range [%p - %p]\n", cur->start, cur->end); + pr_info("\t\t Range [%p - %p]\n", cur->start, cur->end); cur = cur->next; }; - printf("\t Objects:\n"); + pr_info("\t Objects:\n"); while (object) { - printf("\t\t Object [%p - %" PRIu64 "]\n", + pr_info("\t\t Object [%p - %" PRIu64 "]\n", object->start, object->size); object = object->next; }; @@ -796,19 +796,19 @@ void fmm_print(uint32_t gpu_id) int32_t gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id >= 0) { /* Found */ - printf("LDS aperture:\n"); + pr_info("LDS aperture:\n"); aperture_print(&gpu_mem[gpu_mem_id].lds_aperture); - printf("GPUVM aperture:\n"); + pr_info("GPUVM aperture:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].gpuvm_aperture); - printf("Scratch aperture:\n"); + pr_info("Scratch aperture:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_aperture); - printf("Scratch backing memory:\n"); + pr_info("Scratch backing memory:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_physical); } - printf("dGPU aperture:\n"); + pr_info("dGPU aperture:\n"); manageable_aperture_print(&svm.dgpu_aperture); - printf("dGPU alt aperture:\n"); + pr_info("dGPU alt aperture:\n"); manageable_aperture_print(&svm.dgpu_alt_aperture); } @@ -1533,7 +1533,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) KFD_IOC_CACHE_POLICY_COHERENT, alt_base, alt_size); if (err != 0) { - fprintf(stderr, "Error! Failed to set alt aperture for GPU [0x%x]\n", + pr_err("Failed to set alt aperture for GPU [0x%x]\n", gpu_mem[gpu_mem_id].gpu_id); ret = HSAKMT_STATUS_ERROR; } @@ -1911,10 +1911,10 @@ static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_ #ifdef DEBUG_PRINT_APERTURE device_id_array_size /= sizeof(uint32_t); - printf("device id array size %d\n", device_id_array_size); + pr_info("device id array size %d\n", device_id_array_size); for (uint32_t i = 0 ; i < device_id_array_size; i++) - printf("%d . 0x%x\n", (i+1), device_id_array[i]); + pr_info("%d . 0x%x\n", (i+1), device_id_array[i]); #endif } @@ -2191,8 +2191,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); if (ret != HSAKMT_STATUS_SUCCESS) { - fprintf(stderr, - "Unable to find vm_size for dGPU, assuming 64GB.\n"); + pr_err("Unable to find vm_size for dGPU, assuming 64GB.\n"); max_vm_limit_in_gb = 64; } max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; @@ -2216,15 +2215,13 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi munmap(ret_addr, len); } if (!ret_addr) { - fprintf(stderr, - "Failed to reserve %uGB for SVM ...\n", + pr_err("Failed to reserve %uGB for SVM ...\n", (unsigned int)(len >> 30)); continue; } if ((HSAuint64)ret_addr + min_vm_size - 1 > max_vm_limit) { /* addressable size is less than the minimum */ - fprintf(stderr, - "Got %uGB for SVM at %p with only %dGB usable ...\n", + pr_warn("Got %uGB for SVM at %p with only %dGB usable ...\n", (unsigned int)(len >> 30), ret_addr, (int)(((HSAint64)max_vm_limit - (HSAint64)ret_addr) >> 30)); @@ -2237,8 +2234,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi } if (!found) { - fprintf(stderr, - "Failed to reserve SVM address range. Giving up.\n"); + pr_err("Failed to reserve SVM address range. Giving up.\n"); return HSAKMT_STATUS_ERROR; } @@ -2421,8 +2417,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if ((gpu_id_array_size != object->registered_device_id_array_size) || memcmp(object->registered_device_id_array, gpu_id_array, gpu_id_array_size)) { - fprintf(stderr, - "Error. Changing nodes in a registered addr.\n"); + pr_err("Cannot change nodes in a registered addr.\n"); return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; } else return HSAKMT_STATUS_SUCCESS; diff --git a/src/memory.c b/src/memory.c index c4785b9ec1..fcf201a3ad 100644 --- a/src/memory.c +++ b/src/memory.c @@ -172,7 +172,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, CHECK_KFD_OPEN(); if (!MemoryAddress) { - fprintf(stderr, "FIXME: freeing NULL pointer\n"); + pr_err("FIXME: freeing NULL pointer\n"); return HSAKMT_STATUS_ERROR; } @@ -308,7 +308,7 @@ static uint64_t convertHsaToKfdRange(HsaMemoryRange *HsaRange) { if (sizeof(struct kfd_memory_range) != sizeof(HsaMemoryRange)) { - fprintf(stderr, "Struct size mismatch in thunk. Cannot cast Hsa Range to KFD IOCTL range\n"); + pr_err("Struct size mismatch in thunk. Cannot cast Hsa Range to KFD IOCTL range\n"); return 0; } return (uint64_t) HsaRange; @@ -395,7 +395,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress, CHECK_KFD_OPEN(); if (!MemoryAddress) { - fprintf(stderr, "FIXME: mapping NULL pointer\n"); + pr_err("FIXME: mapping NULL pointer\n"); return HSAKMT_STATUS_ERROR; } @@ -419,7 +419,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(void *MemoryAddress, HSAKMT_STATUS ret; if (!MemoryAddress) { - fprintf(stderr, "FIXME: mapping NULL pointer\n"); + pr_err("FIXME: mapping NULL pointer\n"); return HSAKMT_STATUS_ERROR; } @@ -443,7 +443,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) if (!MemoryAddress) { /* Workaround for runtime bug */ - fprintf(stderr, "FIXME: Unmapping NULL pointer\n"); + pr_err("FIXME: Unmapping NULL pointer\n"); return HSAKMT_STATUS_SUCCESS; } diff --git a/src/openclose.c b/src/openclose.c index 204b31c360..3a1a17b941 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -141,7 +141,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) goto init_doorbell_failed; if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) - printf("Insufficient Memory. Debugging unavailable\n"); + pr_warn("Insufficient Memory. Debugging unavailable\n"); mask = umask(0); /* save the current umask */ /* We don't want the existing umask to mask out S_IWOTH */ @@ -151,11 +151,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) 0666); umask(mask); /* restore the original umask */ if (amd_hsa_thunk_lock_fd < 0) - fprintf(stderr, - "Profiling of privileged counters is not available\n"); + pr_warn("Profiling of privileged counters is not available\n"); if (init_counter_props(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) - fprintf(stderr, "Profiling is not available\n"); + pr_warn("Profiling is not available\n"); } else { kfd_open_count++; result = HSAKMT_STATUS_SUCCESS; diff --git a/src/perfctr.c b/src/perfctr.c index f54599cd46..29801a686f 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -385,8 +385,7 @@ static HSAKMT_STATUS open_perf_event_fd(struct perf_trace_block *block) return HSAKMT_STATUS_INVALID_HANDLE; if (getuid()) { - fprintf(stderr, - "Error. Must be root to open perf_event.\n"); + pr_err("Must be root to open perf_event.\n"); return HSAKMT_STATUS_ERROR; } @@ -560,7 +559,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace(HSAuint32 NodeId, return HSAKMT_STATUS_INVALID_NODE_UNIT; if (NumberOfCounters > MAX_COUNTERS) { - fprintf(stderr, "Error: MAX_COUNTERS is too small for %d.\n", + pr_err("MAX_COUNTERS is too small for %d.\n", NumberOfCounters); return HSAKMT_STATUS_NO_MEMORY; } @@ -588,11 +587,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace(HSAuint32 NodeId, continue; concurrent_limit = get_block_concurrent_limit(NodeId, i); if (!concurrent_limit) { - fprintf(stderr, "Invalid block ID: %d\n", i); + pr_err("Invalid block ID: %d\n", i); return HSAKMT_STATUS_INVALID_PARAMETER; } if (num_counters[i] > concurrent_limit) { - fprintf(stderr, "Counters exceed the limit.\n"); + pr_err("Counters exceed the limit.\n"); return HSAKMT_STATUS_INVALID_PARAMETER; } num_blocks++; diff --git a/src/pmc_table.c b/src/pmc_table.c index 50ff81a521..e4c81f2949 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1628,7 +1628,7 @@ static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) continue; if (snprintf(path, len, "%s/%s", sysfs_amdiommu_event_path, dent->d_name) >= len) { - fprintf(stderr, "Increase path length.\n"); + pr_err("Increase path length.\n"); ret = HSAKMT_STATUS_NO_MEMORY; goto out; } @@ -1644,8 +1644,7 @@ static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) } if (num > max_counter_id) /* This should never happen. If it does, check IOMMU driver. */ - fprintf(stderr, - "Error: max_counter_id %d is set too small.\n", + pr_err("max_counter_id %d is set too small.\n", max_counter_id); else { counter_id[num] = 1; @@ -1671,7 +1670,7 @@ static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) "/sys/devices/virtual/kfd/kfd/topology/nodes", 0, /* IOMMU is in node 0. Change this if NUMA is introduced to APU. */ "perf/iommu/max_concurrent") >= len) { - fprintf(stderr, "Increase path length\n"); + pr_err("Increase path length\n"); ret = HSAKMT_STATUS_NO_MEMORY; goto out; }; diff --git a/src/topology.c b/src/topology.c index 2456a9c197..40d3316661 100644 --- a/src/topology.c +++ b/src/topology.c @@ -593,7 +593,7 @@ static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, fd = fopen(PROC_CPUINFO_PATH, "r"); if (!fd) { - printf("Failed to open [%s]. Unable to get CPU Model Name", + pr_err("Failed to open [%s]. Unable to get CPU Model Name", PROC_CPUINFO_PATH); return HSAKMT_STATUS_ERROR; } @@ -674,7 +674,7 @@ static void topology_set_processor_vendor(void) fd = fopen(PROC_CPUINFO_PATH, "r"); if (!fd) { - printf("Failed to open [%s]. Setting Processor Vendor to %s", + pr_err("Failed to open [%s]. Setting Processor Vendor to %s", PROC_CPUINFO_PATH, supported_processor_vendor_name[GENUINE_INTEL]); processor_vendor = GENUINE_INTEL; return; @@ -694,7 +694,7 @@ static void topology_set_processor_vendor(void) } } fclose(fd); - printf("Failed to get Processor Vendor. Setting to %s", + pr_err("Failed to get Processor Vendor. Setting to %s", supported_processor_vendor_name[GENUINE_INTEL]); processor_vendor = GENUINE_INTEL; } @@ -814,8 +814,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if ((sscanf(envvar, "%u.%u.%u%c", &major, &minor, &step, &dummy) != 3) || (major > 63 || minor > 255 || step > 255)) { - fprintf(stderr, - "HSA_OVERRIDE_GFX_VERSION %s is invalid\n", + pr_err("HSA_OVERRIDE_GFX_VERSION %s is invalid\n", envvar); ret = HSAKMT_STATUS_ERROR; goto err; @@ -840,7 +839,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, /* Is APU node */ ret = topology_get_cpu_model_name(props, true); if (ret != HSAKMT_STATUS_SUCCESS) { - printf("Failed to get APU Model Name from %s\n", PROC_CPUINFO_PATH); + pr_err("Failed to get APU Model Name from %s\n", PROC_CPUINFO_PATH); ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ } } else { @@ -859,7 +858,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if (!props->NumFComputeCores || !props->DeviceId) { ret = topology_get_cpu_model_name(props, false); if (ret != HSAKMT_STATUS_SUCCESS) { - printf("Failed to get CPU Model Name from %s\n", PROC_CPUINFO_PATH); + pr_err("Failed to get CPU Model Name from %s\n", PROC_CPUINFO_PATH); ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ } } else { @@ -1005,7 +1004,7 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list */ CPU_ZERO(&orig_cpuset); if (sched_getaffinity(0, sizeof(cpu_set_t), &orig_cpuset) != 0) { - printf("Failed to get CPU affinity\n"); + pr_err("Failed to get CPU affinity\n"); free(p_temp_cpu_ci_list); ret = HSAKMT_STATUS_ERROR; goto exit; @@ -1272,19 +1271,18 @@ static HsaIoLinkProperties *topology_get_free_io_link_slot_for_node(uint32_t nod HsaIoLinkProperties *props; if (node_id >= sys_props->NumNodes) { - fprintf(stderr, "Invalid node [%d]\n", node_id); + pr_err("Invalid node [%d]\n", node_id); return NULL; } props = nodes[node_id].link; if (!props) { - fprintf(stderr, "No io_link reported for Node [%d]\n", node_id); + pr_err("No io_link reported for Node [%d]\n", node_id); return NULL; } if (nodes[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { - fprintf(stderr, "No more space for io_link for Node [%d]\n", - node_id); + pr_err("No more space for io_link for Node [%d]\n", node_id); return NULL; } @@ -1470,8 +1468,7 @@ static void topology_create_indirect_gpu_links(const HsaSystemProperties *sys_pr continue; if (topology_add_io_link_for_node(i, sys_props, nodes, type, j, weight, true) != HSAKMT_STATUS_SUCCESS) - fprintf(stderr, - "Fail to add IO link %d->%d\n", i, j); + pr_err("Fail to add IO link %d->%d\n", i, j); } } } @@ -1611,7 +1608,7 @@ HSAKMT_STATUS topology_drop_snapshot(void) HSAKMT_STATUS err; if (!!_system != !!node) { - printf("Probable inconsistency?\n"); + pr_warn("Probably inconsistency?\n"); err = HSAKMT_STATUS_SUCCESS; goto out; } From 4d0697bf6554c2b826fafc5b77f9874148d71606 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 26 May 2017 16:12:09 -0400 Subject: [PATCH 0261/1247] Honor ReadOnly bit in HsaMemFlags Change-Id: I456cde81384bf0f4bf055711d94b731179706d28 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 32562f363e..6776a8ef5b 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -710,7 +710,6 @@ static vm_object_t *fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, args.flags = flags | KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED | - KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; args.va_addr = (uint64_t)mem; if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && @@ -862,6 +861,22 @@ static void fmm_release_scratch(uint32_t gpu_id) gpu_mem[gpu_mem_id].scratch_physical.limit = NULL; } +static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) +{ + uint32_t ioc_flags = 0; + + if (flags.ui32.AQLQueueMemory) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; + if (flags.ui32.ReadOnly) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_READONLY; + /* TODO: Since, ROCr interfaces doesn't allow caller to set page + * permissions, mark all user allocations with exec permission. + * Check for flags.ui32.ExecuteAccess once ROCr is ready. + */ + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS; + return ioc_flags; +} + #define SCRATCH_ALIGN 0x10000 void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { @@ -992,13 +1007,13 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla if (flags.ui32.HostAccess) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC; + ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); + if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { aperture = &svm.dgpu_aperture; offset = 0; - if (flags.ui32.AQLQueueMemory) { + if (flags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; - } } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; offset = GPUVM_APP_OFFSET; @@ -1138,10 +1153,9 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, aperture = &svm.dgpu_alt_aperture; /* coherent */ ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; } - if (flags.ui32.AQLQueueMemory) { + ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); + if (flags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; - } /* Paged memory is allocated as a userptr mapping, non-paged * memory is allocated from KFD @@ -2354,7 +2368,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, - &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR, &obj); + &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | + KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS, &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; From dc6ece67fdec622f7047bde03af476e8a3a61e9a Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 30 Jun 2017 15:55:40 -0400 Subject: [PATCH 0262/1247] Fix fd leak if application forks If the application forks, close the fd inherited from the parent. Change-Id: I48e4157d5f0d6f04d07ecb23b719a23934687cdb Signed-off-by: Harish Kasiviswanathan --- src/openclose.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/openclose.c b/src/openclose.c index 3a1a17b941..abd977a72f 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -68,6 +68,7 @@ static void clear_after_fork(void) clear_events_page(); fmm_clear_all_mem(); destroy_device_debugging_memory(); + close(kfd_fd); kfd_open_count = 0; } From 4fbffcdd9c36443f3ae33a2e298c4afd2ab463f8 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Fri, 30 Jun 2017 13:00:53 -0500 Subject: [PATCH 0263/1247] Always allocate space for control stack at beginning of save area Hardware block testing is done with the workgroup state offset initialized to the control stack size on all ASICs. MEC microcode assumes this space is available when the workgroup state offset is reset after a context restore event. Fixes context save area overrun when the full save area is used. Change-Id: I8eeb62f97140c6fe409fe78b4497d833584feea8 Signed-off-by: Jay Cornwall --- src/queues.c | 38 +++++--------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/src/queues.c b/src/queues.c index 983ab8705d..efc0f1fe62 100644 --- a/src/queues.c +++ b/src/queues.c @@ -122,7 +122,6 @@ struct queue { void *ctx_save_restore; uint32_t ctx_save_restore_size; uint32_t ctl_stack_size; - void *ctl_stack_copy; const struct device_info *dev_info; /* This queue structure is allocated from GPU with page aligned size * but only small bytes are used. We use the extra space in the end for @@ -337,11 +336,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) ctl_stack_size = cu_num * WAVES_PER_CU_VI * 8 + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU_VI; q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size); - q->ctx_save_restore_size = PAGE_ALIGN_UP(wg_data_size); - - if (q->dev_info->asic_family < CHIP_VEGA10) - /* GFX8 chips store ctl-stack with WG data */ - q->ctx_save_restore_size += q->ctl_stack_size; + q->ctx_save_restore_size = q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size); return true; } @@ -423,8 +418,6 @@ static void free_queue(struct queue *q) free_exec_aligned_memory(q->ctx_save_restore, q->ctx_save_restore_size, PAGE_SIZE, q->dev_info->asic_family); - if (q->ctl_stack_copy) - free(q->ctl_stack_copy); free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } @@ -461,11 +454,6 @@ static int handle_concrete_asic(struct queue *q, return HSAKMT_STATUS_NO_MEMORY; args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; - if (IS_SOC15(q->dev_info->asic_family)) { - q->ctl_stack_copy = malloc(q->ctl_stack_size); - if (q->ctl_stack_copy == NULL) - return HSAKMT_STATUS_NO_MEMORY; - } } } @@ -706,7 +694,6 @@ hsaKmtGetQueueInfo( { struct queue *q = PORT_UINT64_TO_VPTR(QueueId); struct kfd_ioctl_get_queue_wave_state_args args; - uintptr_t ctl_stack_base_addr = 0; CHECK_KFD_OPEN(); @@ -718,30 +705,15 @@ hsaKmtGetQueueInfo( memset(&args, 0, sizeof(args)); args.queue_id = q->queue_id; - - if (IS_SOC15(q->dev_info->asic_family)) { - /* From SOC15 onwards the control stack is kept in the MQD. - * The save area is at the beginning of the allocated space. - * Request a copy of the control stack from the KFD. - */ - ctl_stack_base_addr = (uintptr_t)q->ctl_stack_copy; - QueueInfo->UserContextSaveArea = q->ctx_save_restore; - args.ctl_stack_address = ctl_stack_base_addr; - } else { - /* Before SOC15 the control stack is already in userspace. - * The save area immediately follows the control stack. - */ - ctl_stack_base_addr = (uintptr_t)q->ctx_save_restore; - QueueInfo->UserContextSaveArea = (void *) - ((uintptr_t)q->ctx_save_restore + q->ctl_stack_size); - args.ctl_stack_address = 0; - } + args.ctl_stack_address = (uintptr_t)q->ctx_save_restore; if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_QUEUE_WAVE_STATE, &args) < 0) return HSAKMT_STATUS_ERROR; - QueueInfo->ControlStackTop = (void *)(ctl_stack_base_addr + + QueueInfo->ControlStackTop = (void *)(args.ctl_stack_address + q->ctl_stack_size - args.ctl_stack_used_size); + QueueInfo->UserContextSaveArea = (void *) + (args.ctl_stack_address + q->ctl_stack_size); QueueInfo->SaveAreaSizeInBytes = args.save_area_used_size; QueueInfo->ControlStackUsedInBytes = args.ctl_stack_used_size; QueueInfo->NumCUAssigned = q->cu_mask_count; From ac468f676c889c8697ecd44d27de39c15f3cb822 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 4 Jul 2017 11:51:17 -0400 Subject: [PATCH 0264/1247] Replace lock file with shared memory Performance counters have limited slots for concurrent profiling. We need a mechanism to synchronize the slots access across different processes. Lock file file was first used for this access control. It reveals a RedHat bug that /var/lock, symbolic linked to /run/lock, is not writable by others. To avoid this bug and to simplify the code, POSIX shared memory is created to replace the lock file usage. Access of the shared memory is controlled by semaphores. Change-Id: I1e13c17f0e042fdfe6657afe8b3c88db7e84d292 Signed-off-by: Amber Lin --- src/openclose.c | 23 +-------- src/perfctr.c | 135 +++++++++++++++++++++++++++++------------------- 2 files changed, 82 insertions(+), 76 deletions(-) diff --git a/src/openclose.c b/src/openclose.c index abd977a72f..caaa6d924c 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -36,8 +36,6 @@ #include "fmm.h" static const char kfd_device_name[] = "/dev/kfd"; -static const char tmp_file[] = "/var/lock/.amd_hsa_thunk_lock"; -int amd_hsa_thunk_lock_fd; static pid_t parent_pid = -1; int hsakmt_debug_level; @@ -102,7 +100,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) HSAKMT_STATUS result; int fd; HsaSystemProperties sys_props; - mode_t mask; pthread_mutex_lock(&hsakmt_mutex); @@ -115,7 +112,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (kfd_open_count == 0) { init_debug_level(); - amd_hsa_thunk_lock_fd = 0; fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); @@ -144,18 +140,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) pr_warn("Insufficient Memory. Debugging unavailable\n"); - mask = umask(0); /* save the current umask */ - /* We don't want the existing umask to mask out S_IWOTH */ - umask(0001); - amd_hsa_thunk_lock_fd = open(tmp_file, - O_CREAT | O_RDWR, - 0666); - umask(mask); /* restore the original umask */ - if (amd_hsa_thunk_lock_fd < 0) - pr_warn("Profiling of privileged counters is not available\n"); - if (init_counter_props(sys_props.NumNodes) != - HSAKMT_STATUS_SUCCESS) - pr_warn("Profiling is not available\n"); + init_counter_props(sys_props.NumNodes); } else { kfd_open_count++; result = HSAKMT_STATUS_SUCCESS; @@ -188,12 +173,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void) destroy_process_doorbells(); fmm_destroy_process_apertures(); close(kfd_fd); - - if (amd_hsa_thunk_lock_fd > 0) { - close(amd_hsa_thunk_lock_fd); - unlink(tmp_file); - } - } result = HSAKMT_STATUS_SUCCESS; diff --git a/src/perfctr.c b/src/perfctr.c index 29801a686f..221b0a1edc 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -34,6 +34,9 @@ #include #include #include +#include +#include +#include #define BITS_PER_BYTE CHAR_BIT @@ -66,7 +69,7 @@ enum perf_trace_action { PERF_TRACE_ACTION__RELEASE }; -struct perf_lockf_tbl { +struct perf_shared_table { uint32_t magic4cc; uint32_t iommu_slots_left; }; @@ -82,10 +85,13 @@ struct perf_counts_values { }; }; -extern int amd_hsa_thunk_lock_fd; - static HsaCounterProperties **counter_props; static unsigned int counter_props_count; +static const char shmem_name[] = "/hsakmt_shared_mem"; +static int shmem_fd; +static const char sem_name[] = "hsakmt_semaphore"; +static sem_t *sem; +struct perf_shared_table *shared_table; static ssize_t readn(int fd, void *buf, size_t n) { @@ -108,57 +114,95 @@ static ssize_t readn(int fd, void *buf, size_t n) return n; } -static HSAKMT_STATUS init_lockf_perf_section(void) +static HSAKMT_STATUS init_shared_region(void) { - struct perf_lockf_tbl lockf_tbl; - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - - if (amd_hsa_thunk_lock_fd <= 0) - return HSAKMT_STATUS_UNAVAILABLE; - - if (lockf(amd_hsa_thunk_lock_fd, F_TLOCK, sizeof(lockf_tbl))) + sem = sem_open(sem_name, O_CREAT, 0666, 1); + if (sem == SEM_FAILED) return HSAKMT_STATUS_ERROR; - memset(&lockf_tbl, 0, sizeof(lockf_tbl)); - if (readn(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) { - ret = HSAKMT_STATUS_ERROR; - goto out; + shmem_fd = shm_open(shmem_name, O_CREAT | O_RDWR, 0666); + if (shmem_fd < 0) + goto exit_1; + + if (ftruncate(shmem_fd, sizeof(struct perf_shared_table)) < 0) + goto exit_2; + + shared_table = mmap(NULL, sizeof(*shared_table), + PROT_READ | PROT_WRITE, MAP_SHARED, shmem_fd, 0); + if (shared_table == MAP_FAILED) + goto exit_2; + + return HSAKMT_STATUS_SUCCESS; + +exit_2: + shm_unlink(shmem_name); + shmem_fd = 0; +exit_1: + sem_close(sem); + sem_unlink(sem_name); + sem = NULL; + return HSAKMT_STATUS_ERROR; +} + +static void destroy_shared_region(void) +{ + if (shared_table && shared_table != MAP_FAILED) + munmap(shared_table, sizeof(*shared_table)); + + if (shmem_fd > 0) + shm_unlink(shmem_name); + if (sem && sem != SEM_FAILED) { + sem_close(sem); + sem_unlink(sem_name); } - /* If the magic number exists, the lock file table has been +} + +static void init_perf_shared_table(void) +{ + sem_wait(sem); + + /* If the magic number exists, the perf shared table has been * initialized by another process and is in use. Don't overwrite it. */ - if (lockf_tbl.magic4cc == HSA_PERF_MAGIC4CC) - goto out; - /* write the perf content */ - lockf_tbl.magic4cc = HSA_PERF_MAGIC4CC; - lockf_tbl.iommu_slots_left = - pmc_table_get_max_concurrent(PERFCOUNTER_BLOCKID__IOMMUV2); - if (write(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) - ret = HSAKMT_STATUS_ERROR; -out: - /* unlock the perf section */ - if (lockf(amd_hsa_thunk_lock_fd, F_ULOCK, sizeof(lockf_tbl))) - ret = HSAKMT_STATUS_ERROR; + if (shared_table->magic4cc == HSA_PERF_MAGIC4CC) { + sem_post(sem); + return; + } - return ret; + /* write the perf content */ + shared_table->magic4cc = HSA_PERF_MAGIC4CC; + shared_table->iommu_slots_left = + pmc_table_get_max_concurrent(PERFCOUNTER_BLOCKID__IOMMUV2); + + sem_post(sem); } HSAKMT_STATUS init_counter_props(unsigned int NumNodes) { counter_props = calloc(NumNodes, sizeof(struct HsaCounterProperties *)); - if (!counter_props) + if (!counter_props) { + pr_warn("Profiling is not available.\n"); return HSAKMT_STATUS_NO_MEMORY; + } counter_props_count = NumNodes; alloc_pmc_blocks(); - return init_lockf_perf_section(); + if (init_shared_region() != HSAKMT_STATUS_SUCCESS) { + pr_warn("Profiling of privileged blocks is not available.\n"); + return HSAKMT_STATUS_ERROR; + } + init_perf_shared_table(); + + return HSAKMT_STATUS_SUCCESS; } void destroy_counter_props(void) { unsigned int i; + destroy_shared_region(); + if (!counter_props) return; @@ -274,29 +318,18 @@ static HSAuint32 get_block_concurrent_limit(uint32_t node_id, static HSAKMT_STATUS update_block_slots(enum perf_trace_action action, uint32_t block_id, uint32_t num_slots) { - struct perf_lockf_tbl lockf_tbl; uint32_t *slots_left; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - if (amd_hsa_thunk_lock_fd <= 0) + if (shmem_fd <= 0) + return HSAKMT_STATUS_UNAVAILABLE; + if (!sem || sem == SEM_FAILED) return HSAKMT_STATUS_UNAVAILABLE; - if (lockf(amd_hsa_thunk_lock_fd, F_TLOCK, sizeof(lockf_tbl)) < 0) - return HSAKMT_STATUS_ERROR; - - if (lseek(amd_hsa_thunk_lock_fd, 0, SEEK_SET)) { - ret = HSAKMT_STATUS_ERROR; - goto out; - } - - if (readn(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) - != sizeof(lockf_tbl)) { - ret = HSAKMT_STATUS_ERROR; - goto out; - } + sem_wait(sem); if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) - slots_left = &lockf_tbl.iommu_slots_left; + slots_left = &shared_table->iommu_slots_left; else { ret = HSAKMT_STATUS_UNAVAILABLE; goto out; @@ -321,14 +354,8 @@ static HSAKMT_STATUS update_block_slots(enum perf_trace_action action, break; } - if (ret == HSAKMT_STATUS_SUCCESS) { - if (write(amd_hsa_thunk_lock_fd, &lockf_tbl, sizeof(lockf_tbl)) < 0) - ret = HSAKMT_STATUS_ERROR; - } out: - /* unlock the perf section */ - if (lockf(amd_hsa_thunk_lock_fd, F_ULOCK, sizeof(lockf_tbl))) - ret = HSAKMT_STATUS_ERROR; + sem_post(sem); return ret; } From dc2c52be78b2f3f9a93fb75b9f654e6e359f6dc6 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 17 Jul 2017 11:16:10 -0400 Subject: [PATCH 0265/1247] Align imported graphics buffers Imported graphics buffers are most likely images. Align them for tiled image access. 64KB seems to do the trick. This fixes VM faults with OpenCL graphics interop. Change-Id: I7f60e205d93fff9407e0d00d3dbb02cc4990b863 Signed-off-by: Felix Kuehling --- src/fmm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 6776a8ef5b..de46fe77c1 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2463,6 +2463,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, uint64_t offset; int r; HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; + static const uint64_t IMAGE_ALIGN = 64*1024; if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -2503,7 +2504,8 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, if (!aperture_is_valid(aperture->base, aperture->limit)) goto error_free_metadata; pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area(aperture, infoArgs.size, offset); + mem = aperture_allocate_area_aligned(aperture, infoArgs.size, offset, + MAX(aperture->align, IMAGE_ALIGN)); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) goto error_free_metadata; From c7bd7733e56510b72ab5f8235ca88aea52021cbb Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 29 Jun 2017 14:33:08 -0400 Subject: [PATCH 0266/1247] Align large buffers to BigK or huge-page boundary This should allow us to take advantage of BigK fragments and huge pages and improve TLB efficiency for VRAM allocations. Huge pages only work with 4-level page tables (gfx900 and up). BigK fragments work on older GPUs. Change-Id: I02e1fbf74de554e16fdaf44e44d03b47df45c3b0 Signed-off-by: Felix Kuehling --- src/fmm.c | 13 +++++++++++-- src/libhsakmt.h | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index de46fe77c1..4ee9d71278 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -539,11 +539,20 @@ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, vm_area_t *cur, *next; void *start; - MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); - if (align < app->align) align = app->align; + /* Huge-page and Big-K TLB optimizations require proper alignment */ + if (MemorySizeInBytes >= GPU_HUGE_PAGE_SIZE) { + if (align < GPU_HUGE_PAGE_SIZE) + align = GPU_HUGE_PAGE_SIZE; + } else if (MemorySizeInBytes >= GPU_BIGK_PAGE_SIZE) { + if (align < GPU_BIGK_PAGE_SIZE) + align = GPU_BIGK_PAGE_SIZE; + } + + MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); + /* Find a big enough "hole" in the address space */ cur = NULL; next = app->vm_ranges; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 7c4cc19f8c..8dd9d39ae2 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -55,6 +55,12 @@ extern int PAGE_SHIFT; /* VI HW bug requires this virtual address alignment */ #define TONGA_PAGE_SIZE 0x8000 +/* 64KB BigK fragment size for TLB efficiency */ +#define GPU_BIGK_PAGE_SIZE (1 << 16) + +/* 2MB huge page size for 4-level page tables on Vega10 and later GPUs */ +#define GPU_HUGE_PAGE_SIZE (2 << 20) + #define CHECK_PAGE_MULTIPLE(x) \ do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) From c48ff6b4826f88233ca908f7ecad5a4faf629ff8 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 29 Jun 2017 12:43:14 -0400 Subject: [PATCH 0267/1247] Make HSA_DISABLE_CACHE work on gfx900 Change-Id: I624390bfa70b2ff4cefd1bbdf8d960b7121f22bb Signed-off-by: Felix Kuehling --- src/fmm.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 4ee9d71278..6c5f23f379 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -44,7 +44,8 @@ .guard_pages = 1, \ .vm_ranges = NULL, \ .vm_objects = NULL, \ - .fmm_mutex = PTHREAD_MUTEX_INITIALIZER \ + .fmm_mutex = PTHREAD_MUTEX_INITIALIZER, \ + .is_coherent = false \ } struct vm_object { @@ -96,6 +97,7 @@ typedef struct { vm_area_t *vm_ranges; vm_object_t *vm_objects; pthread_mutex_t fmm_mutex; + bool is_coherent; } manageable_aperture_t; typedef struct { @@ -1028,6 +1030,9 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla offset = GPUVM_APP_OFFSET; } + if (aperture->is_coherent) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + mem = __fmm_allocate_device(gpu_id, size, aperture, offset, &mmap_offset, ioc_flags, &vm_obj); @@ -1156,13 +1161,15 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, size = MemorySizeInBytes; ioc_flags = 0; - if (flags.ui32.CoarseGrain) { + if (flags.ui32.CoarseGrain) aperture = &svm.dgpu_aperture; - } else { - aperture = &svm.dgpu_alt_aperture; /* coherent */ + else + aperture = &svm.dgpu_alt_aperture; /* always coherent */ + + if (aperture->is_coherent) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - } ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); + if (flags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; @@ -1548,7 +1555,9 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) alt_size = (alt_size + 0xffff) & ~0xffffULL; svm.dgpu_alt_aperture.base = (void *)alt_base; svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); + svm.dgpu_alt_aperture.is_coherent = true; svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); + svm.dgpu_aperture.is_coherent = !!disableCache; err = fmm_set_memory_policy(gpu_mem[gpu_mem_id].gpu_id, disableCache ? KFD_IOC_CACHE_POLICY_COHERENT : From a1353acd85f01dab0c960990d3f038c3bdbfbad7 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Tue, 18 Jul 2017 13:33:30 -0500 Subject: [PATCH 0268/1247] Update the ROCt CMakeList.txt files to build both runtime and devel packages. Change-Id: I01b6e4e5db91dd5f56ffea54c548e10f1f4aae5d --- CMakeLists.txt | 78 +++++++++++++++++++-------------- Makefile | 18 -------- README.md | 32 ++++++++------ hsakmt-dev.txt | 75 ++++++++++++++++++++++++++++++++ src/Makefile | 116 ------------------------------------------------- 5 files changed, 139 insertions(+), 180 deletions(-) delete mode 100644 Makefile create mode 100644 hsakmt-dev.txt delete mode 100644 src/Makefile diff --git a/CMakeLists.txt b/CMakeLists.txt index bc8ce505bb..21aab02cd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,23 +25,24 @@ cmake_minimum_required ( VERSION 3.5.0 ) -set ( HSAKMT_NAME "hsakmt" ) -set ( HSAKMT_COMPONENT "lib${HSAKMT_NAME}" ) -set ( HSAKMT_TARGET "${HSAKMT_NAME}" ) +set ( HSAKMT "hsakmt" ) +set ( HSAKMT_PACKAGE "hsakmt-roct" ) +set ( HSAKMT_COMPONENT "lib${HSAKMT}" ) +set ( HSAKMT_TARGET "${HSAKMT}" ) project ( ${HSAKMT_TARGET} ) -list ( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules" ) - +## Include common cmake modules include ( utils ) -## Get the package version. The defaults to 1.0.0. +## Setup the package version. get_version ( "1.0.0" ) set ( BUILD_VERSION_MAJOR ${VERSION_MAJOR} ) set ( BUILD_VERSION_MINOR ${VERSION_MINOR} ) set ( BUILD_VERSION_PATCH ${VERSION_PATCH} ) set ( LIB_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) + if ( DEFINED VERSION_BUILD AND NOT ${VERSION_BUILD} STREQUAL "" ) message ( "VERSION BUILD DEFINED ${VERSION_BUILD}" ) set ( BUILD_VERSION_PATCH "${BUILD_VERSION_PATCH}-${VERSION_BUILD}" ) @@ -57,22 +58,22 @@ set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Werror -Wno-unused-parameter -Wform set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) ## Linker Flags -set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.$(VERSION_MAJOR) -Wl,-z,nodelete" ) +set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.$(PROJECT_VERSION_MAJOR) -Wl,-z,nodelete" ) ## Source files -set ( HSAKMT_SRC ${HSAKMT_SRC} src/debug.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/events.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/fmm.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/globals.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/libhsakmt.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/memory.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/openclose.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/perfctr.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/pmc_table.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/queues.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/time.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/topology.c ) -set ( HSAKMT_SRC ${HSAKMT_SRC} src/version.c ) +set ( HSAKMT_SRC "src/debug.c" + "src/events.c" + "src/fmm.c" + "src/globals.c" + "src/libhsakmt.c" + "src/memory.c" + "src/openclose.c" + "src/perfctr.c" + "src/pmc_table.c" + "src/queues.c" + "src/time.c" + "src/topology.c" + "src/version.c") ## Include paths include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/include ) @@ -94,28 +95,39 @@ if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} ${HSAKMT_COMPONENT}.so ) endif () -## Create symlinks for packaging and install -add_custom_target ( ${HSAKMT_COMPONENT} ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../${HSAKMT_COMPONENT}/include/${HSAKMT_COMPONENT} ${HSAKMT_COMPONENT}-link ) -add_custom_target ( ${HSAKMT_COMPONENT}.so-link ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../${HSAKMT_COMPONENT}/lib/${HSAKMT_COMPONENT}.so ${HSAKMT_COMPONENT}.so-link ) +## Define default variable and variables for the optional build target hsakmt-dev +set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) +set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) +set ( CPACK_PACKAGING_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default packaging prefix." ) +set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) -## Set the install targets +## Specify build, install and package targets hsakmt-dev +set ( DEV_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/hsakmt-roct-dev ) +configure_file ( hsakmt-dev.txt ${DEV_BUILD_DIR}/CMakeLists.txt @ONLY ) +add_custom_target ( build-dev + COMMAND ${CMAKE_COMMAND} + -DSOURCE_DIR="${SOURCE_DIR}" + -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" + -DCPACK_PACKAGING_INSTALL_PREFIX="${CPACK_PACKAGING_INSTALL_PREFIX}" + COMMAND rm -rf *.deb *.rpm *.tar.gz + COMMAND make package + WORKING_DIRECTORY ${DEV_BUILD_DIR} ) +## Remember to remove the 'ALL' dependency and the package-copy target in final version +add_custom_target ( install-dev ALL DEPENDS build-dev COMMAND make install WORKING_DIRECTORY ${DEV_BUILD_DIR} ) +add_custom_target ( package-dev ALL DEPENDS install-dev COMMAND make package WORKING_DIRECTORY ${DEV_BUILD_DIR} ) +add_custom_target ( package-copy ALL DEPENDS package-dev COMMAND ${CMAKE_COMMAND} -E copy hsakmt-roct-dev-* .. WORKING_DIRECTORY ${DEV_BUILD_DIR} ) + +## Add the install directives for the runtime library. install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}-link DESTINATION include RENAME ${HSAKMT_COMPONENT} ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_COMPONENT}.so-link DESTINATION lib RENAME ${HSAKMT_COMPONENT}.so ) -## Packaging directives -set ( CPACK_PACKAGE_NAME "hsakmt-roct-dev" ) +## Add the packaging directives for the runtime library. +set ( CPACK_PACKAGE_NAME ${HSAKMT_PACKAGE} ) set ( CPACK_PACKAGE_VENDOR "AMD" ) set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT library for AMD KFD support" ) -set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md" ) # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) diff --git a/Makefile b/Makefile deleted file mode 100644 index c306e90756..0000000000 --- a/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -.PHONY: all clean lnx64a lnx - -all: lnx64a - -deb: - $(MAKE) -C src deb - -rpm: - $(MAKE) -C src rpm - -lnx64a: - $(MAKE) -C src lnx64a - -lnx: - $(MAKE) -C src lnx - -clean: - $(MAKE) -C src clean diff --git a/README.md b/README.md index 304a7f929a..2d5d9bf819 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,32 @@ -### Boltzmann Thunk Library +### ROCt Library -This repository includes the user-mode API interfaces used to interact with the Boltzmann KFD driver. Currently supported agents include only the AMD/ATI Fiji family of discrete GPUs. +This repository includes the user-mode API interfaces used to interact with the ROCk driver. Currently supported agents include only the AMD/ATI Fiji family of discrete GPUs. -#### Boltzmann Kernel Driver +#### ROCk Driver -The thunk is not a standalone product and requires that you have the correct KFD installed. We recommend reading the full compatibility and installation details which are available in the ROCK github: -https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver +The ROCt library is not a standalone product and requires that you have the correct ROCk driver set installed. We recommend reading the full compatibility and installation details which are available in the ROCk github: -#### Binaries for Ubuntu and Fedora - -For deb and rpm binaries, please check the packages/ folder in the ROCK repository: https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver #### Building the Thunk -A simple make-based system is available for building thunk. The following are the supported targets: +A simple cmake-based system is available for building thunk. To build the runtime from the the ROCT-Thunk-Interface directory, execute: ```bash + mkdir -p build + cd build + cmake .. make - make deb - make rpm - make clean +``` + +If the hsakmt-roct and hsakmt-roct-dev packages are desired: + +```bash + mkdir -p build + cd build + cmake .. + make package + make package-dev ``` #### Disclaimer @@ -29,4 +35,4 @@ The information contained herein is for informational purposes only, and is subj AMD, the AMD Arrow logo, and combinations thereof are trademarks of Advanced Micro Devices, Inc. Other product names used in this publication are for identification purposes only and may be trademarks of their respective companies. -Copyright (c) 2014-2016 Advanced Micro Devices, Inc. All rights reserved. +Copyright (c) 2014-2017 Advanced Micro Devices, Inc. All rights reserved. diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt new file mode 100644 index 0000000000..a818598399 --- /dev/null +++ b/hsakmt-dev.txt @@ -0,0 +1,75 @@ +################################################################################ +## +## Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved. +## +## MIT LICENSE: +## Permission is hereby granted, free of charge, to any person obtaining a copy of +## this software and associated documentation files (the "Software"), to deal in +## the Software without restriction, including without limitation the rights to +## use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +## of the Software, and to permit persons to whom the Software is furnished to do +## so, subject to the following conditions: +## +## The above copyright notice and this permission notice shall be included in all +## copies or substantial portions of the Software. +## +## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +## SOFTWARE. +## +################################################################################ + +cmake_minimum_required ( VERSION 3.5.0 ) + +set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-dev" ) + +project ( ${HSAKMT_DEV_PACKAGE} ) + +## Set the runtime package name. +set ( HSAKMT_PACKAGE @HSAKMT_PACKAGE@ ) + +## Setup the package version. +set ( BUILD_VERSION_MAJOR @BUILD_VERSION_MAJOR@ ) +set ( BUILD_VERSION_MINOR @BUILD_VERSION_MINOR@ ) +set ( BUILD_VERSION_PATCH @BUILD_VERSION_PATCH@ ) + +## Verbose output. +set ( CMAKE_VERBOSE_MAKEFILE on ) + +## Create symlinks for packaging and install +add_custom_target ( ${HSAKMT_DEV_PACKAGE} ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/include/libhsakmt libhsakmt-link ) +add_custom_target ( ${HSAKMT_COMPONENT}.so-link ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/lib/libhsakmt.so libhsakmt.so-link ) + +## Set the install targets +install ( FILES ${SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt ) +install ( FILES ${SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt ) +install ( FILES ${SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt ) +install ( FILES ${SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt-link DESTINATION include RENAME libhsakmt ) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.so-link DESTINATION lib RENAME libhsakmt.so ) + +## Set the default generator types for the devel package. +set ( CPACK_GENERATOR "DEB;RPM;TGZ" CACHE STRING "Default packaging generators." ) + +## Packaging directives +set ( CPACK_PACKAGE_NAME ${HSAKMT_DEV_PACKAGE} ) +set ( CPACK_PACKAGE_VENDOR "AMD" ) +set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) +set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) +set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) +set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) +set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT development package." ) +set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) + +# Debian package specific variables +set ( CPACK_DEBIAN_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) +set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) + +# RPM package specific variables +set ( CPACK_RPM_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} = ${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) + +include ( CPack ) diff --git a/src/Makefile b/src/Makefile deleted file mode 100644 index 0ecf9d932f..0000000000 --- a/src/Makefile +++ /dev/null @@ -1,116 +0,0 @@ -# Our directory -THUNK_ROOT ?= $(abspath ../) - -# Include directories -INCLUDE_DIR := ../include -INCLUDES += $(INCLUDE_DIR) -CFLAGS += $(foreach DIR,$(INCLUDES),-I$(DIR)) - -LIB_NAME = libhsakmt.so -LIB_MAJOR_VER = 1 - -PACKAGE_MAJOR_VER = 2 -PACKAGE_MINOR_VER = 0 -PACKAGE_PATCH_VER ?= 0 -PACKAGE_VER = $(PACKAGE_MAJOR_VER).$(PACKAGE_MINOR_VER).$(PACKAGE_PATCH_VER) -export PACKAGE_VER -DEB_BUILD_ARCH = $(shell dpkg --print-architecture | sed 's/ \*$i//g') - -# Compiler options -CFLAGS += -fPIC # Position-independent code required to build shared library -CFLAGS += -W -Wall -Wextra -Werror -Wno-unused-parameter -CFLAGS += -Wformat-security -Wswitch-default -Wundef \ - -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual \ - -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes \ - -Wmissing-declarations -Wredundant-decls \ - -Wunreachable-code -CFLAGS += -std=gnu99 -pthread -fvisibility=hidden -O2 - -# If this is not a release build, add debug-specific flags -ifneq ($(REL),1) - CFLAGS += -ggdb -endif - -LDFLAGS += -lrt -pthread -lpci -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) -Wl,-z,nodelete - -OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ - events.o openclose.o queues.o topology.o fmm.o pmc_table.o \ - libhsakmt.o - -.PHONY: all lnx lnx64a clean - -# Default target -all: lnx64a - -# To relocate build output, set "O=target/dir/" -# otherwise it defaults to a local directory - -ifdef O - BUILD_ROOT := $(O) -else - BUILD_ROOT := ../build -endif -BUILD_ROOT := $(abspath $(BUILD_ROOT)) - -BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) -TARGET = $(addprefix $(BUILDDIR)/,$(OBJS)) - -$(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER): $(TARGET) - gcc -shared -o $@ $^ $(LDFLAGS) - -$(BUILDDIR)/$(LIB_NAME): $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) - @ln -sf $(LIB_NAME).$(LIB_MAJOR_VER) $(BUILDDIR)/$(LIB_NAME) - -lnx: CFLAGS += -m32 -lnx: LDFLAGS += -m32 -lnx: $(BUILDDIR)/$(LIB_NAME) - -PACKAGE_DIR = $(BUILDDIR)/libhsakmt -DEBIAN_DIR = ../DEBIAN -LDCONF_FILE = $(DEBIAN_DIR)/x86_64-libhsakmt.conf -LDCONF_INSTALL_DIR = $(PACKAGE_DIR)/etc/ld.so.conf.d -ROCM_ROOT = $(PACKAGE_DIR)/opt/rocm -ROCM_LIB = $(ROCM_ROOT)/lib -ROCM_INCLUDE = $(ROCM_ROOT)/include -INSTALL_LIB_DIR = libhsakmt/lib -INSTALL_INCLUDE_DIR = libhsakmt/include/libhsakmt - -export PACKAGE_DIR - -package-common: lnx64a - @mkdir -p $(PACKAGE_DIR) - - @mkdir -p $(LDCONF_INSTALL_DIR) - @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) - - @mkdir -p $(ROCM_ROOT)/$(INSTALL_LIB_DIR) - @cp $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_ROOT)/$(INSTALL_LIB_DIR) - - @mkdir -p $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) - @cp -r $(INCLUDE_DIR)/* $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) - - @mkdir -p $(ROCM_LIB) - @ln -fs ../$(INSTALL_LIB_DIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_LIB)/$(LIB_NAME) - - @mkdir -p $(ROCM_INCLUDE) - ln -fs ../$(INSTALL_INCLUDE_DIR) $(ROCM_INCLUDE)/ - -deb: package-common - @mkdir -p $(PACKAGE_DIR)/DEBIAN - @sed 's/\$$version/$(PACKAGE_VER)/g;s/\$$arch/$(DEB_BUILD_ARCH)/g' $(DEBIAN_DIR)/control > $(PACKAGE_DIR)/DEBIAN/control - - @fakeroot dpkg-deb --build $(PACKAGE_DIR) \ - $(BUILDDIR)/hsakmt-dev-$(PACKAGE_VER)-${DEB_BUILD_ARCH}.deb - -rpm: package-common - @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec - -lnx64a: $(BUILDDIR)/$(LIB_NAME) - -clean: - rm -rf $(BUILD_ROOT) -#Rule -$(BUILDDIR)/%.o: %.c ../include/hsakmt.h ../include/hsakmttypes.h ../include/linux/kfd_ioctl.h - @echo Compiling $^ - @mkdir -p $(dir $@) - gcc $(CFLAGS) -c $< -o $@ From e93d3de0a1b57dfce2c0dca4355b1165e9d85654 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Wed, 19 Jul 2017 17:16:17 -0500 Subject: [PATCH 0269/1247] Final changes to roct CMakeLists.txt file for devel package. Change-Id: Ie0ce0c5cd8e7811f67e92439d1df1612eabefdfa --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 21aab02cd7..8c954d20bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,10 +112,10 @@ add_custom_target ( build-dev COMMAND rm -rf *.deb *.rpm *.tar.gz COMMAND make package WORKING_DIRECTORY ${DEV_BUILD_DIR} ) -## Remember to remove the 'ALL' dependency and the package-copy target in final version -add_custom_target ( install-dev ALL DEPENDS build-dev COMMAND make install WORKING_DIRECTORY ${DEV_BUILD_DIR} ) -add_custom_target ( package-dev ALL DEPENDS install-dev COMMAND make package WORKING_DIRECTORY ${DEV_BUILD_DIR} ) -add_custom_target ( package-copy ALL DEPENDS package-dev COMMAND ${CMAKE_COMMAND} -E copy hsakmt-roct-dev-* .. WORKING_DIRECTORY ${DEV_BUILD_DIR} ) + +## Custom targets for the devel package +add_custom_target ( install-dev DEPENDS build-dev COMMAND make install WORKING_DIRECTORY ${DEV_BUILD_DIR} ) +add_custom_target ( package-dev DEPENDS build-dev COMMAND make package WORKING_DIRECTORY ${DEV_BUILD_DIR} ) ## Add the install directives for the runtime library. install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) From ee22d8076007b74810f898a1012e73e5b8fccbf3 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Thu, 20 Jul 2017 09:17:54 -0500 Subject: [PATCH 0270/1247] Update README.md to include new build instructions. Change-Id: I72ca67d3016c99682cfe745bfd74c722ea181a61 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d5d9bf819..80e26fb506 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ A simple cmake-based system is available for building thunk. To build the runtim ```bash mkdir -p build cd build - cmake .. + cmake -DCMAKE_MODULE_PATH=../cmake_modules .. make ``` From d563e2cb1dd002d970c8e4b2530f151787feac3c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 20 Jul 2017 21:03:31 -0400 Subject: [PATCH 0271/1247] Update image alignment to 256KB Needed for some tiling formats. Change-Id: Icd460edaa77ccbeb3c98bc74b574ca5517db22af Signed-off-by: Felix Kuehling --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 6c5f23f379..d6e67701ed 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2481,7 +2481,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, uint64_t offset; int r; HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; - static const uint64_t IMAGE_ALIGN = 64*1024; + static const uint64_t IMAGE_ALIGN = 256*1024; if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; From e46743b1dd874f73935e873f15c8de986cc148f2 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 19 Jul 2017 16:04:42 -0400 Subject: [PATCH 0272/1247] Workaround cpuid issue under Valgrind Topology uses cpuid to get CPU cache information. However when running under Valgrind, data returned from cpuid are not from the processor we set affinity to. Instead they are all from one specific processor. For a quick workaround so other teams can continue their work, this patch will report CPU cache from that specific processor and ignore others. Change-Id: I5cfac2329dac277f3dbde1be92fa26e085465401 Signed-off-by: Amber Lin --- src/topology.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 40d3316661..ba26af9101 100644 --- a/src/topology.c +++ b/src/topology.c @@ -58,6 +58,7 @@ typedef struct { static HsaSystemProperties *_system = NULL; static node_t *node = NULL; +static int is_valgrind; static int processor_vendor; /* Supported System Vendors */ @@ -407,6 +408,12 @@ static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) cpu_cacheinfo_t *this_cpu, *cpu2; uint32_t index; + /* FixMe: cpuid under Valgrind doesn't return data from the processor we set + * affinity to. We can't use that data to calculate siblings. + */ + if (is_valgrind) + return; + for (n = 0; n < cpu_ci_list->len; n++) { this_cpu = cpu_ci_list + n; for (index = 0; index < this_cpu->num_caches; index++) { @@ -1036,8 +1043,10 @@ err: /* restore affinity to original */ sched_setaffinity(0, sizeof(cpu_set_t), &orig_cpuset); exit: - if (ret != HSAKMT_STATUS_SUCCESS) + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_warn("Topology fails to create cpu cache list\n"); topology_destroy_temp_cpu_cache_list(*temp_cpu_ci_list); + } return ret; } @@ -1074,6 +1083,16 @@ static HSAKMT_STATUS topology_get_cpu_cache_props(node_t *tbl, this_cpu->num_caches - this_cpu->num_duplicated_caches; } + /* FixMe: cpuid under Valgrind doesn't return data from the processor we set + * affinity to. All the data come from one specific processor. We'll report + * this one processor's cache and ignore others. + */ + if (is_valgrind) { + this_cpu = cpu_ci_list; + tbl->node.NumCaches = this_cpu->num_caches; + apicid_low = apicid_max = this_cpu->apicid; + } + tbl->cache = calloc( sizeof(HsaCacheProperties) * tbl->node.NumCaches, 1); if (!tbl->cache) { @@ -1481,8 +1500,15 @@ HSAKMT_STATUS topology_take_snapshot(void) void *cpu_ci_list = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; struct pci_access *pacc; + char *envvar; topology_set_processor_vendor(); + envvar = getenv("HSA_RUNNING_UNDER_VALGRIND"); + if (envvar && !strcmp(envvar, "1")) + is_valgrind = 1; + else + is_valgrind = 0; + retry: ret = topology_sysfs_get_generation(&gen_start); if (ret != HSAKMT_STATUS_SUCCESS) From 20f0de71df10807ee7af3eb2ef72fd28bff72589 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 24 Jul 2017 14:46:13 -0400 Subject: [PATCH 0273/1247] Fix inconsistent calling of validate_nodeid Change-Id: I3e8e65a5629059abdde89832b619cd8bf1f2b36c Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 4 ++-- src/perfctr.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index d6e67701ed..cbec8c9a6c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2595,8 +2595,8 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, return HSAKMT_STATUS_INVALID_PARAMETER; r = validate_nodeid(obj->node_id, &gpu_id); - if (r < 0) - return HSAKMT_STATUS_ERROR; + if (r != HSAKMT_STATUS_SUCCESS) + return r; exportArgs.handle = obj->handle; exportArgs.gpu_id = gpu_id; diff --git a/src/perfctr.c b/src/perfctr.c index 221b0a1edc..5a0d6bd84a 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -492,7 +492,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId, if (!CounterProperties) return HSAKMT_STATUS_INVALID_PARAMETER; - if (validate_nodeid(NodeId, &gpu_id) != 0) + if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (counter_props[NodeId]) { @@ -706,7 +706,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcUnregisterTrace(HSAuint32 NodeId, if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; - if (validate_nodeid(NodeId, &gpu_id) != 0) + if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) return HSAKMT_STATUS_INVALID_NODE_UNIT; trace = (struct perf_trace *)PORT_UINT64_TO_VPTR(TraceId); From 186527d0b7b9eb5e7ddb7dbb11f16799ac92d086 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 24 Jul 2017 15:12:24 -0400 Subject: [PATCH 0274/1247] Support IPC sharing of non paged system memory Non paged system memory is allocated with node id 0. However, since a gpu node is required for allocating system memory via KFD, the first dgpu is used. In hsaKmtShareMemory() if system memory use the same (first) dgpu. Change-Id: I85789a89a4e4f7888e3826826401ea89ce4d1718 Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index cbec8c9a6c..5695ece5b4 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2597,7 +2597,14 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, r = validate_nodeid(obj->node_id, &gpu_id); if (r != HSAKMT_STATUS_SUCCESS) return r; - + if (!gpu_id && is_dgpu) { + /* Sharing non paged system memory. Use first dgpu which was + * used during allocation. See fmm_allocate_host_gpu() + */ + r = find_first_dgpu(&gpu_id); + if (r != HSAKMT_STATUS_SUCCESS) + return r; + } exportArgs.handle = obj->handle; exportArgs.gpu_id = gpu_id; From 2c2de075a981c3fc0bac63619de9a78e675be9e5 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sun, 30 Jul 2017 08:21:35 -0500 Subject: [PATCH 0275/1247] Add back GNU Makefiles. Change-Id: I4a367655a905a85d4c29980aa2da8ac28db73d10 --- Makefile | 18 ++++++++ src/Makefile | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 Makefile create mode 100644 src/Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..c306e90756 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +.PHONY: all clean lnx64a lnx + +all: lnx64a + +deb: + $(MAKE) -C src deb + +rpm: + $(MAKE) -C src rpm + +lnx64a: + $(MAKE) -C src lnx64a + +lnx: + $(MAKE) -C src lnx + +clean: + $(MAKE) -C src clean diff --git a/src/Makefile b/src/Makefile new file mode 100644 index 0000000000..615fd9b10c --- /dev/null +++ b/src/Makefile @@ -0,0 +1,116 @@ +# Our directory +THUNK_ROOT ?= $(abspath ../) + +# Include directories +INCLUDE_DIR := ../include +INCLUDES += $(INCLUDE_DIR) +CFLAGS += $(foreach DIR,$(INCLUDES),-I$(DIR)) + +LIB_NAME = libhsakmt.so +LIB_MAJOR_VER = 1 + +PACKAGE_MAJOR_VER = 1 +PACKAGE_MINOR_VER = 0 +PACKAGE_PATCH_VER ?= 0 +PACKAGE_VER = $(PACKAGE_MAJOR_VER).$(PACKAGE_MINOR_VER).$(PACKAGE_PATCH_VER) +export PACKAGE_VER +DEB_BUILD_ARCH = $(shell dpkg --print-architecture | sed 's/ \*$i//g') + +# Compiler options +CFLAGS += -fPIC # Position-independent code required to build shared library +CFLAGS += -W -Wall -Wextra -Werror -Wno-unused-parameter +CFLAGS += -Wformat-security -Wswitch-default -Wundef \ + -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual \ + -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes \ + -Wmissing-declarations -Wredundant-decls \ + -Wunreachable-code +CFLAGS += -std=gnu99 -pthread -fvisibility=hidden -O2 + +# If this is not a release build, add debug-specific flags +ifneq ($(REL),1) + CFLAGS += -ggdb +endif + +LDFLAGS += -lrt -pthread -lpci -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) -Wl,-z,nodelete + +OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ + events.o openclose.o queues.o topology.o fmm.o pmc_table.o \ + libhsakmt.o + +.PHONY: all lnx lnx64a clean + +# Default target +all: lnx64a + +# To relocate build output, set "O=target/dir/" +# otherwise it defaults to a local directory + +ifdef O + BUILD_ROOT := $(O) +else + BUILD_ROOT := ../build +endif +BUILD_ROOT := $(abspath $(BUILD_ROOT)) + +BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) +TARGET = $(addprefix $(BUILDDIR)/,$(OBJS)) + +$(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER): $(TARGET) + gcc -shared -o $@ $^ $(LDFLAGS) + +$(BUILDDIR)/$(LIB_NAME): $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) + @ln -sf $(LIB_NAME).$(LIB_MAJOR_VER) $(BUILDDIR)/$(LIB_NAME) + +lnx: CFLAGS += -m32 +lnx: LDFLAGS += -m32 +lnx: $(BUILDDIR)/$(LIB_NAME) + +PACKAGE_DIR = $(BUILDDIR)/libhsakmt +DEBIAN_DIR = ../DEBIAN +LDCONF_FILE = $(DEBIAN_DIR)/x86_64-libhsakmt.conf +LDCONF_INSTALL_DIR = $(PACKAGE_DIR)/etc/ld.so.conf.d +ROCM_ROOT = $(PACKAGE_DIR)/opt/rocm +ROCM_LIB = $(ROCM_ROOT)/lib +ROCM_INCLUDE = $(ROCM_ROOT)/include +INSTALL_LIB_DIR = libhsakmt/lib +INSTALL_INCLUDE_DIR = libhsakmt/include/libhsakmt + +export PACKAGE_DIR + +package-common: lnx64a + @mkdir -p $(PACKAGE_DIR) + + @mkdir -p $(LDCONF_INSTALL_DIR) + @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) + + @mkdir -p $(ROCM_ROOT)/$(INSTALL_LIB_DIR) + @cp $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_ROOT)/$(INSTALL_LIB_DIR) + + @mkdir -p $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) + @cp -r $(INCLUDE_DIR)/* $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) + + @mkdir -p $(ROCM_LIB) + @ln -fs ../$(INSTALL_LIB_DIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_LIB)/$(LIB_NAME) + + @mkdir -p $(ROCM_INCLUDE) + ln -fs ../$(INSTALL_INCLUDE_DIR) $(ROCM_INCLUDE)/ + +deb: package-common + @mkdir -p $(PACKAGE_DIR)/DEBIAN + @sed 's/\$$version/$(PACKAGE_VER)/g;s/\$$arch/$(DEB_BUILD_ARCH)/g' $(DEBIAN_DIR)/control > $(PACKAGE_DIR)/DEBIAN/control + + @fakeroot dpkg-deb --build $(PACKAGE_DIR) \ + $(BUILDDIR)/hsakmt-dev-$(PACKAGE_VER)-${DEB_BUILD_ARCH}.deb + +rpm: package-common + @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec + +lnx64a: $(BUILDDIR)/$(LIB_NAME) + +clean: + rm -rf $(BUILD_ROOT) +#Rule +$(BUILDDIR)/%.o: %.c ../include/hsakmt.h ../include/hsakmttypes.h ../include/linux/kfd_ioctl.h + @echo Compiling $^ + @mkdir -p $(dir $@) + gcc $(CFLAGS) -c $< -o $@ From 81768305771cd7623443f79b4d6b6f0afb9f21ce Mon Sep 17 00:00:00 2001 From: ozeng Date: Mon, 31 Jul 2017 19:25:59 -0400 Subject: [PATCH 0276/1247] CMakeLists changes to make thunk buildable on CentOS 6.9 Removed Werror CFLAGS for lower version of gcc. there will be some warning message on lower gcc version but build is ok. Signed-off-by: Oak Zeng Change-Id: Icf556625cb870c4ad73e1d89f3d4ade3a96e821f --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c954d20bb..bb283f1e27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,7 +53,11 @@ set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUIL set ( CMAKE_VERBOSE_MAKEFILE on ) ## Compiler flags -set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Werror -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden -O2" ) +set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden -O2" ) +if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") + set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror" ) +endif () + set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) From d0e287201160cf369ee7476cfef111cde5f0dff3 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 24 May 2017 17:39:07 -0400 Subject: [PATCH 0277/1247] Add gfx902 support Change-Id: Iefc6d1bea0d1d2ea8768867c53f16cdf1279d38f Signed-off-by: Yong Zhao --- src/libhsakmt.h | 7 ++++--- src/pmc_table.c | 1 + src/queues.c | 9 ++++++++- src/topology.c | 4 +++- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 8dd9d39ae2..ac42d60888 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -96,10 +96,11 @@ enum asic_family_type { CHIP_FIJI, CHIP_POLARIS10, CHIP_POLARIS11, - CHIP_VEGA10 + CHIP_VEGA10, + CHIP_RAVEN }; -#define IS_DGPU(chip) (((chip) >= CHIP_TONGA && (chip) <= CHIP_VEGA10) || \ - (chip) == CHIP_HAWAII) +#define IS_DGPU(chip) ((chip) != CHIP_KAVERI && (chip) != CHIP_CARRIZO && \ + (chip) != CHIP_RAVEN) #define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); diff --git a/src/pmc_table.c b/src/pmc_table.c index e4c81f2949..1bb4ba04bf 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1739,6 +1739,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, *block = polaris_blocks[block_id]; break; case CHIP_VEGA10: + case CHIP_RAVEN: *block = vega_blocks[block_id]; break; default: diff --git a/src/queues.c b/src/queues.c index efc0f1fe62..7765e76a36 100644 --- a/src/queues.c +++ b/src/queues.c @@ -98,6 +98,12 @@ struct device_info vega10_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +struct device_info raven_device_info = { + .asic_family = CHIP_RAVEN, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -106,7 +112,8 @@ static struct device_info *dev_lookup_table[] = { [CHIP_FIJI] = &fiji_device_info, [CHIP_POLARIS10] = &polaris10_device_info, [CHIP_POLARIS11] = &polaris11_device_info, - [CHIP_VEGA10] = &vega10_device_info + [CHIP_VEGA10] = &vega10_device_info, + [CHIP_RAVEN] = &raven_device_info }; struct device_id { diff --git a/src/topology.c b/src/topology.c index ba26af9101..784f005957 100644 --- a/src/topology.c +++ b/src/topology.c @@ -171,7 +171,9 @@ static struct hsa_gfxip_table { { 0x6867, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6868, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 } + { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + /* Raven */ + { 0x15DD, 9, 0, 0, 0, "Raven", CHIP_RAVEN }, }; enum cache_type { From 78e683acf48d670fcc9eeef6f144ab3503caa8e5 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 5 Aug 2017 00:45:50 -0400 Subject: [PATCH 0278/1247] Changes to run on old kernels Fall back to older apertures API and old events page size if the new APIs fail. This allows running on current upstream kernels (with only minor fixes) on gfx801 and enables testing of further changes during upstreaming. Change-Id: I9d86d4f576e52fcbb5bc158d80f1bf41261e4e87 Signed-off-by: Felix Kuehling --- src/events.c | 12 ++++++++++-- src/fmm.c | 50 +++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/src/events.c b/src/events.c index d934391c6d..876bc846d8 100644 --- a/src/events.c +++ b/src/events.c @@ -51,6 +51,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, bool ManualReset, bool IsSignaled, HsaEvent **Event) { + unsigned int event_limit = KFD_SIGNAL_EVENT_LIMIT; + CHECK_KFD_OPEN(); if (EventDesc->EventType >= HSA_EVENTTYPE_MAXID) @@ -94,8 +96,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, e->EventId = args.event_id; if (!events_page && args.event_page_offset > 0) { - events_page = mmap(NULL, KFD_SIGNAL_EVENT_LIMIT * 8, PROT_WRITE | PROT_READ, + events_page = mmap(NULL, event_limit * 8, PROT_WRITE | PROT_READ, MAP_SHARED, kfd_fd, args.event_page_offset); + if (events_page == MAP_FAILED) { + /* old kernels only support 256 events */ + event_limit = 256; + events_page = mmap(NULL, PAGE_SIZE, PROT_WRITE | PROT_READ, + MAP_SHARED, kfd_fd, args.event_page_offset); + } if (events_page == MAP_FAILED) { events_page = NULL; pthread_mutex_unlock(&hsakmt_mutex); @@ -106,7 +114,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, pthread_mutex_unlock(&hsakmt_mutex); - if (args.event_page_offset > 0 && args.event_slot_index < KFD_SIGNAL_EVENT_LIMIT) + if (args.event_page_offset > 0 && args.event_slot_index < event_limit) e->EventData.HWData2 = (HSAuint64)&events_page[args.event_slot_index]; e->EventData.EventType = EventDesc->EventType; diff --git a/src/fmm.c b/src/fmm.c index 5695ece5b4..0b73dd413f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1382,14 +1382,45 @@ static uint32_t get_vm_alignment(uint32_t device_id) return MAX(PAGE_SIZE, page_size); } +static HSAKMT_STATUS get_process_apertures( + struct kfd_process_device_apertures *process_apertures, + uint32_t *num_of_nodes) +{ + struct kfd_ioctl_get_process_apertures_new_args args_new = { + .kfd_process_device_apertures_ptr = (uintptr_t)process_apertures, + .num_of_nodes = *num_of_nodes + }; + struct kfd_ioctl_get_process_apertures_args args_old; + + if (!kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW, + (void *)&args_new)) { + *num_of_nodes = args_new.num_of_nodes; + return HSAKMT_STATUS_SUCCESS; + } + + /* New IOCTL failed, try the old one in case we're running on + * a really old kernel */ + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, + (void *)&args_old)) + return HSAKMT_STATUS_ERROR; + + if (args_old.num_of_nodes < *num_of_nodes) + *num_of_nodes = args_old.num_of_nodes; + + memcpy(process_apertures, args_old.process_apertures, + sizeof(*process_apertures) * *num_of_nodes); + + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { - struct kfd_ioctl_get_process_apertures_new_args args; uint32_t i = 0; int32_t gpu_mem_id = 0; uint32_t gpu_id; HsaNodeProperties props; struct kfd_process_device_apertures *process_apertures; + uint32_t num_of_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; unsigned int guardPages = 1; @@ -1463,31 +1494,28 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * required since Number of nodes is already known. Kernel will fill in * the apertures in kfd_process_device_apertures_ptr */ - process_apertures = malloc(gpu_mem_count * sizeof(struct kfd_process_device_apertures)); + num_of_nodes = gpu_mem_count; + process_apertures = malloc(num_of_nodes * sizeof(struct kfd_process_device_apertures)); if (!process_apertures) { ret = HSAKMT_STATUS_NO_MEMORY; goto sysfs_parse_failed; } - args.kfd_process_device_apertures_ptr = (uintptr_t)process_apertures; - args.num_of_nodes = gpu_mem_count; - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW, (void *)&args)) { - ret = HSAKMT_STATUS_ERROR; + ret = get_process_apertures(process_apertures, &num_of_nodes); + if (ret != HSAKMT_STATUS_SUCCESS) goto get_aperture_ioctl_failed; - } all_gpu_id_array_size = 0; all_gpu_id_array = NULL; - if (args.num_of_nodes > 0) { - all_gpu_id_array = malloc(sizeof(uint32_t) * args.num_of_nodes); + if (num_of_nodes > 0) { + all_gpu_id_array = malloc(sizeof(uint32_t) * num_of_nodes); if (!all_gpu_id_array) { ret = HSAKMT_STATUS_NO_MEMORY; goto get_aperture_ioctl_failed; } } - for (i = 0 ; i < args.num_of_nodes ; i++) { + for (i = 0 ; i < num_of_nodes ; i++) { /* Map Kernel process device data node i <--> gpu_mem_id which * indexes into gpu_mem[] based on gpu_id */ From f1a5248cf240c6733574a3b9eb9e6d8b2cb4b3b3 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 15 Aug 2017 17:13:55 -0400 Subject: [PATCH 0279/1247] Simplify memory maps Simplify fmm_map_to_gpu_nodes code. Also fix a memory leak in this change. Change-Id: I3487338b78c915de44588d0206bac4c53e728c60 Signed-off-by: Amber Lin --- src/fmm.c | 43 ++++++++++++++++++++----------------------- src/fmm.h | 2 +- src/memory.c | 9 +++++++-- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 0b73dd413f..381898d35f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1677,8 +1677,10 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA return err; } +/* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, - void *address, uint64_t size, vm_object_t *obj) + void *address, uint64_t size, vm_object_t *obj, + uint32_t *nodes_to_map, uint32_t nodes_array_size) { struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; @@ -1705,11 +1707,17 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, } args.handle = object->handle; - if (object->registered_device_id_array_size > 0) { + if (nodes_to_map) { + /* If specified, map the requested */ + args.device_ids_array_ptr = (uint64_t)nodes_to_map; + args.device_ids_array_size = nodes_array_size; + } else if (object->registered_device_id_array_size > 0) { + /* otherwise map all registered */ args.device_ids_array_ptr = (uint64_t)object->registered_device_id_array; args.device_ids_array_size = object->registered_device_id_array_size; } else { + /* not specified, not registered: map all GPUs */ args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; args.device_ids_array_size = all_gpu_id_array_size; } @@ -1800,7 +1808,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* map to GPU */ - ret = _fmm_map_to_gpu_gtt(aperture, address, size, NULL); + ret = _fmm_map_to_gpu_gtt(aperture, address, size, NULL, NULL, 0); if (ret != 0) __fmm_release(mem, aperture); @@ -1902,7 +1910,7 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size, obj); + ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size, obj, NULL, 0); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -1940,12 +1948,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) (address <= svm.dgpu_aperture.limit)) /* map it */ return _fmm_map_to_gpu_gtt(&svm.dgpu_aperture, - address, size, NULL); + address, size, NULL, NULL, 0); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* map it */ return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture, - address, size, NULL); + address, size, NULL, NULL, 0); /* * If address isn't an SVM memory address, we assume that this @@ -2848,7 +2856,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) */ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, - uint32_t *nodes_to_map, uint32_t nodes_to_map_size, + uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address) { manageable_aperture_t *aperture; @@ -2860,7 +2868,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; - if ((nodes_to_map_size > 0 && !nodes_to_map) || !address) + if (!num_of_nodes || !nodes_to_map || !address) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -2907,7 +2915,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, registered_node_id_array = object->registered_device_id_array; registered_node_id_array_size = object->registered_device_id_array_size; } - for (i = 0 ; i < nodes_to_map_size / sizeof(uint32_t); i++) { + for (i = 0 ; i < num_of_nodes; i++) { temp_node = nodes_to_map[i]; found = false; for (j = 0 ; j < registered_node_id_array_size / sizeof(uint32_t); j++) { @@ -2933,7 +2941,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, for (i = 0 ; i < object->mapped_device_id_array_size / sizeof(uint32_t); i++) { temp_node = object->mapped_device_id_array[i]; found = false; - for (j = 0 ; j < nodes_to_map_size / sizeof(uint32_t); j++) { + for (j = 0 ; j < num_of_nodes; j++) { if (temp_node == nodes_to_map[j]) { found = true; break; @@ -2952,19 +2960,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return ret; } - /* Keep registered device id array and size */ - temp_node_id_array = object->registered_device_id_array; - temp_node_id_array_size = object->registered_device_id_array_size; - /* Change registered device id array and size to nodes array/size that we want to map */ - object->registered_device_id_array = nodes_to_map; - object->registered_device_id_array_size = nodes_to_map_size; - - if (nodes_to_map_size > 0) - retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object); - - /* Restore old registered device id array */ - object->registered_device_id_array = temp_node_id_array; - object->registered_device_id_array_size = temp_node_id_array_size; + retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, + nodes_to_map, num_of_nodes * sizeof(uint32_t)); pthread_mutex_unlock(&aperture->fmm_mutex); diff --git a/src/fmm.h b/src/fmm.h index f3870842fa..3bc0fb2040 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -85,5 +85,5 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo uint32_t *gpu_id_array, uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, - uint32_t *nodes_to_map, uint32_t nodes_to_map_size, uint64_t *gpuvm_address); + uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address); #endif /* FMM_H_ */ diff --git a/src/memory.c b/src/memory.c index fcf201a3ad..76b2153efa 100644 --- a/src/memory.c +++ b/src/memory.c @@ -433,8 +433,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(void *MemoryAddress, if (ret != HSAKMT_STATUS_SUCCESS) return ret; - return fmm_map_to_gpu_nodes(MemoryAddress, MemorySizeInBytes, - gpu_id_array, NumberOfNodes * sizeof(uint32_t), AlternateVAGPU); + ret = fmm_map_to_gpu_nodes(MemoryAddress, MemorySizeInBytes, + gpu_id_array, NumberOfNodes, AlternateVAGPU); + + if (gpu_id_array) + free(gpu_id_array); + + return ret; } HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) From 00250f7686373b12e21acc77ee452a57e4178e76 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Fri, 18 Aug 2017 16:50:54 -0400 Subject: [PATCH 0280/1247] Fix hsaKmtQueryPointerInfo for scratch memory Change-Id: I35a0f1a81f8b0ac6e99c2e1572829eb32d3bc95b Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 46 +++++++++++++++++++++++----------------------- src/fmm.h | 1 - 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 381898d35f..11963b66ec 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -656,6 +656,23 @@ static manageable_aperture_t *fmm_get_aperture(HsaApertureInfo info) return NULL; } } + +static manageable_aperture_t *fmm_is_scratch_aperture(const void *address) +{ + uint32_t i; + + for (i = 0; i < gpu_mem_count; i++) { + if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) + continue; + + if ((address >= gpu_mem[i].scratch_physical.base) && + (address <= gpu_mem[i].scratch_physical.limit)) + return &gpu_mem[i].scratch_physical; + + } + return NULL; +} + static manageable_aperture_t *fmm_find_aperture(const void *address, HsaApertureInfo *info) { @@ -666,8 +683,12 @@ static manageable_aperture_t *fmm_find_aperture(const void *address, if (is_dgpu) { if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { - aperture = &svm.dgpu_aperture; - _info.type = HSA_APERTURE_DGPU; + + aperture = fmm_is_scratch_aperture(address); + if (!aperture) { + aperture = &svm.dgpu_aperture; + _info.type = HSA_APERTURE_DGPU; + } } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { aperture = &svm.dgpu_alt_aperture; @@ -753,27 +774,6 @@ err_object_allocation_failed: return NULL; } -bool fmm_is_inside_some_aperture(void *address) -{ - uint32_t i; - - for (i = 0; i < gpu_mem_count; i++) { - if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) - continue; - if ((address >= gpu_mem[i].lds_aperture.base) && - (address <= gpu_mem[i].lds_aperture.limit)) - return true; - if ((address >= gpu_mem[i].gpuvm_aperture.base) && - (address <= gpu_mem[i].gpuvm_aperture.limit)) - return true; - if ((address >= gpu_mem[i].scratch_aperture.base) && - (address <= gpu_mem[i].scratch_aperture.limit)) - return true; - } - - return false; -} - #ifdef DEBUG_PRINT_APERTURE static void aperture_print(aperture_t *app) { diff --git a/src/fmm.h b/src/fmm.h index 3bc0fb2040..c874f6fe01 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -54,7 +54,6 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_ void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); void fmm_print(uint32_t node); -bool fmm_is_inside_some_aperture(void *address); void fmm_release(void *address); int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); int fmm_unmap_from_gpu(void *address); From 73707766ef22564eb1735448c62bfded0f835d7a Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 17 Aug 2017 15:38:06 -0400 Subject: [PATCH 0281/1247] Add debugging message on memory Add pr_debug to all memory APIs and pr_err to some failure cases. Change-Id: I8b519a1228cc19e6c04118fd87432e7f48f3cbf9 Signed-off-by: Amber Lin --- src/fmm.c | 4 +--- src/memory.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 11963b66ec..abb2bdd47a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2283,7 +2283,7 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi munmap(ret_addr, len); } if (!ret_addr) { - pr_err("Failed to reserve %uGB for SVM ...\n", + pr_warn("Failed to reserve %uGB for SVM ...\n", (unsigned int)(len >> 30)); continue; } @@ -2871,7 +2871,6 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, if (!num_of_nodes || !nodes_to_map || !address) return HSAKMT_STATUS_INVALID_PARAMETER; - /* Find object by address */ if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) @@ -2882,7 +2881,6 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, else { aperture = &svm.dgpu_aperture; userptr = true; - } pthread_mutex_lock(&aperture->fmm_mutex); diff --git a/src/memory.c b/src/memory.c index 76b2153efa..75b392a90a 100644 --- a/src/memory.c +++ b/src/memory.c @@ -46,6 +46,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, CHECK_KFD_OPEN(); + pr_debug("[%s] node %d; default %d; alternate %d\n", + __func__, Node, DefaultPolicy, AlternatePolicy); + if (is_dgpu) /* This is a legacy API useful on Kaveri only. On dGPU * the alternate aperture is setup and used @@ -114,9 +117,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, CHECK_KFD_OPEN(); + pr_debug("[%s] node %d\n", __func__, PreferredNode); + result = validate_nodeid(PreferredNode, &gpu_id); - if (result != HSAKMT_STATUS_SUCCESS) + if (result != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode); return result; + } page_size = PageSizeFromFlags(MemFlags.ui32.PageSize); @@ -127,8 +134,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, MemFlags); - if (!(*MemoryAddress)) + if (!(*MemoryAddress)) { + pr_err("[%s] failed to allocate %lu bytes from host\n", + __func__, SizeInBytes); return HSAKMT_STATUS_ERROR; + } return HSAKMT_STATUS_SUCCESS; } @@ -136,16 +146,22 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, if (gpu_id && MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes, MemFlags); - if (!(*MemoryAddress)) + if (!(*MemoryAddress)) { + pr_err("[%s] failed to allocate %lu bytes from device\n", + __func__, SizeInBytes); return HSAKMT_STATUS_NO_MEMORY; + } return HSAKMT_STATUS_SUCCESS; } if (MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_scratch(gpu_id, SizeInBytes); - if (!(*MemoryAddress)) + if (!(*MemoryAddress)) { + pr_err("[%s] failed to allocate %lu bytes from scratch\n", + __func__, SizeInBytes); return HSAKMT_STATUS_NO_MEMORY; + } return HSAKMT_STATUS_SUCCESS; } @@ -157,8 +173,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, MemFlags); - if (!(*MemoryAddress)) + if (!(*MemoryAddress)) { + pr_err("[%s] failed to allocate %lu bytes from paged\n", + __func__, SizeInBytes); return HSAKMT_STATUS_ERROR; + } return HSAKMT_STATUS_SUCCESS; } @@ -171,6 +190,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, { CHECK_KFD_OPEN(); + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + if (!MemoryAddress) { pr_err("FIXME: freeing NULL pointer\n"); return HSAKMT_STATUS_ERROR; @@ -185,6 +206,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress, { CHECK_KFD_OPEN(); + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + if (!is_dgpu) /* TODO: support mixed APU and dGPU configurations */ return HSAKMT_STATUS_SUCCESS; @@ -202,6 +225,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, uint32_t *gpu_id_array; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + pr_debug("[%s] address %p number of nodes %lu\n", + __func__, MemoryAddress, NumberOfNodes); + if (!is_dgpu) /* TODO: support mixed APU and dGPU configurations */ return HSAKMT_STATUS_NOT_SUPPORTED; @@ -229,6 +255,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes(HSAuint64 GraphicsRe uint32_t *gpu_id_array; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + pr_debug("[%s] number of nodes %lu\n", __func__, NumberOfNodes); + ret = validate_nodeid_array(&gpu_id_array, NumberOfNodes, NodeArray); @@ -249,6 +277,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtShareMemory(void *MemoryAddress, { CHECK_KFD_OPEN(); + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + if (!SharedMemoryHandle) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -261,6 +291,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandle(const HsaSharedMemoryHandle * { CHECK_KFD_OPEN(); + pr_debug("[%s] handle %p\n", __func__, SharedMemoryHandle); + return hsaKmtRegisterSharedHandleToNodes(SharedMemoryHandle, MemoryAddress, SizeInBytes, @@ -279,6 +311,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterSharedHandleToNodes(const HsaSharedMemoryH uint32_t *gpu_id_array = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + pr_debug("[%s] handle %p number of nodes %lu\n", + __func__, SharedMemoryHandle, NumberOfNodes); + if (!SharedMemoryHandle) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -323,6 +358,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, { struct kfd_ioctl_cross_memory_copy_args args; + pr_debug("[%s]\n", __func__); + if (!LocalMemoryArray || !RemoteMemoryArray || LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) return HSAKMT_STATUS_ERROR; @@ -356,6 +393,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, { struct kfd_ioctl_cross_memory_copy_args args; + pr_debug("[%s]\n", __func__); + if (!LocalMemoryArray || !RemoteMemoryArray || LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) return HSAKMT_STATUS_ERROR; @@ -385,6 +424,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDeregisterMemory(void *MemoryAddress) { CHECK_KFD_OPEN(); + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + return fmm_deregister_memory(MemoryAddress); } @@ -394,6 +435,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress, { CHECK_KFD_OPEN(); + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + if (!MemoryAddress) { pr_err("FIXME: mapping NULL pointer\n"); return HSAKMT_STATUS_ERROR; @@ -418,6 +461,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(void *MemoryAddress, uint32_t *gpu_id_array; HSAKMT_STATUS ret; + pr_debug("[%s] address %p number of nodes %lu\n", + __func__, MemoryAddress, NumberOfNodes); + if (!MemoryAddress) { pr_err("FIXME: mapping NULL pointer\n"); return HSAKMT_STATUS_ERROR; @@ -446,6 +492,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapMemoryToGPU(void *MemoryAddress) { CHECK_KFD_OPEN(); + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + if (!MemoryAddress) { /* Workaround for runtime bug */ pr_err("FIXME: Unmapping NULL pointer\n"); @@ -477,7 +525,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapGraphicHandle(HSAuint32 NodeId, HSAuint64 FlatMemoryAddress, HSAuint64 SizeInBytes) { - return hsaKmtUnmapMemoryToGPU(PORT_UINT64_TO_VPTR(FlatMemoryAddress)); } @@ -487,6 +534,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig * uint32_t gpu_id; HSAKMT_STATUS result; + pr_debug("[%s] node %d\n", __func__, NodeId); + result = validate_nodeid(NodeId, &gpu_id); if (result != HSAKMT_STATUS_SUCCESS) return result; @@ -514,6 +563,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig * HSAKMT_STATUS HSAKMTAPI hsaKmtQueryPointerInfo(const void *Pointer, HsaPointerInfo *PointerInfo) { + pr_debug("[%s] pointer %p\n", __func__, Pointer); + if (!PointerInfo) return HSAKMT_STATUS_INVALID_PARAMETER; return fmm_get_mem_info(Pointer, PointerInfo); @@ -522,5 +573,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtQueryPointerInfo(const void *Pointer, HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryUserData(const void *Pointer, void *UserData) { + pr_debug("[%s] pointer %p\n", __func__, Pointer); + return fmm_set_mem_user_data(Pointer, UserData); } From 369902bf5b3a957179200c378f46a90300456d42 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 18 Aug 2017 10:15:21 -0400 Subject: [PATCH 0282/1247] IOMMU path in sysfs is renamed IOMMU path in sysfs was amd_iommu. After implementing multiple devices support, the path is replaced with amd_iommu_. Current Thunk spec is not clear about how to support multiple instances in one block. There is no products having multiple IOMMUs yet at this point. This patch changes the path to support both amd_iommu and amd_iommu_0 for Carizo. Change-Id: I3beea2fc78d96296232226191501a02ccf20d6b1 Signed-off-by: Amber Lin --- src/perfctr.c | 11 +++++++++-- src/pmc_table.c | 14 ++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 5a0d6bd84a..bf05880ca5 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -365,9 +365,16 @@ static unsigned int get_perf_event_type(enum perf_block_id block_id) FILE *file = NULL; unsigned int type = 0; - if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) - file = fopen("/sys/bus/event_source/devices/amd_iommu/type", + if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) { + /* Starting from kernel 4.12, amd_iommu_0 is used */ + file = fopen("/sys/bus/event_source/devices/amd_iommu_0/type", "r"); + if (!file) + file = fopen(/* kernel 4.11 and older */ + "/sys/bus/event_source/devices/amd_iommu/type", + "r"); + } + if (!file) return 0; diff --git a/src/pmc_table.c b/src/pmc_table.c index 1bb4ba04bf..5f1cb3c877 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1601,8 +1601,8 @@ static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) { DIR *dir; struct dirent *dent; - const char sysfs_amdiommu_event_path[] = - "/sys/bus/event_source/devices/amd_iommu/events"; + const char *sysfs_amdiommu_event_path = + "/sys/bus/event_source/devices/amd_iommu_0/events"; /* Counter source in IOMMU's Counter Bank Addressing register is 8 bits, * so the biggest counter number/id possible is 0xff. */ @@ -1619,8 +1619,14 @@ static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) memset(block, 0, sizeof(struct perf_counter_block)); dir = opendir(sysfs_amdiommu_event_path); - if (!dir) - goto out; + if (!dir) { + /* Before kernel 4.12, amd_iommu is the path */ + sysfs_amdiommu_event_path = + "/sys/bus/event_source/devices/amd_iommu/events"; + dir = opendir(sysfs_amdiommu_event_path); + if (!dir) + goto out; + } memset(counter_id, 0, max_counter_id + 1); while ((dent = readdir(dir))) { From 395ecaa9852baff347d01c8cbb151b76aaeb5f6a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 21 Aug 2017 19:58:35 -0400 Subject: [PATCH 0283/1247] Fix two more device ID array bugs Use the mapped_device_id_array size when allocating temp_node_id_array for unmapping queues in fmm_map_to_gpu_nodes. registered_device_id_array size may be 0. Also, this temporary array is small enough to allocate it on the stack. Malloc and free are overkill here. Fix potential memory leak when registering the same device ID array multiple times. Change-Id: I83f09fd0925d9de7cf11bf72ba0ebb77273f587d Signed-off-by: Felix Kuehling --- src/fmm.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index abb2bdd47a..233ff2acaa 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2488,8 +2488,13 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, gpu_id_array, gpu_id_array_size)) { pr_err("Cannot change nodes in a registered addr.\n"); return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; - } else + } else { + /* Delete the new array, keep the existing one. */ + if (gpu_id_array) + free(gpu_id_array); + return HSAKMT_STATUS_SUCCESS; + } } if (gpu_id_array_size > 0) { @@ -2863,7 +2868,6 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, vm_object_t *object = NULL; uint32_t i, j, temp_node; bool found, userptr = false; - uint32_t *temp_node_id_array, temp_node_id_array_size; uint32_t *registered_node_id_array, registered_node_id_array_size; HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; @@ -2930,12 +2934,9 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, /* Unmap buffer from all nodes that have this buffer mapped that are not included on nodes_to_map array */ if (object->mapped_device_id_array_size > 0) { - temp_node_id_array = (uint32_t *)malloc(object->registered_device_id_array_size); - if (!temp_node_id_array) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_NO_MEMORY; - } - temp_node_id_array_size = 0; + uint32_t temp_node_id_array[object->mapped_device_id_array_size]; + uint32_t temp_node_id_array_size = 0; + for (i = 0 ; i < object->mapped_device_id_array_size / sizeof(uint32_t); i++) { temp_node = object->mapped_device_id_array[i]; found = false; @@ -2953,7 +2954,6 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, ret = _fmm_unmap_from_gpu(aperture, address, temp_node_id_array, temp_node_id_array_size, object); - free(temp_node_id_array); if (ret != HSAKMT_STATUS_SUCCESS) return ret; } From 65d680c035b380ebe6374fdab3ce6e6259a5c43d Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 23 Aug 2017 17:17:09 -0400 Subject: [PATCH 0284/1247] Set guard page as disabled as default Due to max_mmap_count issue, set default of guard page as disabled. Change-Id: Ic9dfe69b621733e9fac86831b008a122994a67e7 Signed-off-by: Amber Lin --- src/fmm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 233ff2acaa..c4f3da5b7a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1423,7 +1423,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; - unsigned int guardPages = 1; + unsigned int guardPages = 0; struct pci_access *pacc; /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ @@ -1443,10 +1443,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) checkUserptr = getenv("HSA_CHECK_USERPTR"); svm.check_userptr = (checkUserptr && strcmp(checkUserptr, "0")); - /* Specify number of guard pages for SVM apertures, default is 1 */ + /* Specify number of guard pages for SVM apertures, default is 0 */ guardPagesStr = getenv("HSA_SVM_GUARD_PAGES"); if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1) - guardPages = 1; + guardPages = 0; /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than From 52598cf37e249ba5c425899aef6ea1e453e679c7 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 28 Aug 2017 14:57:20 -0400 Subject: [PATCH 0285/1247] Generalize size-dependent virtual address alignment Support all fragment sizes up to 2MB by aligning buffers according to their size. Change-Id: I82b7ef8be6f1507d941e5c97edb6618adf8c66de Signed-off-by: Felix Kuehling --- src/fmm.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index c4f3da5b7a..115137a201 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -544,14 +544,11 @@ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, if (align < app->align) align = app->align; - /* Huge-page and Big-K TLB optimizations require proper alignment */ - if (MemorySizeInBytes >= GPU_HUGE_PAGE_SIZE) { - if (align < GPU_HUGE_PAGE_SIZE) - align = GPU_HUGE_PAGE_SIZE; - } else if (MemorySizeInBytes >= GPU_BIGK_PAGE_SIZE) { - if (align < GPU_BIGK_PAGE_SIZE) - align = GPU_BIGK_PAGE_SIZE; - } + /* Align big buffers to the next power-of-2 up to huge page + * size for flexible fragment size TLB optimizations + */ + while (align < GPU_HUGE_PAGE_SIZE && MemorySizeInBytes >= (align << 1)) + align <<= 1; MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); From a81b29890c6bbd3e4545fd7f94804688e96714cc Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 28 Aug 2017 16:35:18 -0400 Subject: [PATCH 0286/1247] Fix mmap when the count reaches the max Applications may try to allocate lots host memory and reaches the mmap limit (/proc/sys/vm/max_map_count). When Applications fails to allocate memory and calls hsaKmtFreeMemory to release the memory, Thunk fails to reduce the maps count so the following hsaKmtAllocMemory calls continue to fail, which doesn't make sense to the application. This patch checks the mmap to NORESERVE return value. If it fails and the error number is ENOMEM, reduce the map count by munmap and map it again immediately. Change-Id: I127cb479dfd86b199172eef269d59426f23859ea Signed-off-by: Amber Lin --- src/fmm.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 115137a201..063944b39c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1266,6 +1266,7 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args; vm_object_t *object; + void *mmap_ret; if (!address) return; @@ -1290,8 +1291,19 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { /* Remove any CPU mapping, but keep the address range reserved */ - mmap(address, object->size, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0); + mmap_ret = mmap(address, object->size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + if (mmap_ret == MAP_FAILED && errno == ENOMEM) { + /* When mmap count reaches max_map_count, any mmap will + * fail. Reduce the count with munmap then map it as + * NORESERVE immediately. + */ + munmap(address, object->size); + mmap(address, object->size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + } } aperture_release_area(aperture, address, object->size); From a74f6896ea492a746f23fe0147cf152c9cf45109 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 29 Aug 2017 15:10:05 -0400 Subject: [PATCH 0287/1247] Revert "Set guard page as disabled as default" This reverts commit 65d680c035b380ebe6374fdab3ce6e6259a5c43d. Change-Id: I09b7e7915ec4759cab57d5863089a2c4a44dfacd Signed-off-by: Amber Lin --- src/fmm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 063944b39c..5756df4e62 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1432,7 +1432,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; - unsigned int guardPages = 0; + unsigned int guardPages = 1; struct pci_access *pacc; /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ @@ -1452,10 +1452,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) checkUserptr = getenv("HSA_CHECK_USERPTR"); svm.check_userptr = (checkUserptr && strcmp(checkUserptr, "0")); - /* Specify number of guard pages for SVM apertures, default is 0 */ + /* Specify number of guard pages for SVM apertures, default is 1 */ guardPagesStr = getenv("HSA_SVM_GUARD_PAGES"); if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1) - guardPages = 0; + guardPages = 1; /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than From 117fa5034b82296fce795744226d7a10fe84355f Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 30 Aug 2017 17:03:22 -0400 Subject: [PATCH 0288/1247] Fix PmcGetCounterProperties Blocks inside of HsaCounterProperties structure is not a fixed size. It varies with number of counters in the block -- size of Counters in HsaCounterBlockProperties is different in every block. Current implementation assumes fixed size and the next block will overwrite the previous block's Counters. This patch change the array implementation to using a pointer so it'll move the next block to the correction position. Change-Id: I72800f4db5f2a68215fba477a61ca07ec99054bf Signed-off-by: Amber Lin --- src/perfctr.c | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index bf05880ca5..cc04469b94 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -306,11 +306,13 @@ static HSAuint32 get_block_concurrent_limit(uint32_t node_id, HSAuint32 block_id) { uint32_t i; + HsaCounterBlockProperties *block = &counter_props[node_id]->Blocks[0]; - for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) - if (counter_props[node_id]->Blocks[i].Counters[0].BlockIndex == - block_id) - return counter_props[node_id]->Blocks[i].NumConcurrent; + for (i = 0; i < PERFCOUNTER_BLOCKID__MAX; i++) { + if (block->Counters[0].BlockIndex == block_id) + return block->NumConcurrent; + block = (HsaCounterBlockProperties *)&block->Counters[block->NumCounters]; + } return 0; } @@ -491,7 +493,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId, uint32_t total_concurrent = 0; struct perf_counter_block block = {0}; uint32_t total_blocks = 0; - uint32_t entry; + HsaCounterBlockProperties *block_prop; if (!counter_props) return HSAKMT_STATUS_NO_MEMORY; @@ -519,8 +521,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId, } counter_props_size = sizeof(HsaCounterProperties) + - sizeof(HsaCounterBlockProperties)*(total_blocks-1) + - sizeof(HsaCounter)*(total_counters-1); + sizeof(HsaCounterBlockProperties) * (total_blocks - 1) + + sizeof(HsaCounter) * (total_counters - total_blocks); counter_props[NodeId] = malloc(counter_props_size); if (!counter_props[NodeId]) @@ -529,36 +531,34 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId, counter_props[NodeId]->NumBlocks = total_blocks; counter_props[NodeId]->NumConcurrent = total_concurrent; - entry = 0; + block_prop = &counter_props[NodeId]->Blocks[0]; for (block_id = 0; block_id < PERFCOUNTER_BLOCKID__MAX; block_id++) { rc = get_block_properties(NodeId, block_id, &block); if (rc != HSAKMT_STATUS_SUCCESS) { free(counter_props[NodeId]); + counter_props[NodeId] = NULL; return rc; } if (!block.num_of_slots) /* not a valid block */ continue; - blockid2uuid(block_id, - &counter_props[NodeId]->Blocks[entry].BlockId); - counter_props[NodeId]->Blocks[entry].NumCounters = - block.num_of_counters; - counter_props[NodeId]->Blocks[entry].NumConcurrent = - block.num_of_slots; - + blockid2uuid(block_id, &block_prop->BlockId); + block_prop->NumCounters = block.num_of_counters; + block_prop->NumConcurrent = block.num_of_slots; for (i = 0; i < block.num_of_counters; i++) { - counter_props[NodeId]->Blocks[entry].Counters[i].BlockIndex = block_id; - counter_props[NodeId]->Blocks[entry].Counters[i].CounterId = block.counter_ids[i]; - counter_props[NodeId]->Blocks[entry].Counters[i].CounterSizeInBits = block.counter_size_in_bits; - counter_props[NodeId]->Blocks[entry].Counters[i].CounterMask = block.counter_mask; - counter_props[NodeId]->Blocks[entry].Counters[i].Flags.ui32.Global = 1; + block_prop->Counters[i].BlockIndex = block_id; + block_prop->Counters[i].CounterId = block.counter_ids[i]; + block_prop->Counters[i].CounterSizeInBits = block.counter_size_in_bits; + block_prop->Counters[i].CounterMask = block.counter_mask; + block_prop->Counters[i].Flags.ui32.Global = 1; if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) - counter_props[NodeId]->Blocks[entry].Counters[i].Type = HSA_PROFILE_TYPE_PRIVILEGED_IMMEDIATE; + block_prop->Counters[i].Type = HSA_PROFILE_TYPE_PRIVILEGED_IMMEDIATE; else - counter_props[NodeId]->Blocks[entry].Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; + block_prop->Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; } - entry++; + + block_prop = (HsaCounterBlockProperties *)&block_prop->Counters[block_prop->NumCounters]; } *CounterProperties = counter_props[NodeId]; From d9e71260c4dc1c87684bf1afdc5b97bc646203b9 Mon Sep 17 00:00:00 2001 From: oak zeng Date: Tue, 3 Oct 2017 20:25:38 -0400 Subject: [PATCH 0289/1247] Print debug message for GPU vm fault analysis Change-Id: Ia6dac9d3f5c35a7d0e41de9b54c06596d00c7946 Signed-off-by: Oak Zeng Reviewed-by: Felix Kuehling --- src/events.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/events.c b/src/events.c index 876bc846d8..daea86219d 100644 --- a/src/events.c +++ b/src/events.c @@ -229,6 +229,66 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event, return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); } +//Analysis memory exception data, print debug messages +static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * + memory_exception_data) +{ + HSAKMT_STATUS ret; + HsaPointerInfo info; + const uint64_t addr = memory_exception_data->va; + uint32_t node_id = 0; + unsigned int i; + + gpuid_to_nodeid(memory_exception_data->gpu_id, &node_id); + pr_err("Memory exception on virtual address 0x%lx, ", addr); + pr_err("node id %d : ", node_id); + if (memory_exception_data->failure.NotPresent) + pr_err("Page not present\n"); + else if (memory_exception_data->failure.ReadOnly) + pr_err("Writing to readonly page\n"); + else if (memory_exception_data->failure.NoExecute) + pr_err("Execute to none-executable page\n"); + + ret = fmm_get_mem_info((const void *)addr, &info); + + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("Address does not belong to a known buffer\n"); + return; + } + + pr_err("GPU address 0x%lx, node id %d, size in byte 0x%lx\n", + info.GPUAddress, info.Node, info.SizeInBytes); + switch (info.Type) { + case HSA_POINTER_REGISTERED_GRAPHICS: + pr_err("Memory is registered graphics buffer\n"); + break; + case HSA_POINTER_REGISTERED_USER: + pr_err("Memory is registered user pointer\n"); + pr_err("CPU address of the memory is %p\n", info.CPUAddress); + break; + case HSA_POINTER_ALLOCATED: + pr_err("Memory is allocated using hsaKmtAllocMemory\n"); + pr_err("CPU address of the memory is %p\n", info.CPUAddress); + break; + default: + pr_err("Invalid memory type %d\n", info.Type); + break; + } + + if (info.RegisteredNodes) { + pr_err("Memory is registered to node id: "); + for (i = 0; i < info.NRegisteredNodes; i++) + pr_err("%d ", info.RegisteredNodes[i]); + } + pr_err("\n"); + if (info.MappedNodes) { + pr_err("Memory is mapped to node id: "); + for (i = 0; i < info.NMappedNodes; i++) + pr_err("%d ", info.MappedNodes[i]); + } + pr_err("\n"); +} + HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], HSAuint32 NumEvents, bool WaitOnAll, @@ -274,6 +334,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], Events[i]->EventData.EventData.MemoryAccessFault.Failure.NoExecute = event_data[i].memory_exception_data.failure.NoExecute; Events[i]->EventData.EventData.MemoryAccessFault.Failure.Imprecise = event_data[i].memory_exception_data.failure.imprecise; Events[i]->EventData.EventData.MemoryAccessFault.Flags = HSA_EVENTID_MEMORY_FATAL_PROCESS; + analysis_memory_exception(&event_data[i].memory_exception_data); } } } From df4d8a0010f505620e8abcdebb53b8c8932f8fae Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 5 Oct 2017 17:50:20 -0400 Subject: [PATCH 0290/1247] Revise gfx902 GFX version to 9.0.3 Change-Id: I6c16726ac9d096dc4ab127fb266eed105a4f9c87 Signed-off-by: Yong Zhao --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 784f005957..ef22517bb3 100644 --- a/src/topology.c +++ b/src/topology.c @@ -173,7 +173,7 @@ static struct hsa_gfxip_table { { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, /* Raven */ - { 0x15DD, 9, 0, 0, 0, "Raven", CHIP_RAVEN }, + { 0x15DD, 9, 0, 3, 0, "Raven", CHIP_RAVEN }, }; enum cache_type { From 8126ddc77edd20e1ef202a6a77b56d91bb6fd9a1 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 5 Oct 2017 16:01:26 -0400 Subject: [PATCH 0291/1247] Update kfd_ioctl.h Kernel file has been changed recently, so we update the file in thunk. Change-Id: I359a389fa9d91641114c7fb75f420ee6b16f467a Signed-off-by: Yong Zhao --- include/linux/kfd_ioctl.h | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index b318a8fd92..ae79f16e1d 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -208,14 +208,7 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_IOC_WAIT_RESULT_TIMEOUT 1 #define KFD_IOC_WAIT_RESULT_FAIL 2 -/* - * The added 512 is because, currently, 8*(4096/256) signal events are - * reserved for debugger events, and we want to provide at least 4K signal - * events for EOP usage. - * We add 512 to make the allocated size (KFD_SIGNAL_EVENT_LIMIT * 8) be - * page aligned. - */ -#define KFD_SIGNAL_EVENT_LIMIT (4096 + 512) +#define KFD_SIGNAL_EVENT_LIMIT 4096 struct kfd_ioctl_create_event_args { uint64_t event_page_offset; /* from KFD */ @@ -520,7 +513,7 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x1F, struct kfd_ioctl_cross_memory_copy_args) -#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ +#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ AMDKFD_IOWR(0x20, struct kfd_ioctl_get_queue_wave_state_args) #define AMDKFD_COMMAND_START 0x01 From 5815d9de9b0b6e6c02309ebe998718f6e10bac8b Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 12 Oct 2017 14:22:42 -0400 Subject: [PATCH 0292/1247] Fix endless loop Fix a while loop that can cause forever loop when cpuid instruction doesn't work properly. Change-Id: Iefa49d23b40c994eb4369621974a7d3c4067e47a Signed-off-by: Amber Lin --- src/topology.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index ef22517bb3..0a5b5d8cab 100644 --- a/src/topology.c +++ b/src/topology.c @@ -354,7 +354,8 @@ static int cpuid_find_num_cache_leaves(uint32_t op) do { ++idx; cpuid_count(op, idx, &eax.full, &ebx.full, &ecx, &edx); - } while (eax.split.type != CACHE_TYPE_NULL); + /* Modern systems have cache levels up to 3. */ + } while (eax.split.type != CACHE_TYPE_NULL && idx < 4); return idx; } From 4815d9b8877fbb952a7835c9972dd357e98d590b Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Wed, 11 Oct 2017 23:34:44 -0700 Subject: [PATCH 0293/1247] Build fixes for gcc 7.2 v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/perfctr.c: In function ‘destroy_shared_region’: src/perfctr.c:154:10: error: logical ‘and’ of equal expressions [-Werror=logical-op] if (sem && sem != SEM_FAILED) { ^~ src/perfctr.c: In function ‘update_block_slots’: src/perfctr.c:323:11: error: logical ‘or’ of equal expressions [-Werror=logical-op] if (!sem || sem == SEM_FAILED) ^~ v2: - Initialize and reset sem to SEM_FAILED. Change-Id: Id70361079b715c4946b13e4460e4fd85d9542c46 Signed-off-by: Felix Kuehling --- src/perfctr.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index cc04469b94..8040d76152 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -90,7 +90,7 @@ static unsigned int counter_props_count; static const char shmem_name[] = "/hsakmt_shared_mem"; static int shmem_fd; static const char sem_name[] = "hsakmt_semaphore"; -static sem_t *sem; +static sem_t *sem = SEM_FAILED; struct perf_shared_table *shared_table; static ssize_t readn(int fd, void *buf, size_t n) @@ -140,7 +140,7 @@ exit_2: exit_1: sem_close(sem); sem_unlink(sem_name); - sem = NULL; + sem = SEM_FAILED; return HSAKMT_STATUS_ERROR; } @@ -151,9 +151,10 @@ static void destroy_shared_region(void) if (shmem_fd > 0) shm_unlink(shmem_name); - if (sem && sem != SEM_FAILED) { + if (sem != SEM_FAILED) { sem_close(sem); sem_unlink(sem_name); + sem = SEM_FAILED; } } @@ -325,7 +326,7 @@ static HSAKMT_STATUS update_block_slots(enum perf_trace_action action, if (shmem_fd <= 0) return HSAKMT_STATUS_UNAVAILABLE; - if (!sem || sem == SEM_FAILED) + if (sem == SEM_FAILED) return HSAKMT_STATUS_UNAVAILABLE; sem_wait(sem); From e2ed9cf79a3ec462c170ec564391b288d3068ad9 Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Wed, 11 Oct 2017 16:57:38 -0700 Subject: [PATCH 0294/1247] Don't mark heap memory as executable v3 Marking heap memory as executable using mprotect() is not allowed by SELinux. mprotect() calls that try to do this will fail on systems with SELinux enabled. This is also a security risk, so it should be fixed even on systems that allow this. Any memory we want to mark as executable must be allocated using mmap(). See https://www.akkadia.org/drepper/selinux-mem.html The two places where we try to mark heap memory as executable both use posix_memalign() to allocate the heap memory. In both cases, the alignment value passed into this function is always equal to PAGE_SIZE, which means that they are safe to replace with mmap(), which guarantees alignment to PAGE_SIZE. In this case PAGE_SIZE has been set to sysconf(_SC_PAGESIZE); v2: - Use MAP_PRIVATE instead of MAP_SHARED. This matches the behavior of memory allocated by posix_memalign() - Ignore alignment hints instead of returning error when we can't accommodate them. - Drop alignment parameter of allocate_exec_aligned_memory() since the only alignment supported is sysconf(_SC_PAGESIZE). - Remove extra parameter from fmm_release(). - Add error path to fmm_allocate_host_cpu() for when mmap fails. v3: - Avoid use after free. Change-Id: I7d51279790d9700bc3fa761c44bfde1c1936019b Signed-off-by: Felix Kuehling --- src/fmm.c | 34 +++++++++++++++++----------------- src/queues.c | 33 ++++++++++++++++----------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 5756df4e62..cc5c4cb1a6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1110,26 +1110,22 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, HsaMemFlags flags) { - int err; - HSAuint64 page_size; void *mem = NULL; vm_object_t *vm_obj; + int mmap_prot = PROT_READ | PROT_WRITE; - page_size = PageSizeFromFlags(flags.ui32.PageSize); - err = posix_memalign(&mem, page_size, MemorySizeInBytes); - if (err != 0) + if (flags.ui32.ExecuteAccess) + mmap_prot |= PROT_EXEC; + + /* mmap will return a pointer with alignment equal to + * sysconf(_SC_PAGESIZE). + */ + mem = mmap(NULL, MemorySizeInBytes, mmap_prot, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + + if (mem == MAP_FAILED) return NULL; - if (flags.ui32.ExecuteAccess) { - err = mprotect(mem, MemorySizeInBytes, - PROT_READ | PROT_WRITE | PROT_EXEC); - - if (err != 0) { - free(mem); - return NULL; - } - } - pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); vm_obj = aperture_allocate_object(&cpuvm_aperture, mem, 0, MemorySizeInBytes, flags.Value); @@ -1354,14 +1350,18 @@ void fmm_release(void *address) * refers to the system memory */ if (!found) { + uint64_t size = 0; /* Release the vm object in CPUVM */ pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); object = vm_find_object_by_address(&cpuvm_aperture, address, 0); - if (object) + if (object) { + size = object->size; vm_remove_object(&cpuvm_aperture, object); + } pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); /* Free the memory from the system */ - free(address); + if (size) + munmap(address, size); } } diff --git a/src/queues.c b/src/queues.c index 7765e76a36..c698c5ca6e 100644 --- a/src/queues.c +++ b/src/queues.c @@ -309,21 +309,20 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, return status; } -static void *allocate_exec_aligned_memory_cpu(uint32_t size, uint32_t align) +static void *allocate_exec_aligned_memory_cpu(uint32_t size) { void *ptr; - int retval; - retval = posix_memalign(&ptr, align, size); - if (retval != 0) - return NULL; + /* mmap will return a pointer with alignment equal to + * sysconf(_SC_PAGESIZE). + * + * MAP_ANONYMOUS initializes the memory to zero. + */ + ptr = mmap(NULL, size, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - retval = mprotect(ptr, size, PROT_READ | PROT_WRITE | PROT_EXEC); - if (retval != 0) { - free(ptr); + if (ptr == MAP_FAILED) return NULL; - } - memset(ptr, 0, size); return ptr; } @@ -395,15 +394,17 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) hsaKmtFreeMemory(addr, size); } +/* + * Allocates memory aligned to sysconf(_SC_PAGESIZE) + */ static void *allocate_exec_aligned_memory(uint32_t size, - uint32_t align, enum asic_family_type type, uint32_t NodeId) { if (IS_DGPU(type)) - return allocate_exec_aligned_memory_gpu(size, align, NodeId, + return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId, false); - return allocate_exec_aligned_memory_cpu(size, align); + return allocate_exec_aligned_memory_cpu(size); } static void free_exec_aligned_memory(void *addr, uint32_t size, uint32_t align, @@ -412,7 +413,7 @@ static void free_exec_aligned_memory(void *addr, uint32_t size, uint32_t align, if (IS_DGPU(type)) free_exec_aligned_memory_gpu(addr, size, align); else - free(addr); + munmap(addr, size); } static void free_queue(struct queue *q) @@ -439,7 +440,6 @@ static int handle_concrete_asic(struct queue *q, if (dev_info->eop_buffer_size > 0) { q->eop_buffer = allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, - PAGE_SIZE, dev_info->asic_family, NodeId); if (!q->eop_buffer) @@ -454,7 +454,6 @@ static int handle_concrete_asic(struct queue *q, args->ctl_stack_size = q->ctl_stack_size; q->ctx_save_restore = allocate_exec_aligned_memory(q->ctx_save_restore_size, - PAGE_SIZE, dev_info->asic_family, NodeId); if (!q->ctx_save_restore) @@ -506,7 +505,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, dev_info = get_device_info_by_dev_id(dev_id); struct queue *q = allocate_exec_aligned_memory(sizeof(*q), - PAGE_SIZE, dev_info->asic_family, + dev_info->asic_family, NodeId); if (!q) return HSAKMT_STATUS_NO_MEMORY; From cb4814eadcca190f64ffd2978801b7b72bac1055 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 6 Oct 2017 19:31:24 -0400 Subject: [PATCH 0295/1247] Make system memory allocations NUMA aware Use mbind to specify the NUMA node for system memory allocation. This only works with HSA_USERPTR_FOR_PAGED_MEM=1. Change-Id: I88e7815d5a5aefcc4c22358c1a4a1635d7677ef3 Signed-off-by: Felix Kuehling --- CMakeLists.txt | 2 +- src/fmm.c | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb283f1e27..bd8f32e447 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,7 +91,7 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${BUILD_VERSION_MAJOR}" ) target_link_libraries ( ${HSAKMT_TARGET} - pthread rt pci + pthread rt pci numa ) ## If the library is a release, strip the target library diff --git a/src/fmm.c b/src/fmm.c index cc5c4cb1a6..97ece30411 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -34,6 +34,13 @@ #include #include #include +#include +#ifndef MPOL_F_STATIC_NODES +/* Bug in numaif.h, this should be defined in there. Definition copied + * from linux/mempolicy.h. + */ +#define MPOL_F_STATIC_NODES (1 << 15) +#endif #define NON_VALID_GPU_ID 0 @@ -1170,6 +1177,10 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, * memory is allocated from KFD */ if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { + const unsigned int bits_per_long = sizeof(unsigned long) * 8; + unsigned long node_mask[node_id / bits_per_long + 1]; + int mode = MPOL_F_STATIC_NODES; + /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, size, 0); @@ -1177,6 +1188,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, if (!mem) return NULL; + /* Bind to NUMA node */ + memset(node_mask, 0, sizeof(node_mask)); + node_mask[node_id / bits_per_long] = 1UL << (node_id % bits_per_long); + mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; + if (mbind(mem, MemorySizeInBytes, mode, node_mask, node_id+1, 0)) + pr_warn("Failed to set NUMA policy for %lu pages at %p\n", + MemorySizeInBytes >> 12, mem); + /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) @@ -1286,6 +1305,9 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { + /* Reset NUMA policy */ + mbind(address, object->size, MPOL_DEFAULT, NULL, 0, 0); + /* Remove any CPU mapping, but keep the address range reserved */ mmap_ret = mmap(address, object->size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, From 3b852b44371dae249135b1c9a54231bd275c5ecc Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 27 Sep 2017 16:53:57 -0400 Subject: [PATCH 0296/1247] Add SVM aperture on gfx902 Because of HW design change, GPUVM aperture is no longer needed on GFX9 APUs. However, on APUs some functionalities still depend on GPUVM aperture, so we choose to use SVM aperture instead to assume the functionality of previous GPUVM aperture. Change-Id: Ife7f0d598dd7989f2bcf7cdf3466d5a68703ca60 Signed-off-by: Yong Zhao --- src/fmm.c | 22 ++++++++++++++-------- src/libhsakmt.h | 1 + src/topology.c | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 97ece30411..add26aa4b1 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -703,12 +703,18 @@ static manageable_aperture_t *fmm_find_aperture(const void *address, _info.type = HSA_APERTURE_DGPU; } } else { /* APU */ - for (i = 0; i < gpu_mem_count; i++) { - if ((address >= gpu_mem[i].gpuvm_aperture.base) && - (address <= gpu_mem[i].gpuvm_aperture.limit)) { - aperture = &gpu_mem[i].gpuvm_aperture; - _info.type = HSA_APERTURE_GPUVM; - _info.idx = i; + if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { + aperture = &svm.dgpu_aperture; + _info.type = HSA_APERTURE_DGPU; + } else { + /* gpuvm_aperture */ + for (i = 0; i < gpu_mem_count; i++) { + if ((address >= gpu_mem[i].gpuvm_aperture.base) && + (address <= gpu_mem[i].gpuvm_aperture.limit)) { + aperture = &gpu_mem[i].gpuvm_aperture; + _info.type = HSA_APERTURE_GPUVM; + _info.idx = i; + } } } if (!aperture) { @@ -1577,7 +1583,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_id].scratch_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].scratch_limit); - if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { uintptr_t alt_base; uint64_t alt_size; int err; @@ -2582,7 +2588,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, gpu_mem_id = gpu_mem_find_by_gpu_id(infoArgs.gpu_id); if (gpu_mem_id < 0) goto error_free_metadata; - if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { aperture = &svm.dgpu_aperture; aperture_base = NULL; offset = 0; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index ac42d60888..142379010e 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -115,6 +115,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties uint32_t *gpu_id, struct pci_access* pacc); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); bool topology_is_dgpu(uint16_t device_id); +bool topology_is_svm_needed(uint16_t device_id); HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, enum asic_family_type *asic); diff --git a/src/topology.c b/src/topology.c index 0a5b5d8cab..64db33f910 100644 --- a/src/topology.c +++ b/src/topology.c @@ -590,6 +590,21 @@ bool topology_is_dgpu(uint16_t device_id) return false; } +bool topology_is_svm_needed(uint16_t device_id) +{ + const struct hsa_gfxip_table *hsa_gfxip; + + if (topology_is_dgpu(device_id)) + return true; + + hsa_gfxip = find_hsa_gfxip_device(device_id); + + if (hsa_gfxip && hsa_gfxip->asic_family >= CHIP_VEGA10) + return true; + + return false; +} + static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, bool is_apu) { From 383f275aa760a85b278816bcb4cc5e6ee9def6fe Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Fri, 21 Apr 2017 20:13:07 -0400 Subject: [PATCH 0297/1247] cmake: Use pkg-config to find libpci Change-Id: I1ab4397d88a2bd48ce0d6f2d3c33efcf47bc442f Signed-off-by: Jan Vesely Signed-off-by: Felix Kuehling --- CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd8f32e447..e485adfc9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,8 +90,11 @@ add_library ( ${HSAKMT_TARGET} SHARED ${HSAKMT_SRC} ) set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" ) set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${BUILD_VERSION_MAJOR}" ) +find_package(PkgConfig) +pkg_check_modules(PC_LIBPCI REQUIRED libpci) +include_directories ( ${PC_LIBPCI_INCLUDEDIR} ) target_link_libraries ( ${HSAKMT_TARGET} - pthread rt pci numa + pthread rt numa ${PC_LIBPCI_LIBRARIES} ) ## If the library is a release, strip the target library From d8a8f8873767f218eafad1903455da7bd8bec335 Mon Sep 17 00:00:00 2001 From: Jan Vesely Date: Fri, 21 Apr 2017 20:14:03 -0400 Subject: [PATCH 0298/1247] cmake: make sure there are no undefined symbols Change-Id: Id5a268d7e512f71c1a65af598543eb60ae6c3596 Signed-off-by: Jan Vesely Signed-off-by: Felix Kuehling --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e485adfc9f..6bc2aee14e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ endif () set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) ## Linker Flags -set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.$(PROJECT_VERSION_MAJOR) -Wl,-z,nodelete" ) +set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.$(PROJECT_VERSION_MAJOR) -Wl,-z,nodelete -Wl,-no-undefined" ) ## Source files set ( HSAKMT_SRC "src/debug.c" From aece2f8fc2825fde02c3d281a6104c104904fa63 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Wed, 25 Oct 2017 14:10:58 -0500 Subject: [PATCH 0299/1247] Remove make build file. Change-Id: I86abae4c44b6c606fb850eff6d44cdbf30cf59f5 --- src/Makefile | 116 --------------------------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 src/Makefile diff --git a/src/Makefile b/src/Makefile deleted file mode 100644 index 615fd9b10c..0000000000 --- a/src/Makefile +++ /dev/null @@ -1,116 +0,0 @@ -# Our directory -THUNK_ROOT ?= $(abspath ../) - -# Include directories -INCLUDE_DIR := ../include -INCLUDES += $(INCLUDE_DIR) -CFLAGS += $(foreach DIR,$(INCLUDES),-I$(DIR)) - -LIB_NAME = libhsakmt.so -LIB_MAJOR_VER = 1 - -PACKAGE_MAJOR_VER = 1 -PACKAGE_MINOR_VER = 0 -PACKAGE_PATCH_VER ?= 0 -PACKAGE_VER = $(PACKAGE_MAJOR_VER).$(PACKAGE_MINOR_VER).$(PACKAGE_PATCH_VER) -export PACKAGE_VER -DEB_BUILD_ARCH = $(shell dpkg --print-architecture | sed 's/ \*$i//g') - -# Compiler options -CFLAGS += -fPIC # Position-independent code required to build shared library -CFLAGS += -W -Wall -Wextra -Werror -Wno-unused-parameter -CFLAGS += -Wformat-security -Wswitch-default -Wundef \ - -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual \ - -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes \ - -Wmissing-declarations -Wredundant-decls \ - -Wunreachable-code -CFLAGS += -std=gnu99 -pthread -fvisibility=hidden -O2 - -# If this is not a release build, add debug-specific flags -ifneq ($(REL),1) - CFLAGS += -ggdb -endif - -LDFLAGS += -lrt -pthread -lpci -Wl,--version-script=libhsakmt.ver -Wl,-soname=$(LIB_NAME).$(LIB_MAJOR_VER) -Wl,-z,nodelete - -OBJS = debug.o globals.o memory.o perfctr.o time.o version.o \ - events.o openclose.o queues.o topology.o fmm.o pmc_table.o \ - libhsakmt.o - -.PHONY: all lnx lnx64a clean - -# Default target -all: lnx64a - -# To relocate build output, set "O=target/dir/" -# otherwise it defaults to a local directory - -ifdef O - BUILD_ROOT := $(O) -else - BUILD_ROOT := ../build -endif -BUILD_ROOT := $(abspath $(BUILD_ROOT)) - -BUILDDIR = $(BUILD_ROOT)/$(MAKECMDGOALS) -TARGET = $(addprefix $(BUILDDIR)/,$(OBJS)) - -$(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER): $(TARGET) - gcc -shared -o $@ $^ $(LDFLAGS) - -$(BUILDDIR)/$(LIB_NAME): $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) - @ln -sf $(LIB_NAME).$(LIB_MAJOR_VER) $(BUILDDIR)/$(LIB_NAME) - -lnx: CFLAGS += -m32 -lnx: LDFLAGS += -m32 -lnx: $(BUILDDIR)/$(LIB_NAME) - -PACKAGE_DIR = $(BUILDDIR)/libhsakmt -DEBIAN_DIR = ../DEBIAN -LDCONF_FILE = $(DEBIAN_DIR)/x86_64-libhsakmt.conf -LDCONF_INSTALL_DIR = $(PACKAGE_DIR)/etc/ld.so.conf.d -ROCM_ROOT = $(PACKAGE_DIR)/opt/rocm -ROCM_LIB = $(ROCM_ROOT)/lib -ROCM_INCLUDE = $(ROCM_ROOT)/include -INSTALL_LIB_DIR = libhsakmt/lib -INSTALL_INCLUDE_DIR = libhsakmt/include/libhsakmt - -export PACKAGE_DIR - -package-common: lnx64a - @mkdir -p $(PACKAGE_DIR) - - @mkdir -p $(LDCONF_INSTALL_DIR) - @cp -r $(LDCONF_FILE) $(LDCONF_INSTALL_DIR) - - @mkdir -p $(ROCM_ROOT)/$(INSTALL_LIB_DIR) - @cp $(BUILDDIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_ROOT)/$(INSTALL_LIB_DIR) - - @mkdir -p $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) - @cp -r $(INCLUDE_DIR)/* $(ROCM_ROOT)/$(INSTALL_INCLUDE_DIR) - - @mkdir -p $(ROCM_LIB) - @ln -fs ../$(INSTALL_LIB_DIR)/$(LIB_NAME).$(LIB_MAJOR_VER) $(ROCM_LIB)/$(LIB_NAME) - - @mkdir -p $(ROCM_INCLUDE) - ln -fs ../$(INSTALL_INCLUDE_DIR) $(ROCM_INCLUDE)/ - -deb: package-common - @mkdir -p $(PACKAGE_DIR)/DEBIAN - @sed 's/\$$version/$(PACKAGE_VER)/g;s/\$$arch/$(DEB_BUILD_ARCH)/g' $(DEBIAN_DIR)/control > $(PACKAGE_DIR)/DEBIAN/control - - @fakeroot dpkg-deb --build $(PACKAGE_DIR) \ - $(BUILDDIR)/hsakmt-dev-$(PACKAGE_VER)-${DEB_BUILD_ARCH}.deb - -rpm: package-common - @rpmbuild --define '_topdir $(BUILD_ROOT)/rpm' -ba $(THUNK_ROOT)/RPM/libhsakmt.spec - -lnx64a: $(BUILDDIR)/$(LIB_NAME) - -clean: - rm -rf $(BUILD_ROOT) -#Rule -$(BUILDDIR)/%.o: %.c ../include/hsakmt.h ../include/hsakmttypes.h ../include/linux/kfd_ioctl.h - @echo Compiling $^ - @mkdir -p $(dir $@) - gcc $(CFLAGS) -c $< -o $@ From 3501b2f40d07b68ab560ec3deb3ad82f14dfe1fa Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 26 Oct 2017 10:54:15 -0400 Subject: [PATCH 0300/1247] Fix double free on fork after hsaKmtCloseKFD Child process hsaKmtOpenKFD() call must re-initialize global variables copied from parent process. This includes close all file handles, free dynamically malloc buf. Double free issue is because destroy_device_ debugging_memory() free the memory in parent process hsaKmtCloseKFD() but don't reset it to null pointer. As a result, child process free it again. kfd_fd is closed in parent process but don't reset to 0, so child process close it again. Fix: reset kfd_fd to 0 after close, reset is_device_debugged pointer to 0 after free Change-Id: I421b3decbcaa4111298b8e599aa16940d851a58c Signed-off-by: Philip Yang --- src/debug.c | 4 +++- src/openclose.c | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/debug.c b/src/debug.c index 05a21329b9..5f8e95c34a 100644 --- a/src/debug.c +++ b/src/debug.c @@ -47,8 +47,10 @@ HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes) void destroy_device_debugging_memory(void) { - if (is_device_debugged) + if (is_device_debugged) { free(is_device_debugged); + is_device_debugged = NULL; + } } HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId) diff --git a/src/openclose.c b/src/openclose.c index caaa6d924c..403d38de30 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -66,7 +66,10 @@ static void clear_after_fork(void) clear_events_page(); fmm_clear_all_mem(); destroy_device_debugging_memory(); - close(kfd_fd); + if (kfd_fd) { + close(kfd_fd); + kfd_fd = 0; + } kfd_open_count = 0; } @@ -172,7 +175,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void) destroy_device_debugging_memory(); destroy_process_doorbells(); fmm_destroy_process_apertures(); - close(kfd_fd); + if (kfd_fd) { + close(kfd_fd); + kfd_fd = 0; + } } result = HSAKMT_STATUS_SUCCESS; From 55fc06dac3bd12554e7517b364930838e1ece373 Mon Sep 17 00:00:00 2001 From: Shaoyun Liu Date: Tue, 31 Oct 2017 14:47:03 -0400 Subject: [PATCH 0301/1247] Add asic id for gfx906 on emulator On thunk level, gfx906 works same as gfx900 chip Change-Id: I727bd904284616f3b1b9b911e41ad0f19318b3ee Signed-off-by: Shaoyun Liu --- src/topology.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/topology.c b/src/topology.c index 64db33f910..44b6fe4e64 100644 --- a/src/topology.c +++ b/src/topology.c @@ -174,6 +174,8 @@ static struct hsa_gfxip_table { { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, /* Raven */ { 0x15DD, 9, 0, 3, 0, "Raven", CHIP_RAVEN }, + /* Vega20 on emulator, treat it as vega10 */ + { 0x66A0, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, }; enum cache_type { From 68a2d286cadae6623cb505e2d624b43dee86779b Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Fri, 27 Oct 2017 16:17:51 -0400 Subject: [PATCH 0302/1247] Use drm render device to map kfd BOs Previously kfd device is used to map memory for CPU access. However this is not compatible with how TTM handles CPU mapping on eviction - memory won't be unmapped and remapped on restore. This fixes the issue by mmapping memory using DRM render device. This patch requires a coordinated kernel driver change to work. To make it compatible with old kernel driver, some temporary codes are included. Once the coordinated kernel driver is checked in, the temporary codes can be removed. Change-Id: Ie7b304c4a82b7e8d5ab703acb81d66430af4f0bc Signed-off-by: Oak Zeng --- include/hsakmttypes.h | 2 +- src/fmm.c | 21 ++++++++++++++------- src/libhsakmt.h | 1 + src/topology.c | 32 +++++++++++++++++++++++++++++++- 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index eb640d7014..6fc65df068 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -247,7 +247,7 @@ typedef struct _HsaNodeProperties HSAuint64 LocalMemSize; // Local memory size HSAuint32 MaxEngineClockMhzFCompute; // maximum engine clocks for CPU and HSAuint32 MaxEngineClockMhzCCompute; // GPU function, including any boost caopabilities, - + HSAint32 DrmRenderMinor; // DRM render device minor device number HSAuint16 MarketingName[HSA_PUBLIC_NAME_SIZE]; // Public name of the "device" on the node (board or APU name). // Unicode string HSAuint8 AMDName[HSA_PUBLIC_NAME_SIZE]; //CAL Name of the "device", ASCII diff --git a/src/fmm.c b/src/fmm.c index add26aa4b1..67cea1c834 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1056,9 +1056,12 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla } if (mem && flags.ui32.HostAccess) { + int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : + get_drm_render_fd_by_gpu_id(gpu_id); void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd, mmap_offset); + MAP_SHARED | MAP_FIXED, + map_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; @@ -1245,9 +1248,11 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, ioc_flags, &vm_obj); if (mem && flags.ui32.HostAccess) { + int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : + get_drm_render_fd_by_gpu_id(gpu_id); void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd, mmap_offset); + MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; @@ -1259,7 +1264,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, memset(ret, 0, MemorySizeInBytes); mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd, mmap_offset); + MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); } } } @@ -1827,6 +1832,8 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert if (!obj) return -1; } else { + int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : + get_drm_render_fd_by_gpu_id(gpu_id); fmm_allocate_memory_in_device(gpu_id, address, size, @@ -1835,8 +1842,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert KFD_IOC_ALLOC_MEM_FLAGS_GTT); mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, - kfd_fd, mmap_offset); + MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (mmap_ret == MAP_FAILED) { __fmm_release(mem, aperture); return -1; @@ -2753,10 +2759,11 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo pthread_mutex_unlock(&aperture->fmm_mutex); if (importArgs.mmap_offset) { + int map_fd = importArgs.mmap_offset >= (1ULL<<40) ? kfd_fd : + get_drm_render_fd_by_gpu_id(importArgs.gpu_id); void *ret = mmap(reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, kfd_fd, - importArgs.mmap_offset); + MAP_SHARED | MAP_FIXED, map_fd, importArgs.mmap_offset); if (ret == MAP_FAILED) { err = HSAKMT_STATUS_ERROR; goto err_free_obj; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 142379010e..07402ffef5 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -107,6 +107,7 @@ HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint16_t get_device_id_by_node(HSAuint32 node_id); uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); +int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); diff --git a/src/topology.c b/src/topology.c index 44b6fe4e64..2cc2ebdba6 100644 --- a/src/topology.c +++ b/src/topology.c @@ -54,6 +54,7 @@ typedef struct { HsaMemoryProperties *mem; /* node->NumBanks elements */ HsaCacheProperties *cache; HsaIoLinkProperties *link; + int drm_render_fd; } node_t; static HsaSystemProperties *_system = NULL; @@ -239,6 +240,8 @@ free_node(node_t *n) free((n)->cache); if ((n)->link) free((n)->link); + if ((n)->drm_render_fd > 0) + close((n)->drm_render_fd); } static void free_nodes(node_t *temp_nodes, int size) @@ -825,6 +828,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->MaxEngineClockMhzCCompute = (uint32_t)prop_val; else if (strcmp(prop_name, "local_mem_size") == 0) props->LocalMemSize = prop_val; + else if (strcmp(prop_name, "drm_render_minor") == 0) + props->DrmRenderMinor = (int32_t)prop_val; } @@ -1512,6 +1517,16 @@ static void topology_create_indirect_gpu_links(const HsaSystemProperties *sys_pr } } + +static void open_drm_render_device(node_t *n) +{ + int minor = n->node.DrmRenderMinor; + char path[128]; + + sprintf(path, "/dev/dri/renderD%d", minor); + n->drm_render_fd = open(path, O_RDWR | O_CLOEXEC); +} + HSAKMT_STATUS topology_take_snapshot(void) { uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; @@ -1609,7 +1624,7 @@ retry: } } } - + open_drm_render_device(&temp_nodes[i]); } pci_cleanup(pacc); } @@ -1970,6 +1985,21 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) return 0; } +int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id) +{ + unsigned int i; + + if (!node || !_system) + return 0; + + for (i = 0; i < _system->NumNodes; i++) { + if (node[i].gpu_id == gpu_id) + return node[i].drm_render_fd; + } + + return -1; +} + HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray) { From 310e3d7b8b461cfb83fe2b991cf6ef84003f5a03 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 1 Nov 2017 14:17:15 -0400 Subject: [PATCH 0303/1247] Use absolute path on cmake parameter Update build instructions in README.md to use absolute path on cmake parameter, CMAKE_MODULE_PATH. Relative path causes build error. Tested on cmake 3.5.1 ans cmake 3.5.2. Change-Id: I1b8e8deb9f4941580580be8087a94655ae155d02 Signed-off-by: Amber Lin --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 80e26fb506..ed699befd2 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,12 @@ https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver #### Building the Thunk -A simple cmake-based system is available for building thunk. To build the runtime from the the ROCT-Thunk-Interface directory, execute: +A simple cmake-based system is available for building thunk. To build the thunk from the the ROCT-Thunk-Interface directory, execute: ```bash mkdir -p build cd build - cmake -DCMAKE_MODULE_PATH=../cmake_modules .. + cmake -DCMAKE_MODULE_PATH=${PWD}/../cmake_modules .. make ``` From c704ff60b35f0b5db184476c6c38b86c4751d938 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 7 Nov 2017 06:42:40 -0500 Subject: [PATCH 0304/1247] CMakeLists: Make roct-dev dependent on roct Change-Id: Ib7d2927087dcd53da7916951de9d6a71ae6bb21b --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bc2aee14e..39ef33e653 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,7 @@ add_custom_target ( build-dev COMMAND ${CMAKE_COMMAND} -DSOURCE_DIR="${SOURCE_DIR}" -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" + -DCPACK_RPM_PACKAGE_REQUIRES="hsakmt-roct" -DCPACK_PACKAGING_INSTALL_PREFIX="${CPACK_PACKAGING_INSTALL_PREFIX}" COMMAND rm -rf *.deb *.rpm *.tar.gz COMMAND make package From 07110fbd3824a5f14401e65857bf02ea94803726 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 7 Nov 2017 11:46:51 -0500 Subject: [PATCH 0305/1247] Correctly handle max_map_count limit after failed memory allocation Also separated a function for removing CPU mapping and reserving address, as a refactoring of codes Change-Id: I1feb85b0b2ec942487f899ec3192c7c47dd7c7d5 Signed-off-by: Oak Zeng --- src/fmm.c | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 67cea1c834..f3ea06d1da 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1152,6 +1152,26 @@ static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, return mem; } +/* Remove any CPU mapping, but keep the address range reserved */ +static void munmap_and_reserve_address(void *address, uint64_t size) +{ + void *mmap_ret; + + mmap_ret = mmap(address, size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + if (mmap_ret == MAP_FAILED && errno == ENOMEM) { + /* When mmap count reaches max_map_count, any mmap will + * fail. Reduce the count with munmap then map it as + * NORESERVE immediately. + */ + munmap(address, size); + mmap(address, size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + } +} + static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { @@ -1233,12 +1253,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, size); pthread_mutex_unlock(&aperture->fmm_mutex); - /* Remove any CPU mapping, but keep the - * address range reserved - */ - mmap(mem, MemorySizeInBytes, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | - MAP_PRIVATE | MAP_FIXED, -1, 0); + munmap_and_reserve_address(mem, MemorySizeInBytes); return NULL; } } else { @@ -1292,7 +1307,6 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args; vm_object_t *object; - void *mmap_ret; if (!address) return; @@ -1318,21 +1332,7 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) address <= dgpu_shared_aperture_limit) { /* Reset NUMA policy */ mbind(address, object->size, MPOL_DEFAULT, NULL, 0, 0); - - /* Remove any CPU mapping, but keep the address range reserved */ - mmap_ret = mmap(address, object->size, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, - -1, 0); - if (mmap_ret == MAP_FAILED && errno == ENOMEM) { - /* When mmap count reaches max_map_count, any mmap will - * fail. Reduce the count with munmap then map it as - * NORESERVE immediately. - */ - munmap(address, object->size); - mmap(address, object->size, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, - -1, 0); - } + munmap_and_reserve_address(address, object->size); } aperture_release_area(aperture, address, object->size); From d4e6ec0ff00aa9c7c4613e0390467dbc847349d1 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 6 Nov 2017 16:47:47 -0500 Subject: [PATCH 0306/1247] Added "-g" to CFLAGS for debug build Previously even for debug build, -O2 is used. So there wasn't debug information in the debug build. Change-Id: I6334474e007480eb2db191bdfec5a71677c26a52 Signed-off-by: Oak Zeng --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 39ef33e653..0592150e6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,11 +53,16 @@ set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUIL set ( CMAKE_VERBOSE_MAKEFILE on ) ## Compiler flags -set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden -O2" ) +set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden" ) if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror" ) endif () +if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) + set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2" ) +else () + set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g" ) +endif () set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) From 61d1c6ffacd81c9e38a7fe9e213642f0bd37f73c Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 16 Nov 2017 14:10:23 -0500 Subject: [PATCH 0307/1247] Correct command in make package cmake command in making packages was not updated. Signed-off-by: Amber Lin Change-Id: Iafd3d9f4941d782bd77cfd0efafe48a02221b002 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed699befd2..9b8f528abb 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ If the hsakmt-roct and hsakmt-roct-dev packages are desired: ```bash mkdir -p build cd build - cmake .. + cmake -DCMAKE_MODULE_PATH=${PWD}/../cmake_modules .. make package make package-dev ``` From e81d04e11b8474f33753416c4be3f6c773551455 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Fri, 17 Nov 2017 21:35:11 -0600 Subject: [PATCH 0308/1247] Use a default module search path if not already specified. Change-Id: I782f0b758dc908c25abeb7f3536418cb5a48ac5e --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0592150e6a..046be997b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,11 @@ set ( HSAKMT_TARGET "${HSAKMT}" ) project ( ${HSAKMT_TARGET} ) +## Set default module path if not already set +if ( NOT DEFINED CMAKE_MODULE_PATH ) + set ( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/" ) +endif () + ## Include common cmake modules include ( utils ) From f06e88772555c7151c5837a82dc845703b951219 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 20 Nov 2017 16:15:29 -0500 Subject: [PATCH 0309/1247] Properly control lifecycle of ptr info objects Buffer mapping to devices and buffer registration to devices can be changed b/t two pointer info queries. Thus update buffer mapping info and registration info only when mapping and registration changed. This is done by free mapped_node_id_array on mapping to new device and free registered_node_id_array on registration and re-allocate them on next ptr info query. Also uses fmm_mutex to avoid race conditions in case of calling hsaKmtQueryPointerInfo concurrently with calling of buffer mapping or registration Change-Id: Ibc2e20be1fc0147066f873dfa44b21f5015104b7 Signed-off-by: Oak Zeng --- src/fmm.c | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index f3ea06d1da..b375b776fe 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1785,6 +1785,13 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, object->mapped_device_id_array = temp_mapped_id_array; object->mapped_device_id_array_size = args.device_ids_array_size; object->mapping_count = 1; + /* Mapping changed and lifecycle of object->mapped_node_id_array + * terminates here. Free it and allocate on next query + */ + if (object->mapped_node_id_array) { + free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; + } exit_ok: if (!obj) @@ -1906,6 +1913,14 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, object->mapped_device_id_array = temp_mapped_id_array; object->mapped_device_id_array_size = args.device_ids_array_size; + /* Mapping changed and lifecycle of object->mapped_node_id_array + * terminates here. Free it and allocate on next query + */ + if (object->mapped_node_id_array) { + free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; + } + pthread_mutex_unlock(&aperture->fmm_mutex); if (gpuvm_address) { @@ -2516,26 +2531,29 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, /* fall through */ } - if (!object) { - pthread_mutex_lock(&aperture->fmm_mutex); + pthread_mutex_lock(&aperture->fmm_mutex); + if (!object) object = vm_find_object_by_address(aperture, address, 0); + + if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_NOT_SUPPORTED; } - if (!object) - return HSAKMT_STATUS_NOT_SUPPORTED; if (object->registered_device_id_array_size > 0) { /* Multiple registration is allowed, but not changing nodes */ if ((gpu_id_array_size != object->registered_device_id_array_size) || memcmp(object->registered_device_id_array, gpu_id_array, gpu_id_array_size)) { pr_err("Cannot change nodes in a registered addr.\n"); + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_ALREADY_REGISTERED; } else { /* Delete the new array, keep the existing one. */ if (gpu_id_array) free(gpu_id_array); + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } } @@ -2543,8 +2561,17 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (gpu_id_array_size > 0) { object->registered_device_id_array = gpu_id_array; object->registered_device_id_array_size = gpu_id_array_size; + /* Registration of object changed. Lifecycle of object-> + * registered_node_id_array terminates here. Free old one + * and re-allocate on next query + */ + if (object->registered_node_id_array) { + free(object->registered_node_id_array); + object->registered_node_id_array = NULL; + } } + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } @@ -3024,6 +3051,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) aperture = fmm_find_aperture(address, NULL); + pthread_mutex_lock(&aperture->fmm_mutex); vm_obj = vm_find_object_by_address_range(aperture, address); if (!vm_obj) vm_obj = vm_find_object_by_userptr_range(aperture, address); @@ -3051,8 +3079,8 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) vm_obj->registered_node_id_array = (uint32_t *) (uint32_t *)malloc(vm_obj->registered_device_id_array_size); /* vm_obj->registered_node_id_array allocated here will be - * freed whenever the registration is deregistered or the - * memory being freed + * freed whenever the registration is changed (deregistration or + * register to new nodes) or the memory being freed */ for (i = 0; i < info->NRegisteredNodes; i++) gpuid_to_nodeid(vm_obj->registered_device_id_array[i], @@ -3066,7 +3094,8 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) vm_obj->mapped_node_id_array = (uint32_t *)malloc(vm_obj->mapped_device_id_array_size); /* vm_obj->mapped_node_id_array allocated here will be - * freed whenever the mapping is unmapped or memory being freed + * freed whenever the mapping is changed (unmapped or map + * to new nodes) or memory being freed */ for (i = 0; i < info->NMappedNodes; i++) gpuid_to_nodeid(vm_obj->mapped_device_id_array[i], @@ -3085,6 +3114,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) } exit: + pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } From 061db45fe2447a616ab3e8dcf88e95437a331fff Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 20 Nov 2017 10:57:37 -0500 Subject: [PATCH 0310/1247] Fix unconditional unmap in fmm_map_to_gpu_nodes _fmm_unmap_from_gpu is called in fmm_map_to_gpu_nodes to unmap buffer from nodes that is already mapped to but not in the new map nodes list. Previously, the unmap was called unconditionally even though the size of the array to unmap is 0. This fixes the issue by calling the unmap func only when the unmap array size is not 0. Also releases the fmm_mutex on error returns Change-Id: Iadd8383caf7ebb92f02618798c5efd138a352aaa Signed-off-by: Oak Zeng --- src/fmm.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index b375b776fe..1313566c3c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3022,11 +3022,16 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, } temp_node_id_array_size *= sizeof(uint32_t); - ret = _fmm_unmap_from_gpu(aperture, address, - temp_node_id_array, temp_node_id_array_size, - object); - if (ret != HSAKMT_STATUS_SUCCESS) - return ret; + if (temp_node_id_array_size) { + ret = _fmm_unmap_from_gpu(aperture, address, + temp_node_id_array, + temp_node_id_array_size, + object); + if (ret != HSAKMT_STATUS_SUCCESS) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return ret; + } + } } retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, From 6f7b55f2d832b6130fb70c5c9506467ef03725fd Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 21 Nov 2017 11:47:25 -0500 Subject: [PATCH 0311/1247] Add debug message in PMC trace Print data in PMC trace when the debug level is set to 7(pr_debug). Change-Id: I9abbb8f6c3f7962fb637528578c1a58b7784042d Signed-off-by: Amber Lin --- src/perfctr.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/perfctr.c b/src/perfctr.c index 8040d76152..645418d404 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -584,6 +584,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcRegisterTrace(HSAuint32 NodeId, uint64_t *counter_id_ptr; int *fd_ptr; + pr_debug("[%s] Number of counters %d\n", __func__, NumberOfCounters); + if (!counter_props) return HSAKMT_STATUS_NO_MEMORY; @@ -711,6 +713,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcUnregisterTrace(HSAuint32 NodeId, uint32_t gpu_id; struct perf_trace *trace; + pr_debug("[%s] Trace ID 0x%lx\n", __func__, TraceId); + if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -746,6 +750,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess(HSAuint32 NodeId, uint32_t gpu_id, i; int j; + pr_debug("[%s] Trace ID 0x%lx\n", __func__, TraceId); + if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -789,6 +795,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess(HSAuint32 NodeId, struct perf_trace *trace; uint32_t i; + pr_debug("[%s] Trace ID 0x%lx\n", __func__, TraceId); + if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -819,6 +827,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStartTrace(HSATraceId TraceId, int32_t j; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + pr_debug("[%s] Trace ID 0x%lx\n", __func__, TraceId); + if (TraceId == 0 || !TraceBuffer || TraceBufferSizeBytes == 0) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -866,6 +876,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcQueryTrace(HSATraceId TraceId) return HSAKMT_STATUS_INVALID_HANDLE; buf = (uint64_t *)trace->buf; + pr_debug("[%s] Trace buffer(%p): ", __func__, buf); for (i = 0; i < trace->num_blocks; i++) for (j = 0; j < trace->blocks[i].num_counters; j++) { buf_filled += sizeof(uint64_t); @@ -875,8 +886,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcQueryTrace(HSATraceId TraceId) buf); if (ret != HSAKMT_STATUS_SUCCESS) return ret; + pr_debug("%lu_", *buf); buf++; } + pr_debug("\n"); return HSAKMT_STATUS_SUCCESS; } @@ -890,6 +903,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcStopTrace(HSATraceId TraceId) uint32_t i; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + pr_debug("[%s] Trace ID 0x%lx\n", __func__, TraceId); + if (TraceId == 0) return HSAKMT_STATUS_INVALID_PARAMETER; From b4c89c1ea74598bd5874ef7aa45087381f205e4e Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 21 Nov 2017 13:04:58 -0500 Subject: [PATCH 0312/1247] Cleanup fmm.c 1. Initialize pointers to NULL in vm_create_and_init_object 2. Added helper function to add/remove device ids to/from mapped arrary 3. Only map nodes that were not mapped currently 4. Remove unnecessary condition check on object frees Change-Id: I7aed6d40c7464be0d168d5796229af55451e0f34 Signed-off-by: Oak Zeng --- src/fmm.c | 207 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 114 insertions(+), 93 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 1313566c3c..ba22f34508 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -264,6 +264,8 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->next = object->prev = NULL; object->registered_device_id_array_size = 0; object->mapped_device_id_array_size = 0; + object->registered_device_id_array = NULL; + object->mapped_device_id_array = NULL; object->registered_node_id_array = NULL; object->mapped_node_id_array = NULL; object->registration_count = 0; @@ -303,32 +305,19 @@ static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object) vm_object_t *prev; /* Free allocations inside the object */ - if (object->registered_device_id_array_size > 0) { - if (object->mapped_device_id_array == - object->registered_device_id_array) { - object->mapped_device_id_array_size = 0; - object->mapped_device_id_array = NULL; - object->mapped_node_id_array = NULL; - } + if (object->registered_device_id_array) free(object->registered_device_id_array); - object->registered_device_id_array_size = 0; - } - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) { + + if (object->mapped_device_id_array) free(object->mapped_device_id_array); - object->mapped_device_id_array_size = 0; - } + if (object->metadata) free(object->metadata); if (object->registered_node_id_array) free(object->registered_node_id_array); - object->registered_node_id_array = NULL; if (object->mapped_node_id_array) free(object->mapped_node_id_array); - object->mapped_node_id_array = NULL; next = object->next; prev = object->prev; @@ -1719,6 +1708,67 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA return err; } +static bool id_in_array(uint32_t id, uint32_t *ids_array, + uint32_t ids_array_size) +{ + uint32_t i; + + for (i = 0; i < ids_array_size/sizeof(uint32_t); i++) { + if (id == ids_array[i]) + return true; + } + return false; +} + +/* Helper function to remove ids_array from + * obj->mapped_device_id_array + */ +static void remove_device_ids_from_mapped_array(vm_object_t *obj, + uint32_t *ids_array, uint32_t ids_array_size) +{ + uint32_t i = 0, j = 0; + + for (i = 0; i < obj->mapped_device_id_array_size/ + sizeof(uint32_t); i++) { + if (!id_in_array(obj->mapped_device_id_array[i], + ids_array, ids_array_size)) + obj->mapped_device_id_array[j++] = + obj->mapped_device_id_array[i]; + } + + obj->mapped_device_id_array_size = j*sizeof(uint32_t); + if (!j) { + if (obj->mapped_device_id_array) + free(obj->mapped_device_id_array); + + obj->mapped_device_id_array = NULL; + } +} + +/* Helper function to add ids_array to + * obj->mapped_device_id_array + */ +static void add_device_ids_to_mapped_array(vm_object_t *obj, + uint32_t *ids_array, uint32_t ids_array_size) +{ + uint32_t new_array_size; + + /* Remove any potential duplicated ids */ + remove_device_ids_from_mapped_array(obj, ids_array, ids_array_size); + new_array_size = obj->mapped_device_id_array_size + + ids_array_size; + + obj->mapped_device_id_array = (uint32_t *)realloc( + obj->mapped_device_id_array, new_array_size); + + memcpy(&obj->mapped_device_id_array + [obj->mapped_device_id_array_size/sizeof(uint32_t)], + ids_array, ids_array_size); + + obj->mapped_device_id_array_size = new_array_size; +} + + /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, @@ -1726,7 +1776,6 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; - void *temp_mapped_id_array = NULL; if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); @@ -1764,26 +1813,15 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, args.device_ids_array_size = all_gpu_id_array_size; } - temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); - if (!temp_mapped_id_array) - goto err_object_not_found; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - print_device_id_array((void *)args.device_ids_array_ptr, - args.device_ids_array_size); + add_device_ids_to_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); + print_device_id_array((uint32_t *)object->mapped_device_id_array, + object->mapped_device_id_array_size); - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); - - memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, - args.device_ids_array_size); - object->mapped_device_id_array = temp_mapped_id_array; - object->mapped_device_id_array_size = args.device_ids_array_size; object->mapping_count = 1; /* Mapping changed and lifecycle of object->mapped_node_id_array * terminates here. Free it and allocate on next query @@ -1800,7 +1838,6 @@ exit_ok: return 0; err_map_ioctl_failed: - free(temp_mapped_id_array); err_object_not_found: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1871,7 +1908,6 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; - void *temp_mapped_id_array = NULL; /* Check that address space was previously reserved */ if (!vm_find(aperture, address)) @@ -1895,24 +1931,13 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, args.device_ids_array_size = all_gpu_id_array_size; } - temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); - if (!temp_mapped_id_array) - goto err_object_not_found; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); - - memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, - args.device_ids_array_size); - object->mapped_device_id_array = temp_mapped_id_array; - object->mapped_device_id_array_size = args.device_ids_array_size; + add_device_ids_to_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); /* Mapping changed and lifecycle of object->mapped_node_id_array * terminates here. Free it and allocate on next query */ @@ -2096,15 +2121,10 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, if (ret != 0) goto out; - /* Clearing all mapped nodes list */ - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); + remove_device_ids_from_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); - object->mapped_device_id_array = NULL; - object->mapped_device_id_array_size = 0; if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; @@ -2152,15 +2172,10 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, args.device_ids_array_size = object->mapped_device_id_array_size; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); - /* Clearing all mapped nodes list */ - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); + remove_device_ids_from_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); - object->mapped_device_id_array = NULL; - object->mapped_device_id_array_size = 0; if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; @@ -2908,14 +2923,17 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) return HSAKMT_STATUS_SUCCESS; } - if (object->registered_device_id_array_size <= 0) { + if (!object->registered_device_id_array || + object->registered_device_id_array_size <= 0) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } - free(object->registered_device_id_array); - object->registered_device_id_array = NULL; - object->registered_device_id_array_size = 0; + if (object->registered_device_id_array) { + free(object->registered_device_id_array); + object->registered_device_id_array = NULL; + object->registered_device_id_array_size = 0; + } if (object->registered_node_id_array) free(object->registered_node_id_array); object->registered_node_id_array = NULL; @@ -2937,8 +2955,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, { manageable_aperture_t *aperture; vm_object_t *object = NULL; - uint32_t i, j, temp_node; - bool found, userptr = false; + uint32_t i; + bool userptr = false; uint32_t *registered_node_id_array, registered_node_id_array_size; HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; @@ -2989,15 +3007,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, registered_node_id_array_size = object->registered_device_id_array_size; } for (i = 0 ; i < num_of_nodes; i++) { - temp_node = nodes_to_map[i]; - found = false; - for (j = 0 ; j < registered_node_id_array_size / sizeof(uint32_t); j++) { - if (temp_node == registered_node_id_array[j]) { - found = true; - break; - } - } - if (!found) { + if (!id_in_array(nodes_to_map[i], registered_node_id_array, + registered_node_id_array_size)) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } @@ -3009,16 +3020,10 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t temp_node_id_array_size = 0; for (i = 0 ; i < object->mapped_device_id_array_size / sizeof(uint32_t); i++) { - temp_node = object->mapped_device_id_array[i]; - found = false; - for (j = 0 ; j < num_of_nodes; j++) { - if (temp_node == nodes_to_map[j]) { - found = true; - break; - } - } - if (!found) - temp_node_id_array[temp_node_id_array_size++] = temp_node; + if (!id_in_array(object->mapped_device_id_array[i], + nodes_to_map, num_of_nodes)) + temp_node_id_array[temp_node_id_array_size++] = + object->mapped_device_id_array[i]; } temp_node_id_array_size *= sizeof(uint32_t); @@ -3034,8 +3039,24 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, } } - retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, - nodes_to_map, num_of_nodes * sizeof(uint32_t)); + /* Remove already mapped nodes from nodes_to_map + * to generate the final map list + */ + uint32_t map_node_id_array[num_of_nodes]; + uint32_t map_node_id_array_size = 0; + + for (i = 0; i < num_of_nodes; i++) { + if (!id_in_array(nodes_to_map[i], + object->mapped_device_id_array, + object->mapped_device_id_array_size)) + map_node_id_array[map_node_id_array_size++] = + nodes_to_map[i]; + } + + if (map_node_id_array_size) + retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, + map_node_id_array, + map_node_id_array_size * sizeof(uint32_t)); pthread_mutex_unlock(&aperture->fmm_mutex); From 019f7cbd207315f21854d81d9f6a9763ae039669 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 27 Nov 2017 18:13:08 -0500 Subject: [PATCH 0313/1247] More cleanup of fmm.c 1. Renamed _fmm_map_to_gpu to _fmm_map_to_apu_local to reflect the real semantics of this function 2. Renamed _fmm_map_to_gpu_gtt to _fmm_map_to_gpu because this function is used to map both gtt and local memory 3. Call _fmm_map_to_gpu in _fmm_map_to_apu_local to get rid of duplicated codes Change-Id: Id8e3ebfffe0a3c27ebdcac8a8f4dc3738d67d10a Signed-off-by: Oak Zeng --- src/fmm.c | 60 ++++++++++++++++--------------------------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ba22f34508..0f3b4aa969 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1770,7 +1770,7 @@ static void add_device_ids_to_mapped_array(vm_object_t *obj, /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ -static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, +static int _fmm_map_to_gpu(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, uint32_t *nodes_to_map, uint32_t nodes_array_size) { @@ -1895,20 +1895,22 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* map to GPU */ - ret = _fmm_map_to_gpu_gtt(aperture, address, size, NULL, NULL, 0); + ret = _fmm_map_to_gpu(aperture, address, size, NULL, NULL, 0); if (ret != 0) __fmm_release(mem, aperture); return ret; } -static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, +static int _fmm_map_to_apu_local(uint32_t gpu_id, + manageable_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) { - struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; + if (gpuvm_address) + *gpuvm_address = 0; /* Check that address space was previously reserved */ if (!vm_find(aperture, address)) return -1; @@ -1917,37 +1919,15 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (!object) - goto err_object_not_found; - - args.handle = object->handle; - if (object->registered_device_id_array_size > 0 && - object->registered_device_id_array) { - args.device_ids_array_ptr = - (uint64_t)object->registered_device_id_array; - args.device_ids_array_size = object->registered_device_id_array_size; - } else { - args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; - args.device_ids_array_size = all_gpu_id_array_size; + if (!object) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return -1; } - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) - goto err_map_ioctl_failed; - - - add_device_ids_to_mapped_array(object, - (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); - /* Mapping changed and lifecycle of object->mapped_node_id_array - * terminates here. Free it and allocate on next query - */ - if (object->mapped_node_id_array) { - free(object->mapped_node_id_array); - object->mapped_node_id_array = NULL; - } - pthread_mutex_unlock(&aperture->fmm_mutex); + if (_fmm_map_to_gpu(aperture, address, size, object, NULL, 0)) + return -1; + if (gpuvm_address) { *gpuvm_address = (uint64_t)object->start; if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) @@ -1955,12 +1935,6 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, } return 0; - -err_map_ioctl_failed: -err_object_not_found: - pthread_mutex_unlock(&aperture->fmm_mutex); - *gpuvm_address = 0; - return -1; } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, @@ -1993,7 +1967,7 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size, obj, NULL, 0); + ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, obj, NULL, 0); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -2022,7 +1996,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* map it */ - return _fmm_map_to_gpu(gpu_mem[i].gpu_id, + return _fmm_map_to_apu_local(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); } @@ -2030,12 +2004,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) /* map it */ - return _fmm_map_to_gpu_gtt(&svm.dgpu_aperture, + return _fmm_map_to_gpu(&svm.dgpu_aperture, address, size, NULL, NULL, 0); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* map it */ - return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture, + return _fmm_map_to_gpu(&svm.dgpu_alt_aperture, address, size, NULL, NULL, 0); /* @@ -3054,7 +3028,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, } if (map_node_id_array_size) - retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, + retcode = _fmm_map_to_gpu(aperture, address, size, object, map_node_id_array, map_node_id_array_size * sizeof(uint32_t)); From cce57cec26ab5f93ab52966461b5b4f5c768097c Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 28 Nov 2017 15:20:51 -0500 Subject: [PATCH 0314/1247] Cosmetic changes in events.c Change-Id: Idecb8eede8811020b3af51cbc71da74849029c82 Signed-off-by: Oak Zeng --- src/events.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/events.c b/src/events.c index daea86219d..28c55f2c67 100644 --- a/src/events.c +++ b/src/events.c @@ -279,14 +279,14 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * pr_err("Memory is registered to node id: "); for (i = 0; i < info.NRegisteredNodes; i++) pr_err("%d ", info.RegisteredNodes[i]); + pr_err("\n"); } - pr_err("\n"); if (info.MappedNodes) { pr_err("Memory is mapped to node id: "); for (i = 0; i < info.NMappedNodes; i++) pr_err("%d ", info.MappedNodes[i]); + pr_err("\n"); } - pr_err("\n"); } HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], From 52f6a619705ed23fea2554dd67f648086854875b Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Wed, 29 Nov 2017 09:31:35 -0500 Subject: [PATCH 0315/1247] Revert "More cleanup of fmm.c" This reverts commit 019f7cbd207315f21854d81d9f6a9763ae039669. This change caused a regression () Revert temporarily Change-Id: I5af59d319afeb7f0b03e5a09e8397e3853b8b37b Signed-off-by: Oak Zeng --- src/fmm.c | 62 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 0f3b4aa969..ba22f34508 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1770,7 +1770,7 @@ static void add_device_ids_to_mapped_array(vm_object_t *obj, /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ -static int _fmm_map_to_gpu(manageable_aperture_t *aperture, +static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, uint32_t *nodes_to_map, uint32_t nodes_array_size) { @@ -1895,22 +1895,20 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* map to GPU */ - ret = _fmm_map_to_gpu(aperture, address, size, NULL, NULL, 0); + ret = _fmm_map_to_gpu_gtt(aperture, address, size, NULL, NULL, 0); if (ret != 0) __fmm_release(mem, aperture); return ret; } -static int _fmm_map_to_apu_local(uint32_t gpu_id, - manageable_aperture_t *aperture, +static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) { + struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; - if (gpuvm_address) - *gpuvm_address = 0; /* Check that address space was previously reserved */ if (!vm_find(aperture, address)) return -1; @@ -1919,14 +1917,36 @@ static int _fmm_map_to_apu_local(uint32_t gpu_id, /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (!object) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; - } - pthread_mutex_unlock(&aperture->fmm_mutex); + if (!object) + goto err_object_not_found; - if (_fmm_map_to_gpu(aperture, address, size, object, NULL, 0)) - return -1; + args.handle = object->handle; + if (object->registered_device_id_array_size > 0 && + object->registered_device_id_array) { + args.device_ids_array_ptr = + (uint64_t)object->registered_device_id_array; + args.device_ids_array_size = object->registered_device_id_array_size; + } else { + args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; + args.device_ids_array_size = all_gpu_id_array_size; + } + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) + goto err_map_ioctl_failed; + + + add_device_ids_to_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); + /* Mapping changed and lifecycle of object->mapped_node_id_array + * terminates here. Free it and allocate on next query + */ + if (object->mapped_node_id_array) { + free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; + } + + pthread_mutex_unlock(&aperture->fmm_mutex); if (gpuvm_address) { *gpuvm_address = (uint64_t)object->start; @@ -1935,6 +1955,12 @@ static int _fmm_map_to_apu_local(uint32_t gpu_id, } return 0; + +err_map_ioctl_failed: +err_object_not_found: + pthread_mutex_unlock(&aperture->fmm_mutex); + *gpuvm_address = 0; + return -1; } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, @@ -1967,7 +1993,7 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, obj, NULL, 0); + ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size, obj, NULL, 0); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -1996,7 +2022,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* map it */ - return _fmm_map_to_apu_local(gpu_mem[i].gpu_id, + return _fmm_map_to_gpu(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); } @@ -2004,12 +2030,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) /* map it */ - return _fmm_map_to_gpu(&svm.dgpu_aperture, + return _fmm_map_to_gpu_gtt(&svm.dgpu_aperture, address, size, NULL, NULL, 0); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* map it */ - return _fmm_map_to_gpu(&svm.dgpu_alt_aperture, + return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture, address, size, NULL, NULL, 0); /* @@ -3028,7 +3054,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, } if (map_node_id_array_size) - retcode = _fmm_map_to_gpu(aperture, address, size, object, + retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, map_node_id_array, map_node_id_array_size * sizeof(uint32_t)); From f7689d4fef2ca4354a7ad1ed6e6fd64748ba8342 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Wed, 29 Nov 2017 09:31:46 -0500 Subject: [PATCH 0316/1247] Revert "Cleanup fmm.c" This reverts commit b4c89c1ea74598bd5874ef7aa45087381f205e4e. This change caused a regression () Revert temporarily Change-Id: Ic3829264151e37d1f8c6927c6f464006234ba17f Signed-off-by: Oak Zeng --- src/fmm.c | 207 ++++++++++++++++++++++++------------------------------ 1 file changed, 93 insertions(+), 114 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ba22f34508..1313566c3c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -264,8 +264,6 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->next = object->prev = NULL; object->registered_device_id_array_size = 0; object->mapped_device_id_array_size = 0; - object->registered_device_id_array = NULL; - object->mapped_device_id_array = NULL; object->registered_node_id_array = NULL; object->mapped_node_id_array = NULL; object->registration_count = 0; @@ -305,19 +303,32 @@ static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object) vm_object_t *prev; /* Free allocations inside the object */ - if (object->registered_device_id_array) + if (object->registered_device_id_array_size > 0) { + if (object->mapped_device_id_array == + object->registered_device_id_array) { + object->mapped_device_id_array_size = 0; + object->mapped_device_id_array = NULL; + object->mapped_node_id_array = NULL; + } free(object->registered_device_id_array); - - if (object->mapped_device_id_array) + object->registered_device_id_array_size = 0; + } + if (object->mapped_device_id_array && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) { free(object->mapped_device_id_array); - + object->mapped_device_id_array_size = 0; + } if (object->metadata) free(object->metadata); if (object->registered_node_id_array) free(object->registered_node_id_array); + object->registered_node_id_array = NULL; if (object->mapped_node_id_array) free(object->mapped_node_id_array); + object->mapped_node_id_array = NULL; next = object->next; prev = object->prev; @@ -1708,67 +1719,6 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA return err; } -static bool id_in_array(uint32_t id, uint32_t *ids_array, - uint32_t ids_array_size) -{ - uint32_t i; - - for (i = 0; i < ids_array_size/sizeof(uint32_t); i++) { - if (id == ids_array[i]) - return true; - } - return false; -} - -/* Helper function to remove ids_array from - * obj->mapped_device_id_array - */ -static void remove_device_ids_from_mapped_array(vm_object_t *obj, - uint32_t *ids_array, uint32_t ids_array_size) -{ - uint32_t i = 0, j = 0; - - for (i = 0; i < obj->mapped_device_id_array_size/ - sizeof(uint32_t); i++) { - if (!id_in_array(obj->mapped_device_id_array[i], - ids_array, ids_array_size)) - obj->mapped_device_id_array[j++] = - obj->mapped_device_id_array[i]; - } - - obj->mapped_device_id_array_size = j*sizeof(uint32_t); - if (!j) { - if (obj->mapped_device_id_array) - free(obj->mapped_device_id_array); - - obj->mapped_device_id_array = NULL; - } -} - -/* Helper function to add ids_array to - * obj->mapped_device_id_array - */ -static void add_device_ids_to_mapped_array(vm_object_t *obj, - uint32_t *ids_array, uint32_t ids_array_size) -{ - uint32_t new_array_size; - - /* Remove any potential duplicated ids */ - remove_device_ids_from_mapped_array(obj, ids_array, ids_array_size); - new_array_size = obj->mapped_device_id_array_size - + ids_array_size; - - obj->mapped_device_id_array = (uint32_t *)realloc( - obj->mapped_device_id_array, new_array_size); - - memcpy(&obj->mapped_device_id_array - [obj->mapped_device_id_array_size/sizeof(uint32_t)], - ids_array, ids_array_size); - - obj->mapped_device_id_array_size = new_array_size; -} - - /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, @@ -1776,6 +1726,7 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; + void *temp_mapped_id_array = NULL; if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); @@ -1813,15 +1764,26 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, args.device_ids_array_size = all_gpu_id_array_size; } + temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); + if (!temp_mapped_id_array) + goto err_object_not_found; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - add_device_ids_to_mapped_array(object, - (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); - print_device_id_array((uint32_t *)object->mapped_device_id_array, - object->mapped_device_id_array_size); + print_device_id_array((void *)args.device_ids_array_ptr, + args.device_ids_array_size); + if (object->mapped_device_id_array && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + + memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, + args.device_ids_array_size); + object->mapped_device_id_array = temp_mapped_id_array; + object->mapped_device_id_array_size = args.device_ids_array_size; object->mapping_count = 1; /* Mapping changed and lifecycle of object->mapped_node_id_array * terminates here. Free it and allocate on next query @@ -1838,6 +1800,7 @@ exit_ok: return 0; err_map_ioctl_failed: + free(temp_mapped_id_array); err_object_not_found: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1908,6 +1871,7 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; + void *temp_mapped_id_array = NULL; /* Check that address space was previously reserved */ if (!vm_find(aperture, address)) @@ -1931,13 +1895,24 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, args.device_ids_array_size = all_gpu_id_array_size; } + temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); + if (!temp_mapped_id_array) + goto err_object_not_found; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; + if (object->mapped_device_id_array && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + + memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, + args.device_ids_array_size); + object->mapped_device_id_array = temp_mapped_id_array; + object->mapped_device_id_array_size = args.device_ids_array_size; - add_device_ids_to_mapped_array(object, - (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); /* Mapping changed and lifecycle of object->mapped_node_id_array * terminates here. Free it and allocate on next query */ @@ -2121,10 +2096,15 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, if (ret != 0) goto out; - remove_device_ids_from_mapped_array(object, - (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + /* Clearing all mapped nodes list */ + if (object->mapped_device_id_array && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + object->mapped_device_id_array = NULL; + object->mapped_device_id_array_size = 0; if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; @@ -2172,10 +2152,15 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, args.device_ids_array_size = object->mapped_device_id_array_size; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); - remove_device_ids_from_mapped_array(object, - (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + /* Clearing all mapped nodes list */ + if (object->mapped_device_id_array && + object->mapped_device_id_array_size > 0 && + object->mapped_device_id_array != all_gpu_id_array && + object->mapped_device_id_array != object->registered_device_id_array) + free(object->mapped_device_id_array); + object->mapped_device_id_array = NULL; + object->mapped_device_id_array_size = 0; if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; @@ -2923,17 +2908,14 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) return HSAKMT_STATUS_SUCCESS; } - if (!object->registered_device_id_array || - object->registered_device_id_array_size <= 0) { + if (object->registered_device_id_array_size <= 0) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } - if (object->registered_device_id_array) { - free(object->registered_device_id_array); - object->registered_device_id_array = NULL; - object->registered_device_id_array_size = 0; - } + free(object->registered_device_id_array); + object->registered_device_id_array = NULL; + object->registered_device_id_array_size = 0; if (object->registered_node_id_array) free(object->registered_node_id_array); object->registered_node_id_array = NULL; @@ -2955,8 +2937,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, { manageable_aperture_t *aperture; vm_object_t *object = NULL; - uint32_t i; - bool userptr = false; + uint32_t i, j, temp_node; + bool found, userptr = false; uint32_t *registered_node_id_array, registered_node_id_array_size; HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; @@ -3007,8 +2989,15 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, registered_node_id_array_size = object->registered_device_id_array_size; } for (i = 0 ; i < num_of_nodes; i++) { - if (!id_in_array(nodes_to_map[i], registered_node_id_array, - registered_node_id_array_size)) { + temp_node = nodes_to_map[i]; + found = false; + for (j = 0 ; j < registered_node_id_array_size / sizeof(uint32_t); j++) { + if (temp_node == registered_node_id_array[j]) { + found = true; + break; + } + } + if (!found) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } @@ -3020,10 +3009,16 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t temp_node_id_array_size = 0; for (i = 0 ; i < object->mapped_device_id_array_size / sizeof(uint32_t); i++) { - if (!id_in_array(object->mapped_device_id_array[i], - nodes_to_map, num_of_nodes)) - temp_node_id_array[temp_node_id_array_size++] = - object->mapped_device_id_array[i]; + temp_node = object->mapped_device_id_array[i]; + found = false; + for (j = 0 ; j < num_of_nodes; j++) { + if (temp_node == nodes_to_map[j]) { + found = true; + break; + } + } + if (!found) + temp_node_id_array[temp_node_id_array_size++] = temp_node; } temp_node_id_array_size *= sizeof(uint32_t); @@ -3039,24 +3034,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, } } - /* Remove already mapped nodes from nodes_to_map - * to generate the final map list - */ - uint32_t map_node_id_array[num_of_nodes]; - uint32_t map_node_id_array_size = 0; - - for (i = 0; i < num_of_nodes; i++) { - if (!id_in_array(nodes_to_map[i], - object->mapped_device_id_array, - object->mapped_device_id_array_size)) - map_node_id_array[map_node_id_array_size++] = - nodes_to_map[i]; - } - - if (map_node_id_array_size) - retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, - map_node_id_array, - map_node_id_array_size * sizeof(uint32_t)); + retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, + nodes_to_map, num_of_nodes * sizeof(uint32_t)); pthread_mutex_unlock(&aperture->fmm_mutex); From 587d4f4bdf6fa061d9ac09f0241fb7bde86faf2c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 4 Dec 2017 10:23:55 -0500 Subject: [PATCH 0317/1247] Rename fmm_allocate_memory_in_device to fmm_allocate_memory_object. This function name was confusingly similar to fmm_allocate_device and __fmm_allocate_device. The new name reflects its function better: allocate the VM object and the kernel mode buffer object. Change-Id: I6604d228004b4d41e871d4de784786823608b5d6 Signed-off-by: Felix Kuehling --- src/fmm.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 1313566c3c..359e0d092d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -733,7 +733,7 @@ static manageable_aperture_t *fmm_find_aperture(const void *address, /* After allocating the memory, return the vm_object created for this memory. * Return NULL if any failure. */ -static vm_object_t *fmm_allocate_memory_in_device(uint32_t gpu_id, void *mem, +static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t *mmap_offset, @@ -984,7 +984,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, * Now that we have the area reserved, allocate memory in the device * itself */ - obj = fmm_allocate_memory_in_device(gpu_id, mem, + obj = fmm_allocate_memory_object(gpu_id, mem, MemorySizeInBytes, aperture, mmap_offset, flags); if (!obj) { /* @@ -1245,7 +1245,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, /* Create userptr BO */ mmap_offset = (uint64_t)mem; ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_USERPTR; - vm_obj = fmm_allocate_memory_in_device(gpu_id, mem, size, + vm_obj = fmm_allocate_memory_object(gpu_id, mem, size, aperture, &mmap_offset, ioc_flags); if (!vm_obj) { @@ -1833,7 +1833,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); /* allocate object within the scratch backing aperture */ if (!ret && !is_debugger) { - vm_object_t *obj = fmm_allocate_memory_in_device( + vm_object_t *obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, NULL, KFD_IOC_ALLOC_MEM_FLAGS_VRAM); if (!obj) @@ -1841,7 +1841,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert } else { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : get_drm_render_fd_by_gpu_id(gpu_id); - fmm_allocate_memory_in_device(gpu_id, + fmm_allocate_memory_object(gpu_id, address, size, aperture, From 786e470241500d44655d0ae8d3c0f74d71cc8e89 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 30 Nov 2017 10:56:38 -0500 Subject: [PATCH 0318/1247] Revert "Revert "Cleanup fmm.c"" This reverts commit f7689d4fef2ca4354a7ad1ed6e6fd64748ba8342, Plus a bug fix to patch "Cleanup fmm.c": Call id_in_array with correct parameter. The third parameter of id_in_array is size in byte of the array, not the number of array items. Call it correctly. Change-Id: I72d8e2fcc0df32af76c72967386e92c1be18c159 Signed-off-by: Oak Zeng --- src/fmm.c | 212 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 119 insertions(+), 93 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 359e0d092d..860b121e60 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -264,6 +264,8 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->next = object->prev = NULL; object->registered_device_id_array_size = 0; object->mapped_device_id_array_size = 0; + object->registered_device_id_array = NULL; + object->mapped_device_id_array = NULL; object->registered_node_id_array = NULL; object->mapped_node_id_array = NULL; object->registration_count = 0; @@ -303,32 +305,19 @@ static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object) vm_object_t *prev; /* Free allocations inside the object */ - if (object->registered_device_id_array_size > 0) { - if (object->mapped_device_id_array == - object->registered_device_id_array) { - object->mapped_device_id_array_size = 0; - object->mapped_device_id_array = NULL; - object->mapped_node_id_array = NULL; - } + if (object->registered_device_id_array) free(object->registered_device_id_array); - object->registered_device_id_array_size = 0; - } - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) { + + if (object->mapped_device_id_array) free(object->mapped_device_id_array); - object->mapped_device_id_array_size = 0; - } + if (object->metadata) free(object->metadata); if (object->registered_node_id_array) free(object->registered_node_id_array); - object->registered_node_id_array = NULL; if (object->mapped_node_id_array) free(object->mapped_node_id_array); - object->mapped_node_id_array = NULL; next = object->next; prev = object->prev; @@ -1719,6 +1708,71 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA return err; } +static bool id_in_array(uint32_t id, uint32_t *ids_array, + uint32_t ids_array_size) +{ + uint32_t i; + + for (i = 0; i < ids_array_size/sizeof(uint32_t); i++) { + if (id == ids_array[i]) + return true; + } + return false; +} + +/* Helper function to remove ids_array from + * obj->mapped_device_id_array + */ +static void remove_device_ids_from_mapped_array(vm_object_t *obj, + uint32_t *ids_array, uint32_t ids_array_size) +{ + uint32_t i = 0, j = 0; + + if (obj->mapped_device_id_array == ids_array) + goto set_size_and_free; + + for (i = 0; i < obj->mapped_device_id_array_size/ + sizeof(uint32_t); i++) { + if (!id_in_array(obj->mapped_device_id_array[i], + ids_array, ids_array_size)) + obj->mapped_device_id_array[j++] = + obj->mapped_device_id_array[i]; + } + +set_size_and_free: + obj->mapped_device_id_array_size = j*sizeof(uint32_t); + if (!j) { + if (obj->mapped_device_id_array) + free(obj->mapped_device_id_array); + + obj->mapped_device_id_array = NULL; + } +} + +/* Helper function to add ids_array to + * obj->mapped_device_id_array + */ +static void add_device_ids_to_mapped_array(vm_object_t *obj, + uint32_t *ids_array, uint32_t ids_array_size) +{ + uint32_t new_array_size; + + /* Remove any potential duplicated ids */ + remove_device_ids_from_mapped_array(obj, ids_array, ids_array_size); + new_array_size = obj->mapped_device_id_array_size + + ids_array_size; + + obj->mapped_device_id_array = (uint32_t *)realloc( + obj->mapped_device_id_array, new_array_size); + + memcpy(&obj->mapped_device_id_array + [obj->mapped_device_id_array_size/sizeof(uint32_t)], + ids_array, ids_array_size); + + obj->mapped_device_id_array_size = new_array_size; +} + + /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, @@ -1726,7 +1780,6 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; - void *temp_mapped_id_array = NULL; if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); @@ -1764,26 +1817,15 @@ static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, args.device_ids_array_size = all_gpu_id_array_size; } - temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); - if (!temp_mapped_id_array) - goto err_object_not_found; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - print_device_id_array((void *)args.device_ids_array_ptr, - args.device_ids_array_size); + add_device_ids_to_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); + print_device_id_array((uint32_t *)object->mapped_device_id_array, + object->mapped_device_id_array_size); - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); - - memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, - args.device_ids_array_size); - object->mapped_device_id_array = temp_mapped_id_array; - object->mapped_device_id_array_size = args.device_ids_array_size; object->mapping_count = 1; /* Mapping changed and lifecycle of object->mapped_node_id_array * terminates here. Free it and allocate on next query @@ -1800,7 +1842,6 @@ exit_ok: return 0; err_map_ioctl_failed: - free(temp_mapped_id_array); err_object_not_found: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1871,7 +1912,6 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; - void *temp_mapped_id_array = NULL; /* Check that address space was previously reserved */ if (!vm_find(aperture, address)) @@ -1895,24 +1935,13 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, args.device_ids_array_size = all_gpu_id_array_size; } - temp_mapped_id_array = (uint32_t *)malloc(args.device_ids_array_size); - if (!temp_mapped_id_array) - goto err_object_not_found; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) goto err_map_ioctl_failed; - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); - - memcpy(temp_mapped_id_array, (void *)args.device_ids_array_ptr, - args.device_ids_array_size); - object->mapped_device_id_array = temp_mapped_id_array; - object->mapped_device_id_array_size = args.device_ids_array_size; + add_device_ids_to_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); /* Mapping changed and lifecycle of object->mapped_node_id_array * terminates here. Free it and allocate on next query */ @@ -2096,15 +2125,10 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, if (ret != 0) goto out; - /* Clearing all mapped nodes list */ - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); + remove_device_ids_from_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); - object->mapped_device_id_array = NULL; - object->mapped_device_id_array_size = 0; if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; @@ -2152,15 +2176,10 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, args.device_ids_array_size = object->mapped_device_id_array_size; kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); - /* Clearing all mapped nodes list */ - if (object->mapped_device_id_array && - object->mapped_device_id_array_size > 0 && - object->mapped_device_id_array != all_gpu_id_array && - object->mapped_device_id_array != object->registered_device_id_array) - free(object->mapped_device_id_array); + remove_device_ids_from_mapped_array(object, + (uint32_t *)args.device_ids_array_ptr, + args.device_ids_array_size); - object->mapped_device_id_array = NULL; - object->mapped_device_id_array_size = 0; if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; @@ -2908,14 +2927,17 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) return HSAKMT_STATUS_SUCCESS; } - if (object->registered_device_id_array_size <= 0) { + if (!object->registered_device_id_array || + object->registered_device_id_array_size <= 0) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } - free(object->registered_device_id_array); - object->registered_device_id_array = NULL; - object->registered_device_id_array_size = 0; + if (object->registered_device_id_array) { + free(object->registered_device_id_array); + object->registered_device_id_array = NULL; + object->registered_device_id_array_size = 0; + } if (object->registered_node_id_array) free(object->registered_node_id_array); object->registered_node_id_array = NULL; @@ -2937,8 +2959,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, { manageable_aperture_t *aperture; vm_object_t *object = NULL; - uint32_t i, j, temp_node; - bool found, userptr = false; + uint32_t i; + bool userptr = false; uint32_t *registered_node_id_array, registered_node_id_array_size; HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; @@ -2989,15 +3011,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, registered_node_id_array_size = object->registered_device_id_array_size; } for (i = 0 ; i < num_of_nodes; i++) { - temp_node = nodes_to_map[i]; - found = false; - for (j = 0 ; j < registered_node_id_array_size / sizeof(uint32_t); j++) { - if (temp_node == registered_node_id_array[j]) { - found = true; - break; - } - } - if (!found) { + if (!id_in_array(nodes_to_map[i], registered_node_id_array, + registered_node_id_array_size)) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } @@ -3009,16 +3024,11 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t temp_node_id_array_size = 0; for (i = 0 ; i < object->mapped_device_id_array_size / sizeof(uint32_t); i++) { - temp_node = object->mapped_device_id_array[i]; - found = false; - for (j = 0 ; j < num_of_nodes; j++) { - if (temp_node == nodes_to_map[j]) { - found = true; - break; - } - } - if (!found) - temp_node_id_array[temp_node_id_array_size++] = temp_node; + if (!id_in_array(object->mapped_device_id_array[i], + nodes_to_map, + num_of_nodes*sizeof(uint32_t))) + temp_node_id_array[temp_node_id_array_size++] = + object->mapped_device_id_array[i]; } temp_node_id_array_size *= sizeof(uint32_t); @@ -3034,8 +3044,24 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, } } - retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, - nodes_to_map, num_of_nodes * sizeof(uint32_t)); + /* Remove already mapped nodes from nodes_to_map + * to generate the final map list + */ + uint32_t map_node_id_array[num_of_nodes]; + uint32_t map_node_id_array_size = 0; + + for (i = 0; i < num_of_nodes; i++) { + if (!id_in_array(nodes_to_map[i], + object->mapped_device_id_array, + object->mapped_device_id_array_size)) + map_node_id_array[map_node_id_array_size++] = + nodes_to_map[i]; + } + + if (map_node_id_array_size) + retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, + map_node_id_array, + map_node_id_array_size * sizeof(uint32_t)); pthread_mutex_unlock(&aperture->fmm_mutex); From c2dc301792a61ac09bb8ba714f3dbce1248d9002 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 30 Nov 2017 11:06:45 -0500 Subject: [PATCH 0319/1247] Revert "Revert "More cleanup of fmm.c"" This reverts commit 52f6a619705ed23fea2554dd67f648086854875b. Change-Id: I31afe4889794df8cf1e96f5f18771bed75a213d9 Signed-off-by: Oak Zeng --- src/fmm.c | 60 ++++++++++++++++--------------------------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 860b121e60..ceae462d79 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1774,7 +1774,7 @@ static void add_device_ids_to_mapped_array(vm_object_t *obj, /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ -static int _fmm_map_to_gpu_gtt(manageable_aperture_t *aperture, +static int _fmm_map_to_gpu(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, uint32_t *nodes_to_map, uint32_t nodes_array_size) { @@ -1899,20 +1899,22 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* map to GPU */ - ret = _fmm_map_to_gpu_gtt(aperture, address, size, NULL, NULL, 0); + ret = _fmm_map_to_gpu(aperture, address, size, NULL, NULL, 0); if (ret != 0) __fmm_release(mem, aperture); return ret; } -static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, +static int _fmm_map_to_apu_local(uint32_t gpu_id, + manageable_aperture_t *aperture, void *address, uint64_t size, uint64_t *gpuvm_address) { - struct kfd_ioctl_map_memory_to_gpu_args args; vm_object_t *object; + if (gpuvm_address) + *gpuvm_address = 0; /* Check that address space was previously reserved */ if (!vm_find(aperture, address)) return -1; @@ -1921,37 +1923,15 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (!object) - goto err_object_not_found; - - args.handle = object->handle; - if (object->registered_device_id_array_size > 0 && - object->registered_device_id_array) { - args.device_ids_array_ptr = - (uint64_t)object->registered_device_id_array; - args.device_ids_array_size = object->registered_device_id_array_size; - } else { - args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; - args.device_ids_array_size = all_gpu_id_array_size; + if (!object) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return -1; } - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) - goto err_map_ioctl_failed; - - - add_device_ids_to_mapped_array(object, - (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); - /* Mapping changed and lifecycle of object->mapped_node_id_array - * terminates here. Free it and allocate on next query - */ - if (object->mapped_node_id_array) { - free(object->mapped_node_id_array); - object->mapped_node_id_array = NULL; - } - pthread_mutex_unlock(&aperture->fmm_mutex); + if (_fmm_map_to_gpu(aperture, address, size, object, NULL, 0)) + return -1; + if (gpuvm_address) { *gpuvm_address = (uint64_t)object->start; if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) @@ -1959,12 +1939,6 @@ static int _fmm_map_to_gpu(uint32_t gpu_id, manageable_aperture_t *aperture, } return 0; - -err_map_ioctl_failed: -err_object_not_found: - pthread_mutex_unlock(&aperture->fmm_mutex); - *gpuvm_address = 0; - return -1; } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, @@ -1997,7 +1971,7 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu_gtt(aperture, svm_addr, svm_size, obj, NULL, 0); + ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, obj, NULL, 0); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -2026,7 +2000,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= gpu_mem[i].gpuvm_aperture.base) && (address <= gpu_mem[i].gpuvm_aperture.limit)) /* map it */ - return _fmm_map_to_gpu(gpu_mem[i].gpu_id, + return _fmm_map_to_apu_local(gpu_mem[i].gpu_id, &gpu_mem[i].gpuvm_aperture, address, size, gpuvm_address); } @@ -2034,12 +2008,12 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= svm.dgpu_aperture.base) && (address <= svm.dgpu_aperture.limit)) /* map it */ - return _fmm_map_to_gpu_gtt(&svm.dgpu_aperture, + return _fmm_map_to_gpu(&svm.dgpu_aperture, address, size, NULL, NULL, 0); else if ((address >= svm.dgpu_alt_aperture.base) && (address <= svm.dgpu_alt_aperture.limit)) /* map it */ - return _fmm_map_to_gpu_gtt(&svm.dgpu_alt_aperture, + return _fmm_map_to_gpu(&svm.dgpu_alt_aperture, address, size, NULL, NULL, 0); /* @@ -3059,7 +3033,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, } if (map_node_id_array_size) - retcode = _fmm_map_to_gpu_gtt(aperture, address, size, object, + retcode = _fmm_map_to_gpu(aperture, address, size, object, map_node_id_array, map_node_id_array_size * sizeof(uint32_t)); From 0f8377463587123553b21b957f92696045fc525b Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 7 Dec 2017 16:08:10 -0500 Subject: [PATCH 0320/1247] Report gfx902 as GFX 9.0.2 This change is needed to match other higher level components. Change-Id: I45114d23f2ed428dfbbb836061b3020c5ab166ec Signed-off-by: Yong Zhao --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 2cc2ebdba6..5dcceb9edb 100644 --- a/src/topology.c +++ b/src/topology.c @@ -174,7 +174,7 @@ static struct hsa_gfxip_table { { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, /* Raven */ - { 0x15DD, 9, 0, 3, 0, "Raven", CHIP_RAVEN }, + { 0x15DD, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, /* Vega20 on emulator, treat it as vega10 */ { 0x66A0, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, }; From 716755b1de0bf0da68be5f00187a5494b0796719 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 29 Nov 2017 11:48:10 -0500 Subject: [PATCH 0321/1247] Workaround: make mmap memory resident for gfx902 Change-Id: I5f90f316740f7995d54cb083a6d7e05bc4e2966e Signed-off-by: Yong Zhao --- src/fmm.c | 4 ++++ src/queues.c | 3 +++ 2 files changed, 7 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index ceae462d79..44492b63f3 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -933,6 +933,8 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) munmap(aligned_end, VOID_PTRS_SUB(padded_end, aligned_end)); mem = aligned_start; + + memset(mem, 0, aligned_size); } /* Remember scratch backing aperture for later */ @@ -1131,6 +1133,8 @@ static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, if (mem == MAP_FAILED) return NULL; + memset(mem, 0, MemorySizeInBytes); + pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); vm_obj = aperture_allocate_object(&cpuvm_aperture, mem, 0, MemorySizeInBytes, flags.Value); diff --git a/src/queues.c b/src/queues.c index c698c5ca6e..42c86a395f 100644 --- a/src/queues.c +++ b/src/queues.c @@ -323,6 +323,9 @@ static void *allocate_exec_aligned_memory_cpu(uint32_t size) if (ptr == MAP_FAILED) return NULL; + + memset(ptr, 0, size); + return ptr; } From 8bc83e1e9bb251c4ad1bc887c1bda544760975f2 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 18 Dec 2017 12:10:24 -0500 Subject: [PATCH 0322/1247] Update README to include new requirement Latest Thunk requires the user to belong to video group. Add this statement to README.md to notify external users on Github. Change-Id: Id9843abf09de5b63a3b7c3f7b322bc9099c6ff1a Signed-off-by: Amber Lin --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9b8f528abb..5ede88314e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ This repository includes the user-mode API interfaces used to interact with the ROCk driver. Currently supported agents include only the AMD/ATI Fiji family of discrete GPUs. +Starting at 1.7 release, ROCt uses drm render device. This requires the user to belong to video group. Add the user account to video group with "sudo usermod -a -G video _username_" command if the user if not part of video group yet. + #### ROCk Driver The ROCt library is not a standalone product and requires that you have the correct ROCk driver set installed. We recommend reading the full compatibility and installation details which are available in the ROCk github: From 7031a774281ca7c415c5fe38ecd758b5d5164b84 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Thu, 4 Jan 2018 15:40:49 -0500 Subject: [PATCH 0323/1247] Update README to reflect cmake change New CMakeLists.txt sets a default module search so -DCMAKE_MODULE_PATH is no longer required in the command. Change-Id: I95189ce2f36016b7c4929239d0e512851bec5ef6 Signed-off-by: Amber Lin --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ede88314e..2d1d4ea065 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ A simple cmake-based system is available for building thunk. To build the thunk ```bash mkdir -p build cd build - cmake -DCMAKE_MODULE_PATH=${PWD}/../cmake_modules .. + cmake .. make ``` @@ -26,7 +26,7 @@ If the hsakmt-roct and hsakmt-roct-dev packages are desired: ```bash mkdir -p build cd build - cmake -DCMAKE_MODULE_PATH=${PWD}/../cmake_modules .. + cmake .. make package make package-dev ``` From 056ddbbc8292ca3764b1203db0eda4a429ae9630 Mon Sep 17 00:00:00 2001 From: Laurent Morichetti Date: Thu, 25 Jan 2018 13:47:14 -0800 Subject: [PATCH 0324/1247] Silence Valgrind warnings Change-Id: I8803f3d310fccd69d0d04b2464b00dccc40270e3 Signed-off-by: Felix Kuehling --- include/hsakmttypes.h | 4 ++-- src/debug.c | 6 ++---- src/events.c | 23 ++++++----------------- src/fmm.c | 41 +++++++++++++++++++++-------------------- src/memory.c | 16 ++++++++++------ src/queues.c | 20 ++++++-------------- src/time.c | 2 +- src/version.c | 4 +--- 8 files changed, 49 insertions(+), 67 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 6fc65df068..3cb23e1121 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1094,8 +1094,8 @@ typedef struct _HsaPmcTraceRoot typedef struct _HsaGpuTileConfig { - const HSAuint32 *TileConfig; - const HSAuint32 *MacroTileConfig; + HSAuint32 *TileConfig; + HSAuint32 *MacroTileConfig; HSAuint32 NumTileConfigs; HSAuint32 NumMacroTileConfigs; diff --git a/src/debug.c b/src/debug.c index 5f8e95c34a..34dcc8970c 100644 --- a/src/debug.c +++ b/src/debug.c @@ -67,9 +67,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId) if (result != HSAKMT_STATUS_SUCCESS) return result; - struct kfd_ioctl_dbg_register_args args; + struct kfd_ioctl_dbg_register_args args = {0}; - memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER, &args); @@ -96,9 +95,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgUnregister(HSAuint32 NodeId) if (result != HSAKMT_STATUS_SUCCESS) return result; - struct kfd_ioctl_dbg_unregister_args args; + struct kfd_ioctl_dbg_unregister_args args = {0}; - memset(&args, 0, sizeof(args)); args.gpu_id = gpu_id; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER, &args); diff --git a/src/events.c b/src/events.c index 28c55f2c67..01f4db2992 100644 --- a/src/events.c +++ b/src/events.c @@ -65,9 +65,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, memset(e, 0, sizeof(*e)); - struct kfd_ioctl_create_event_args args; - - memset(&args, 0, sizeof(args)); + struct kfd_ioctl_create_event_args args = {0}; args.event_type = EventDesc->EventType; args.node_id = EventDesc->NodeId; @@ -127,9 +125,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, EventDesc->SyncVar.SyncVarSize; if (IsSignaled && !IsSystemEventType(e->EventData.EventType)) { - struct kfd_ioctl_set_event_args set_args; + struct kfd_ioctl_set_event_args set_args = {0}; - memset(&set_args, 0, sizeof(set_args)); set_args.event_id = args.event_id; kmtIoctl(kfd_fd, AMDKFD_IOC_SET_EVENT, &set_args); @@ -147,9 +144,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyEvent(HsaEvent *Event) if (!Event) return HSAKMT_STATUS_INVALID_HANDLE; - struct kfd_ioctl_destroy_event_args args; - - memset(&args, 0, sizeof(args)); + struct kfd_ioctl_destroy_event_args args = {0}; args.event_id = Event->EventId; @@ -173,9 +168,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetEvent(HsaEvent *Event) if (IsSystemEventType(Event->EventData.EventType)) return HSAKMT_STATUS_ERROR; - struct kfd_ioctl_set_event_args args; - - memset(&args, 0, sizeof(args)); + struct kfd_ioctl_set_event_args args = {0}; args.event_id = Event->EventId; @@ -198,9 +191,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtResetEvent(HsaEvent *Event) if (IsSystemEventType(Event->EventData.EventType)) return HSAKMT_STATUS_ERROR; - struct kfd_ioctl_reset_event_args args; - - memset(&args, 0, sizeof(args)); + struct kfd_ioctl_reset_event_args args = {0}; args.event_id = Event->EventId; @@ -306,9 +297,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], event_data[i].kfd_event_data_ext = (uint64_t)(uintptr_t)NULL; } - struct kfd_ioctl_wait_events_args args; - - memset(&args, 0, sizeof(args)); + struct kfd_ioctl_wait_events_args args = {0}; args.wait_for_all = WaitOnAll; args.timeout = Milliseconds; diff --git a/src/fmm.c b/src/fmm.c index 44492b63f3..a7003a624f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -728,8 +728,8 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, uint64_t *mmap_offset, uint32_t flags) { - struct kfd_ioctl_alloc_memory_of_gpu_args args; - struct kfd_ioctl_free_memory_of_gpu_args free_args; + struct kfd_ioctl_alloc_memory_of_gpu_args args = {0}; + struct kfd_ioctl_free_memory_of_gpu_args free_args = {0}; vm_object_t *vm_obj = NULL; if (!mem) @@ -891,7 +891,7 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { manageable_aperture_t *aperture_phy; - struct kfd_ioctl_alloc_memory_of_scratch_args args; + struct kfd_ioctl_alloc_memory_of_scratch_args args = {0}; int32_t gpu_mem_id; void *mem = NULL; uint64_t aligned_size = ALIGN_UP(MemorySizeInBytes, SCRATCH_ALIGN); @@ -1298,7 +1298,7 @@ void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, static void __fmm_release(void *address, manageable_aperture_t *aperture) { - struct kfd_ioctl_free_memory_of_gpu_args args; + struct kfd_ioctl_free_memory_of_gpu_args args = {0}; vm_object_t *object; if (!address) @@ -1394,7 +1394,7 @@ void fmm_release(void *address) static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_policy, uintptr_t alt_base, uint64_t alt_size) { - struct kfd_ioctl_set_memory_policy_args args; + struct kfd_ioctl_set_memory_policy_args args = {0}; args.gpu_id = gpu_id; args.default_policy = default_policy; @@ -1421,12 +1421,11 @@ static HSAKMT_STATUS get_process_apertures( struct kfd_process_device_apertures *process_apertures, uint32_t *num_of_nodes) { - struct kfd_ioctl_get_process_apertures_new_args args_new = { - .kfd_process_device_apertures_ptr = (uintptr_t)process_apertures, - .num_of_nodes = *num_of_nodes - }; + struct kfd_ioctl_get_process_apertures_new_args args_new = {0}; struct kfd_ioctl_get_process_apertures_args args_old; + args_new.kfd_process_device_apertures_ptr = (uintptr_t)process_apertures; + args_new.num_of_nodes = *num_of_nodes; if (!kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES_NEW, (void *)&args_new)) { *num_of_nodes = args_new.num_of_nodes; @@ -1435,6 +1434,8 @@ static HSAKMT_STATUS get_process_apertures( /* New IOCTL failed, try the old one in case we're running on * a really old kernel */ + memset(&args_old, 0, sizeof(args_old)); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_PROCESS_APERTURES, (void *)&args_old)) return HSAKMT_STATUS_ERROR; @@ -1530,7 +1531,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * the apertures in kfd_process_device_apertures_ptr */ num_of_nodes = gpu_mem_count; - process_apertures = malloc(num_of_nodes * sizeof(struct kfd_process_device_apertures)); + process_apertures = calloc(num_of_nodes, sizeof(struct kfd_process_device_apertures)); if (!process_apertures) { ret = HSAKMT_STATUS_NO_MEMORY; goto sysfs_parse_failed; @@ -1782,7 +1783,7 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, uint32_t *nodes_to_map, uint32_t nodes_array_size) { - struct kfd_ioctl_map_memory_to_gpu_args args; + struct kfd_ioctl_map_memory_to_gpu_args args = {0}; vm_object_t *object; if (!obj) @@ -2057,7 +2058,7 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, { vm_object_t *object; int ret = 0; - struct kfd_ioctl_unmap_memory_from_gpu_args args; + struct kfd_ioctl_unmap_memory_from_gpu_args args = {0}; HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); if (!obj) @@ -2124,7 +2125,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, { int32_t gpu_mem_id; vm_object_t *object; - struct kfd_ioctl_unmap_memory_from_gpu_args args; + struct kfd_ioctl_unmap_memory_from_gpu_args args = {0}; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -2245,7 +2246,7 @@ static bool is_dgpu_mem_init = false; static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit) { - struct kfd_ioctl_set_process_dgpu_aperture_args args; + struct kfd_ioctl_set_process_dgpu_aperture_args args = {0}; args.gpu_id = gpu_id; args.dgpu_base = base; @@ -2578,9 +2579,9 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) { - struct kfd_ioctl_get_dmabuf_info_args infoArgs; - struct kfd_ioctl_import_dmabuf_args importArgs; - struct kfd_ioctl_free_memory_of_gpu_args freeArgs; + struct kfd_ioctl_get_dmabuf_info_args infoArgs = {0}; + struct kfd_ioctl_import_dmabuf_args importArgs = {0}; + struct kfd_ioctl_free_memory_of_gpu_args freeArgs = {0}; manageable_aperture_t *aperture; vm_object_t *obj; void *metadata; @@ -2684,7 +2685,7 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, HSAuint32 gpu_id = 0; vm_object_t *obj = NULL; manageable_aperture_t *aperture = NULL; - struct kfd_ioctl_ipc_export_handle_args exportArgs; + struct kfd_ioctl_ipc_export_handle_args exportArgs = {0}; HsaApertureInfo ApeInfo; HsaSharedMemoryStruct *SharedMemoryStruct = to_hsa_shared_memory_struct(SharedMemoryHandle); @@ -2741,8 +2742,8 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo vm_object_t *obj = NULL; void *reservedMem = NULL; manageable_aperture_t *aperture; - struct kfd_ioctl_ipc_import_handle_args importArgs; - struct kfd_ioctl_free_memory_of_gpu_args freeArgs; + struct kfd_ioctl_ipc_import_handle_args importArgs = {0}; + struct kfd_ioctl_free_memory_of_gpu_args freeArgs = {0}; const HsaSharedMemoryStruct *SharedMemoryStruct = to_const_hsa_shared_memory_struct(SharedMemoryHandle); diff --git a/src/memory.c b/src/memory.c index 75b392a90a..22f7c30abe 100644 --- a/src/memory.c +++ b/src/memory.c @@ -40,7 +40,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, void *MemoryAddressAlternate, HSAuint64 MemorySizeInBytes) { - struct kfd_ioctl_set_memory_policy_args args; + struct kfd_ioctl_set_memory_policy_args args = {0}; HSAKMT_STATUS result; uint32_t gpu_id; @@ -74,8 +74,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, CHECK_PAGE_MULTIPLE(MemoryAddressAlternate); CHECK_PAGE_MULTIPLE(MemorySizeInBytes); - memset(&args, 0, sizeof(args)); - args.gpu_id = gpu_id; args.default_policy = (DefaultPolicy == HSA_CACHING_CACHED) ? KFD_IOC_CACHE_POLICY_COHERENT : @@ -356,7 +354,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, HSAuint64 RemoteMemoryArrayCount, HSAuint64 *SizeCopied) { - struct kfd_ioctl_cross_memory_copy_args args; + struct kfd_ioctl_cross_memory_copy_args args = {0}; pr_debug("[%s]\n", __func__); @@ -391,7 +389,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, HSAuint64 RemoteMemoryArrayCount, HSAuint64 *SizeCopied) { - struct kfd_ioctl_cross_memory_copy_args args; + struct kfd_ioctl_cross_memory_copy_args args = {0}; pr_debug("[%s]\n", __func__); @@ -530,7 +528,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUnmapGraphicHandle(HSAuint32 NodeId, HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig *config) { - struct kfd_ioctl_get_tile_config_args args; + struct kfd_ioctl_get_tile_config_args args = {0}; uint32_t gpu_id; HSAKMT_STATUS result; @@ -540,6 +538,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetTileConfig(HSAuint32 NodeId, HsaGpuTileConfig * if (result != HSAKMT_STATUS_SUCCESS) return result; + /* Avoid Valgrind warnings about uninitialized data. Valgrind doesn't + * know that KFD writes this. + */ + memset(config->TileConfig, 0, sizeof(*config->TileConfig) * config->NumTileConfigs); + memset(config->MacroTileConfig, 0, sizeof(*config->MacroTileConfig) * config->NumMacroTileConfigs); + args.gpu_id = gpu_id; args.tile_config_ptr = (uint64_t)config->TileConfig; args.macro_tile_config_ptr = (uint64_t)config->MacroTileConfig; diff --git a/src/queues.c b/src/queues.c index 42c86a395f..92be431afc 100644 --- a/src/queues.c +++ b/src/queues.c @@ -528,9 +528,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, q->cu_mask[i/32] |= (1 << (i % 32)); } - struct kfd_ioctl_create_queue_args args; - - memset(&args, 0, sizeof(args)); + struct kfd_ioctl_create_queue_args args = {0}; args.gpu_id = gpu_id; @@ -618,7 +616,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtUpdateQueue(HSA_QUEUEID QueueId, HSAuint64 QueueSize, HsaEvent *Event) { - struct kfd_ioctl_update_queue_args arg; + struct kfd_ioctl_update_queue_args arg = {0}; struct queue *q = PORT_UINT64_TO_VPTR(QueueId); CHECK_KFD_OPEN(); @@ -648,13 +646,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyQueue(HSA_QUEUEID QueueId) CHECK_KFD_OPEN(); struct queue *q = PORT_UINT64_TO_VPTR(QueueId); - struct kfd_ioctl_destroy_queue_args args; + struct kfd_ioctl_destroy_queue_args args = {0}; if (!q) return HSAKMT_STATUS_INVALID_PARAMETER; - memset(&args, 0, sizeof(args)); - args.queue_id = q->queue_id; int err = kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_QUEUE, &args); @@ -671,14 +667,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetQueueCUMask(HSA_QUEUEID QueueId, HSAuint32 *QueueCUMask) { struct queue *q = PORT_UINT64_TO_VPTR(QueueId); - struct kfd_ioctl_set_cu_mask_args args; + struct kfd_ioctl_set_cu_mask_args args = {0}; CHECK_KFD_OPEN(); if (CUMaskCount == 0 || !QueueCUMask || ((CUMaskCount % 32) != 0)) return HSAKMT_STATUS_INVALID_PARAMETER; - memset(&args, 0, sizeof(args)); args.queue_id = q->queue_id; args.num_cu_mask = CUMaskCount; args.cu_mask_ptr = (uintptr_t)QueueCUMask; @@ -702,7 +697,7 @@ hsaKmtGetQueueInfo( ) { struct queue *q = PORT_UINT64_TO_VPTR(QueueId); - struct kfd_ioctl_get_queue_wave_state_args args; + struct kfd_ioctl_get_queue_wave_state_args args = {0}; CHECK_KFD_OPEN(); @@ -712,7 +707,6 @@ hsaKmtGetQueueInfo( if (q->ctx_save_restore == NULL) return HSAKMT_STATUS_ERROR; - memset(&args, 0, sizeof(args)); args.queue_id = q->queue_id; args.ctl_stack_address = (uintptr_t)q->ctx_save_restore; @@ -739,7 +733,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node, void *TrapBufferBaseAddress, HSAuint64 TrapBufferSizeInBytes) { - struct kfd_ioctl_set_trap_handler_args args; + struct kfd_ioctl_set_trap_handler_args args = {0}; HSAKMT_STATUS result; uint32_t gpu_id; @@ -749,8 +743,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node, if (result != HSAKMT_STATUS_SUCCESS) return result; - memset(&args, 0, sizeof(args)); - args.gpu_id = gpu_id; args.tba_addr = (uintptr_t)TrapHandlerBaseAddress; args.tma_addr = (uintptr_t)TrapBufferBaseAddress; diff --git a/src/time.c b/src/time.c index 56e5484850..96729a1074 100644 --- a/src/time.c +++ b/src/time.c @@ -31,7 +31,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetClockCounters(HSAuint32 NodeId, { HSAKMT_STATUS result; uint32_t gpu_id; - struct kfd_ioctl_get_clock_counters_args args; + struct kfd_ioctl_get_clock_counters_args args = {0}; int err; CHECK_KFD_OPEN(); diff --git a/src/version.c b/src/version.c index 0b7c7c86e8..2de79d0b08 100644 --- a/src/version.c +++ b/src/version.c @@ -32,9 +32,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetVersion(HsaVersionInfo *VersionInfo) { CHECK_KFD_OPEN(); - struct kfd_ioctl_get_version_args args; - - memset(&args, 0, sizeof(args)); + struct kfd_ioctl_get_version_args args = {0}; if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1) return HSAKMT_STATUS_ERROR; From 55bb61ff9c41aeafa1a4eae2c35e648fe298a6eb Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 2 Feb 2018 12:31:01 -0500 Subject: [PATCH 0325/1247] Revert "Workaround: make mmap memory resident for gfx902" This reverts commit 716755b1de0bf0da68be5f00187a5494b0796719. Change-Id: I9f4f0b6b426aeae4cb652b33cf0d4c0f57270ca5 Signed-off-by: Yong Zhao --- src/fmm.c | 4 ---- src/queues.c | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index a7003a624f..7e5a023f96 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -933,8 +933,6 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) munmap(aligned_end, VOID_PTRS_SUB(padded_end, aligned_end)); mem = aligned_start; - - memset(mem, 0, aligned_size); } /* Remember scratch backing aperture for later */ @@ -1133,8 +1131,6 @@ static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, if (mem == MAP_FAILED) return NULL; - memset(mem, 0, MemorySizeInBytes); - pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); vm_obj = aperture_allocate_object(&cpuvm_aperture, mem, 0, MemorySizeInBytes, flags.Value); diff --git a/src/queues.c b/src/queues.c index 92be431afc..8f035a12d4 100644 --- a/src/queues.c +++ b/src/queues.c @@ -323,9 +323,6 @@ static void *allocate_exec_aligned_memory_cpu(uint32_t size) if (ptr == MAP_FAILED) return NULL; - - memset(ptr, 0, size); - return ptr; } From 7de0199e999b28b6d58d59e6eb35b32c6ec262ff Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 8 Feb 2018 10:04:08 -0500 Subject: [PATCH 0326/1247] CMA: Initialize SizeCopied return parameter UCX test cases are reporting uninitialized values when CMA fails. The application should ideally ignore SizeCopied when the function fails but it doesn't. This is leading to wrong diagnosis. v2: Fill in partial SizeCopied in case of failure Change-Id: I6b7e1c19a8b702ec91ca64201a3dda27bd897877 Signed-off-by: Harish Kasiviswanathan --- src/memory.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/memory.c b/src/memory.c index 22f7c30abe..37e4d3229a 100644 --- a/src/memory.c +++ b/src/memory.c @@ -354,6 +354,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, HSAuint64 RemoteMemoryArrayCount, HSAuint64 *SizeCopied) { + int ret = HSAKMT_STATUS_SUCCESS; struct kfd_ioctl_cross_memory_copy_args args = {0}; pr_debug("[%s]\n", __func__); @@ -371,15 +372,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, args.dst_mem_array_size = LocalMemoryArrayCount; args.bytes_copied = 0; - int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args); - - if (err) - return HSAKMT_STATUS_ERROR; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args)) + ret = HSAKMT_STATUS_ERROR; if (SizeCopied) *SizeCopied = args.bytes_copied; - return HSAKMT_STATUS_SUCCESS; + return ret; } HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, @@ -389,10 +388,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, HSAuint64 RemoteMemoryArrayCount, HSAuint64 *SizeCopied) { + int ret = HSAKMT_STATUS_SUCCESS; struct kfd_ioctl_cross_memory_copy_args args = {0}; pr_debug("[%s]\n", __func__); + if (SizeCopied) + *SizeCopied = 0; + if (!LocalMemoryArray || !RemoteMemoryArray || LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) return HSAKMT_STATUS_ERROR; @@ -406,15 +409,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, args.dst_mem_array_size = RemoteMemoryArrayCount; args.bytes_copied = 0; - int err = kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args); - - if (err) - return HSAKMT_STATUS_ERROR; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args)) + ret = HSAKMT_STATUS_ERROR; if (SizeCopied) *SizeCopied = args.bytes_copied; - return HSAKMT_STATUS_SUCCESS; + return ret; } From 25170c3c57086139cc2166e3e186f60964a033e9 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Fri, 8 Dec 2017 15:59:01 -0500 Subject: [PATCH 0327/1247] Support ptrace access invisible vram Invisible device memory is mmapped as PROT_NONE. Normal CPU access to the memory is still not allowed but struct vm_area_struct will be created for the memory address so ptrace can access the memory via the vma. Change-Id: I07c69208716c920ccce33e6b494b610b61a0a7c1 Signed-off-by: Oak Zeng --- src/fmm.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 7e5a023f96..1b4a0ca6c8 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -150,7 +150,7 @@ typedef struct { */ static gpu_mem_t *gpu_mem; static unsigned int gpu_mem_count; - +static bool hsa_debug; static void *dgpu_shared_aperture_base; static void *dgpu_shared_aperture_limit; @@ -1044,13 +1044,16 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla pthread_mutex_unlock(&aperture->fmm_mutex); } - if (mem && flags.ui32.HostAccess) { + if (mem && (flags.ui32.HostAccess || hsa_debug)) { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : get_drm_render_fd_by_gpu_id(gpu_id); - void *ret = mmap(mem, MemorySizeInBytes, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, - map_fd, mmap_offset); + int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : + PROT_NONE; + int flag = flags.ui32.HostAccess ? MAP_SHARED | MAP_FIXED : + MAP_PRIVATE|MAP_FIXED; + void *ret = mmap(mem, MemorySizeInBytes, prot, flag, + map_fd, mmap_offset); + if (ret == MAP_FAILED) { __fmm_release(mem, aperture); return NULL; @@ -1455,9 +1458,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; + char *hsaDebug; unsigned int guardPages = 1; struct pci_access *pacc; + hsaDebug = getenv("HSA_DEBUG"); + hsa_debug = hsaDebug && strcmp(hsaDebug, "0"); + /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ disableCache = getenv("HSA_DISABLE_CACHE"); if (disableCache && strcmp(disableCache, "0") == 0) From e2c353dc0dd73853b7e5bac8838ce7080cf248d7 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Thu, 22 Feb 2018 13:06:15 -0600 Subject: [PATCH 0328/1247] Allocate EOP queue local to GPU On discrete GPUs place the EOP queue in VRAM. The reader/writer of this queue is the CP and the size is small. Dispatch latency improves through lower read latency in AQL completion phase. Change-Id: Id8351dcddbd21fd7c7d699803c96434c9132db71 Signed-off-by: Jay Cornwall --- src/events.c | 2 +- src/libhsakmt.h | 3 ++- src/queues.c | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/events.c b/src/events.c index 01f4db2992..65330c90ee 100644 --- a/src/events.c +++ b/src/events.c @@ -76,7 +76,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, if (is_dgpu && !events_page) { events_page = allocate_exec_aligned_memory_gpu( - KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true); + KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true, false); if (!events_page) { pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 07402ffef5..48435c477b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -123,7 +123,8 @@ HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId, bool NonPaged); + uint32_t NodeId, bool NonPaged, + bool DeviceLocal); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes); void destroy_process_doorbells(void); diff --git a/src/queues.c b/src/queues.c index 8f035a12d4..613180cedb 100644 --- a/src/queues.c +++ b/src/queues.c @@ -350,7 +350,8 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) } void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, - uint32_t NodeId, bool nonPaged) + uint32_t NodeId, bool nonPaged, + bool DeviceLocal) { void *mem; HSAuint64 gpu_va; @@ -358,14 +359,14 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, HSAKMT_STATUS ret; flags.Value = 0; - flags.ui32.HostAccess = 1; + flags.ui32.HostAccess = !DeviceLocal; flags.ui32.ExecuteAccess = 1; flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; size = ALIGN_UP(size, align); - ret = hsaKmtAllocMemory(0, size, flags, &mem); + ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : 0, size, flags, &mem); if (ret != HSAKMT_STATUS_SUCCESS) return NULL; @@ -399,11 +400,12 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) */ static void *allocate_exec_aligned_memory(uint32_t size, enum asic_family_type type, - uint32_t NodeId) + uint32_t NodeId, + bool DeviceLocal) { if (IS_DGPU(type)) return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId, - false); + DeviceLocal, DeviceLocal); return allocate_exec_aligned_memory_cpu(size); } @@ -441,7 +443,7 @@ static int handle_concrete_asic(struct queue *q, q->eop_buffer = allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, dev_info->asic_family, - NodeId); + NodeId, true); if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; @@ -455,7 +457,7 @@ static int handle_concrete_asic(struct queue *q, q->ctx_save_restore = allocate_exec_aligned_memory(q->ctx_save_restore_size, dev_info->asic_family, - NodeId); + NodeId, false); if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; @@ -506,7 +508,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, struct queue *q = allocate_exec_aligned_memory(sizeof(*q), dev_info->asic_family, - NodeId); + NodeId, false); if (!q) return HSAKMT_STATUS_NO_MEMORY; From 105291849f6e3cd4f98563f129e9f4e22e88bd48 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 22 Feb 2018 15:39:41 -0500 Subject: [PATCH 0329/1247] Close shmem file handle, to fix file handle leak kfdtest hsaKmtOpenKFD failed after 1019 loop if using --gtest_loop=-1, because default max open file handle limit is 1024. Found shmem file handle is not closed from lsof output. Change-Id: I474de2bae6c03e879a219dedf5f18639118b73e5 Signed-off-by: Philip Yang --- src/perfctr.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/perfctr.c b/src/perfctr.c index 645418d404..73bab19f45 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -149,8 +149,11 @@ static void destroy_shared_region(void) if (shared_table && shared_table != MAP_FAILED) munmap(shared_table, sizeof(*shared_table)); - if (shmem_fd > 0) + if (shmem_fd > 0) { + close(shmem_fd); shm_unlink(shmem_name); + } + if (sem != SEM_FAILED) { sem_close(sem); sem_unlink(sem_name); From 2c426a026a78d05f565262a73dfb6b457c93807f Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 26 Feb 2018 18:17:13 -0500 Subject: [PATCH 0330/1247] Turn off the verbose building message Change-Id: If4ebdb6f87fde9c3cc76b16c57e862bfb972ed5e Signed-off-by: Yong Zhao --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 046be997b7..63319bf91d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,7 @@ endif () set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) ## Verbose output. -set ( CMAKE_VERBOSE_MAKEFILE on ) +#set ( CMAKE_VERBOSE_MAKEFILE on ) ## Compiler flags set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden" ) From 15e525af45369d3b8558ffbe329b80fc2f622ba9 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 26 Feb 2018 18:18:00 -0500 Subject: [PATCH 0331/1247] Add pkg config support in the hsakmt-roct-dev package Change-Id: Ida6b3083bfc9405ef9b6b8e426dc7dc51d61a811 Signed-off-by: Yong Zhao --- CMakeLists.txt | 3 +++ hsakmt-dev.txt | 2 ++ libhsakmt.pc.in | 11 +++++++++++ 3 files changed, 16 insertions(+) create mode 100644 libhsakmt.pc.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 63319bf91d..c91f342ba6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,4 +156,7 @@ set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libpci3" ) set ( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) +# CPACK_PACKAGING_INSTALL_PREFIX is needed in libhsakmt.pc.in +configure_file ( libhsakmt.pc.in ${DEV_BUILD_DIR}/libhsakmt.pc @ONLY ) + include ( CPack ) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index a818598399..3fd4d9bbdb 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -51,6 +51,8 @@ install ( FILES ${SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/includ install ( FILES ${SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux ) install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt-link DESTINATION include RENAME libhsakmt ) install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.so-link DESTINATION lib RENAME libhsakmt.so ) +install ( FILES libhsakmt.pc DESTINATION libhsakmt ) + ## Set the default generator types for the devel package. set ( CPACK_GENERATOR "DEB;RPM;TGZ" CACHE STRING "Default packaging generators." ) diff --git a/libhsakmt.pc.in b/libhsakmt.pc.in new file mode 100644 index 0000000000..01d748bbc6 --- /dev/null +++ b/libhsakmt.pc.in @@ -0,0 +1,11 @@ +prefix=@CPACK_PACKAGING_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${prefix}/lib +includedir=${prefix}/include/libhsakmt + +Name: libhsakmt +Description: HSA Kernel Mode Thunk library for AMD KFD support +Version: @LIB_VERSION_STRING@ + +Libs: -L${libdir} -lhsakmt +Cflags: -I${includedir} From c5cfb7e25b93e364e2c8a75f54d7680b044889b8 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 9 Mar 2018 14:28:47 -0500 Subject: [PATCH 0332/1247] Move dGPU memory aperture initialization Define dgpu_mem_init before it's used and keep the code close to the rest of the aperture initialization code. Change-Id: I14ad11a364524a15affee9186b1298ba7d56d2c9 Signed-off-by: Felix Kuehling --- src/fmm.c | 280 +++++++++++++++++++++++++++--------------------------- 1 file changed, 139 insertions(+), 141 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 1b4a0ca6c8..18cbac672a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -211,8 +211,6 @@ static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( } extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); -static HSAKMT_STATUS dgpu_mem_init(uint32_t node_id, void **base, void **limit); -static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit); static void __fmm_release(void *address, manageable_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, @@ -1448,6 +1446,145 @@ static HSAKMT_STATUS get_process_apertures( return HSAKMT_STATUS_SUCCESS; } +/* Tonga dGPU specific functions */ +static bool is_dgpu_mem_init; + +static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit) +{ + struct kfd_ioctl_set_process_dgpu_aperture_args args = {0}; + + args.gpu_id = gpu_id; + args.dgpu_base = base; + args.dgpu_limit = limit; + + return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE, &args); +} + +static void *reserve_address(void *addr, unsigned long long int len) +{ + void *ret_addr; + + if (len <= 0) + return NULL; + + ret_addr = mmap(addr, len, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); + if (ret_addr == MAP_FAILED) + return NULL; + + return ret_addr; +} + +#define ADDRESS_RANGE_LIMIT_MASK 0xFFFFFFFFFF +#define AMDGPU_SYSFS_VM_SIZE "/sys/module/amdgpu/parameters/vm_size" + +/* + * TODO: Provide a cleaner interface via topology + */ +static HSAKMT_STATUS get_dgpu_vm_limit(uint32_t *vm_size_in_gb) +{ + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + fd = fopen(AMDGPU_SYSFS_VM_SIZE, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + if (fscanf(fd, "%ul", vm_size_in_gb) != 1) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + +err: + fclose(fd); + return ret; +} + +#define DGPU_APERTURE_ADDR_MIN 0x1000000 /* Leave at least 16MB for kernel */ +#define DGPU_APERTURE_ADDR_INC 0x200000 /* Search in huge-page increments */ + +static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limit) +{ + bool found; + HSAKMT_STATUS ret; + void *addr, *ret_addr; + HSAuint64 len, vm_limit, max_vm_limit, min_vm_size; + uint32_t max_vm_limit_in_gb; + + if (is_dgpu_mem_init) { + if (base) + *base = dgpu_shared_aperture_base; + if (limit) + *limit = dgpu_shared_aperture_limit; + return HSAKMT_STATUS_SUCCESS; + } + + ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("Unable to find vm_size for dGPU, assuming 64GB.\n"); + max_vm_limit_in_gb = 64; + } + max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; + min_vm_size = (HSAuint64)4 << 30; + + found = false; + + for (len = max_vm_limit+1; !found && len >= min_vm_size; len >>= 1) { + for (addr = (void *)DGPU_APERTURE_ADDR_MIN, ret_addr = NULL; + (HSAuint64)addr + (len >> 1) < max_vm_limit; + addr = (void *)((HSAuint64)addr + DGPU_APERTURE_ADDR_INC)) { + ret_addr = reserve_address(addr, len); + if (!ret_addr) + break; + if ((HSAuint64)ret_addr + (len >> 1) < max_vm_limit) + /* At least half the returned address + * space is GPU addressable, we'll + * take it + */ + break; + munmap(ret_addr, len); + } + if (!ret_addr) { + pr_warn("Failed to reserve %uGB for SVM ...\n", + (unsigned int)(len >> 30)); + continue; + } + if ((HSAuint64)ret_addr + min_vm_size - 1 > max_vm_limit) { + /* addressable size is less than the minimum */ + pr_warn("Got %uGB for SVM at %p with only %dGB usable ...\n", + (unsigned int)(len >> 30), ret_addr, + (int)(((HSAint64)max_vm_limit - + (HSAint64)ret_addr) >> 30)); + munmap(ret_addr, len); + continue; + } else { + found = true; + break; + } + } + + if (!found) { + pr_err("Failed to reserve SVM address range. Giving up.\n"); + return HSAKMT_STATUS_ERROR; + } + + vm_limit = (HSAuint64)ret_addr + len - 1; + if (vm_limit > max_vm_limit) { + /* trim the tail that's not GPU-addressable */ + munmap((void *)(max_vm_limit + 1), vm_limit - max_vm_limit); + vm_limit = max_vm_limit; + } + + if (base) + *base = ret_addr; + dgpu_shared_aperture_base = ret_addr; + if (limit) + *limit = (void *)vm_limit; + dgpu_shared_aperture_limit = (void *)vm_limit; + is_dgpu_mem_init = true; + + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i = 0; @@ -2244,145 +2381,6 @@ int fmm_unmap_from_gpu(void *address) return 0; } -/* Tonga dGPU specific functions */ -static bool is_dgpu_mem_init = false; - -static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit) -{ - struct kfd_ioctl_set_process_dgpu_aperture_args args = {0}; - - args.gpu_id = gpu_id; - args.dgpu_base = base; - args.dgpu_limit = limit; - - return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE, &args); -} - -static void *reserve_address(void *addr, unsigned long long int len) -{ - void *ret_addr; - - if (len <= 0) - return NULL; - - ret_addr = mmap(addr, len, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); - if (ret_addr == MAP_FAILED) - return NULL; - - return ret_addr; -} - -#define ADDRESS_RANGE_LIMIT_MASK 0xFFFFFFFFFF -#define AMDGPU_SYSFS_VM_SIZE "/sys/module/amdgpu/parameters/vm_size" - -/* - * TODO: Provide a cleaner interface via topology - */ -static HSAKMT_STATUS get_dgpu_vm_limit(uint32_t *vm_size_in_gb) -{ - FILE *fd; - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - - fd = fopen(AMDGPU_SYSFS_VM_SIZE, "r"); - if (!fd) - return HSAKMT_STATUS_ERROR; - if (fscanf(fd, "%ul", vm_size_in_gb) != 1) { - ret = HSAKMT_STATUS_ERROR; - goto err; - } - -err: - fclose(fd); - return ret; -} - -#define DGPU_APERTURE_ADDR_MIN 0x1000000 /* Leave at least 16MB for kernel */ -#define DGPU_APERTURE_ADDR_INC 0x200000 /* Search in huge-page increments */ - -static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limit) -{ - bool found; - HSAKMT_STATUS ret; - void *addr, *ret_addr; - HSAuint64 len, vm_limit, max_vm_limit, min_vm_size; - uint32_t max_vm_limit_in_gb; - - if (is_dgpu_mem_init) { - if (base) - *base = dgpu_shared_aperture_base; - if (limit) - *limit = dgpu_shared_aperture_limit; - return HSAKMT_STATUS_SUCCESS; - } - - ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("Unable to find vm_size for dGPU, assuming 64GB.\n"); - max_vm_limit_in_gb = 64; - } - max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; - min_vm_size = (HSAuint64)4 << 30; - - found = false; - - for (len = max_vm_limit+1; !found && len >= min_vm_size; len >>= 1) { - for (addr = (void *)DGPU_APERTURE_ADDR_MIN, ret_addr = NULL; - (HSAuint64)addr + (len >> 1) < max_vm_limit; - addr = (void *)((HSAuint64)addr + DGPU_APERTURE_ADDR_INC)) { - ret_addr = reserve_address(addr, len); - if (!ret_addr) - break; - if ((HSAuint64)ret_addr + (len >> 1) < max_vm_limit) - /* At least half the returned address - * space is GPU addressable, we'll - * take it - */ - break; - munmap(ret_addr, len); - } - if (!ret_addr) { - pr_warn("Failed to reserve %uGB for SVM ...\n", - (unsigned int)(len >> 30)); - continue; - } - if ((HSAuint64)ret_addr + min_vm_size - 1 > max_vm_limit) { - /* addressable size is less than the minimum */ - pr_warn("Got %uGB for SVM at %p with only %dGB usable ...\n", - (unsigned int)(len >> 30), ret_addr, - (int)(((HSAint64)max_vm_limit - - (HSAint64)ret_addr) >> 30)); - munmap(ret_addr, len); - continue; - } else { - found = true; - break; - } - } - - if (!found) { - pr_err("Failed to reserve SVM address range. Giving up.\n"); - return HSAKMT_STATUS_ERROR; - } - - vm_limit = (HSAuint64)ret_addr + len - 1; - if (vm_limit > max_vm_limit) { - /* trim the tail that's not GPU-addressable */ - munmap((void *)(max_vm_limit + 1), vm_limit - max_vm_limit); - vm_limit = max_vm_limit; - } - - if (base) - *base = ret_addr; - dgpu_shared_aperture_base = ret_addr; - if (limit) - *limit = (void *)vm_limit; - dgpu_shared_aperture_limit = (void *)vm_limit; - is_dgpu_mem_init = true; - - return HSAKMT_STATUS_SUCCESS; -} - bool fmm_get_handle(void *address, uint64_t *handle) { uint32_t i; From 85e1a9bf5ee29ec629d07e8aa6eaaa2eb2133b2a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 24 Jan 2018 23:02:48 -0500 Subject: [PATCH 0333/1247] Rework SVM aperture initialization Query GPUVM aperture limits of all dGPUs to determine SVM aperture base and limit. This depends on a recent KFD change that reports the GPUVM apurture limits for dGPUs in the AMDKFD_IOC_GET_PROCESS_APERTURES_NEW ioctl (drm/amdkfd: Simplify dGPU SVM aperture handling). Only initialize SVM aperture once, instead of once per GPU. Don't call AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE. It's not needed any more and will not be upstreamed. Change-Id: Ib3389e8ba18505ba15fc33f45fe8a57e690a565d Signed-off-by: Felix Kuehling --- src/fmm.c | 253 +++++++++++++++++++++++++++--------------------------- 1 file changed, 127 insertions(+), 126 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 18cbac672a..aa24d2f6e4 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1446,20 +1446,6 @@ static HSAKMT_STATUS get_process_apertures( return HSAKMT_STATUS_SUCCESS; } -/* Tonga dGPU specific functions */ -static bool is_dgpu_mem_init; - -static int set_dgpu_aperture(uint32_t gpu_id, uint64_t base, uint64_t limit) -{ - struct kfd_ioctl_set_process_dgpu_aperture_args args = {0}; - - args.gpu_id = gpu_id; - args.dgpu_base = base; - args.dgpu_limit = limit; - - return kmtIoctl(kfd_fd, AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE, &args); -} - static void *reserve_address(void *addr, unsigned long long int len) { void *ret_addr; @@ -1475,85 +1461,77 @@ static void *reserve_address(void *addr, unsigned long long int len) return ret_addr; } -#define ADDRESS_RANGE_LIMIT_MASK 0xFFFFFFFFFF -#define AMDGPU_SYSFS_VM_SIZE "/sys/module/amdgpu/parameters/vm_size" - -/* - * TODO: Provide a cleaner interface via topology +/* Managed SVM aperture limits: only reserve up to 40 bits (1TB, what + * GFX8 supports). Need to find at least 4GB of usable address space. */ -static HSAKMT_STATUS get_dgpu_vm_limit(uint32_t *vm_size_in_gb) +#define SVM_RESERVATION_LIMIT ((1ULL << 40) - 1) +#define SVM_MIN_VM_SIZE (4ULL << 30) +#define IS_CANONICAL_ADDR(a) ((a) < (1ULL << 47)) + +static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, + HSAuint32 align, HSAuint32 guard_pages, + bool disable_cache) { - FILE *fd; - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - - fd = fopen(AMDGPU_SYSFS_VM_SIZE, "r"); - if (!fd) - return HSAKMT_STATUS_ERROR; - if (fscanf(fd, "%ul", vm_size_in_gb) != 1) { - ret = HSAKMT_STATUS_ERROR; - goto err; - } - -err: - fclose(fd); - return ret; -} - -#define DGPU_APERTURE_ADDR_MIN 0x1000000 /* Leave at least 16MB for kernel */ -#define DGPU_APERTURE_ADDR_INC 0x200000 /* Search in huge-page increments */ - -static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limit) -{ - bool found; - HSAKMT_STATUS ret; + const HSAuint64 ADDR_INC = GPU_HUGE_PAGE_SIZE; + HSAuint64 len, alt_base, alt_size; + bool found = false; void *addr, *ret_addr; - HSAuint64 len, vm_limit, max_vm_limit, min_vm_size; - uint32_t max_vm_limit_in_gb; - if (is_dgpu_mem_init) { - if (base) - *base = dgpu_shared_aperture_base; - if (limit) - *limit = dgpu_shared_aperture_limit; + /* If we already have an SVM aperture initialized (from a + * parent process), keep using it + */ + if (dgpu_shared_aperture_limit) return HSAKMT_STATUS_SUCCESS; + + base = ALIGN_UP(base, GPU_HUGE_PAGE_SIZE); + limit = ((limit + 1) & ~(HSAuint64)(GPU_HUGE_PAGE_SIZE - 1)) - 1; + if (limit > SVM_RESERVATION_LIMIT) + limit = SVM_RESERVATION_LIMIT; + if (base >= limit) { + pr_err("No SVM range compatible with all GPU and software constraints\n"); + return HSAKMT_STATUS_ERROR; } - ret = get_dgpu_vm_limit(&max_vm_limit_in_gb); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("Unable to find vm_size for dGPU, assuming 64GB.\n"); - max_vm_limit_in_gb = 64; - } - max_vm_limit = ((HSAuint64)max_vm_limit_in_gb << 30) - 1; - min_vm_size = (HSAuint64)4 << 30; + /* Try to reserve address space for SVM. + * + * Inner loop: try start addresses in huge-page increments up + * to half the VM size we're trying to reserve + * + * Outer loop: reduce size of the allocation by factor 2 at a + * time and print a warning for every reduction + */ + for (len = limit - base + 1; !found && len >= SVM_MIN_VM_SIZE; + len = (len + 1) >> 1) { + for (addr = (void *)base, ret_addr = NULL; + (HSAuint64)addr + ((len + 1) >> 1) - 1 <= limit; + addr = (void *)((HSAuint64)addr + ADDR_INC)) { + HSAuint64 top = MIN((HSAuint64)addr + len, limit+1); + HSAuint64 size = (top - (HSAuint64)addr) & + ~(HSAuint64)(PAGE_SIZE - 1); - found = false; - - for (len = max_vm_limit+1; !found && len >= min_vm_size; len >>= 1) { - for (addr = (void *)DGPU_APERTURE_ADDR_MIN, ret_addr = NULL; - (HSAuint64)addr + (len >> 1) < max_vm_limit; - addr = (void *)((HSAuint64)addr + DGPU_APERTURE_ADDR_INC)) { - ret_addr = reserve_address(addr, len); + if (size < SVM_MIN_VM_SIZE) + break; + ret_addr = reserve_address(addr, size); if (!ret_addr) break; - if ((HSAuint64)ret_addr + (len >> 1) < max_vm_limit) + if ((HSAuint64)ret_addr + ((len + 1) >> 1) - 1 <= limit) /* At least half the returned address * space is GPU addressable, we'll * take it */ break; - munmap(ret_addr, len); + munmap(ret_addr, size); } if (!ret_addr) { pr_warn("Failed to reserve %uGB for SVM ...\n", (unsigned int)(len >> 30)); continue; } - if ((HSAuint64)ret_addr + min_vm_size - 1 > max_vm_limit) { + if ((HSAuint64)ret_addr + SVM_MIN_VM_SIZE - 1 > limit) { /* addressable size is less than the minimum */ pr_warn("Got %uGB for SVM at %p with only %dGB usable ...\n", (unsigned int)(len >> 30), ret_addr, - (int)(((HSAint64)max_vm_limit - - (HSAint64)ret_addr) >> 30)); + (int)((limit - (HSAint64)ret_addr) >> 30)); munmap(ret_addr, len); continue; } else { @@ -1567,20 +1545,41 @@ static HSAKMT_STATUS dgpu_mem_init(uint32_t gpu_mem_id, void **base, void **limi return HSAKMT_STATUS_ERROR; } - vm_limit = (HSAuint64)ret_addr + len - 1; - if (vm_limit > max_vm_limit) { + base = (HSAuint64)ret_addr; + if (base + len - 1 > limit) /* trim the tail that's not GPU-addressable */ - munmap((void *)(max_vm_limit + 1), vm_limit - max_vm_limit); - vm_limit = max_vm_limit; - } + munmap((void *)(limit + 1), base + len - 1 - limit); + else + limit = base + len - 1; - if (base) - *base = ret_addr; - dgpu_shared_aperture_base = ret_addr; - if (limit) - *limit = (void *)vm_limit; - dgpu_shared_aperture_limit = (void *)vm_limit; - is_dgpu_mem_init = true; + /* init aperture */ + svm.dgpu_aperture.base = dgpu_shared_aperture_base = ret_addr; + svm.dgpu_aperture.limit = dgpu_shared_aperture_limit = (void *)limit; + svm.dgpu_aperture.align = align; + svm.dgpu_aperture.guard_pages = guard_pages; + svm.dgpu_aperture.is_coherent = disable_cache; + + /* Use the first 1/4 of the dGPU aperture as + * alternate aperture for coherent access. + * Base and size must be 64KB aligned. + */ + alt_base = (HSAuint64)svm.dgpu_aperture.base; + alt_size = (VOID_PTRS_SUB(svm.dgpu_aperture.limit, + svm.dgpu_aperture.base) + 1) >> 2; + alt_base = (alt_base + 0xffff) & ~0xffffULL; + alt_size = (alt_size + 0xffff) & ~0xffffULL; + svm.dgpu_alt_aperture.base = (void *)alt_base; + svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); + svm.dgpu_alt_aperture.align = align; + svm.dgpu_alt_aperture.guard_pages = guard_pages; + svm.dgpu_alt_aperture.is_coherent = true; + + svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); + + pr_info("SVM alt (coherent): %12p - %12p\n", + svm.dgpu_alt_aperture.base, svm.dgpu_alt_aperture.limit); + pr_info("SVM (non-coherent): %12p - %12p\n", + svm.dgpu_aperture.base, svm.dgpu_aperture.limit); return HSAKMT_STATUS_SUCCESS; } @@ -1598,6 +1597,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) char *hsaDebug; unsigned int guardPages = 1; struct pci_access *pacc; + uint64_t svm_base = 0, svm_limit = 0; + uint32_t svm_alignment = 0; hsaDebug = getenv("HSA_DEBUG"); hsa_debug = hsaDebug && strcmp(hsaDebug, "0"); @@ -1706,75 +1707,75 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_id].lds_aperture.base = PORT_UINT64_TO_VPTR(process_apertures[i].lds_base); - gpu_mem[gpu_mem_id].lds_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].lds_limit); - gpu_mem[gpu_mem_id].gpuvm_aperture.base = - PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_base); - - gpu_mem[gpu_mem_id].gpuvm_aperture.limit = - PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_limit); - gpu_mem[gpu_mem_id].scratch_aperture.base = PORT_UINT64_TO_VPTR(process_apertures[i].scratch_base); - gpu_mem[gpu_mem_id].scratch_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].scratch_limit); - if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { - uintptr_t alt_base; - uint64_t alt_size; - int err; + if (IS_CANONICAL_ADDR(process_apertures[i].gpuvm_limit)) { uint64_t vm_alignment = get_vm_alignment( gpu_mem[gpu_mem_id].device_id); - dgpu_mem_init(gpu_mem_id, &svm.dgpu_aperture.base, - &svm.dgpu_aperture.limit); - /* Set proper alignment for scratch backing aperture */ gpu_mem[gpu_mem_id].scratch_physical.align = vm_alignment; - /* Set kernel process dgpu aperture. */ - set_dgpu_aperture(process_apertures[i].gpu_id, - (uint64_t)svm.dgpu_aperture.base, - (uint64_t)svm.dgpu_aperture.limit); - svm.dgpu_aperture.align = vm_alignment; - svm.dgpu_aperture.guard_pages = guardPages; - /* Non-canonical per-ASIC GPUVM aperture does * not exist on dGPUs in GPUVM64 address mode */ gpu_mem[gpu_mem_id].gpuvm_aperture.base = NULL; gpu_mem[gpu_mem_id].gpuvm_aperture.limit = NULL; - /* Use the first 1/4 of the dGPU aperture as - * alternate aperture for coherent access. - * Base and size must be 64KB aligned. - */ - alt_base = (uintptr_t)svm.dgpu_aperture.base; - alt_size = (VOID_PTRS_SUB(svm.dgpu_aperture.limit, - svm.dgpu_aperture.base) + 1) >> 2; - alt_base = (alt_base + 0xffff) & ~0xffffULL; - alt_size = (alt_size + 0xffff) & ~0xffffULL; - svm.dgpu_alt_aperture.base = (void *)alt_base; - svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); - svm.dgpu_alt_aperture.is_coherent = true; - svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); - svm.dgpu_aperture.is_coherent = !!disableCache; - err = fmm_set_memory_policy(gpu_mem[gpu_mem_id].gpu_id, - disableCache ? + /* Update SVM aperture limits and alignment */ + if (process_apertures[i].gpuvm_base > svm_base) + svm_base = process_apertures[i].gpuvm_base; + if (process_apertures[i].gpuvm_limit < svm_limit || + svm_limit == 0) + svm_limit = process_apertures[i].gpuvm_limit; + if (vm_alignment > svm_alignment) + svm_alignment = vm_alignment; + } else { + gpu_mem[gpu_mem_id].gpuvm_aperture.base = + PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_base); + gpu_mem[gpu_mem_id].gpuvm_aperture.limit = + PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_limit); + } + } + + if (svm_limit) { + /* At least one GPU uses GPUVM in canonical address + * space. Set up SVM apertures shared by all such GPUs + */ + ret = init_svm_apertures(svm_base, svm_limit, svm_alignment, + guardPages, disableCache); + if (ret != HSAKMT_STATUS_SUCCESS) + goto init_svm_failed; + + for (i = 0 ; i < num_of_nodes ; i++) { + uintptr_t alt_base; + uint64_t alt_size; + int err; + + if (!IS_CANONICAL_ADDR(process_apertures[i].gpuvm_limit)) + continue; + + /* Set memory policy to match the SVM apertures */ + alt_base = (uintptr_t)svm.dgpu_alt_aperture.base; + alt_size = VOID_PTRS_SUB(svm.dgpu_alt_aperture.limit, + svm.dgpu_alt_aperture.base) + 1; + err = fmm_set_memory_policy(process_apertures[i].gpu_id, + svm.dgpu_aperture.is_coherent ? KFD_IOC_CACHE_POLICY_COHERENT : KFD_IOC_CACHE_POLICY_NONCOHERENT, KFD_IOC_CACHE_POLICY_COHERENT, alt_base, alt_size); - if (err != 0) { - pr_err("Failed to set alt aperture for GPU [0x%x]\n", - gpu_mem[gpu_mem_id].gpu_id); + if (err) { + pr_err("Failed to set mem policy for GPU [0x%x]\n", + process_apertures[i].gpu_id); ret = HSAKMT_STATUS_ERROR; } - svm.dgpu_alt_aperture.align = vm_alignment; - svm.dgpu_alt_aperture.guard_pages = guardPages; } } @@ -1784,6 +1785,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) free(process_apertures); return ret; +init_svm_failed: get_aperture_ioctl_failed: invalid_gpu_id: free(process_apertures); @@ -3177,7 +3179,7 @@ void fmm_clear_all_mem(void) fmm_clear_aperture(&gpu_mem[i].scratch_physical); } - if (is_dgpu_mem_init) { + if (dgpu_shared_aperture_limit) { fmm_clear_aperture(&svm.dgpu_aperture); fmm_clear_aperture(&svm.dgpu_alt_aperture); @@ -3196,7 +3198,6 @@ void fmm_clear_all_mem(void) dgpu_shared_aperture_base = NULL; dgpu_shared_aperture_limit = NULL; - is_dgpu_mem_init = false; } } From 19dacdecd3ce2592efa6c046afc342c08868dced Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 1 Mar 2018 16:54:41 -0500 Subject: [PATCH 0334/1247] Update kfd_ioctl.h from kernel This adds new acquire_vm ioctl. Change-Id: Ia6794bfd291706cecdb2d06f4902b324b48577df Signed-off-by: Felix Kuehling --- include/linux/kfd_ioctl.h | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index ae79f16e1d..b0226ea526 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -278,6 +278,11 @@ struct kfd_ioctl_alloc_memory_of_scratch_args { uint32_t pad; }; +struct kfd_ioctl_acquire_vm_args { + uint32_t drm_fd; /* to KFD */ + uint32_t gpu_id; /* to KFD */ +}; + /* Allocation flags: memory types */ #define KFD_IOC_ALLOC_MEM_FLAGS_VRAM (1 << 0) #define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) @@ -361,22 +366,22 @@ struct kfd_ioctl_ipc_import_handle_args { struct kfd_ioctl_get_tile_config_args { /* to KFD: pointer to tile array */ - uint64_t tile_config_ptr; + __u64 tile_config_ptr; /* to KFD: pointer to macro tile array */ - uint64_t macro_tile_config_ptr; + __u64 macro_tile_config_ptr; /* to KFD: array size allocated by user mode * from KFD: array size filled by kernel */ - uint32_t num_tile_configs; + __u32 num_tile_configs; /* to KFD: array size allocated by user mode * from KFD: array size filled by kernel */ - uint32_t num_macro_tile_configs; + __u32 num_macro_tile_configs; - uint32_t gpu_id; /* to KFD */ - uint32_t gb_addr_config; /* from KFD */ - uint32_t num_banks; /* from KFD */ - uint32_t num_ranks; /* from KFD */ + __u32 gpu_id; /* to KFD */ + __u32 gb_addr_config; /* from KFD */ + __u32 num_banks; /* from KFD */ + __u32 num_ranks; /* from KFD */ /* struct size can be extended later if needed * without breaking ABI compatibility */ @@ -516,7 +521,10 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ AMDKFD_IOWR(0x20, struct kfd_ioctl_get_queue_wave_state_args) +#define AMDKFD_IOC_ACQUIRE_VM \ + AMDKFD_IOW(0x21, struct kfd_ioctl_acquire_vm_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x21 +#define AMDKFD_COMMAND_END 0x22 #endif From 1bf93d4e8970a44c4e37b1ee69d3dcdb832a910c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 9 Mar 2018 15:31:38 -0500 Subject: [PATCH 0335/1247] Export microcode version of sDMA Change-Id: I86fa5da5e72af13a2e76e6e3be4667a7220923d5 Signed-off-by: Philip Yang --- include/hsakmttypes.h | 18 +++++++++++++++--- src/topology.c | 11 +++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 3cb23e1121..107703e20d 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -150,10 +150,10 @@ typedef struct _HsaSystemProperties HSAuint32 PlatformRev; // HSA platform revision, reflects Platform Table Revision ID } HsaSystemProperties; -typedef union +typedef union { HSAuint32 Value; - struct + struct { unsigned int uCode : 10; // ucode packet processor version unsigned int Major : 6; // GFXIP Major engine version @@ -162,6 +162,17 @@ typedef union }ui32; } HSA_ENGINE_ID; +typedef union +{ + HSAuint32 Value; + struct + { + unsigned int uCodeSDMA: 10; // ucode version SDMA engine + unsigned int uCodeRes : 10; // ucode version (reserved) + unsigned int Reserved : 12; // Reserved, must be 0 + }; +} HSA_ENGINE_VERSION; + typedef union { HSAuint32 Value; @@ -251,7 +262,8 @@ typedef struct _HsaNodeProperties HSAuint16 MarketingName[HSA_PUBLIC_NAME_SIZE]; // Public name of the "device" on the node (board or APU name). // Unicode string HSAuint8 AMDName[HSA_PUBLIC_NAME_SIZE]; //CAL Name of the "device", ASCII - HSAuint8 Reserved[64]; + HSA_ENGINE_VERSION uCodeEngineVersions; + HSAuint8 Reserved[60]; } HsaNodeProperties; diff --git a/src/topology.c b/src/topology.c index 5dcceb9edb..5c9fdff594 100644 --- a/src/topology.c +++ b/src/topology.c @@ -740,7 +740,6 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, char path[256]; unsigned long long prop_val; uint32_t i, prog, major, minor, step; - uint16_t fw_version = 0; int read_size; const struct hsa_gfxip_table *hsa_gfxip; char namebuf[HSA_PUBLIC_NAME_SIZE]; @@ -815,7 +814,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, else if (strcmp(prop_name, "max_slots_scratch_cu") == 0) props->MaxSlotsScratchCU = (uint32_t)prop_val; else if (strcmp(prop_name, "fw_version") == 0) - fw_version = (uint16_t)prop_val; + props->EngineId.Value = (uint32_t)prop_val & 0x3ff; else if (strcmp(prop_name, "vendor_id") == 0) props->VendorId = (uint32_t)prop_val; else if (strcmp(prop_name, "device_id") == 0) @@ -830,14 +829,10 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->LocalMemSize = prop_val; else if (strcmp(prop_name, "drm_render_minor") == 0) props->DrmRenderMinor = (int32_t)prop_val; - + else if (strcmp(prop_name, "sdma_fw_version") == 0) + props->uCodeEngineVersions.Value = (uint32_t)prop_val & 0x3ff; } - props->EngineId.ui32.uCode = fw_version & 0x3ff; - props->EngineId.ui32.Major = 0; - props->EngineId.ui32.Minor = 0; - props->EngineId.ui32.Stepping = 0; - hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); if (hsa_gfxip) { envvar = getenv("HSA_OVERRIDE_GFX_VERSION"); From 8ac2150e8159e622f614d5e09b768e97294d653b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 1 Mar 2018 17:31:38 -0500 Subject: [PATCH 0336/1247] Let KFD use VM from DRM render node Move opening of DRM render nodes from topology to FMM aperture initialization. Keep the same FDs open for the life time of the process to match how KFD uses the VMs in the FDs. Call acquire_vm ioctl during aperture initialization to let KFD use the VMs from the render nodes. Change-Id: Ie07d57788cbe685b1841cccc00820c12894a0356 Signed-off-by: Felix Kuehling --- src/fmm.c | 104 +++++++++++++++++++++++++++++++++++++++++++------ src/topology.c | 30 -------------- 2 files changed, 93 insertions(+), 41 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index aa24d2f6e4..41e99f98cb 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -29,6 +29,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -124,6 +127,7 @@ typedef struct { * GPU will get a differnt range */ manageable_aperture_t gpuvm_aperture; /* used for GPUVM on APU, outsidethe canonical address range */ + int drm_render_fd; } gpu_mem_t; /* The main structure for dGPU Shared Virtual Memory Management */ @@ -1044,7 +1048,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla if (mem && (flags.ui32.HostAccess || hsa_debug)) { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : - get_drm_render_fd_by_gpu_id(gpu_id); + gpu_mem[gpu_mem_id].drm_render_fd; int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; int flag = flags.ui32.HostAccess ? MAP_SHARED | MAP_FIXED : @@ -1170,12 +1174,12 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, uint64_t mmap_offset; uint32_t ioc_flags; uint64_t size; - int32_t i; + int32_t gpu_mem_id; uint32_t gpu_id; vm_object_t *vm_obj = NULL; - i = find_first_dgpu(&gpu_id); - if (i < 0) + gpu_mem_id = find_first_dgpu(&gpu_id); + if (gpu_mem_id < 0) return NULL; size = MemorySizeInBytes; @@ -1254,7 +1258,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, if (mem && flags.ui32.HostAccess) { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : - get_drm_render_fd_by_gpu_id(gpu_id); + gpu_mem[gpu_mem_id].drm_render_fd; void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); @@ -1446,6 +1450,57 @@ static HSAKMT_STATUS get_process_apertures( return HSAKMT_STATUS_SUCCESS; } +/* The VMs from DRM render nodes are used by KFD for the lifetime of + * the process. Therefore we have to keep using the same FDs for the + * lifetime of the process, even when we close and reopen KFD. There + * are up to 128 render nodes that we cache in this array. + */ +#define DRM_FIRST_RENDER_NODE 128 +#define DRM_LAST_RENDER_NODE 255 +static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE]; + +static int open_drm_render_device(int minor) +{ + char path[128]; + int index, fd; + + if (minor < DRM_FIRST_RENDER_NODE || minor > DRM_LAST_RENDER_NODE) { + pr_err("DRM render minor %d out of range [%d, %d]\n", minor, + DRM_FIRST_RENDER_NODE, DRM_LAST_RENDER_NODE); + return -EINVAL; + } + index = minor - DRM_FIRST_RENDER_NODE; + + /* If the render node was already opened, keep using the same FD */ + if (drm_render_fds[index]) + return drm_render_fds[index]; + + sprintf(path, "/dev/dri/renderD%d", minor); + fd = open(path, O_RDWR | O_CLOEXEC); + if (fd < 0) { + pr_err("Failed to open %s: %s\n", path, strerror(errno)); + return -errno; + } + drm_render_fds[index] = fd; + + return fd; +} + +static HSAKMT_STATUS acquire_vm(uint32_t gpu_id, int fd) +{ + struct kfd_ioctl_acquire_vm_args args; + + args.gpu_id = gpu_id; + args.drm_fd = fd; + pr_info("acquiring VM for %x using %d\n", gpu_id, fd); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_ACQUIRE_VM, (void *)&args)) { + pr_err("AMDKFD_IOC_ACQUIRE_VM failed\n"); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + static void *reserve_address(void *addr, unsigned long long int len) { void *ret_addr; @@ -1648,6 +1703,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) /* Skip non-GPU nodes */ if (gpu_id != 0) { + gpu_mem[gpu_mem_count].drm_render_fd = + open_drm_render_device(props.DrmRenderMinor); + if (gpu_mem[gpu_mem_count].drm_render_fd <= 0) + goto sysfs_parse_failed; + gpu_mem[gpu_mem_count].gpu_id = gpu_id; gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; @@ -1742,6 +1802,12 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_id].gpuvm_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_limit); } + + /* Acquire the VM from the DRM render node for KFD use */ + ret = acquire_vm(gpu_mem[gpu_mem_id].gpu_id, + gpu_mem[gpu_mem_id].drm_render_fd); + if (ret != HSAKMT_STATUS_SUCCESS) + goto acquire_vm_failed; } if (svm_limit) { @@ -1786,6 +1852,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) return ret; init_svm_failed: +acquire_vm_failed: get_aperture_ioctl_failed: invalid_gpu_id: free(process_apertures); @@ -2028,7 +2095,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert return -1; } else { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : - get_drm_render_fd_by_gpu_id(gpu_id); + gpu_mem[gpu_mem_id].drm_render_fd; fmm_allocate_memory_object(gpu_id, address, size, @@ -2787,11 +2854,19 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo pthread_mutex_unlock(&aperture->fmm_mutex); if (importArgs.mmap_offset) { - int map_fd = importArgs.mmap_offset >= (1ULL<<40) ? kfd_fd : - get_drm_render_fd_by_gpu_id(importArgs.gpu_id); - void *ret = mmap(reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, map_fd, importArgs.mmap_offset); + int32_t gpu_mem_id = gpu_mem_find_by_gpu_id(importArgs.gpu_id); + int map_fd; + void *ret; + + if (gpu_mem_id < 0) { + err = HSAKMT_STATUS_ERROR; + goto err_free_obj; + } + map_fd = importArgs.mmap_offset >= (1ULL<<40) ? kfd_fd : + gpu_mem[gpu_mem_id].drm_render_fd; + ret = mmap(reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, map_fd, importArgs.mmap_offset); if (ret == MAP_FAILED) { err = HSAKMT_STATUS_ERROR; goto err_free_obj; @@ -3167,6 +3242,13 @@ void fmm_clear_all_mem(void) uint32_t i; void *map_addr; + /* Close render node FDs. The child process needs to open new ones */ + for (i = 0; i <= DRM_LAST_RENDER_NODE - DRM_FIRST_RENDER_NODE; i++) + if (drm_render_fds[i]) { + close(drm_render_fds[i]); + drm_render_fds[i] = 0; + } + /* Nothing is initialized. */ if (!gpu_mem) return; diff --git a/src/topology.c b/src/topology.c index 5c9fdff594..998e395358 100644 --- a/src/topology.c +++ b/src/topology.c @@ -30,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -54,7 +53,6 @@ typedef struct { HsaMemoryProperties *mem; /* node->NumBanks elements */ HsaCacheProperties *cache; HsaIoLinkProperties *link; - int drm_render_fd; } node_t; static HsaSystemProperties *_system = NULL; @@ -240,8 +238,6 @@ free_node(node_t *n) free((n)->cache); if ((n)->link) free((n)->link); - if ((n)->drm_render_fd > 0) - close((n)->drm_render_fd); } static void free_nodes(node_t *temp_nodes, int size) @@ -1512,16 +1508,6 @@ static void topology_create_indirect_gpu_links(const HsaSystemProperties *sys_pr } } - -static void open_drm_render_device(node_t *n) -{ - int minor = n->node.DrmRenderMinor; - char path[128]; - - sprintf(path, "/dev/dri/renderD%d", minor); - n->drm_render_fd = open(path, O_RDWR | O_CLOEXEC); -} - HSAKMT_STATUS topology_take_snapshot(void) { uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; @@ -1619,7 +1605,6 @@ retry: } } } - open_drm_render_device(&temp_nodes[i]); } pci_cleanup(pacc); } @@ -1980,21 +1965,6 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) return 0; } -int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id) -{ - unsigned int i; - - if (!node || !_system) - return 0; - - for (i = 0; i < _system->NumNodes; i++) { - if (node[i].gpu_id == gpu_id) - return node[i].drm_render_fd; - } - - return -1; -} - HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray) { From aa28484583096bf718182784af663e4116572242 Mon Sep 17 00:00:00 2001 From: Shaoyun Liu Date: Tue, 5 Dec 2017 16:51:42 -0500 Subject: [PATCH 0337/1247] Thunk: Add gfx904 support on libthunk Change-Id: I78bc623f6b86293e2bf9fbe00a646d152faafdc4 Signed-off-by: Shaoyun Liu --- src/topology.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/topology.c b/src/topology.c index 998e395358..9029035e37 100644 --- a/src/topology.c +++ b/src/topology.c @@ -171,6 +171,11 @@ static struct hsa_gfxip_table { { 0x6868, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + /* Vega12 */ + { 0x69A0, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, + { 0x69A1, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, + { 0x69A3, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, + { 0x69Af, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, /* Raven */ { 0x15DD, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, /* Vega20 on emulator, treat it as vega10 */ From dc1bbccc39742e888197c08b48e27ce00e5cf397 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 3 May 2018 12:03:22 -0400 Subject: [PATCH 0338/1247] Use svm aperture for device memory allocate for gfx902 and after APU Change-Id: Ib1d822adde30138a016e010bf581220465a087b9 Signed-off-by: Oak Zeng --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 41e99f98cb..f3398ac245 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1021,7 +1021,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); - if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) { + if (topology_is_svm_needed(get_device_id_by_gpu_id(gpu_id))) { aperture = &svm.dgpu_aperture; offset = 0; if (flags.ui32.AQLQueueMemory) From 2915d521a1d6dc473a4f4fa7806d8c829dd7fd2e Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 1 May 2018 15:13:58 -0400 Subject: [PATCH 0339/1247] Remove redundant dev package build No need to build the package in the build-dev target. This is taken care of by package-dev. Removing the redundant packaging command allows install-dev to work without building a package unnecessarily. Also moved the rm command into the package-dev target. Change-Id: I044871be03ebc5673146b44e4291b48b112f4440 Signed-off-by: Felix Kuehling --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c91f342ba6..7f0781b0c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,13 +127,14 @@ add_custom_target ( build-dev -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" -DCPACK_RPM_PACKAGE_REQUIRES="hsakmt-roct" -DCPACK_PACKAGING_INSTALL_PREFIX="${CPACK_PACKAGING_INSTALL_PREFIX}" - COMMAND rm -rf *.deb *.rpm *.tar.gz - COMMAND make package WORKING_DIRECTORY ${DEV_BUILD_DIR} ) ## Custom targets for the devel package add_custom_target ( install-dev DEPENDS build-dev COMMAND make install WORKING_DIRECTORY ${DEV_BUILD_DIR} ) -add_custom_target ( package-dev DEPENDS build-dev COMMAND make package WORKING_DIRECTORY ${DEV_BUILD_DIR} ) +add_custom_target ( package-dev DEPENDS build-dev + COMMAND rm -rf *.deb *.rpm *.tar.gz + COMMAND make package + WORKING_DIRECTORY ${DEV_BUILD_DIR} ) ## Add the install directives for the runtime library. install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) From 43f119bcbcc569cc9c42d8b762a2c92696517f90 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 16 May 2018 14:53:09 -0400 Subject: [PATCH 0340/1247] Improve the code readablity The main point is to move update_ctx_save_restore_size() out of if() condition. Change-Id: I58a1a4f3edca2d1c510fdd0e31e59b5c41e92a14 Signed-off-by: Yong Zhao --- src/queues.c | 53 +++++++++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/src/queues.c b/src/queues.c index 613180cedb..2a11d605c1 100644 --- a/src/queues.c +++ b/src/queues.c @@ -437,32 +437,39 @@ static int handle_concrete_asic(struct queue *q, uint32_t NodeId) { const struct device_info *dev_info = q->dev_info; + bool ret; - if (dev_info) { - if (dev_info->eop_buffer_size > 0) { - q->eop_buffer = - allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, - dev_info->asic_family, - NodeId, true); - if (!q->eop_buffer) - return HSAKMT_STATUS_NO_MEMORY; + if (!dev_info) + return HSAKMT_STATUS_SUCCESS; - args->eop_buffer_address = (uintptr_t)q->eop_buffer; - args->eop_buffer_size = dev_info->eop_buffer_size; - } - if (args->queue_type != KFD_IOC_QUEUE_TYPE_SDMA && - update_ctx_save_restore_size(NodeId, q) == true) { - args->ctx_save_restore_size = q->ctx_save_restore_size; - args->ctl_stack_size = q->ctl_stack_size; - q->ctx_save_restore = - allocate_exec_aligned_memory(q->ctx_save_restore_size, - dev_info->asic_family, - NodeId, false); - if (!q->ctx_save_restore) - return HSAKMT_STATUS_NO_MEMORY; + if (dev_info->eop_buffer_size > 0) { + q->eop_buffer = + allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, + dev_info->asic_family, + NodeId, true); + if (!q->eop_buffer) + return HSAKMT_STATUS_NO_MEMORY; - args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; - } + args->eop_buffer_address = (uintptr_t)q->eop_buffer; + args->eop_buffer_size = dev_info->eop_buffer_size; + } + + if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) + return HSAKMT_STATUS_SUCCESS; + + ret = update_ctx_save_restore_size(NodeId, q); + + if (ret) { + args->ctx_save_restore_size = q->ctx_save_restore_size; + args->ctl_stack_size = q->ctl_stack_size; + q->ctx_save_restore = + allocate_exec_aligned_memory(q->ctx_save_restore_size, + dev_info->asic_family, + NodeId, false); + if (!q->ctx_save_restore) + return HSAKMT_STATUS_NO_MEMORY; + + args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; } return HSAKMT_STATUS_SUCCESS; From ec440fb428f065b5eb839c2e628fd081821837cc Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 16 May 2018 15:30:23 -0400 Subject: [PATCH 0341/1247] Stop allocating eop buffer for SDMA queues Change-Id: I9a4eaee05588292a797eb424503dd7b793c1408c Signed-off-by: Yong Zhao --- src/queues.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/queues.c b/src/queues.c index 2a11d605c1..a87aa2d2a2 100644 --- a/src/queues.c +++ b/src/queues.c @@ -439,7 +439,7 @@ static int handle_concrete_asic(struct queue *q, const struct device_info *dev_info = q->dev_info; bool ret; - if (!dev_info) + if (!dev_info || args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) return HSAKMT_STATUS_SUCCESS; if (dev_info->eop_buffer_size > 0) { @@ -454,9 +454,6 @@ static int handle_concrete_asic(struct queue *q, args->eop_buffer_size = dev_info->eop_buffer_size; } - if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) - return HSAKMT_STATUS_SUCCESS; - ret = update_ctx_save_restore_size(NodeId, q); if (ret) { From 93d07cf9164eca4ff193e22b37f9a52cf0eca633 Mon Sep 17 00:00:00 2001 From: Shaoyun Liu Date: Mon, 19 Mar 2018 11:30:09 -0400 Subject: [PATCH 0342/1247] Thunk: Add gfx906 support on thunk Signed-off-by: Shaoyun Liu Conflicts: src/topology.c Change-Id: I692d9295a954d4eda08eba301312014f7b3969cb --- src/libhsakmt.h | 3 ++- src/pmc_table.c | 1 + src/queues.c | 8 ++++++++ src/topology.c | 9 +++++++-- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 48435c477b..2c215d7789 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -97,7 +97,8 @@ enum asic_family_type { CHIP_POLARIS10, CHIP_POLARIS11, CHIP_VEGA10, - CHIP_RAVEN + CHIP_RAVEN, + CHIP_VEGA20 }; #define IS_DGPU(chip) ((chip) != CHIP_KAVERI && (chip) != CHIP_CARRIZO && \ (chip) != CHIP_RAVEN) diff --git a/src/pmc_table.c b/src/pmc_table.c index 5f1cb3c877..7c964821a2 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1745,6 +1745,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, *block = polaris_blocks[block_id]; break; case CHIP_VEGA10: + case CHIP_VEGA20: case CHIP_RAVEN: *block = vega_blocks[block_id]; break; diff --git a/src/queues.c b/src/queues.c index a87aa2d2a2..4e26811ede 100644 --- a/src/queues.c +++ b/src/queues.c @@ -104,6 +104,13 @@ struct device_info raven_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +struct device_info vega20_device_info = { + .asic_family = CHIP_VEGA20, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + + static struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -113,6 +120,7 @@ static struct device_info *dev_lookup_table[] = { [CHIP_POLARIS10] = &polaris10_device_info, [CHIP_POLARIS11] = &polaris11_device_info, [CHIP_VEGA10] = &vega10_device_info, + [CHIP_VEGA20] = &vega20_device_info, [CHIP_RAVEN] = &raven_device_info }; diff --git a/src/topology.c b/src/topology.c index 9029035e37..c409bef14e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -178,8 +178,13 @@ static struct hsa_gfxip_table { { 0x69Af, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, /* Raven */ { 0x15DD, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, - /* Vega20 on emulator, treat it as vega10 */ - { 0x66A0, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + /* Vega20 */ + { 0x66A0, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + { 0x66A1, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + { 0x66A2, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + { 0x66A3, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + { 0x66A7, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + { 0x66AF, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, }; enum cache_type { From 571e2cf7e4f7eb92d3b67fb7b3435a4469dbfdf1 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 15 Aug 2017 19:18:03 -0400 Subject: [PATCH 0343/1247] Update KFD-Thunk ioctl ABI to match upstream - Clean up and renumber scratch memory ioctl - Renumber get_tile_config ioctl - Renumber set_trap_handler ioctl - Update KFD_IOC_ALLOC_MEM_FLAGS - Renumber GPUVM memory management ioctls - Remove unused SEP_PROCESS_DGPU_APERTURE ioctl - Update memory management ioctls Replace device_ids_array_size (in bytes) with n_devices. Fix error handling and use n_success to update device_id arrays in objects. This commit breaks the ABI and requires a corresponding KFD change. Change-Id: Ibf0af5a5188e817c886eab388d1533130fc18293 Signed-off-by: Felix Kuehling --- include/linux/kfd_ioctl.h | 171 +++++++++++++++++++++----------------- src/fmm.c | 90 ++++++++++---------- 2 files changed, 144 insertions(+), 117 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index b0226ea526..35d4c385cf 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -271,9 +271,8 @@ struct kfd_ioctl_wait_events_args { uint32_t wait_result; /* from KFD */ }; -struct kfd_ioctl_alloc_memory_of_scratch_args { +struct kfd_ioctl_set_scratch_backing_va_args { uint64_t va_addr; /* to KFD */ - uint64_t size; /* to KFD */ uint32_t gpu_id; /* to KFD */ uint32_t pad; }; @@ -289,46 +288,73 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) /* Allocation flags: attributes/access options */ -#define KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED (1 << 31) -#define KFD_IOC_ALLOC_MEM_FLAGS_READONLY (1 << 30) +#define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31) +#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30) #define KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC (1 << 29) #define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28) #define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) -#define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS (1 << 26) -#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 25) +#define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26) +/* Allocate memory for later SVM (shared virtual memory) mapping. + * + * @va_addr: virtual address of the memory to be allocated + * all later mappings on all GPUs will use this address + * @size: size in bytes + * @handle: buffer handle returned to user mode, used to refer to + * this allocation for mapping, unmapping and freeing + * @mmap_offset: for CPU-mapping the allocation by mmapping a render node + * for userptrs this is overloaded to specify the CPU address + * @gpu_id: device identifier + * @flags: memory type and attributes. See KFD_IOC_ALLOC_MEM_FLAGS above + */ struct kfd_ioctl_alloc_memory_of_gpu_args { - uint64_t va_addr; /* to KFD */ - uint64_t size; /* to KFD */ - uint64_t handle; /* from KFD */ - uint64_t mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ - uint32_t gpu_id; /* to KFD */ - uint32_t flags; + __u64 va_addr; /* to KFD */ + __u64 size; /* to KFD */ + __u64 handle; /* from KFD */ + __u64 mmap_offset; /* to KFD (userptr), from KFD (mmap offset) */ + __u32 gpu_id; /* to KFD */ + __u32 flags; }; +/* Free memory allocated with kfd_ioctl_alloc_memory_of_gpu + * + * @handle: memory handle returned by alloc + */ struct kfd_ioctl_free_memory_of_gpu_args { - uint64_t handle; /* to KFD */ + __u64 handle; /* to KFD */ }; +/* Map memory to one or more GPUs + * + * @handle: memory handle returned by alloc + * @device_ids_array_ptr: array of gpu_ids (__u32 per device) + * @n_devices: number of devices in the array + * @n_success: number of devices mapped successfully + * + * @n_success returns information to the caller how many devices from + * the start of the array have mapped the buffer successfully. It can + * be passed into a subsequent retry call to skip those devices. For + * the first call the caller should initialize it to 0. + * + * If the ioctl completes with return code 0 (success), n_success == + * n_devices. + */ struct kfd_ioctl_map_memory_to_gpu_args { - uint64_t handle; /* to KFD */ - uint64_t device_ids_array_ptr; /* to KFD */ - uint32_t device_ids_array_size; /* to KFD */ - uint32_t pad; + __u64 handle; /* to KFD */ + __u64 device_ids_array_ptr; /* to KFD */ + __u32 n_devices; /* to KFD */ + __u32 n_success; /* to/from KFD */ }; +/* Unmap memory from one or more GPUs + * + * same arguments as for mapping + */ struct kfd_ioctl_unmap_memory_from_gpu_args { - uint64_t handle; /* to KFD */ - uint64_t device_ids_array_ptr; /* to KFD */ - uint32_t device_ids_array_size; /* to KFD */ - uint32_t pad; -}; - -struct kfd_ioctl_set_process_dgpu_aperture_args { - uint64_t dgpu_base; - uint64_t dgpu_limit; - uint32_t gpu_id; - uint32_t pad; + __u64 handle; /* to KFD */ + __u64 device_ids_array_ptr; /* to KFD */ + __u32 n_devices; /* to KFD */ + __u32 n_success; /* to/from KFD */ }; struct kfd_ioctl_get_dmabuf_info_args { @@ -472,59 +498,56 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_DBG_WAVE_CONTROL \ AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args) -#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU \ - AMDKFD_IOWR(0x11, struct kfd_ioctl_alloc_memory_of_gpu_args) - -#define AMDKFD_IOC_FREE_MEMORY_OF_GPU \ - AMDKFD_IOWR(0x12, struct kfd_ioctl_free_memory_of_gpu_args) - -#define AMDKFD_IOC_MAP_MEMORY_TO_GPU \ - AMDKFD_IOWR(0x13, struct kfd_ioctl_map_memory_to_gpu_args) - -#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ - AMDKFD_IOWR(0x14, struct kfd_ioctl_unmap_memory_from_gpu_args) - -#define AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH \ - AMDKFD_IOWR(0x15, struct kfd_ioctl_alloc_memory_of_scratch_args) - -#define AMDKFD_IOC_SET_CU_MASK \ - AMDKFD_IOW(0x16, struct kfd_ioctl_set_cu_mask_args) - -#define AMDKFD_IOC_SET_PROCESS_DGPU_APERTURE \ - AMDKFD_IOW(0x17, \ - struct kfd_ioctl_set_process_dgpu_aperture_args) - -#define AMDKFD_IOC_SET_TRAP_HANDLER \ - AMDKFD_IOW(0x18, struct kfd_ioctl_set_trap_handler_args) - -#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ - AMDKFD_IOWR(0x19, struct kfd_ioctl_get_process_apertures_new_args) - -#define AMDKFD_IOC_GET_DMABUF_INFO \ - AMDKFD_IOWR(0x1A, struct kfd_ioctl_get_dmabuf_info_args) - -#define AMDKFD_IOC_IMPORT_DMABUF \ - AMDKFD_IOWR(0x1B, struct kfd_ioctl_import_dmabuf_args) +#define AMDKFD_IOC_SET_SCRATCH_BACKING_VA \ + AMDKFD_IOWR(0x11, struct kfd_ioctl_set_scratch_backing_va_args) #define AMDKFD_IOC_GET_TILE_CONFIG \ - AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_tile_config_args) + AMDKFD_IOWR(0x12, struct kfd_ioctl_get_tile_config_args) -#define AMDKFD_IOC_IPC_IMPORT_HANDLE \ - AMDKFD_IOWR(0x1D, struct kfd_ioctl_ipc_import_handle_args) +#define AMDKFD_IOC_SET_TRAP_HANDLER \ + AMDKFD_IOW(0x13, struct kfd_ioctl_set_trap_handler_args) -#define AMDKFD_IOC_IPC_EXPORT_HANDLE \ - AMDKFD_IOWR(0x1E, struct kfd_ioctl_ipc_export_handle_args) - -#define AMDKFD_IOC_CROSS_MEMORY_COPY \ - AMDKFD_IOWR(0x1F, struct kfd_ioctl_cross_memory_copy_args) - -#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_get_queue_wave_state_args) +#define AMDKFD_IOC_GET_PROCESS_APERTURES_NEW \ + AMDKFD_IOWR(0x14, \ + struct kfd_ioctl_get_process_apertures_new_args) #define AMDKFD_IOC_ACQUIRE_VM \ - AMDKFD_IOW(0x21, struct kfd_ioctl_acquire_vm_args) + AMDKFD_IOW(0x15, struct kfd_ioctl_acquire_vm_args) + +#define AMDKFD_IOC_ALLOC_MEMORY_OF_GPU \ + AMDKFD_IOWR(0x16, struct kfd_ioctl_alloc_memory_of_gpu_args) + +#define AMDKFD_IOC_FREE_MEMORY_OF_GPU \ + AMDKFD_IOW(0x17, struct kfd_ioctl_free_memory_of_gpu_args) + +#define AMDKFD_IOC_MAP_MEMORY_TO_GPU \ + AMDKFD_IOWR(0x18, struct kfd_ioctl_map_memory_to_gpu_args) + +#define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \ + AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args) + +#define AMDKFD_IOC_SET_CU_MASK \ + AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args) + +#define AMDKFD_IOC_GET_QUEUE_WAVE_STATE \ + AMDKFD_IOWR(0x1B, struct kfd_ioctl_get_queue_wave_state_args) + +#define AMDKFD_IOC_GET_DMABUF_INFO \ + AMDKFD_IOWR(0x1C, struct kfd_ioctl_get_dmabuf_info_args) + +#define AMDKFD_IOC_IMPORT_DMABUF \ + AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args) + +#define AMDKFD_IOC_IPC_IMPORT_HANDLE \ + AMDKFD_IOWR(0x1E, struct kfd_ioctl_ipc_import_handle_args) + +#define AMDKFD_IOC_IPC_EXPORT_HANDLE \ + AMDKFD_IOWR(0x1F, struct kfd_ioctl_ipc_export_handle_args) + +#define AMDKFD_IOC_CROSS_MEMORY_COPY \ + AMDKFD_IOWR(0x20, struct kfd_ioctl_cross_memory_copy_args) #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x22 +#define AMDKFD_COMMAND_END 0x21 #endif diff --git a/src/fmm.c b/src/fmm.c index f3398ac245..e550a9e957 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -742,7 +742,6 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, args.size = ALIGN_UP(MemorySizeInBytes, aperture->align); args.flags = flags | - KFD_IOC_ALLOC_MEM_FLAGS_NONPAGED | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; args.va_addr = (uint64_t)mem; if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && @@ -879,13 +878,13 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) if (flags.ui32.AQLQueueMemory) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; - if (flags.ui32.ReadOnly) - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_READONLY; + if (!flags.ui32.ReadOnly) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE; /* TODO: Since, ROCr interfaces doesn't allow caller to set page * permissions, mark all user allocations with exec permission. * Check for flags.ui32.ExecuteAccess once ROCr is ready. */ - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS; + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; return ioc_flags; } @@ -893,7 +892,7 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) { manageable_aperture_t *aperture_phy; - struct kfd_ioctl_alloc_memory_of_scratch_args args = {0}; + struct kfd_ioctl_set_scratch_backing_va_args args = {0}; int32_t gpu_mem_id; void *mem = NULL; uint64_t aligned_size = ALIGN_UP(MemorySizeInBytes, SCRATCH_ALIGN); @@ -941,12 +940,11 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) aperture_phy->base = mem; aperture_phy->limit = VOID_PTR_ADD(mem, aligned_size-1); - /* Allocate memory from amdkfd (just programs SH_HIDDEN_PRIVATE_BASE) */ + /* Program SH_HIDDEN_PRIVATE_BASE */ args.gpu_id = gpu_id; - args.size = MemorySizeInBytes; args.va_addr = ((uint64_t)mem) >> 16; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_SCRATCH, &args)) { + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_SCRATCH_BACKING_VA, &args)) { fmm_release_scratch(gpu_id); return NULL; } @@ -1082,6 +1080,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Use fine-grained aperture */ aperture = &svm.dgpu_alt_aperture; ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, @@ -1994,6 +1993,7 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, { struct kfd_ioctl_map_memory_to_gpu_args args = {0}; vm_object_t *object; + int ret = 0; if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); @@ -2002,8 +2002,10 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, if (!object) { /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); - if (!object) + if (!object) { + ret = -EINVAL; goto err_object_not_found; + } } /* For a memory region that is registered by user pointer, changing @@ -2019,24 +2021,25 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, if (nodes_to_map) { /* If specified, map the requested */ args.device_ids_array_ptr = (uint64_t)nodes_to_map; - args.device_ids_array_size = nodes_array_size; + args.n_devices = nodes_array_size / sizeof(uint32_t); } else if (object->registered_device_id_array_size > 0) { /* otherwise map all registered */ args.device_ids_array_ptr = (uint64_t)object->registered_device_id_array; - args.device_ids_array_size = object->registered_device_id_array_size; + args.n_devices = object->registered_device_id_array_size / + sizeof(uint32_t); } else { /* not specified, not registered: map all GPUs */ args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; - args.device_ids_array_size = all_gpu_id_array_size; + args.n_devices = all_gpu_id_array_size / sizeof(uint32_t); } + args.n_success = 0; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args)) - goto err_map_ioctl_failed; + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args); add_device_ids_to_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_success * sizeof(uint32_t)); print_device_id_array((uint32_t *)object->mapped_device_id_array, object->mapped_device_id_array_size); @@ -2050,16 +2053,11 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, } exit_ok: - if (!obj) - pthread_mutex_unlock(&aperture->fmm_mutex); - - return 0; - -err_map_ioctl_failed: err_object_not_found: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; + + return ret; } static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, @@ -2089,19 +2087,18 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* allocate object within the scratch backing aperture */ if (!ret && !is_debugger) { vm_object_t *obj = fmm_allocate_memory_object( - gpu_id, address, size, aperture, - NULL, KFD_IOC_ALLOC_MEM_FLAGS_VRAM); + gpu_id, address, size, aperture, NULL, + KFD_IOC_ALLOC_MEM_FLAGS_VRAM | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); if (!obj) return -1; } else { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_mem[gpu_mem_id].drm_render_fd; - fmm_allocate_memory_object(gpu_id, - address, - size, - aperture, - &mmap_offset, - KFD_IOC_ALLOC_MEM_FLAGS_GTT); + fmm_allocate_memory_object( + gpu_id, address, size, aperture, &mmap_offset, + KFD_IOC_ALLOC_MEM_FLAGS_GTT | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); @@ -2292,10 +2289,11 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, args.handle = object->handle; if (device_ids_array && device_ids_array_size > 0) { args.device_ids_array_ptr = (uint64_t)device_ids_array; - args.device_ids_array_size = device_ids_array_size; + args.n_devices = device_ids_array_size / sizeof(uint32_t); } else if (object->mapped_device_id_array_size > 0) { args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; - args.device_ids_array_size = object->mapped_device_id_array_size; + args.n_devices = object->mapped_device_id_array_size / + sizeof(uint32_t); } else { /* * When unmap exits here it should return failing error code as the user tried to @@ -2305,17 +2303,16 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, ret = 0; goto out; } + args.n_success = 0; print_device_id_array((void *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_devices * sizeof(uint32_t)); ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); - if (ret != 0) - goto out; remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_success * sizeof(uint32_t)); if (object->mapped_node_id_array) free(object->mapped_node_id_array); @@ -2335,6 +2332,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, int32_t gpu_mem_id; vm_object_t *object; struct kfd_ioctl_unmap_memory_from_gpu_args args = {0}; + int ret; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -2348,8 +2346,10 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, /* Find the object to retrieve the handle and size */ object = vm_find_object_by_address(aperture, address, 0); - if (!object) + if (!object) { + ret = -EINVAL; goto err; + } if (!object->mapped_device_id_array || object->mapped_device_id_array_size == 0) { @@ -2357,21 +2357,24 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, return 0; } - /* unmap from GPU */ args.handle = object->handle; args.device_ids_array_ptr = (uint64_t)object->mapped_device_id_array; - args.device_ids_array_size = object->mapped_device_id_array_size; - kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + args.n_devices = object->mapped_device_id_array_size / sizeof(uint32_t); + args.n_success = 0; + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, - args.device_ids_array_size); + args.n_success * sizeof(uint32_t)); if (object->mapped_node_id_array) free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; + if (ret) + goto err; + pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ @@ -2381,7 +2384,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, err: pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; + return ret; } static int _fmm_unmap_from_gpu_userptr(void *addr) @@ -2549,7 +2552,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | - KFD_IOC_ALLOC_MEM_FLAGS_EXECUTE_ACCESS, &obj); + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | + KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE, &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; From 7495e742575fc5e315392d3901d89e9a4b206b8c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 23 May 2018 18:16:15 -0400 Subject: [PATCH 0344/1247] Cosmetic changes to kfd_ioctl.h Make it more similar with upstream. Change-Id: I982ccfd4045d96e3c30bc84d38d0e03db8de9b08 Signed-off-by: Felix Kuehling --- include/linux/kfd_ioctl.h | 354 +++++++++++++++++++------------------- src/events.c | 2 +- 2 files changed, 178 insertions(+), 178 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 35d4c385cf..fdedc63c30 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -30,8 +30,8 @@ #define KFD_IOCTL_MINOR_VERSION 2 struct kfd_ioctl_get_version_args { - uint32_t major_version; /* from KFD */ - uint32_t minor_version; /* from KFD */ + __u32 major_version; /* from KFD */ + __u32 minor_version; /* from KFD */ }; /* For kfd_ioctl_create_queue_args.queue_type. */ @@ -43,51 +43,51 @@ struct kfd_ioctl_get_version_args { #define KFD_MAX_QUEUE_PRIORITY 15 struct kfd_ioctl_create_queue_args { - uint64_t ring_base_address; /* to KFD */ - uint64_t write_pointer_address; /* from KFD */ - uint64_t read_pointer_address; /* from KFD */ - uint64_t doorbell_offset; /* from KFD */ + __u64 ring_base_address; /* to KFD */ + __u64 write_pointer_address; /* from KFD */ + __u64 read_pointer_address; /* from KFD */ + __u64 doorbell_offset; /* from KFD */ - uint32_t ring_size; /* to KFD */ - uint32_t gpu_id; /* to KFD */ - uint32_t queue_type; /* to KFD */ - uint32_t queue_percentage; /* to KFD */ - uint32_t queue_priority; /* to KFD */ - uint32_t queue_id; /* from KFD */ + __u32 ring_size; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 queue_type; /* to KFD */ + __u32 queue_percentage; /* to KFD */ + __u32 queue_priority; /* to KFD */ + __u32 queue_id; /* from KFD */ - uint64_t eop_buffer_address; /* to KFD */ - uint64_t eop_buffer_size; /* to KFD */ - uint64_t ctx_save_restore_address; /* to KFD */ - uint32_t ctx_save_restore_size; /* to KFD */ - uint32_t ctl_stack_size; /* to KFD */ + __u64 eop_buffer_address; /* to KFD */ + __u64 eop_buffer_size; /* to KFD */ + __u64 ctx_save_restore_address; /* to KFD */ + __u32 ctx_save_restore_size; /* to KFD */ + __u32 ctl_stack_size; /* to KFD */ }; struct kfd_ioctl_destroy_queue_args { - uint32_t queue_id; /* to KFD */ - uint32_t pad; + __u32 queue_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_update_queue_args { - uint64_t ring_base_address; /* to KFD */ + __u64 ring_base_address; /* to KFD */ - uint32_t queue_id; /* to KFD */ - uint32_t ring_size; /* to KFD */ - uint32_t queue_percentage; /* to KFD */ - uint32_t queue_priority; /* to KFD */ + __u32 queue_id; /* to KFD */ + __u32 ring_size; /* to KFD */ + __u32 queue_percentage; /* to KFD */ + __u32 queue_priority; /* to KFD */ }; struct kfd_ioctl_set_cu_mask_args { - uint32_t queue_id; /* to KFD */ - uint32_t num_cu_mask; /* to KFD */ - uint64_t cu_mask_ptr; /* to KFD */ + __u32 queue_id; /* to KFD */ + __u32 num_cu_mask; /* to KFD */ + __u64 cu_mask_ptr; /* to KFD */ }; struct kfd_ioctl_get_queue_wave_state_args { - uint64_t ctl_stack_address; /* to KFD */ - uint32_t ctl_stack_used_size; /* from KFD */ - uint32_t save_area_used_size; /* from KFD */ - uint32_t queue_id; /* to KFD */ - uint32_t pad; + __u64 ctl_stack_address; /* to KFD */ + __u32 ctl_stack_used_size; /* from KFD */ + __u32 save_area_used_size; /* from KFD */ + __u32 queue_id; /* to KFD */ + __u32 pad; }; /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ @@ -95,20 +95,13 @@ struct kfd_ioctl_get_queue_wave_state_args { #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 struct kfd_ioctl_set_memory_policy_args { - uint64_t alternate_aperture_base; /* to KFD */ - uint64_t alternate_aperture_size; /* to KFD */ + __u64 alternate_aperture_base; /* to KFD */ + __u64 alternate_aperture_size; /* to KFD */ - uint32_t gpu_id; /* to KFD */ - uint32_t default_policy; /* to KFD */ - uint32_t alternate_policy; /* to KFD */ - uint32_t pad; -}; - -struct kfd_ioctl_set_trap_handler_args { - uint64_t tba_addr; - uint64_t tma_addr; - uint32_t gpu_id; /* to KFD */ - uint32_t pad; + __u32 gpu_id; /* to KFD */ + __u32 default_policy; /* to KFD */ + __u32 alternate_policy; /* to KFD */ + __u32 pad; }; /* @@ -119,52 +112,52 @@ struct kfd_ioctl_set_trap_handler_args { */ struct kfd_ioctl_get_clock_counters_args { - uint64_t gpu_clock_counter; /* from KFD */ - uint64_t cpu_clock_counter; /* from KFD */ - uint64_t system_clock_counter; /* from KFD */ - uint64_t system_clock_freq; /* from KFD */ + __u64 gpu_clock_counter; /* from KFD */ + __u64 cpu_clock_counter; /* from KFD */ + __u64 system_clock_counter; /* from KFD */ + __u64 system_clock_freq; /* from KFD */ - uint32_t gpu_id; /* to KFD */ - uint32_t pad; + __u32 gpu_id; /* to KFD */ + __u32 pad; }; -#define NUM_OF_SUPPORTED_GPUS 7 - struct kfd_process_device_apertures { - uint64_t lds_base; /* from KFD */ - uint64_t lds_limit; /* from KFD */ - uint64_t scratch_base; /* from KFD */ - uint64_t scratch_limit; /* from KFD */ - uint64_t gpuvm_base; /* from KFD */ - uint64_t gpuvm_limit; /* from KFD */ - uint32_t gpu_id; /* from KFD */ - uint32_t pad; + __u64 lds_base; /* from KFD */ + __u64 lds_limit; /* from KFD */ + __u64 scratch_base; /* from KFD */ + __u64 scratch_limit; /* from KFD */ + __u64 gpuvm_base; /* from KFD */ + __u64 gpuvm_limit; /* from KFD */ + __u32 gpu_id; /* from KFD */ + __u32 pad; }; -/* This IOCTL and the limited NUM_OF_SUPPORTED_GPUS is deprecated. Use - * kfd_ioctl_get_process_apertures_new instead, which supports - * arbitrary numbers of GPUs. +/* + * AMDKFD_IOC_GET_PROCESS_APERTURES is deprecated. Use + * AMDKFD_IOC_GET_PROCESS_APERTURES_NEW instead, which supports an + * unlimited number of GPUs. */ +#define NUM_OF_SUPPORTED_GPUS 7 struct kfd_ioctl_get_process_apertures_args { struct kfd_process_device_apertures process_apertures[NUM_OF_SUPPORTED_GPUS];/* from KFD */ /* from KFD, should be in the range [1 - NUM_OF_SUPPORTED_GPUS] */ - uint32_t num_of_nodes; - uint32_t pad; + __u32 num_of_nodes; + __u32 pad; }; struct kfd_ioctl_get_process_apertures_new_args { /* User allocated. Pointer to struct kfd_process_device_apertures * filled in by Kernel */ - uint64_t kfd_process_device_apertures_ptr; + __u64 kfd_process_device_apertures_ptr; /* to KFD - indicates amount of memory present in * kfd_process_device_apertures_ptr * from KFD - Number of entries filled by KFD. */ - uint32_t num_of_nodes; - uint32_t pad; + __u32 num_of_nodes; + __u32 pad; }; #define MAX_ALLOWED_NUM_POINTS 100 @@ -172,83 +165,83 @@ struct kfd_ioctl_get_process_apertures_new_args { #define MAX_ALLOWED_WAC_BUFF_SIZE 128 struct kfd_ioctl_dbg_register_args { - uint32_t gpu_id; /* to KFD */ - uint32_t pad; + __u32 gpu_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_dbg_unregister_args { - uint32_t gpu_id; /* to KFD */ - uint32_t pad; + __u32 gpu_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_dbg_address_watch_args { - uint64_t content_ptr; /* a pointer to the actual content */ - uint32_t gpu_id; /* to KFD */ - uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ + __u64 content_ptr; /* a pointer to the actual content */ + __u32 gpu_id; /* to KFD */ + __u32 buf_size_in_bytes; /*including gpu_id and buf_size */ }; struct kfd_ioctl_dbg_wave_control_args { - uint64_t content_ptr; /* a pointer to the actual content */ - uint32_t gpu_id; /* to KFD */ - uint32_t buf_size_in_bytes; /*including gpu_id and buf_size */ + __u64 content_ptr; /* a pointer to the actual content */ + __u32 gpu_id; /* to KFD */ + __u32 buf_size_in_bytes; /*including gpu_id and buf_size */ }; /* Matching HSA_EVENTTYPE */ -#define KFD_IOC_EVENT_SIGNAL 0 -#define KFD_IOC_EVENT_NODECHANGE 1 -#define KFD_IOC_EVENT_DEVICESTATECHANGE 2 -#define KFD_IOC_EVENT_HW_EXCEPTION 3 -#define KFD_IOC_EVENT_SYSTEM_EVENT 4 -#define KFD_IOC_EVENT_DEBUG_EVENT 5 -#define KFD_IOC_EVENT_PROFILE_EVENT 6 -#define KFD_IOC_EVENT_QUEUE_EVENT 7 -#define KFD_IOC_EVENT_MEMORY 8 +#define KFD_IOC_EVENT_SIGNAL 0 +#define KFD_IOC_EVENT_NODECHANGE 1 +#define KFD_IOC_EVENT_DEVICESTATECHANGE 2 +#define KFD_IOC_EVENT_HW_EXCEPTION 3 +#define KFD_IOC_EVENT_SYSTEM_EVENT 4 +#define KFD_IOC_EVENT_DEBUG_EVENT 5 +#define KFD_IOC_EVENT_PROFILE_EVENT 6 +#define KFD_IOC_EVENT_QUEUE_EVENT 7 +#define KFD_IOC_EVENT_MEMORY 8 -#define KFD_IOC_WAIT_RESULT_COMPLETE 0 -#define KFD_IOC_WAIT_RESULT_TIMEOUT 1 -#define KFD_IOC_WAIT_RESULT_FAIL 2 +#define KFD_IOC_WAIT_RESULT_COMPLETE 0 +#define KFD_IOC_WAIT_RESULT_TIMEOUT 1 +#define KFD_IOC_WAIT_RESULT_FAIL 2 -#define KFD_SIGNAL_EVENT_LIMIT 4096 +#define KFD_SIGNAL_EVENT_LIMIT 4096 struct kfd_ioctl_create_event_args { - uint64_t event_page_offset; /* from KFD */ - uint32_t event_trigger_data; /* from KFD - signal events only */ - uint32_t event_type; /* to KFD */ - uint32_t auto_reset; /* to KFD */ - uint32_t node_id; /* to KFD - only valid for certain + __u64 event_page_offset; /* from KFD */ + __u32 event_trigger_data; /* from KFD - signal events only */ + __u32 event_type; /* to KFD */ + __u32 auto_reset; /* to KFD */ + __u32 node_id; /* to KFD - only valid for certain event types */ - uint32_t event_id; /* from KFD */ - uint32_t event_slot_index; /* from KFD */ + __u32 event_id; /* from KFD */ + __u32 event_slot_index; /* from KFD */ }; struct kfd_ioctl_destroy_event_args { - uint32_t event_id; /* to KFD */ - uint32_t pad; + __u32 event_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_set_event_args { - uint32_t event_id; /* to KFD */ - uint32_t pad; + __u32 event_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_reset_event_args { - uint32_t event_id; /* to KFD */ - uint32_t pad; + __u32 event_id; /* to KFD */ + __u32 pad; }; struct kfd_memory_exception_failure { - uint32_t NotPresent; /* Page not present or supervisor privilege */ - uint32_t ReadOnly; /* Write access to a read-only page */ - uint32_t NoExecute; /* Execute access to a page marked NX */ - uint32_t imprecise; /* Can't determine the exact fault address */ + __u32 NotPresent; /* Page not present or supervisor privilege */ + __u32 ReadOnly; /* Write access to a read-only page */ + __u32 NoExecute; /* Execute access to a page marked NX */ + __u32 imprecise; /* Can't determine the exact fault address */ }; /* memory exception data */ struct kfd_hsa_memory_exception_data { struct kfd_memory_exception_failure failure; - uint64_t va; - uint32_t gpu_id; - uint32_t pad; + __u64 va; + __u32 gpu_id; + __u32 pad; }; /* Event data */ @@ -256,30 +249,60 @@ struct kfd_event_data { union { struct kfd_hsa_memory_exception_data memory_exception_data; }; /* From KFD */ - uint64_t kfd_event_data_ext; /* pointer to an extension structure - for future exception types */ - uint32_t event_id; /* to KFD */ - uint32_t pad; + __u64 kfd_event_data_ext; /* pointer to an extension structure + for future exception types */ + __u32 event_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_wait_events_args { - uint64_t events_ptr; /* pointed to struct + __u64 events_ptr; /* pointed to struct kfd_event_data array, to KFD */ - uint32_t num_events; /* to KFD */ - uint32_t wait_for_all; /* to KFD */ - uint32_t timeout; /* to KFD */ - uint32_t wait_result; /* from KFD */ + __u32 num_events; /* to KFD */ + __u32 wait_for_all; /* to KFD */ + __u32 timeout; /* to KFD */ + __u32 wait_result; /* from KFD */ }; struct kfd_ioctl_set_scratch_backing_va_args { - uint64_t va_addr; /* to KFD */ - uint32_t gpu_id; /* to KFD */ - uint32_t pad; + __u64 va_addr; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + +struct kfd_ioctl_get_tile_config_args { + /* to KFD: pointer to tile array */ + __u64 tile_config_ptr; + /* to KFD: pointer to macro tile array */ + __u64 macro_tile_config_ptr; + /* to KFD: array size allocated by user mode + * from KFD: array size filled by kernel + */ + __u32 num_tile_configs; + /* to KFD: array size allocated by user mode + * from KFD: array size filled by kernel + */ + __u32 num_macro_tile_configs; + + __u32 gpu_id; /* to KFD */ + __u32 gb_addr_config; /* from KFD */ + __u32 num_banks; /* from KFD */ + __u32 num_ranks; /* from KFD */ + /* struct size can be extended later if needed + * without breaking ABI compatibility + */ +}; + +struct kfd_ioctl_set_trap_handler_args { + __u64 tba_addr; /* to KFD */ + __u64 tma_addr; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_acquire_vm_args { - uint32_t drm_fd; /* to KFD */ - uint32_t gpu_id; /* to KFD */ + __u32 drm_fd; /* to KFD */ + __u32 gpu_id; /* to KFD */ }; /* Allocation flags: memory types */ @@ -358,64 +381,41 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { }; struct kfd_ioctl_get_dmabuf_info_args { - uint64_t size; /* from KFD */ - uint64_t metadata_ptr; /* to KFD */ - uint32_t metadata_size; /* to KFD (space allocated by user) + __u64 size; /* from KFD */ + __u64 metadata_ptr; /* to KFD */ + __u32 metadata_size; /* to KFD (space allocated by user) * from KFD (actual metadata size) */ - uint32_t gpu_id; /* from KFD */ - uint32_t flags; /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */ - uint32_t dmabuf_fd; /* to KFD */ + __u32 gpu_id; /* from KFD */ + __u32 flags; /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */ + __u32 dmabuf_fd; /* to KFD */ }; struct kfd_ioctl_import_dmabuf_args { - uint64_t va_addr; /* to KFD */ - uint64_t handle; /* from KFD */ - uint32_t gpu_id; /* to KFD */ - uint32_t dmabuf_fd; /* to KFD */ + __u64 va_addr; /* to KFD */ + __u64 handle; /* from KFD */ + __u32 gpu_id; /* to KFD */ + __u32 dmabuf_fd; /* to KFD */ }; struct kfd_ioctl_ipc_export_handle_args { - uint64_t handle; /* to KFD */ - uint32_t share_handle[4]; /* from KFD */ - uint32_t gpu_id; /* to KFD */ - uint32_t pad; + __u64 handle; /* to KFD */ + __u32 share_handle[4]; /* from KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_ipc_import_handle_args { - uint64_t handle; /* from KFD */ - uint64_t va_addr; /* to KFD */ - uint64_t mmap_offset; /* from KFD */ - uint32_t share_handle[4]; /* to KFD */ - uint32_t gpu_id; /* to KFD */ - uint32_t pad; -}; - -struct kfd_ioctl_get_tile_config_args { - /* to KFD: pointer to tile array */ - __u64 tile_config_ptr; - /* to KFD: pointer to macro tile array */ - __u64 macro_tile_config_ptr; - /* to KFD: array size allocated by user mode - * from KFD: array size filled by kernel - */ - __u32 num_tile_configs; - /* to KFD: array size allocated by user mode - * from KFD: array size filled by kernel - */ - __u32 num_macro_tile_configs; - + __u64 handle; /* from KFD */ + __u64 va_addr; /* to KFD */ + __u64 mmap_offset; /* from KFD */ + __u32 share_handle[4]; /* to KFD */ __u32 gpu_id; /* to KFD */ - __u32 gb_addr_config; /* from KFD */ - __u32 num_banks; /* from KFD */ - __u32 num_ranks; /* from KFD */ - /* struct size can be extended later if needed - * without breaking ABI compatibility - */ + __u32 pad; }; struct kfd_memory_range { - uint64_t va_addr; - uint64_t size; + __u64 va_addr; + __u64 size; }; /* flags definitions @@ -429,19 +429,19 @@ struct kfd_memory_range { struct kfd_ioctl_cross_memory_copy_args { /* to KFD: Process ID of the remote process */ - uint32_t pid; + __u32 pid; /* to KFD: See above definition */ - uint32_t flags; + __u32 flags; /* to KFD: Source GPU VM range */ - uint64_t src_mem_range_array; + __u64 src_mem_range_array; /* to KFD: Size of above array */ - uint64_t src_mem_array_size; + __u64 src_mem_array_size; /* to KFD: Destination GPU VM range */ - uint64_t dst_mem_range_array; + __u64 dst_mem_range_array; /* to KFD: Size of above array */ - uint64_t dst_mem_array_size; + __u64 dst_mem_array_size; /* from KFD: Total amount of bytes copied */ - uint64_t bytes_copied; + __u64 bytes_copied; }; #define AMDKFD_IOCTL_BASE 'K' diff --git a/src/events.c b/src/events.c index 65330c90ee..2e9d126fed 100644 --- a/src/events.c +++ b/src/events.c @@ -81,7 +81,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; } - fmm_get_handle(events_page, &args.event_page_offset); + fmm_get_handle(events_page, (uint64_t *)&args.event_page_offset); } if (kmtIoctl(kfd_fd, AMDKFD_IOC_CREATE_EVENT, &args) != 0) { From 04627449655e596c015119e36e78eb0c2841b485 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 25 Jan 2018 21:18:33 -0500 Subject: [PATCH 0345/1247] Add fallback for GPUVM doorbell mapping Upstream KFD doesn't support mapping doorbells to GPUVM yet. Fall back to the old method. Change-Id: I452a6fc59b88329b833844e3914c480c2f13c82d Signed-off-by: Felix Kuehling --- src/queues.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/queues.c b/src/queues.c index 4e26811ede..c84195c5a0 100644 --- a/src/queues.c +++ b/src/queues.c @@ -304,9 +304,16 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, get_doorbell_map_info(get_device_id_by_node(NodeId), &doorbells[NodeId]); - if (doorbells[NodeId].use_gpuvm) + if (doorbells[NodeId].use_gpuvm) { status = map_doorbell_dgpu(NodeId, gpu_id, doorbell_offset); - else + if (status != HSAKMT_STATUS_SUCCESS) { + /* Fall back to the old method if KFD doesn't + * support doorbells in GPUVM + */ + doorbells[NodeId].use_gpuvm = false; + status = map_doorbell_apu(NodeId, gpu_id, doorbell_offset); + } + } else status = map_doorbell_apu(NodeId, gpu_id, doorbell_offset); if (status != HSAKMT_STATUS_SUCCESS) From 7a8566dc0342152b3b7c4ec7b2c7a87e813af013 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 11 Jun 2018 14:08:57 -0400 Subject: [PATCH 0346/1247] Improve the return value for hsaKmtOpenKFD() When KFD is already opened, Opening it again should return HSAKMT_STATUS_KERNEL_ALREADY_OPENED to align with the specification. Change-Id: Ib10a2d2c48781600bea7d072557d03ccb1a2bc19 Signed-off-by: Yong Zhao --- src/openclose.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openclose.c b/src/openclose.c index 403d38de30..92848f2b91 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -146,7 +146,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) init_counter_props(sys_props.NumNodes); } else { kfd_open_count++; - result = HSAKMT_STATUS_SUCCESS; + result = HSAKMT_STATUS_KERNEL_ALREADY_OPENED; } pthread_mutex_unlock(&hsakmt_mutex); From 9434223752e531a246793ce9bcad46cb30bcc855 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 28 May 2018 15:56:01 -0400 Subject: [PATCH 0347/1247] Clean up cmake install and package * Use GNUInstallDirs * Install headers in $prefix/include directly, drop symlink * Install libraries in $prefix/lib directly, drop symlink * Move LICENSE.md from hsakmt-roct-dev to hsakmt-roct Change-Id: I43562f15cc03029be53e9ec18c337824d8116659 Signed-off-by: Slava Grigorev Signed-off-by: Felix Kuehling --- CMakeLists.txt | 5 ++++- DEBIAN/postinst | 2 +- DEBIAN/x86_64-libhsakmt.conf | 2 +- RPM/rpm_post | 2 +- hsakmt-dev.txt | 13 +++---------- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f0781b0c9..cb2605b725 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,7 @@ endif () ## Include common cmake modules include ( utils ) +include ( GNUInstallDirs ) ## Setup the package version. get_version ( "1.0.0" ) @@ -137,7 +138,8 @@ add_custom_target ( package-dev DEPENDS build-dev WORKING_DIRECTORY ${DEV_BUILD_DIR} ) ## Add the install directives for the runtime library. -install ( TARGETS ${HSAKMT_TARGET} LIBRARY DESTINATION libhsakmt/lib COMPONENT ${HSAKMT_COMPONENT} ) +install ( TARGETS ${HSAKMT_TARGET} DESTINATION ${CMAKE_INSTALL_LIBDIR} ) +install ( FILES ${SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt ) ## Add the packaging directives for the runtime library. set ( CPACK_PACKAGE_NAME ${HSAKMT_PACKAGE} ) @@ -147,6 +149,7 @@ set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT library for AMD KFD support" ) +set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) diff --git a/DEBIAN/postinst b/DEBIAN/postinst index b6a16c013a..9510db5d1e 100755 --- a/DEBIAN/postinst +++ b/DEBIAN/postinst @@ -3,7 +3,7 @@ set -e do_ldconfig() { - echo /opt/rocm/libhsakmt/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig + echo /opt/rocm/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig } case "$1" in diff --git a/DEBIAN/x86_64-libhsakmt.conf b/DEBIAN/x86_64-libhsakmt.conf index cfa6dc46b8..a4551d4820 100644 --- a/DEBIAN/x86_64-libhsakmt.conf +++ b/DEBIAN/x86_64-libhsakmt.conf @@ -1 +1 @@ -/opt/rocm/libhsakmt/lib +/opt/rocm/lib diff --git a/RPM/rpm_post b/RPM/rpm_post index 7585d83dea..abd8ec2a6e 100644 --- a/RPM/rpm_post +++ b/RPM/rpm_post @@ -1 +1 @@ -echo /opt/rocm/libhsakmt/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig +echo /opt/rocm/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index 3fd4d9bbdb..79924fde39 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -29,6 +29,8 @@ set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-dev" ) project ( ${HSAKMT_DEV_PACKAGE} ) +include ( GNUInstallDirs ) + ## Set the runtime package name. set ( HSAKMT_PACKAGE @HSAKMT_PACKAGE@ ) @@ -40,18 +42,9 @@ set ( BUILD_VERSION_PATCH @BUILD_VERSION_PATCH@ ) ## Verbose output. set ( CMAKE_VERBOSE_MAKEFILE on ) -## Create symlinks for packaging and install -add_custom_target ( ${HSAKMT_DEV_PACKAGE} ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/include/libhsakmt libhsakmt-link ) -add_custom_target ( ${HSAKMT_COMPONENT}.so-link ALL WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../libhsakmt/lib/libhsakmt.so libhsakmt.so-link ) - ## Set the install targets -install ( FILES ${SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt ) -install ( FILES ${SOURCE_DIR}/include/hsakmt.h DESTINATION libhsakmt/include/libhsakmt ) -install ( FILES ${SOURCE_DIR}/include/hsakmttypes.h DESTINATION libhsakmt/include/libhsakmt ) -install ( FILES ${SOURCE_DIR}/include/linux/kfd_ioctl.h DESTINATION libhsakmt/include/libhsakmt/linux ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt-link DESTINATION include RENAME libhsakmt ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.so-link DESTINATION lib RENAME libhsakmt.so ) install ( FILES libhsakmt.pc DESTINATION libhsakmt ) +install ( DIRECTORY ${SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_COMPONENT} ) ## Set the default generator types for the devel package. From 8ee5647814c81ff4667f7679d8efceb1c3e0159e Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Wed, 20 Jun 2018 13:30:06 +0800 Subject: [PATCH 0348/1247] THUNK: fix deregister memory issues __fmm_release actually fails to find the object if address is not pagesize aligned. And the caller did not notice this as __fmm_release has no err code return. So to fix this, move the object lookup in caller, and use vm-object instead. Also fmm_release will pass up the error code. Change-Id: Ib8ea1ea5ae844844fd20e8e01f0fdb841d218f2c Signed-off-by: xinhui pan --- src/fmm.c | 87 +++++++++++++++++++++++++--------------------------- src/fmm.h | 2 +- src/memory.c | 3 +- 3 files changed, 43 insertions(+), 49 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index e550a9e957..7ddbf7b80c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -215,7 +215,7 @@ static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( } extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); -static void __fmm_release(void *address, manageable_aperture_t *aperture); +static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address); @@ -1055,7 +1055,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla map_fd, mmap_offset); if (ret == MAP_FAILED) { - __fmm_release(mem, aperture); + __fmm_release(vm_obj, aperture); return NULL; } } @@ -1108,7 +1108,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, MAP_SHARED | MAP_FIXED, kfd_fd, doorbell_offset); if (ret == MAP_FAILED) { - __fmm_release(mem, aperture); + __fmm_release(vm_obj, aperture); return NULL; } } @@ -1262,7 +1262,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (ret == MAP_FAILED) { - __fmm_release(mem, aperture); + __fmm_release(vm_obj, aperture); return NULL; } @@ -1296,23 +1296,13 @@ void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, return fmm_allocate_host_cpu(MemorySizeInBytes, flags); } -static void __fmm_release(void *address, manageable_aperture_t *aperture) +static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args = {0}; - vm_object_t *object; + void *address; - if (!address) + if (!object) return; - - pthread_mutex_lock(&aperture->fmm_mutex); - - /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); - if (!object) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return; - } - /* If memory is user memory and it's still GPU mapped, munmap * would cause an eviction. If the restore happens quickly * enough, restore would also fail with an error message. So @@ -1321,6 +1311,8 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); + address = object->start; + if (address >= dgpu_shared_aperture_base && address <= dgpu_shared_aperture_limit) { /* Reset NUMA policy */ @@ -1334,48 +1326,52 @@ static void __fmm_release(void *address, manageable_aperture_t *aperture) pthread_mutex_unlock(&aperture->fmm_mutex); } -void fmm_release(void *address) +HSAKMT_STATUS fmm_release(void *address) { uint32_t i; - bool found = false; vm_object_t *object; + manageable_aperture_t *aperture = NULL; - for (i = 0; i < gpu_mem_count && !found; i++) { + for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if (address >= gpu_mem[i].scratch_physical.base && address <= gpu_mem[i].scratch_physical.limit) { fmm_release_scratch(gpu_mem[i].gpu_id); - return; + return HSAKMT_STATUS_SUCCESS; } if (address >= gpu_mem[i].gpuvm_aperture.base && address <= gpu_mem[i].gpuvm_aperture.limit) { - found = true; - __fmm_release(address, &gpu_mem[i].gpuvm_aperture); - fmm_print(gpu_mem[i].gpu_id); + aperture = &gpu_mem[i].gpuvm_aperture; + break; } } - if (!found) { + if (!aperture) { if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { - found = true; - __fmm_release(address, &svm.dgpu_aperture); - fmm_print(gpu_mem[i].gpu_id); + aperture = &svm.dgpu_aperture; } else if (address >= svm.dgpu_alt_aperture.base && address <= svm.dgpu_alt_aperture.limit) { - found = true; - __fmm_release(address, &svm.dgpu_alt_aperture); - fmm_print(gpu_mem[i].gpu_id); + aperture = &svm.dgpu_alt_aperture; } } - /* - * If memory address isn't inside of any defined GPU aperture - it - * refers to the system memory - */ - if (!found) { + if (aperture) { + pthread_mutex_lock(&aperture->fmm_mutex); + object = vm_find_object_by_address(aperture, address, 0); + pthread_mutex_unlock(&aperture->fmm_mutex); + if (!object) + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + __fmm_release(object, aperture); + if (i < gpu_mem_count) + fmm_print(gpu_mem[i].gpu_id); + } else { + /* + * If memory address isn't inside of any defined GPU aperture - it + * refers to the system memory + */ uint64_t size = 0; /* Release the vm object in CPUVM */ pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); @@ -1389,6 +1385,7 @@ void fmm_release(void *address) if (size) munmap(address, size); } + return HSAKMT_STATUS_SUCCESS; } static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_policy, @@ -2064,11 +2061,11 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert void *address, uint64_t size) { int32_t gpu_mem_id; - void *mem = NULL; int ret; bool is_debugger = 0; void *mmap_ret = NULL; uint64_t mmap_offset = 0; + vm_object_t *obj; /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); @@ -2086,7 +2083,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); /* allocate object within the scratch backing aperture */ if (!ret && !is_debugger) { - vm_object_t *obj = fmm_allocate_memory_object( + obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, NULL, KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); @@ -2095,7 +2092,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert } else { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_mem[gpu_mem_id].drm_render_fd; - fmm_allocate_memory_object( + obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_GTT | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); @@ -2103,7 +2100,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (mmap_ret == MAP_FAILED) { - __fmm_release(mem, aperture); + __fmm_release(obj, aperture); return -1; } } @@ -2112,7 +2109,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* map to GPU */ ret = _fmm_map_to_gpu(aperture, address, size, NULL, NULL, 0); if (ret != 0) - __fmm_release(mem, aperture); + __fmm_release(obj, aperture); return ret; } @@ -2378,7 +2375,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ - __fmm_release(address, aperture); + __fmm_release(object, aperture); return 0; @@ -2904,7 +2901,6 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) { manageable_aperture_t *aperture; vm_object_t *obj; - void *svm_addr; aperture = &svm.dgpu_aperture; @@ -2915,11 +2911,10 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } - svm_addr = obj->start; pthread_mutex_unlock(&aperture->fmm_mutex); /* Destroy BO */ - __fmm_release(svm_addr, aperture); + __fmm_release(obj, aperture); return HSAKMT_STATUS_SUCCESS; } @@ -2984,7 +2979,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) * userptrs means releasing the BO. */ pthread_mutex_unlock(&aperture->fmm_mutex); - __fmm_release(address, aperture); + __fmm_release(object, aperture); return HSAKMT_STATUS_SUCCESS; } diff --git a/src/fmm.h b/src/fmm.h index c874f6fe01..2154f5250b 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -54,7 +54,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_ void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); void fmm_print(uint32_t node); -void fmm_release(void *address); +HSAKMT_STATUS fmm_release(void *address); int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); int fmm_unmap_from_gpu(void *address); bool fmm_get_handle(void *address, uint64_t *handle); diff --git a/src/memory.c b/src/memory.c index 37e4d3229a..54f262a448 100644 --- a/src/memory.c +++ b/src/memory.c @@ -195,8 +195,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, return HSAKMT_STATUS_ERROR; } - fmm_release(MemoryAddress); - return HSAKMT_STATUS_SUCCESS; + return fmm_release(MemoryAddress); } HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress, From 4e766615d7c17dd6c522875afbb7eab471e6281d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 25 Jun 2018 14:23:16 -0400 Subject: [PATCH 0349/1247] Fixup previous commit Add back missing pthread_mutex_lock. Handle all error cases in fmm_release. Change-Id: I8efa561ddadfd769cede5bf86300215ba3fb3dd1 Signed-off-by: Felix Kuehling --- src/fmm.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 7ddbf7b80c..32c9b5b592 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1303,6 +1303,9 @@ static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) if (!object) return; + + pthread_mutex_lock(&aperture->fmm_mutex); + /* If memory is user memory and it's still GPU mapped, munmap * would cause an eviction. If the restore happens quickly * enough, restore would also fail with an error message. So @@ -1329,7 +1332,7 @@ static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) HSAKMT_STATUS fmm_release(void *address) { uint32_t i; - vm_object_t *object; + vm_object_t *object = NULL; manageable_aperture_t *aperture = NULL; for (i = 0; i < gpu_mem_count; i++) { @@ -1362,9 +1365,8 @@ HSAKMT_STATUS fmm_release(void *address) pthread_mutex_lock(&aperture->fmm_mutex); object = vm_find_object_by_address(aperture, address, 0); pthread_mutex_unlock(&aperture->fmm_mutex); - if (!object) - return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; - __fmm_release(object, aperture); + if (object) + __fmm_release(object, aperture); if (i < gpu_mem_count) fmm_print(gpu_mem[i].gpu_id); } else { @@ -1385,7 +1387,10 @@ HSAKMT_STATUS fmm_release(void *address) if (size) munmap(address, size); } - return HSAKMT_STATUS_SUCCESS; + + return object ? + HSAKMT_STATUS_SUCCESS : + HSAKMT_STATUS_MEMORY_NOT_REGISTERED; } static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_policy, From fb551a44af8433f50ebe76fb8a71b823745cfa26 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 25 Jun 2018 14:25:29 -0400 Subject: [PATCH 0350/1247] Fix compiler warning on Fedora 28 Avoid warnings of the type error: 'strncpy' specified bound 64 equals destination size With the destination being 0-initialized, subtracting 1 from the destination buffer size will ensure that the destination will be a 0-terminated string, even when it's truncated. Change-Id: I7c3a90482065ce4d020db215e3e41348de51a083 Signed-off-by: Felix Kuehling --- src/topology.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/topology.c b/src/topology.c index c409bef14e..32fa642b1d 100644 --- a/src/topology.c +++ b/src/topology.c @@ -669,7 +669,7 @@ static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, if (props->CComputeIdLo == apic_id) { /* Retrieve the CAL name of CPU node */ if (!is_apu) - strncpy((char *)props->AMDName, cpu_model_name, sizeof(props->AMDName)); + strncpy((char *)props->AMDName, cpu_model_name, sizeof(props->AMDName)-1); /* Convert from UTF8 to UTF16 */ for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = cpu_model_name[i]; @@ -867,7 +867,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, } /* Retrieve the CAL name of the node */ - strncpy((char *)props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName)); + strncpy((char *)props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName)-1); if (props->NumCPUCores) { /* Is APU node */ ret = topology_get_cpu_model_name(props, true); From 77ec699460d088f5e4790cef2f21bfd79a478aa1 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 27 Jun 2018 18:51:33 -0400 Subject: [PATCH 0351/1247] Fix a bug that fmm_init_process_apertures() returns incorrect value If opening drm render device fails (usually when the user is not a member of video group), fmm_init_process_apertures() still returns success, resulting in weird segfault in a later stage. Change-Id: Ifbde4481629988944ad7f384d59753c88e287fa9 Signed-off-by: Yong Zhao --- src/fmm.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 32c9b5b592..9aeca436af 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1704,11 +1704,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) /* Skip non-GPU nodes */ if (gpu_id != 0) { - gpu_mem[gpu_mem_count].drm_render_fd = - open_drm_render_device(props.DrmRenderMinor); - if (gpu_mem[gpu_mem_count].drm_render_fd <= 0) + int fd = open_drm_render_device(props.DrmRenderMinor); + if (fd <= 0) { + ret = HSAKMT_STATUS_ERROR; goto sysfs_parse_failed; + } + gpu_mem[gpu_mem_count].drm_render_fd = fd; gpu_mem[gpu_mem_count].gpu_id = gpu_id; gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; From 5972fac417cacd204a071aa916c2bbd01d20be59 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 15 Jun 2018 19:51:15 -0400 Subject: [PATCH 0352/1247] Rename two variable names in doorbells structure There were two doorbells, one embedded in another, which are very confusing. Change the member variable name to mapping to differentiate them. Also, rename doorbells_mutex to just mutext for brevity. Change-Id: Iaa14a1a3ee09449a9089fc1fb39c916fdf32fb44 Signed-off-by: Yong Zhao --- src/queues.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/queues.c b/src/queues.c index c84195c5a0..0055e5c5c6 100644 --- a/src/queues.c +++ b/src/queues.c @@ -149,8 +149,8 @@ struct queue { struct process_doorbells { bool use_gpuvm; uint32_t size; - void *doorbells; - pthread_mutex_t doorbells_mutex; + void *mapping; + pthread_mutex_t mutex; }; static unsigned int num_doorbells; @@ -171,8 +171,8 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) for (i = 0; i < NumNodes; i++) { doorbells[i].use_gpuvm = false; doorbells[i].size = 0; - doorbells[i].doorbells = NULL; - pthread_mutex_init(&doorbells[i].doorbells_mutex, NULL); + doorbells[i].mapping = NULL; + pthread_mutex_init(&doorbells[i].mutex, NULL); } num_doorbells = NumNodes; @@ -218,10 +218,10 @@ void destroy_process_doorbells(void) continue; if (doorbells[i].use_gpuvm) { - fmm_unmap_from_gpu(doorbells[i].doorbells); - fmm_release(doorbells[i].doorbells); + fmm_unmap_from_gpu(doorbells[i].mapping); + fmm_release(doorbells[i].mapping); } else - munmap(doorbells[i].doorbells, doorbells[i].size); + munmap(doorbells[i].mapping, doorbells[i].size); } free(doorbells); @@ -244,7 +244,7 @@ void clear_process_doorbells(void) continue; if (!doorbells[i].use_gpuvm) - munmap(doorbells[i].doorbells, doorbells[i].size); + munmap(doorbells[i].mapping, doorbells[i].size); } free(doorbells); @@ -263,7 +263,7 @@ static HSAKMT_STATUS map_doorbell_apu(HSAuint32 NodeId, HSAuint32 gpu_id, if (ptr == MAP_FAILED) return HSAKMT_STATUS_ERROR; - doorbells[NodeId].doorbells = ptr; + doorbells[NodeId].mapping = ptr; return HSAKMT_STATUS_SUCCESS; } @@ -285,7 +285,7 @@ static HSAKMT_STATUS map_doorbell_dgpu(HSAuint32 NodeId, HSAuint32 gpu_id, return HSAKMT_STATUS_ERROR; } - doorbells[NodeId].doorbells = ptr; + doorbells[NodeId].mapping = ptr; return HSAKMT_STATUS_SUCCESS; } @@ -295,9 +295,9 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, { HSAKMT_STATUS status = HSAKMT_STATUS_SUCCESS; - pthread_mutex_lock(&doorbells[NodeId].doorbells_mutex); + pthread_mutex_lock(&doorbells[NodeId].mutex); if (doorbells[NodeId].size) { - pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); + pthread_mutex_unlock(&doorbells[NodeId].mutex); return HSAKMT_STATUS_SUCCESS; } @@ -319,7 +319,7 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, if (status != HSAKMT_STATUS_SUCCESS) doorbells[NodeId].size = 0; - pthread_mutex_unlock(&doorbells[NodeId].doorbells_mutex); + pthread_mutex_unlock(&doorbells[NodeId].mutex); return status; } @@ -620,7 +620,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, } QueueResource->QueueId = PORT_VPTR_TO_UINT64(q); - QueueResource->Queue_DoorBell = VOID_PTR_ADD(doorbells[NodeId].doorbells, + QueueResource->Queue_DoorBell = VOID_PTR_ADD(doorbells[NodeId].mapping, doorbell_offset); return HSAKMT_STATUS_SUCCESS; From 4eaaf9694decda426ebc27382cbcc6c3a5d59e2d Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 29 Jun 2018 14:25:08 -0400 Subject: [PATCH 0353/1247] Simplify if else logic for hsaKmtAllocMemory() The new logic is easier to follow. Change-Id: I69759a45c5dedaefeff831a2367253d3a4486bd3 Signed-off-by: Yong Zhao --- src/memory.c | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/src/memory.c b/src/memory.c index 54f262a448..8b1b07599b 100644 --- a/src/memory.c +++ b/src/memory.c @@ -128,30 +128,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, if (!MemoryAddress || !SizeInBytes || (SizeInBytes & (page_size-1))) return HSAKMT_STATUS_INVALID_PARAMETER; - if (gpu_id == 0 && !MemFlags.ui32.Scratch) { - *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, - MemFlags); - - if (!(*MemoryAddress)) { - pr_err("[%s] failed to allocate %lu bytes from host\n", - __func__, SizeInBytes); - return HSAKMT_STATUS_ERROR; - } - - return HSAKMT_STATUS_SUCCESS; - } - - if (gpu_id && MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { - *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes, MemFlags); - - if (!(*MemoryAddress)) { - pr_err("[%s] failed to allocate %lu bytes from device\n", - __func__, SizeInBytes); - return HSAKMT_STATUS_NO_MEMORY; - } - - return HSAKMT_STATUS_SUCCESS; - } if (MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_scratch(gpu_id, SizeInBytes); @@ -164,15 +140,16 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, return HSAKMT_STATUS_SUCCESS; } - /* Backwards compatibility hack: Allocate system memory if app - * asks for paged memory from a GPU node. - */ - if (gpu_id && !MemFlags.ui32.NonPaged && !MemFlags.ui32.Scratch) { + /* GPU allocated system memory */ + if (!gpu_id || !MemFlags.ui32.NonPaged) { + /* Backwards compatibility hack: Allocate system memory if app + * asks for paged memory from a GPU node. + */ *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, MemFlags); if (!(*MemoryAddress)) { - pr_err("[%s] failed to allocate %lu bytes from paged\n", + pr_err("[%s] failed to allocate %lu bytes from host\n", __func__, SizeInBytes); return HSAKMT_STATUS_ERROR; } @@ -180,7 +157,17 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, return HSAKMT_STATUS_SUCCESS; } - return HSAKMT_STATUS_INVALID_PARAMETER; + /* GPU allocated VRAM */ + *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes, MemFlags); + + if (!(*MemoryAddress)) { + pr_err("[%s] failed to allocate %lu bytes from device\n", + __func__, SizeInBytes); + return HSAKMT_STATUS_NO_MEMORY; + } + + return HSAKMT_STATUS_SUCCESS; + } HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, From 89e35574e36c2b48f84d3cc0a90d3a2d07e35985 Mon Sep 17 00:00:00 2001 From: Slava Grigorev Date: Fri, 29 Jun 2018 16:06:45 -0400 Subject: [PATCH 0354/1247] Fix 'strncpy' truncating warnings when compiling with gcc 8 Change-Id: Ib145bab9450281da05f70dea34433b83438a756b Signed-off-by: Slava Grigorev --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 32fa642b1d..86b812e021 100644 --- a/src/topology.c +++ b/src/topology.c @@ -669,7 +669,7 @@ static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, if (props->CComputeIdLo == apic_id) { /* Retrieve the CAL name of CPU node */ if (!is_apu) - strncpy((char *)props->AMDName, cpu_model_name, sizeof(props->AMDName)-1); + strncpy((char *)props->AMDName, cpu_model_name, sizeof(props->AMDName)); /* Convert from UTF8 to UTF16 */ for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = cpu_model_name[i]; From 4839882fc8cca2f037ac9af545d41839b8400767 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 3 Jul 2018 20:36:44 -0400 Subject: [PATCH 0355/1247] Set the write permission according to the flag when allocating host cpu mem Change-Id: I758c2b5b1799e968fa852646e1494fabb68c782d Signed-off-by: Yong Zhao --- src/fmm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 9aeca436af..5de949ec82 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1121,11 +1121,14 @@ static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, { void *mem = NULL; vm_object_t *vm_obj; - int mmap_prot = PROT_READ | PROT_WRITE; + int mmap_prot = PROT_READ; if (flags.ui32.ExecuteAccess) mmap_prot |= PROT_EXEC; + if (!flags.ui32.ReadOnly) + mmap_prot |= PROT_WRITE; + /* mmap will return a pointer with alignment equal to * sysconf(_SC_PAGESIZE). */ From d3228f363e458e93bf3b5823206814275aaeeff4 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 5 Jul 2018 15:32:14 -0400 Subject: [PATCH 0356/1247] Fix wrong loop termination condition Compare with gpu_mem_count instead of deprecated NUM_OF_SUPPORTED_GPUS to prevent overflows in case no dGPUs are present. Change-Id: I71fcb7503ba4c20bffadbdb04cefc4e4027a7df7 Signed-off-by: Felix Kuehling --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 5de949ec82..7b9ccd37e4 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -223,11 +223,11 @@ static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_ static int32_t find_first_dgpu(HSAuint32 *gpu_id) { - int32_t i; + uint32_t i; *gpu_id = NON_VALID_GPU_ID; - for (i = 0; i < NUM_OF_SUPPORTED_GPUS; i++) { + for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) continue; if (!topology_is_dgpu(gpu_mem[i].device_id)) From ab9017715f318592db12b02231dc7d2d8de32c89 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Thu, 28 Jun 2018 18:24:54 +0800 Subject: [PATCH 0357/1247] use rbtree instead of vm_objects list simple test of mapping many system memory to gpu. before [ RUN ] KFDMemoryTest.MMap [ ] Using ISA for GFXIP 9.0 [ ] successfully register/map 32GB system memory to gpu [ OK ] KFDMemoryTest.MMap (36932 ms) after [ RUN ] KFDMemoryTest.MMap [ ] Using ISA for GFXIP 9.0 [ ] successfully register/map 32GB system memory to gpu [ OK ] KFDMemoryTest.MMap (11441 ms) So there is 11s VS 36s improvement. Looks like we can do something similar with vm_area too. Change-Id: I0349aacdeddec3534016d28176f0fabf632c61fc Signed-off-by: xinhui pan --- CMakeLists.txt | 1 + LICENSE.md | 27 ++++ src/fmm.c | 275 +++++++++++++++++++++------------- src/rbtree.c | 374 +++++++++++++++++++++++++++++++++++++++++++++++ src/rbtree.h | 92 ++++++++++++ src/rbtree_amd.h | 155 ++++++++++++++++++++ 6 files changed, 820 insertions(+), 104 deletions(-) create mode 100644 src/rbtree.c create mode 100644 src/rbtree.h create mode 100644 src/rbtree_amd.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cb2605b725..85de508202 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,6 +88,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/queues.c" "src/time.c" "src/topology.c" + "src/rbtree.c" "src/version.c") ## Include paths diff --git a/LICENSE.md b/LICENSE.md index f9b6bddadc..f6ac3f7ce6 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -21,3 +21,30 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This product contains software provided by Nginx, Inc. and its contributors. + +Copyright (C) 2002-2018 Igor Sysoev +Copyright (C) 2011-2018 Nginx, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. diff --git a/src/fmm.c b/src/fmm.c index 7b9ccd37e4..4bf6096259 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -38,6 +38,7 @@ #include #include #include +#include "rbtree.h" #ifndef MPOL_F_STATIC_NODES /* Bug in numaif.h, this should be defined in there. Definition copied * from linux/mempolicy.h. @@ -53,11 +54,25 @@ .align = 0, \ .guard_pages = 1, \ .vm_ranges = NULL, \ - .vm_objects = NULL, \ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER, \ .is_coherent = false \ } +#define container_of(ptr, type, member) ({ \ + char *__mptr = (void *)(ptr); \ + ((type *)(__mptr - offsetof(type, member))); }) + +#define rb_entry(ptr, type, member) \ + container_of(ptr, type, member) + +#define vm_object_entry(n, is_userptr) ({ \ + (is_userptr) == 0 ? \ + rb_entry(n, vm_object_t, node) : \ + rb_entry(n, vm_object_t, user_node); }) + +#define vm_object_tree(app, is_userptr) \ + ((is_userptr) ? &(app)->user_tree : &(app)->tree) + struct vm_object { void *start; void *userptr; @@ -68,8 +83,9 @@ struct vm_object { */ uint64_t handle; /* opaque */ uint32_t node_id; - struct vm_object *next; - struct vm_object *prev; + rbtree_node_t node; + rbtree_node_t user_node; + uint32_t flags; /* memory allocation flags */ /* Registered nodes to map on SVM mGPU */ uint32_t *registered_device_id_array; @@ -105,7 +121,8 @@ typedef struct { uint64_t align; uint32_t guard_pages; vm_area_t *vm_ranges; - vm_object_t *vm_objects; + rbtree_t tree; + rbtree_t user_tree; pthread_mutex_t fmm_mutex; bool is_coherent; } manageable_aperture_t; @@ -263,7 +280,6 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->userptr_size = 0; object->size = size; object->handle = handle; - object->next = object->prev = NULL; object->registered_device_id_array_size = 0; object->mapped_device_id_array_size = 0; object->registered_device_id_array = NULL; @@ -276,6 +292,8 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->metadata = NULL; object->user_data = NULL; object->is_imported_kfd_bo = false; + object->node.key = rbtree_key((unsigned long)start, size); + object->user_node.key = rbtree_key(0, 0); } return object; @@ -303,9 +321,6 @@ static void vm_remove_area(manageable_aperture_t *app, vm_area_t *area) static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object) { - vm_object_t *next; - vm_object_t *prev; - /* Free allocations inside the object */ if (object->registered_device_id_array) free(object->registered_device_id_array); @@ -321,16 +336,9 @@ static void vm_remove_object(manageable_aperture_t *app, vm_object_t *object) if (object->mapped_node_id_array) free(object->mapped_node_id_array); - next = object->next; - prev = object->prev; - - if (!prev) /* The first element */ - app->vm_objects = next; - else - prev->next = next; - - if (next) /* If not the last element */ - next->prev = prev; + rbtree_delete(&app->tree, &object->node); + if (object->userptr) + rbtree_delete(&app->user_tree, &object->user_node); free(object); } @@ -347,19 +355,6 @@ static void vm_add_area_after(vm_area_t *after_this, vm_area_t *new_area) next->prev = new_area; } -static void vm_add_object_before(vm_object_t *before_this, - vm_object_t *new_object) -{ - vm_object_t *prev = before_this->prev; - - before_this->prev = new_object; - new_object->next = before_this; - - new_object->prev = prev; - if (prev) - prev->next = new_object; -} - static void vm_split_area(manageable_aperture_t *app, vm_area_t *area, void *address, uint64_t MemorySizeInBytes) { @@ -377,87 +372,129 @@ static void vm_split_area(manageable_aperture_t *app, vm_area_t *area, vm_add_area_after(area, new_area); } -static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app, - const void *address, uint64_t size) +static vm_object_t *vm_find_object_by_address_userptr(manageable_aperture_t *app, + const void *address, uint64_t size, int is_userptr) { - vm_object_t *cur = app->vm_objects; + vm_object_t *cur = NULL; - size = ALIGN_UP(size, app->align); + if (is_userptr == 0) + size = ALIGN_UP(size, app->align); + rbtree_t *tree = vm_object_tree(app, is_userptr); + rbtree_key_t key = rbtree_key((unsigned long)address, size); + void *start; + uint64_t s; - /* Look up the appropriate address range containing the given address */ - while (cur) { - if (cur->start == address && (cur->size == size || size == 0)) - break; - cur = cur->next; + /* rbtree_lookup_nearest(,,,RIGHT) will return a node with + * its size >= key.size and its address >= key.address + * if there are two nodes with format(address, size), + * (0x100, 16) and (0x110, 8). the key is (0x100, 0), + * then node (0x100, 16) will be returned. + */ + rbtree_node_t *n = rbtree_lookup_nearest(tree, &key, LKP_ALL, RIGHT); + + if (n) { + cur = vm_object_entry(n, is_userptr); + if (is_userptr == 0) { + start = cur->start; + s = cur->size; + } else { + start = cur->userptr; + s = cur->userptr_size; + } + + if (start != address) + return NULL; + + if (size) + return size == s ? cur : NULL; + + /* size is 0, make sure there is only one node whose address == key.address*/ + key = rbtree_key((unsigned long)address, (unsigned long)-1); + rbtree_node_t *rn = rbtree_lookup_nearest(tree, &key, LKP_ALL, LEFT); + + if (rn != n) + return NULL; } return cur; /* NULL if not found */ } + +static vm_object_t *vm_find_object_by_address_userptr_range(manageable_aperture_t *app, + const void *address, int is_userptr) +{ + vm_object_t *cur = NULL; + rbtree_t *tree = vm_object_tree(app, is_userptr); + rbtree_key_t key = rbtree_key((unsigned long)address, 0); + rbtree_node_t *ln = rbtree_lookup_nearest(tree, &key, + LKP_ALL, LEFT); + rbtree_node_t *rn = rbtree_lookup_nearest(tree, &key, + LKP_ALL, RIGHT); + void *start; + uint64_t size; + int bad = 0; + +loop: + while (ln) { + cur = vm_object_entry(ln, is_userptr); + if (is_userptr == 0) { + start = cur->start; + size = cur->size; + } else { + start = cur->userptr; + size = cur->userptr_size; + } + + if (address >= start && + (uint64_t)address < ((uint64_t)start + size)) + break; + + cur = NULL; + + if (ln == rn) + break; + + ln = rbtree_next(tree, ln); + } + + if (cur == NULL && bad == 0) { + /* As there is area overlap, say, (address, size) like + * (0x100, 32), (0x108, 8), and the key.address is 0x118 + * The lookup above only find (0x108, 8), but the correct node should + * be (0x100, 16). So try to walk though the tree to find the node. + */ + rn = ln; + key = rbtree_key(0, 0); + ln = rbtree_lookup_nearest(tree, &key, LKP_ALL, RIGHT); + bad = 1; + goto loop; + } + + return cur; /* NULL if not found */ +} + +static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app, + const void *address, uint64_t size) +{ + return vm_find_object_by_address_userptr(app, address, size, 0); +} + static vm_object_t *vm_find_object_by_address_range(manageable_aperture_t *app, const void *address) { - vm_object_t *cur = app->vm_objects; - - while (cur) { - if (address >= cur->start && - (uint64_t)address < ((uint64_t)cur->start + cur->size)) - break; - cur = cur->next; - } - - return cur; /* NULL if not found */ + return vm_find_object_by_address_userptr_range(app, address, 0); } static vm_object_t *vm_find_object_by_userptr(manageable_aperture_t *app, const void *address, HSAuint64 size) { - vm_object_t *cur = app->vm_objects, *obj; - uint32_t found = 0; - - /* Look up the userptr that matches the address. If size is specified, - * the size needs to match too. - */ - while (cur) { - if ((cur->userptr == address) && - ((cur->userptr_size == size) || !size)) { - found = 1; - break; - } - cur = cur->next; - } - - /* If size is not specified, we need to ensure the vm_obj found is the - * only obj having this address. - */ - if (found && !size) { - obj = cur->next; - while (obj) { - if (obj->userptr == address) { - cur = NULL; - break; - } - obj = obj->next; - } - } - - return cur; /* NULL if any look-up failure */ + return vm_find_object_by_address_userptr(app, address, size, 1); } static vm_object_t *vm_find_object_by_userptr_range(manageable_aperture_t *app, const void *address) { - vm_object_t *cur = app->vm_objects; - - /* Look up the appropriate address range containing the given address */ - while (cur) { - if (address >= cur->userptr && - (uint64_t)address < (uint64_t)cur->userptr + cur->userptr_size) - break; - cur = cur->next; - } - - return cur; /* NULL if not found */ + return vm_find_object_by_address_userptr_range(app, address, 1); } static vm_area_t *vm_find(manageable_aperture_t *app, void *address) @@ -615,12 +652,7 @@ static vm_object_t *aperture_allocate_object(manageable_aperture_t *app, if (!new_object) return NULL; - /* check for non-empty list */ - if (app->vm_objects) - /* Add it before the first element */ - vm_add_object_before(app->vm_objects, new_object); - - app->vm_objects = new_object; /* Update head */ + rbtree_insert(&app->tree, &new_object->node); return new_object; } @@ -784,7 +816,8 @@ static void aperture_print(aperture_t *app) static void manageable_aperture_print(manageable_aperture_t *app) { vm_area_t *cur = app->vm_ranges; - vm_object_t *object = app->vm_objects; + rbtree_node_t *n = rbtree_node_any(&app->tree, LEFT); + vm_object_t *object; pr_info("\t Base: %p\n", app->base); pr_info("\t Limit: %p\n", app->limit); @@ -794,11 +827,12 @@ static void manageable_aperture_print(manageable_aperture_t *app) cur = cur->next; }; pr_info("\t Objects:\n"); - while (object) { + while (n) { + object = vm_object_entry(n, 0); pr_info("\t\t Object [%p - %" PRIu64 "]\n", object->start, object->size); - object = object->next; - }; + n = rbtree_next(&app->tree, n); + } } void fmm_print(uint32_t gpu_id) @@ -834,6 +868,7 @@ static void fmm_release_scratch(uint32_t gpu_id) uint64_t size; vm_object_t *obj; manageable_aperture_t *aperture; + rbtree_node_t *n; gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) @@ -846,7 +881,9 @@ static void fmm_release_scratch(uint32_t gpu_id) if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { /* unmap and remove all remaining objects */ pthread_mutex_lock(&aperture->fmm_mutex); - while ((obj = aperture->vm_objects)) { + while ((n = rbtree_node_any(&aperture->tree, MID))) { + obj = vm_object_entry(n, 0); + void *obj_addr = obj->start; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1643,6 +1680,30 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, return HSAKMT_STATUS_SUCCESS; } +static void fmm_init_rbtree(void) +{ + static int once; + int i = gpu_mem_count; + + if (once++ == 0) { + rbtree_init(&svm.dgpu_aperture.tree); + rbtree_init(&svm.dgpu_aperture.user_tree); + rbtree_init(&svm.dgpu_alt_aperture.tree); + rbtree_init(&svm.dgpu_alt_aperture.user_tree); + rbtree_init(&cpuvm_aperture.tree); + rbtree_init(&cpuvm_aperture.user_tree); + } + + while (i--) { + rbtree_init(&gpu_mem[i].scratch_aperture.tree); + rbtree_init(&gpu_mem[i].scratch_aperture.user_tree); + rbtree_init(&gpu_mem[i].scratch_physical.tree); + rbtree_init(&gpu_mem[i].scratch_physical.user_tree); + rbtree_init(&gpu_mem[i].gpuvm_aperture.tree); + rbtree_init(&gpu_mem[i].gpuvm_aperture.user_tree); + } +} + HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i = 0; @@ -1854,6 +1915,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) cpuvm_aperture.align = PAGE_SIZE; cpuvm_aperture.limit = (void *)0x7FFFFFFFFFFF; /* 2^47 - 1 */ + fmm_init_rbtree(); + free(process_apertures); return ret; @@ -2570,6 +2633,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj gpuid_to_nodeid(gpu_id, &obj->node_id); obj->userptr_size = size; obj->registration_count = 1; + obj->user_node.key = rbtree_key((unsigned long)addr, size); + rbtree_insert(&aperture->user_tree, &obj->user_node); pthread_mutex_unlock(&aperture->fmm_mutex); } else return HSAKMT_STATUS_ERROR; @@ -3234,8 +3299,10 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) static void fmm_clear_aperture(manageable_aperture_t *app) { - while (app->vm_objects) - vm_remove_object(app, app->vm_objects); + rbtree_node_t *n; + + while ((n = rbtree_node_any(&app->tree, MID))) + vm_remove_object(app, vm_object_entry(n, 0)); while (app->vm_ranges) vm_remove_area(app, app->vm_ranges); diff --git a/src/rbtree.c b/src/rbtree.c new file mode 100644 index 0000000000..86e2d59f7b --- /dev/null +++ b/src/rbtree.c @@ -0,0 +1,374 @@ +/* + * Copyright (C) 2002-2018 Igor Sysoev + * Copyright (C) 2011-2018 Nginx, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "rbtree.h" + +static inline void rbtree_left_rotate(rbtree_node_t **root, + rbtree_node_t *sentinel, rbtree_node_t *node); +static inline void rbtree_right_rotate(rbtree_node_t **root, + rbtree_node_t *sentinel, rbtree_node_t *node); + +static void +rbtree_insert_value(rbtree_node_t *temp, rbtree_node_t *node, + rbtree_node_t *sentinel) +{ + rbtree_node_t **p; + + for ( ;; ) { + + p = rbtree_key_compare(LKP_ALL, &node->key, &temp->key) < 0 ? + &temp->left : &temp->right; + + if (*p == sentinel) { + break; + } + + temp = *p; + } + + *p = node; + node->parent = temp; + node->left = sentinel; + node->right = sentinel; + rbt_red(node); +} + + +void +rbtree_insert(rbtree_t *tree, rbtree_node_t *node) +{ + rbtree_node_t **root, *temp, *sentinel; + + /* a binary tree insert */ + + root = &tree->root; + sentinel = &tree->sentinel; + + if (*root == sentinel) { + node->parent = NULL; + node->left = sentinel; + node->right = sentinel; + rbt_black(node); + *root = node; + + return; + } + + rbtree_insert_value(*root, node, sentinel); + + /* re-balance tree */ + + while (node != *root && rbt_is_red(node->parent)) { + + if (node->parent == node->parent->parent->left) { + temp = node->parent->parent->right; + + if (rbt_is_red(temp)) { + rbt_black(node->parent); + rbt_black(temp); + rbt_red(node->parent->parent); + node = node->parent->parent; + + } else { + if (node == node->parent->right) { + node = node->parent; + rbtree_left_rotate(root, sentinel, node); + } + + rbt_black(node->parent); + rbt_red(node->parent->parent); + rbtree_right_rotate(root, sentinel, node->parent->parent); + } + + } else { + temp = node->parent->parent->left; + + if (rbt_is_red(temp)) { + rbt_black(node->parent); + rbt_black(temp); + rbt_red(node->parent->parent); + node = node->parent->parent; + + } else { + if (node == node->parent->left) { + node = node->parent; + rbtree_right_rotate(root, sentinel, node); + } + + rbt_black(node->parent); + rbt_red(node->parent->parent); + rbtree_left_rotate(root, sentinel, node->parent->parent); + } + } + } + + rbt_black(*root); +} + + +void +rbtree_delete(rbtree_t *tree, rbtree_node_t *node) +{ + unsigned int red; + rbtree_node_t **root, *sentinel, *subst, *temp, *w; + + /* a binary tree delete */ + + root = &tree->root; + sentinel = &tree->sentinel; + + if (node->left == sentinel) { + temp = node->right; + subst = node; + + } else if (node->right == sentinel) { + temp = node->left; + subst = node; + + } else { + subst = rbtree_min(node->right, sentinel); + + if (subst->left != sentinel) { + temp = subst->left; + } else { + temp = subst->right; + } + } + + if (subst == *root) { + *root = temp; + rbt_black(temp); + + return; + } + + red = rbt_is_red(subst); + + if (subst == subst->parent->left) { + subst->parent->left = temp; + + } else { + subst->parent->right = temp; + } + + if (subst == node) { + + temp->parent = subst->parent; + + } else { + + if (subst->parent == node) { + temp->parent = subst; + + } else { + temp->parent = subst->parent; + } + + subst->left = node->left; + subst->right = node->right; + subst->parent = node->parent; + rbt_copy_color(subst, node); + + if (node == *root) { + *root = subst; + + } else { + if (node == node->parent->left) { + node->parent->left = subst; + } else { + node->parent->right = subst; + } + } + + if (subst->left != sentinel) { + subst->left->parent = subst; + } + + if (subst->right != sentinel) { + subst->right->parent = subst; + } + } + + if (red) { + return; + } + + /* a delete fixup */ + + while (temp != *root && rbt_is_black(temp)) { + + if (temp == temp->parent->left) { + w = temp->parent->right; + + if (rbt_is_red(w)) { + rbt_black(w); + rbt_red(temp->parent); + rbtree_left_rotate(root, sentinel, temp->parent); + w = temp->parent->right; + } + + if (rbt_is_black(w->left) && rbt_is_black(w->right)) { + rbt_red(w); + temp = temp->parent; + + } else { + if (rbt_is_black(w->right)) { + rbt_black(w->left); + rbt_red(w); + rbtree_right_rotate(root, sentinel, w); + w = temp->parent->right; + } + + rbt_copy_color(w, temp->parent); + rbt_black(temp->parent); + rbt_black(w->right); + rbtree_left_rotate(root, sentinel, temp->parent); + temp = *root; + } + + } else { + w = temp->parent->left; + + if (rbt_is_red(w)) { + rbt_black(w); + rbt_red(temp->parent); + rbtree_right_rotate(root, sentinel, temp->parent); + w = temp->parent->left; + } + + if (rbt_is_black(w->left) && rbt_is_black(w->right)) { + rbt_red(w); + temp = temp->parent; + + } else { + if (rbt_is_black(w->left)) { + rbt_black(w->right); + rbt_red(w); + rbtree_left_rotate(root, sentinel, w); + w = temp->parent->left; + } + + rbt_copy_color(w, temp->parent); + rbt_black(temp->parent); + rbt_black(w->left); + rbtree_right_rotate(root, sentinel, temp->parent); + temp = *root; + } + } + } + + rbt_black(temp); +} + + +static inline void +rbtree_left_rotate(rbtree_node_t **root, rbtree_node_t *sentinel, + rbtree_node_t *node) +{ + rbtree_node_t *temp; + + temp = node->right; + node->right = temp->left; + + if (temp->left != sentinel) { + temp->left->parent = node; + } + + temp->parent = node->parent; + + if (node == *root) { + *root = temp; + + } else if (node == node->parent->left) { + node->parent->left = temp; + + } else { + node->parent->right = temp; + } + + temp->left = node; + node->parent = temp; +} + + +static inline void +rbtree_right_rotate(rbtree_node_t **root, rbtree_node_t *sentinel, + rbtree_node_t *node) +{ + rbtree_node_t *temp; + + temp = node->left; + node->left = temp->right; + + if (temp->right != sentinel) { + temp->right->parent = node; + } + + temp->parent = node->parent; + + if (node == *root) { + *root = temp; + + } else if (node == node->parent->right) { + node->parent->right = temp; + + } else { + node->parent->left = temp; + } + + temp->right = node; + node->parent = temp; +} + + +rbtree_node_t * +rbtree_next(rbtree_t *tree, rbtree_node_t *node) +{ + rbtree_node_t *root, *sentinel, *parent; + + sentinel = &tree->sentinel; + + if (node->right != sentinel) { + return rbtree_min(node->right, sentinel); + } + + root = tree->root; + + for ( ;; ) { + parent = node->parent; + + if (node == root) { + return NULL; + } + + if (node == parent->left) { + return parent; + } + + node = parent; + } +} diff --git a/src/rbtree.h b/src/rbtree.h new file mode 100644 index 0000000000..965c8fe773 --- /dev/null +++ b/src/rbtree.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2002-2018 Igor Sysoev + * Copyright (C) 2011-2018 Nginx, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _RBTREE_H_ +#define _RBTREE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "rbtree_amd.h" + +typedef struct rbtree_node_s rbtree_node_t; + +struct rbtree_node_s { + rbtree_key_t key; + rbtree_node_t *left; + rbtree_node_t *right; + rbtree_node_t *parent; + unsigned char color; + unsigned char data; +}; + +typedef struct rbtree_s rbtree_t; + +struct rbtree_s { + rbtree_node_t *root; + rbtree_node_t sentinel; +}; + +#define rbtree_init(tree) \ + rbtree_sentinel_init(&(tree)->sentinel); \ + (tree)->root = &(tree)->sentinel; + +void rbtree_insert(rbtree_t *tree, rbtree_node_t *node); +void rbtree_delete(rbtree_t *tree, rbtree_node_t *node); +rbtree_node_t *rbtree_next(rbtree_t *tree, + rbtree_node_t *node); + +#define rbt_red(node) ((node)->color = 1) +#define rbt_black(node) ((node)->color = 0) +#define rbt_is_red(node) ((node)->color) +#define rbt_is_black(node) (!rbt_is_red(node)) +#define rbt_copy_color(n1, n2) (n1->color = n2->color) + +/* a sentinel must be black */ + +#define rbtree_sentinel_init(node) rbt_black(node) + +static inline rbtree_node_t * +rbtree_min(rbtree_node_t *node, rbtree_node_t *sentinel) +{ + while (node->left != sentinel) { + node = node->left; + } + + return node; +} + +#include "rbtree_amd.h" + +#endif diff --git a/src/rbtree_amd.h b/src/rbtree_amd.h new file mode 100644 index 0000000000..5990404b7d --- /dev/null +++ b/src/rbtree_amd.h @@ -0,0 +1,155 @@ +/* + * Copyright © 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef _RBTREE_AMD_H_ +#define _RBTREE_AMD_H_ + +typedef struct rbtree_key_s rbtree_key_t; +struct rbtree_key_s { +#define ADDR_BIT 0 +#define SIZE_BIT 1 + unsigned long addr; + unsigned long size; +}; +#define BIT(x) (1<<(x)) +#define LKP_ALL (BIT(ADDR_BIT) | BIT(SIZE_BIT)) +#define LKP_ADDR (BIT(ADDR_BIT)) +#define LKP_ADDR_SIZE (BIT(ADDR_BIT) | BIT(SIZE_BIT)) + +static inline rbtree_key_t +rbtree_key(unsigned long addr, unsigned long size) +{ + return (rbtree_key_t){addr, size}; +} + +/* + * compare addr, size one by one + */ +static inline int +rbtree_key_compare(unsigned int type, rbtree_key_t *key1, rbtree_key_t *key2) +{ + if ((type & 1 << ADDR_BIT) && (key1->addr != key2->addr)) + return key1->addr > key2->addr ? 1 : -1; + + if ((type & 1 << SIZE_BIT) && (key1->size != key2->size)) + return key1->size > key2->size ? 1 : -1; + + return 0; +} +#endif /*_RBTREE_AMD_H_*/ + +/*inlcude this file again with RBTREE_HELPER defined*/ +#ifndef RBTREE_HELPER +#define RBTREE_HELPER +#else +#ifndef _RBTREE_AMD_H_HELPER_ +#define _RBTREE_AMD_H_HELPER_ +static inline rbtree_node_t * +rbtree_max(rbtree_node_t *node, rbtree_node_t *sentinel) +{ + while (node->right != sentinel) + node = node->right; + + return node; +} + +#define LEFT 0 +#define RIGHT 1 +#define MID 2 +static inline rbtree_node_t * +rbtree_min_max(rbtree_t *tree, int lr) +{ + rbtree_node_t *sentinel = &tree->sentinel; + rbtree_node_t *node = tree->root; + + if (node == sentinel) + return NULL; + + if (lr == LEFT) + node = rbtree_min(node, sentinel); + else if (lr == RIGHT) + node = rbtree_max(node, sentinel); + + return node; +} + +static inline rbtree_node_t * +rbtree_node_any(rbtree_t *tree, int lmr) +{ + rbtree_node_t *sentinel = &tree->sentinel; + rbtree_node_t *node = tree->root; + + if (node == sentinel) + return NULL; + + if (lmr == MID) + return node; + + return rbtree_min_max(tree, lmr); +} + +static inline rbtree_node_t * +rbtree_lookup_nearest(rbtree_t *rbtree, rbtree_key_t *key, + unsigned int type, int lr) +{ + int rc; + rbtree_node_t *node, *sentinel, *n = NULL; + + node = rbtree->root; + sentinel = &rbtree->sentinel; + + while (node != sentinel) { + rc = rbtree_key_compare(type, key, &node->key); + + if (rc < 0) { + if (lr == RIGHT) + n = node; + node = node->left; + continue; + } + + if (rc > 0) { + if (lr == LEFT) + n = node; + node = node->right; + continue; + } + + return node; + } + + return n; +} + +static inline rbtree_node_t * +rbtree_lookup(rbtree_t *rbtree, rbtree_key_t *key, + unsigned int type) +{ + return rbtree_lookup_nearest(rbtree, key, type, -1); +} +#endif /*_RBTREE_AMD_H_HELPER_*/ + +#endif /*RBTREE_HELPER*/ + From 63f2a0d28053881a471e0ca8379503effbc64f3e Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Tue, 10 Jul 2018 19:49:00 -0500 Subject: [PATCH 0358/1247] Fix git describe command to retrieve version tags correctly. Change-Id: I33282e8130d092e2f56b2f5947946d3c0ee22c60 --- cmake_modules/utils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake_modules/utils.cmake b/cmake_modules/utils.cmake index 5b4e73ce23..096b3de7b7 100644 --- a/cmake_modules/utils.cmake +++ b/cmake_modules/utils.cmake @@ -93,7 +93,7 @@ function ( get_version DEFAULT_VERSION_STRING ) if ( GIT ) - execute_process ( COMMAND git describe --dirty --long --match [0-9]* + execute_process ( COMMAND git describe --tags --dirty --long WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_TAG_STRING OUTPUT_STRIP_TRAILING_WHITESPACE From 3719ec2557620e1d4e74ed83f3f5f4b127339e78 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 25 Jul 2018 19:43:51 -0400 Subject: [PATCH 0359/1247] RPM: Add /opt/rocm/lib64 to lib path CentOS+cmake suffix the "lib" folder with 64 when building 64-bit packages Fixes: 9434223752e531a24: Clean up cmake install and package Change-Id: If029afd6e198b812116f3d3b3921b5486a1be355 --- RPM/rpm_post | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RPM/rpm_post b/RPM/rpm_post index abd8ec2a6e..fb4436af29 100644 --- a/RPM/rpm_post +++ b/RPM/rpm_post @@ -1 +1 @@ -echo /opt/rocm/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig +echo -e "/opt/rocm/lib\n/opt/rocm/lib64" > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig From 6df62c78b829a1469fe6dcbbb7e40861c200e5a9 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 23 Jul 2018 14:45:44 -0400 Subject: [PATCH 0360/1247] kfdtest: Add kfdtest source code The code is a snapshot up to this commit around July 31 2018. commit b00fadff36a3 Author: xinhui pan Date: Mon Jul 30 09:53:03 2018 +0800 kfdtest: skip MMapLarge test on apu Change-Id: I40e9a5a18e5c8f075e5290bb80532f1a3f689058 Signed-off-by: Yong Zhao --- tests/kfdtest/.gitignore | 0 tests/kfdtest/CMakeLists.txt | 108 + tests/kfdtest/LICENSE.kfdtest | 85 + tests/kfdtest/gtest-1.6.0/gtest-all.cpp | 9132 +++++++ tests/kfdtest/gtest-1.6.0/gtest/gtest.h | 19640 ++++++++++++++++ tests/kfdtest/include/amdp2ptest.h | 76 + tests/kfdtest/include/asic_reg/gfx_7_2_d.h | 2557 ++ tests/kfdtest/include/asic_reg/gfx_7_2_enum.h | 6280 +++++ .../include/asic_reg/gfx_7_2_sh_mask.h | 18444 +++++++++++++++ tests/kfdtest/include/kfd_pm4_opcodes.h | 107 + tests/kfdtest/include/pm4_pkt_struct_ai.h | 160 + tests/kfdtest/include/pm4_pkt_struct_ci.h | 129 + tests/kfdtest/include/pm4_pkt_struct_common.h | 275 + tests/kfdtest/include/sdma_pkt_struct.h | 316 + tests/kfdtest/kfdtest.exclude | 142 + tests/kfdtest/sp3/README.txt | 6 + .../sp3/lib_helper/AMD_opensource_license.txt | 23 + .../kfdtest/sp3/lib_helper/CMakeLists_sp3.txt | 75 + tests/kfdtest/sp3/lib_helper/build_sp3.sh | 57 + tests/kfdtest/sp3/sp3.h | 590 + tests/kfdtest/src/AqlQueue.cpp | 52 + tests/kfdtest/src/AqlQueue.hpp | 50 + tests/kfdtest/src/BasePacket.cpp | 37 + tests/kfdtest/src/BasePacket.hpp | 57 + tests/kfdtest/src/BaseQueue.cpp | 200 + tests/kfdtest/src/BaseQueue.hpp | 134 + tests/kfdtest/src/Dispatch.cpp | 221 + tests/kfdtest/src/Dispatch.hpp | 75 + tests/kfdtest/src/GoogleTestExtension.cpp | 79 + tests/kfdtest/src/GoogleTestExtension.hpp | 71 + tests/kfdtest/src/IndirectBuffer.cpp | 51 + tests/kfdtest/src/IndirectBuffer.hpp | 60 + tests/kfdtest/src/IsaGenerator.cpp | 118 + tests/kfdtest/src/IsaGenerator.hpp | 53 + tests/kfdtest/src/IsaGenerator_Gfx72.cpp | 123 + tests/kfdtest/src/IsaGenerator_Gfx72.hpp | 49 + tests/kfdtest/src/IsaGenerator_Gfx8.cpp | 128 + tests/kfdtest/src/IsaGenerator_Gfx8.hpp | 49 + tests/kfdtest/src/IsaGenerator_Gfx9.cpp | 113 + tests/kfdtest/src/IsaGenerator_Gfx9.hpp | 49 + tests/kfdtest/src/KFDBaseComponentTest.cpp | 117 + tests/kfdtest/src/KFDBaseComponentTest.hpp | 58 + tests/kfdtest/src/KFDCWSRTest.cpp | 184 + tests/kfdtest/src/KFDCWSRTest.hpp | 47 + tests/kfdtest/src/KFDDBGTest.cpp | 176 + tests/kfdtest/src/KFDDBGTest.hpp | 45 + tests/kfdtest/src/KFDEventTest.cpp | 285 + tests/kfdtest/src/KFDEventTest.hpp | 44 + tests/kfdtest/src/KFDEvictTest.cpp | 454 + tests/kfdtest/src/KFDEvictTest.hpp | 70 + tests/kfdtest/src/KFDExceptionTest.cpp | 355 + tests/kfdtest/src/KFDExceptionTest.hpp | 65 + tests/kfdtest/src/KFDGraphicsInterop.cpp | 317 + tests/kfdtest/src/KFDGraphicsInterop.hpp | 58 + tests/kfdtest/src/KFDIPCTest.cpp | 731 + tests/kfdtest/src/KFDIPCTest.hpp | 134 + tests/kfdtest/src/KFDLocalMemoryTest.cpp | 528 + tests/kfdtest/src/KFDLocalMemoryTest.hpp | 45 + tests/kfdtest/src/KFDMemoryTest.cpp | 1628 ++ tests/kfdtest/src/KFDMemoryTest.hpp | 48 + tests/kfdtest/src/KFDOpenCloseKFDTest.cpp | 88 + tests/kfdtest/src/KFDOpenCloseKFDTest.hpp | 43 + tests/kfdtest/src/KFDPMTest.cpp | 122 + tests/kfdtest/src/KFDPMTest.hpp | 46 + tests/kfdtest/src/KFDPNPTest.cpp | 99 + tests/kfdtest/src/KFDPNPTest.hpp | 47 + tests/kfdtest/src/KFDPerfCounters.cpp | 281 + tests/kfdtest/src/KFDPerfCounters.hpp | 44 + tests/kfdtest/src/KFDQMTest.cpp | 1566 ++ tests/kfdtest/src/KFDQMTest.hpp | 56 + tests/kfdtest/src/KFDTestFlags.hpp | 65 + tests/kfdtest/src/KFDTestMain.cpp | 92 + tests/kfdtest/src/KFDTestUtil.cpp | 536 + tests/kfdtest/src/KFDTestUtil.hpp | 185 + tests/kfdtest/src/KFDTopologyTest.cpp | 220 + tests/kfdtest/src/KFDTopologyTest.hpp | 40 + tests/kfdtest/src/LinuxOSWrapper.cpp | 240 + tests/kfdtest/src/OSWrapper.hpp | 117 + tests/kfdtest/src/PM4Packet.cpp | 302 + tests/kfdtest/src/PM4Packet.hpp | 221 + tests/kfdtest/src/PM4Queue.cpp | 71 + tests/kfdtest/src/PM4Queue.hpp | 51 + tests/kfdtest/src/RDMATest.cpp | 118 + tests/kfdtest/src/RDMATest.hpp | 45 + tests/kfdtest/src/RDMAUtil.cpp | 69 + tests/kfdtest/src/RDMAUtil.hpp | 37 + tests/kfdtest/src/SDMAPacket.cpp | 189 + tests/kfdtest/src/SDMAPacket.hpp | 143 + tests/kfdtest/src/SDMAQueue.cpp | 78 + tests/kfdtest/src/SDMAQueue.hpp | 50 + 90 files changed, 70621 insertions(+) create mode 100644 tests/kfdtest/.gitignore create mode 100644 tests/kfdtest/CMakeLists.txt create mode 100644 tests/kfdtest/LICENSE.kfdtest create mode 100644 tests/kfdtest/gtest-1.6.0/gtest-all.cpp create mode 100644 tests/kfdtest/gtest-1.6.0/gtest/gtest.h create mode 100644 tests/kfdtest/include/amdp2ptest.h create mode 100644 tests/kfdtest/include/asic_reg/gfx_7_2_d.h create mode 100644 tests/kfdtest/include/asic_reg/gfx_7_2_enum.h create mode 100644 tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h create mode 100644 tests/kfdtest/include/kfd_pm4_opcodes.h create mode 100644 tests/kfdtest/include/pm4_pkt_struct_ai.h create mode 100644 tests/kfdtest/include/pm4_pkt_struct_ci.h create mode 100644 tests/kfdtest/include/pm4_pkt_struct_common.h create mode 100644 tests/kfdtest/include/sdma_pkt_struct.h create mode 100644 tests/kfdtest/kfdtest.exclude create mode 100644 tests/kfdtest/sp3/README.txt create mode 100644 tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt create mode 100644 tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt create mode 100755 tests/kfdtest/sp3/lib_helper/build_sp3.sh create mode 100644 tests/kfdtest/sp3/sp3.h create mode 100644 tests/kfdtest/src/AqlQueue.cpp create mode 100644 tests/kfdtest/src/AqlQueue.hpp create mode 100644 tests/kfdtest/src/BasePacket.cpp create mode 100644 tests/kfdtest/src/BasePacket.hpp create mode 100644 tests/kfdtest/src/BaseQueue.cpp create mode 100644 tests/kfdtest/src/BaseQueue.hpp create mode 100644 tests/kfdtest/src/Dispatch.cpp create mode 100644 tests/kfdtest/src/Dispatch.hpp create mode 100644 tests/kfdtest/src/GoogleTestExtension.cpp create mode 100644 tests/kfdtest/src/GoogleTestExtension.hpp create mode 100644 tests/kfdtest/src/IndirectBuffer.cpp create mode 100644 tests/kfdtest/src/IndirectBuffer.hpp create mode 100644 tests/kfdtest/src/IsaGenerator.cpp create mode 100644 tests/kfdtest/src/IsaGenerator.hpp create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx72.cpp create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx72.hpp create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx8.cpp create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx8.hpp create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx9.cpp create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx9.hpp create mode 100644 tests/kfdtest/src/KFDBaseComponentTest.cpp create mode 100644 tests/kfdtest/src/KFDBaseComponentTest.hpp create mode 100644 tests/kfdtest/src/KFDCWSRTest.cpp create mode 100644 tests/kfdtest/src/KFDCWSRTest.hpp create mode 100644 tests/kfdtest/src/KFDDBGTest.cpp create mode 100644 tests/kfdtest/src/KFDDBGTest.hpp create mode 100644 tests/kfdtest/src/KFDEventTest.cpp create mode 100644 tests/kfdtest/src/KFDEventTest.hpp create mode 100644 tests/kfdtest/src/KFDEvictTest.cpp create mode 100644 tests/kfdtest/src/KFDEvictTest.hpp create mode 100644 tests/kfdtest/src/KFDExceptionTest.cpp create mode 100644 tests/kfdtest/src/KFDExceptionTest.hpp create mode 100644 tests/kfdtest/src/KFDGraphicsInterop.cpp create mode 100644 tests/kfdtest/src/KFDGraphicsInterop.hpp create mode 100644 tests/kfdtest/src/KFDIPCTest.cpp create mode 100644 tests/kfdtest/src/KFDIPCTest.hpp create mode 100644 tests/kfdtest/src/KFDLocalMemoryTest.cpp create mode 100644 tests/kfdtest/src/KFDLocalMemoryTest.hpp create mode 100644 tests/kfdtest/src/KFDMemoryTest.cpp create mode 100644 tests/kfdtest/src/KFDMemoryTest.hpp create mode 100644 tests/kfdtest/src/KFDOpenCloseKFDTest.cpp create mode 100644 tests/kfdtest/src/KFDOpenCloseKFDTest.hpp create mode 100644 tests/kfdtest/src/KFDPMTest.cpp create mode 100644 tests/kfdtest/src/KFDPMTest.hpp create mode 100644 tests/kfdtest/src/KFDPNPTest.cpp create mode 100644 tests/kfdtest/src/KFDPNPTest.hpp create mode 100644 tests/kfdtest/src/KFDPerfCounters.cpp create mode 100644 tests/kfdtest/src/KFDPerfCounters.hpp create mode 100644 tests/kfdtest/src/KFDQMTest.cpp create mode 100644 tests/kfdtest/src/KFDQMTest.hpp create mode 100644 tests/kfdtest/src/KFDTestFlags.hpp create mode 100644 tests/kfdtest/src/KFDTestMain.cpp create mode 100644 tests/kfdtest/src/KFDTestUtil.cpp create mode 100644 tests/kfdtest/src/KFDTestUtil.hpp create mode 100644 tests/kfdtest/src/KFDTopologyTest.cpp create mode 100644 tests/kfdtest/src/KFDTopologyTest.hpp create mode 100644 tests/kfdtest/src/LinuxOSWrapper.cpp create mode 100644 tests/kfdtest/src/OSWrapper.hpp create mode 100644 tests/kfdtest/src/PM4Packet.cpp create mode 100644 tests/kfdtest/src/PM4Packet.hpp create mode 100644 tests/kfdtest/src/PM4Queue.cpp create mode 100644 tests/kfdtest/src/PM4Queue.hpp create mode 100644 tests/kfdtest/src/RDMATest.cpp create mode 100644 tests/kfdtest/src/RDMATest.hpp create mode 100644 tests/kfdtest/src/RDMAUtil.cpp create mode 100644 tests/kfdtest/src/RDMAUtil.hpp create mode 100644 tests/kfdtest/src/SDMAPacket.cpp create mode 100644 tests/kfdtest/src/SDMAPacket.hpp create mode 100644 tests/kfdtest/src/SDMAQueue.cpp create mode 100644 tests/kfdtest/src/SDMAQueue.hpp diff --git a/tests/kfdtest/.gitignore b/tests/kfdtest/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt new file mode 100644 index 0000000000..fe7fed8132 --- /dev/null +++ b/tests/kfdtest/CMakeLists.txt @@ -0,0 +1,108 @@ +# +# Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# + +# If environment variable DRM_DIR or THUNK_LIB_OUT is set, the script +# will pick up the corresponding libraries from those pathes. + +cmake_minimum_required(VERSION 2.8 FATAL_ERROR) + +project(KFDTest) + +#set ( CMAKE_VERBOSE_MAKEFILE on ) + +find_package(PkgConfig) + +if( DEFINED ENV{DRM_DIR} ) +#assume header files and libraries are under the same path + set ( DRM_DIR $ENV{DRM_DIR} ) + set ( DRM_INCLUDE_DIRS ${DRM_DIR}/include ) + link_directories(${DRM_DIR}/lib64) + set ( DRM_LIBRARIES drm ) + set ( DRM_AMDGPU_LIBRARIES drm_amdgpu ) +else() +# The module name passed to pkg_check_modules() is determined by the +# name of file *.pc + pkg_check_modules(DRM REQUIRED libdrm) + pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) + include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) +endif() + + +if( DEFINED ENV{THUNK_LIB_OUT} ) + set ( THUNK_LIB_OUT $ENV{THUNK_LIB_OUT} ) + message ( " THUNK_LIB_OUT environment variable is set" ) +else() + set ( ENV{PKG_CONFIG_PATH} /opt/rocm/libhsakmt/ ) + pkg_check_modules(HSAKMT libhsakmt) + if( NOT HSAKMT_FOUND ) + set ( THUNK_LIB_OUT $ENV{OUT_DIR} ) + endif() +endif() + +if( DEFINED THUNK_LIB_OUT ) + set ( HSAKMT_INCLUDE_DIRS ${THUNK_LIB_OUT}/include ) + set ( HSAKMT_LIBRARY_DIRS ${THUNK_LIB_OUT}/lib ) + set ( HSAKMT_LIBRARIES hsakmt ) +endif() + +message ( "Find thunk lib at ${HSAKMT_LIBRARY_DIRS}" ) + +set ( SP3_DIR ${PROJECT_SOURCE_DIR}/sp3 ) + +include_directories(${PROJECT_SOURCE_DIR}/gtest-1.6.0) +include_directories(${PROJECT_SOURCE_DIR}/include) +include_directories(${SP3_DIR}) + +include_directories(${HSAKMT_INCLUDE_DIRS}) +include_directories(${DRM_INCLUDE_DIRS}) + +aux_source_directory(${PROJECT_SOURCE_DIR}/gtest-1.6.0 SRC_FILES) +aux_source_directory(${PROJECT_SOURCE_DIR}/src SRC_FILES) + +message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) +#message( STATUS "SRC_FILES: ") +#foreach(file ${SRC_FILES}) +# message(STATUS "${file}") +#endforeach() + +#add_definitions(-Wall -std=c++11) + +if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") +set ( CMAKE_CXX_FLAGS "-std=gnu++11" ) +endif() +if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) + set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2" ) +else () + set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g" ) +endif () + +# link_directories() has to be put before add_executable() +# The modules found by pkg_check_modules() in the default pkg config +# path do not need to use link_directories() here. +link_directories(${HSAKMT_LIBRARY_DIRS}) +link_directories(${SP3_DIR}) + +add_executable(kfdtest ${SRC_FILES}) + +target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LIBRARIES} ${DRM_AMDGPU_LIBRARIES} pthread m stdc++ rt amdsp3) + diff --git a/tests/kfdtest/LICENSE.kfdtest b/tests/kfdtest/LICENSE.kfdtest new file mode 100644 index 0000000000..5dfea070f5 --- /dev/null +++ b/tests/kfdtest/LICENSE.kfdtest @@ -0,0 +1,85 @@ +KFDTest - KFD unit tests LICENSE +/* The following license applies to all parts of kfdtest except the + * SP3 shader assembler. See separate license terms below. + */ + +Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. + +MIT LICENSE: +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +----------------------------------------------- + +libamdsp3 - AMD SP3 shader assembler library LICENSE +/* The following license applies to the SP3 shader assembler, which is + * named as libamdsp3.a and will be built into the kfdtest binary. + */ + +Copyright (C) 2018 Advanced Micro Devices, Inc. All rights reserved. + +REDISTRIBUTION: Permission is hereby granted, free of any license fees, +to any person obtaining a copy of this program (the "Software"), to +install, reproduce, copy and distribute copies, in binary form only, of +the Software and to permit persons to whom the Software is provided to +do the same, provided that the following conditions are met: + +No reverse engineering, decompilation, or disassembly of this Software +is permitted. + +Redistributions must reproduce the above copyright notice, this +permission notice, and the following disclaimers and notices in the +Software documentation and/or other materials provided with the +Software. + +DISCLAIMER: THE USE OF THE SOFTWARE IS AT YOUR SOLE RISK. THE SOFTWARE +IS PROVIDED "AS IS" AND WITHOUT WARRANTY OF ANY KIND AND COPYRIGHT +HOLDER AND ITS LICENSORS EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS AND +IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. +COPYRIGHT HOLDER AND ITS LICENSORS DO NOT WARRANT THAT THE SOFTWARE WILL +MEET YOUR REQUIREMENTS, OR THAT THE OPERATION OF THE SOFTWARE WILL BE +UNINTERRUPTED OR ERROR-FREE. THE ENTIRE RISK ASSOCIATED WITH THE USE OF +THE SOFTWARE IS ASSUMED BY YOU. FURTHERMORE, COPYRIGHT HOLDER AND ITS +LICENSORS DO NOT WARRANT OR MAKE ANY REPRESENTATIONS REGARDING THE USE +OR THE RESULTS OF THE USE OF THE SOFTWARE IN TERMS OF ITS CORRECTNESS, +ACCURACY, RELIABILITY, CURRENTNESS, OR OTHERWISE. + +DISCLAIMER: UNDER NO CIRCUMSTANCES INCLUDING NEGLIGENCE, SHALL COPYRIGHT +HOLDER AND ITS LICENSORS OR ITS DIRECTORS, OFFICERS, EMPLOYEES OR AGENTS +("AUTHORIZED REPRESENTATIVES") BE LIABLE FOR ANY INCIDENTAL, INDIRECT, +SPECIAL OR CONSEQUENTIAL DAMAGES (INCLUDING DAMAGES FOR LOSS OF BUSINESS +PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, AND THE +LIKE) ARISING OUT OF THE USE, MISUSE OR INABILITY TO USE THE SOFTWARE, +BREACH OR DEFAULT, INCLUDING THOSE ARISING FROM INFRINGEMENT OR ALLEGED +INFRINGEMENT OF ANY PATENT, TRADEMARK, COPYRIGHT OR OTHER INTELLECTUAL +PROPERTY RIGHT EVEN IF COPYRIGHT HOLDER AND ITS AUTHORIZED +REPRESENTATIVES HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. IN +NO EVENT SHALL COPYRIGHT HOLDER OR ITS AUTHORIZED REPRESENTATIVES TOTAL +LIABILITY FOR ALL DAMAGES, LOSSES, AND CAUSES OF ACTION (WHETHER IN +CONTRACT, TORT (INCLUDING NEGLIGENCE) OR OTHERWISE) EXCEED THE AMOUNT OF +US$10. + +Notice: The Software is subject to United States export laws and +regulations. You agree to comply with all domestic and international +export laws and regulations that apply to the Software, including but +not limited to the Export Administration Regulations administered by the +U.S. Department of Commerce and International Traffic in Arm Regulations +administered by the U.S. Department of State. These laws include +restrictions on destinations, end users and end use. diff --git a/tests/kfdtest/gtest-1.6.0/gtest-all.cpp b/tests/kfdtest/gtest-1.6.0/gtest-all.cpp new file mode 100644 index 0000000000..617eb99166 --- /dev/null +++ b/tests/kfdtest/gtest-1.6.0/gtest-all.cpp @@ -0,0 +1,9132 @@ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// +// Google C++ Testing Framework (Google Test) +// +// Sometimes it's desirable to build Google Test by compiling a single file. +// This file serves this purpose. + +// This line ensures that gtest.h can be compiled on its own, even +// when it's fused. +#include "gtest/gtest.h" + +// The following lines pull in the real gtest *.cc files. +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) + +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// Utilities for testing Google Test itself and code that uses Google Test +// (e.g. frameworks built on top of Google Test). + +#ifndef GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#define GTEST_INCLUDE_GTEST_GTEST_SPI_H_ + + +namespace testing { + +// This helper class can be used to mock out Google Test failure reporting +// so that we can test Google Test or code that builds on Google Test. +// +// An object of this class appends a TestPartResult object to the +// TestPartResultArray object given in the constructor whenever a Google Test +// failure is reported. It can either intercept only failures that are +// generated in the same thread that created this object or it can intercept +// all generated failures. The scope of this mock object can be controlled with +// the second argument to the two arguments constructor. +class GTEST_API_ ScopedFakeTestPartResultReporter + : public TestPartResultReporterInterface { + public: + // The two possible mocking modes of this object. + enum InterceptMode { + INTERCEPT_ONLY_CURRENT_THREAD, // Intercepts only thread local failures. + INTERCEPT_ALL_THREADS // Intercepts all failures. + }; + + // The c'tor sets this object as the test part result reporter used + // by Google Test. The 'result' parameter specifies where to report the + // results. This reporter will only catch failures generated in the current + // thread. DEPRECATED + explicit ScopedFakeTestPartResultReporter(TestPartResultArray* result); + + // Same as above, but you can choose the interception scope of this object. + ScopedFakeTestPartResultReporter(InterceptMode intercept_mode, + TestPartResultArray* result); + + // The d'tor restores the previous test part result reporter. + virtual ~ScopedFakeTestPartResultReporter(); + + // Appends the TestPartResult object to the TestPartResultArray + // received in the constructor. + // + // This method is from the TestPartResultReporterInterface + // interface. + virtual void ReportTestPartResult(const TestPartResult& result); + private: + void Init(); + + const InterceptMode intercept_mode_; + TestPartResultReporterInterface* old_reporter_; + TestPartResultArray* const result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedFakeTestPartResultReporter); +}; + +namespace internal { + +// A helper class for implementing EXPECT_FATAL_FAILURE() and +// EXPECT_NONFATAL_FAILURE(). Its destructor verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +class GTEST_API_ SingleFailureChecker { + public: + // The constructor remembers the arguments. + SingleFailureChecker(const TestPartResultArray* results, + TestPartResult::Type type, + const string& substr); + ~SingleFailureChecker(); + private: + const TestPartResultArray* const results_; + const TestPartResult::Type type_; + const string substr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SingleFailureChecker); +}; + +} // namespace internal + +} // namespace testing + +// A set of macros for testing Google Test assertions or code that's expected +// to generate Google Test fatal failures. It verifies that the given +// statement will cause exactly one fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_FATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_FATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - 'statement' cannot reference local non-static variables or +// non-static members of the current object. +// - 'statement' cannot return a value. +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. The AcceptsMacroThatExpandsToUnprotectedComma test in +// gtest_unittest.cc will fail to compile if we do that. +#define EXPECT_FATAL_FAILURE(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_FATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do { \ + class GTestExpectFatalFailureHelper {\ + public:\ + static void Execute() { statement; }\ + };\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kFatalFailure, (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ALL_THREADS, >est_failures);\ + GTestExpectFatalFailureHelper::Execute();\ + }\ + } while (::testing::internal::AlwaysFalse()) + +// A macro for testing Google Test assertions or code that's expected to +// generate Google Test non-fatal failures. It asserts that the given +// statement will cause exactly one non-fatal Google Test failure with 'substr' +// being part of the failure message. +// +// There are two different versions of this macro. EXPECT_NONFATAL_FAILURE only +// affects and considers failures generated in the current thread and +// EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS does the same but for all threads. +// +// 'statement' is allowed to reference local variables and members of +// the current object. +// +// The verification of the assertion is done correctly even when the statement +// throws an exception or aborts the current function. +// +// Known restrictions: +// - You cannot stream a failure message to this macro. +// +// Note that even though the implementations of the following two +// macros are much alike, we cannot refactor them to use a common +// helper macro, due to some peculiarity in how the preprocessor +// works. If we do that, the code won't compile when the user gives +// EXPECT_NONFATAL_FAILURE() a statement that contains a macro that +// expands to code containing an unprotected comma. The +// AcceptsMacroThatExpandsToUnprotectedComma test in gtest_unittest.cc +// catches that. +// +// For the same reason, we have to write +// if (::testing::internal::AlwaysTrue()) { statement; } +// instead of +// GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) +// to avoid an MSVC warning on unreachable code. +#define EXPECT_NONFATAL_FAILURE(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter:: \ + INTERCEPT_ONLY_CURRENT_THREAD, >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#define EXPECT_NONFATAL_FAILURE_ON_ALL_THREADS(statement, substr) \ + do {\ + ::testing::TestPartResultArray gtest_failures;\ + ::testing::internal::SingleFailureChecker gtest_checker(\ + >est_failures, ::testing::TestPartResult::kNonFatalFailure, \ + (substr));\ + {\ + ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ + ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS,\ + >est_failures);\ + if (::testing::internal::AlwaysTrue()) { statement; }\ + }\ + } while (::testing::internal::AlwaysFalse()) + +#endif // GTEST_INCLUDE_GTEST_GTEST_SPI_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include // NOLINT +#include +#include + +#if GTEST_OS_LINUX + +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +# include // NOLINT +# include // NOLINT +# include // NOLINT +// Declares vsnprintf(). This header is not available on Windows. +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include + +#elif GTEST_OS_SYMBIAN +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +#elif GTEST_OS_ZOS +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT + +// On z/OS we additionally need strings.h for strcasecmp. +# include // NOLINT + +#elif GTEST_OS_WINDOWS_MOBILE // We are on Windows CE. + +# include // NOLINT + +#elif GTEST_OS_WINDOWS // We are on Windows proper. + +# include // NOLINT +# include // NOLINT +# include // NOLINT +# include // NOLINT + +# if GTEST_OS_WINDOWS_MINGW +// MinGW has gettimeofday() but not _ftime64(). +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +// TODO(kenton@google.com): There are other ways to get the time on +// Windows, like GetTickCount() or GetSystemTimeAsFileTime(). MinGW +// supports these. consider using them instead. +# define GTEST_HAS_GETTIMEOFDAY_ 1 +# include // NOLINT +# endif // GTEST_OS_WINDOWS_MINGW + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT + +#else + +// Assume other platforms have gettimeofday(). +// TODO(kenton@google.com): Use autoconf to detect availability of +// gettimeofday(). +# define GTEST_HAS_GETTIMEOFDAY_ 1 + +// cpplint thinks that the header is already included, so we want to +// silence it. +# include // NOLINT +# include // NOLINT + +#endif // GTEST_OS_LINUX + +#if GTEST_HAS_EXCEPTIONS +# include +#endif + +#if GTEST_CAN_STREAM_RESULTS_ +# include // NOLINT +# include // NOLINT +#endif + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick is to +// prevent a user from accidentally including gtest-internal-inl.h in +// his code. +#define GTEST_IMPLEMENTATION_ 1 +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Utility functions and classes used by the Google C++ testing framework. +// +// Author: wan@google.com (Zhanyong Wan) +// +// This file contains purely Google Test's internal implementation. Please +// DO NOT #INCLUDE IT IN A USER PROGRAM. + +#ifndef GTEST_SRC_GTEST_INTERNAL_INL_H_ +#define GTEST_SRC_GTEST_INTERNAL_INL_H_ + +// GTEST_IMPLEMENTATION_ is defined to 1 iff the current translation unit is +// part of Google Test's implementation; otherwise it's undefined. +#if !GTEST_IMPLEMENTATION_ +// A user is trying to include this from his code - just say no. +# error "gtest-internal-inl.h is part of Google Test's internal implementation." +# error "It must not be included except by Google Test itself." +#endif // GTEST_IMPLEMENTATION_ + +#ifndef _WIN32_WCE +# include +#endif // !_WIN32_WCE +#include +#include // For strtoll/_strtoul64/malloc/free. +#include // For memmove. + +#include +#include +#include + + +#if GTEST_OS_WINDOWS +# include // NOLINT +#endif // GTEST_OS_WINDOWS + + +namespace testing { + +// Declares the flags. +// +// We don't want the users to modify this flag in the code, but want +// Google Test's own unit tests to be able to access it. Therefore we +// declare it here as opposed to in gtest.h. +GTEST_DECLARE_bool_(death_test_use_fork); + +namespace internal { + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +GTEST_API_ extern const TypeId kTestTypeIdInGoogleTest; + +// Names of the flags (needed for parsing Google Test flags). +const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests"; +const char kBreakOnFailureFlag[] = "break_on_failure"; +const char kCatchExceptionsFlag[] = "catch_exceptions"; +const char kColorFlag[] = "color"; +const char kFilterFlag[] = "filter"; +const char kListTestsFlag[] = "list_tests"; +const char kOutputFlag[] = "output"; +const char kPrintTimeFlag[] = "print_time"; +const char kRandomSeedFlag[] = "random_seed"; +const char kRepeatFlag[] = "repeat"; +const char kShuffleFlag[] = "shuffle"; +const char kStackTraceDepthFlag[] = "stack_trace_depth"; +const char kStreamResultToFlag[] = "stream_result_to"; +const char kThrowOnFailureFlag[] = "throw_on_failure"; + +// A valid random seed must be in [1, kMaxRandomSeed]. +const int kMaxRandomSeed = 99999; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +GTEST_API_ extern bool g_help_flag; + +// Returns the current time in milliseconds. +GTEST_API_ TimeInMillis GetTimeInMillis(); + +// Returns true iff Google Test should use colors in the output. +GTEST_API_ bool ShouldUseColor(bool stdout_is_tty); + +// Formats the given time in milliseconds as seconds. +GTEST_API_ std::string FormatTimeInMillisAsSeconds(TimeInMillis ms); + +// Parses a string for an Int32 flag, in the form of "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +GTEST_API_ bool ParseInt32Flag( + const char* str, const char* flag, Int32* value); + +// Returns a random seed in range [1, kMaxRandomSeed] based on the +// given --gtest_random_seed flag value. +inline int GetRandomSeedFromFlag(Int32 random_seed_flag) { + const unsigned int raw_seed = (random_seed_flag == 0) ? + static_cast(GetTimeInMillis()) : + static_cast(random_seed_flag); + + // Normalizes the actual seed to range [1, kMaxRandomSeed] such that + // it's easy to type. + const int normalized_seed = + static_cast((raw_seed - 1U) % + static_cast(kMaxRandomSeed)) + 1; + return normalized_seed; +} + +// Returns the first valid random seed after 'seed'. The behavior is +// undefined if 'seed' is invalid. The seed after kMaxRandomSeed is +// considered to be 1. +inline int GetNextRandomSeed(int seed) { + GTEST_CHECK_(1 <= seed && seed <= kMaxRandomSeed) + << "Invalid random seed " << seed << " - must be in [1, " + << kMaxRandomSeed << "]."; + const int next_seed = seed + 1; + return (next_seed > kMaxRandomSeed) ? 1 : next_seed; +} + +// This class saves the values of all Google Test flags in its c'tor, and +// restores them in its d'tor. +class GTestFlagSaver { + public: + // The c'tor. + GTestFlagSaver() { + also_run_disabled_tests_ = GTEST_FLAG(also_run_disabled_tests); + break_on_failure_ = GTEST_FLAG(break_on_failure); + catch_exceptions_ = GTEST_FLAG(catch_exceptions); + color_ = GTEST_FLAG(color); + death_test_style_ = GTEST_FLAG(death_test_style); + death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); + filter_ = GTEST_FLAG(filter); + internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); + list_tests_ = GTEST_FLAG(list_tests); + output_ = GTEST_FLAG(output); + print_time_ = GTEST_FLAG(print_time); + random_seed_ = GTEST_FLAG(random_seed); + repeat_ = GTEST_FLAG(repeat); + shuffle_ = GTEST_FLAG(shuffle); + stack_trace_depth_ = GTEST_FLAG(stack_trace_depth); + stream_result_to_ = GTEST_FLAG(stream_result_to); + throw_on_failure_ = GTEST_FLAG(throw_on_failure); + } + + // The d'tor is not virtual. DO NOT INHERIT FROM THIS CLASS. + ~GTestFlagSaver() { + GTEST_FLAG(also_run_disabled_tests) = also_run_disabled_tests_; + GTEST_FLAG(break_on_failure) = break_on_failure_; + GTEST_FLAG(catch_exceptions) = catch_exceptions_; + GTEST_FLAG(color) = color_; + GTEST_FLAG(death_test_style) = death_test_style_; + GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; + GTEST_FLAG(filter) = filter_; + GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; + GTEST_FLAG(list_tests) = list_tests_; + GTEST_FLAG(output) = output_; + GTEST_FLAG(print_time) = print_time_; + GTEST_FLAG(random_seed) = random_seed_; + GTEST_FLAG(repeat) = repeat_; + GTEST_FLAG(shuffle) = shuffle_; + GTEST_FLAG(stack_trace_depth) = stack_trace_depth_; + GTEST_FLAG(stream_result_to) = stream_result_to_; + GTEST_FLAG(throw_on_failure) = throw_on_failure_; + } + private: + // Fields for saving the original values of flags. + bool also_run_disabled_tests_; + bool break_on_failure_; + bool catch_exceptions_; + String color_; + String death_test_style_; + bool death_test_use_fork_; + String filter_; + String internal_run_death_test_; + bool list_tests_; + String output_; + bool print_time_; + bool pretty_; + internal::Int32 random_seed_; + internal::Int32 repeat_; + bool shuffle_; + internal::Int32 stack_trace_depth_; + String stream_result_to_; + bool throw_on_failure_; +} GTEST_ATTRIBUTE_UNUSED_; + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// The output buffer str must containt at least 32 characters. +// The function returns the address of the output buffer. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. +GTEST_API_ char* CodePointToUtf8(UInt32 code_point, char* str); + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +GTEST_API_ String WideStringToUtf8(const wchar_t* str, int num_chars); + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded(); + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (e.g., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +GTEST_API_ bool ShouldShard(const char* total_shards_str, + const char* shard_index_str, + bool in_subprocess_for_death_test); + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error and +// and aborts. +GTEST_API_ Int32 Int32FromEnvOrDie(const char* env_var, Int32 default_val); + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +GTEST_API_ bool ShouldRunTestOnShard( + int total_shards, int shard_index, int test_id); + +// STL container utilities. + +// Returns the number of elements in the given container that satisfy +// the given predicate. +template +inline int CountIf(const Container& c, Predicate predicate) { + // Implemented as an explicit loop since std::count_if() in libCstd on + // Solaris has a non-standard signature. + int count = 0; + for (typename Container::const_iterator it = c.begin(); it != c.end(); ++it) { + if (predicate(*it)) + ++count; + } + return count; +} + +// Applies a function/functor to each element in the container. +template +void ForEach(const Container& c, Functor functor) { + std::for_each(c.begin(), c.end(), functor); +} + +// Returns the i-th element of the vector, or default_value if i is not +// in range [0, v.size()). +template +inline E GetElementOr(const std::vector& v, int i, E default_value) { + return (i < 0 || i >= static_cast(v.size())) ? default_value : v[i]; +} + +// Performs an in-place shuffle of a range of the vector's elements. +// 'begin' and 'end' are element indices as an STL-style range; +// i.e. [begin, end) are shuffled, where 'end' == size() means to +// shuffle to the end of the vector. +template +void ShuffleRange(internal::Random* random, int begin, int end, + std::vector* v) { + const int size = static_cast(v->size()); + GTEST_CHECK_(0 <= begin && begin <= size) + << "Invalid shuffle range start " << begin << ": must be in range [0, " + << size << "]."; + GTEST_CHECK_(begin <= end && end <= size) + << "Invalid shuffle range finish " << end << ": must be in range [" + << begin << ", " << size << "]."; + + // Fisher-Yates shuffle, from + // http://en.wikipedia.org/wiki/Fisher-Yates_shuffle + for (int range_width = end - begin; range_width >= 2; range_width--) { + const int last_in_range = begin + range_width - 1; + const int selected = begin + random->Generate(range_width); + std::swap((*v)[selected], (*v)[last_in_range]); + } +} + +// Performs an in-place shuffle of the vector's elements. +template +inline void Shuffle(internal::Random* random, std::vector* v) { + ShuffleRange(random, 0, static_cast(v->size()), v); +} + +// A function for deleting an object. Handy for being used as a +// functor. +template +static void Delete(T* x) { + delete x; +} + +// A predicate that checks the key of a TestProperty against a known key. +// +// TestPropertyKeyIs is copyable. +class TestPropertyKeyIs { + public: + // Constructor. + // + // TestPropertyKeyIs has NO default constructor. + explicit TestPropertyKeyIs(const char* key) + : key_(key) {} + + // Returns true iff the test name of test property matches on key_. + bool operator()(const TestProperty& test_property) const { + return String(test_property.key()).Compare(key_) == 0; + } + + private: + String key_; +}; + +// Class UnitTestOptions. +// +// This class contains functions for processing options the user +// specifies when running the tests. It has only static members. +// +// In most cases, the user can specify an option using either an +// environment variable or a command line flag. E.g. you can set the +// test filter using either GTEST_FILTER or --gtest_filter. If both +// the variable and the flag are present, the latter overrides the +// former. +class GTEST_API_ UnitTestOptions { + public: + // Functions for processing the gtest_output flag. + + // Returns the output format, or "" for normal printed output. + static String GetOutputFormat(); + + // Returns the absolute path of the requested output file, or the + // default (test_detail.xml in the original working directory) if + // none was explicitly specified. + static String GetAbsolutePathToOutputFile(); + + // Functions for processing the gtest_filter flag. + + // Returns true iff the wildcard pattern matches the string. The + // first ':' or '\0' character in pattern marks the end of it. + // + // This recursive algorithm isn't very efficient, but is clear and + // works well enough for matching test names, which are short. + static bool PatternMatchesString(const char *pattern, const char *str); + + // Returns true iff the user-specified filter matches the test case + // name and the test name. + static bool FilterMatchesTest(const String &test_case_name, + const String &test_name); + +#if GTEST_OS_WINDOWS + // Function for supporting the gtest_catch_exception flag. + + // Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the + // given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. + // This function is useful as an __except condition. + static int GTestShouldProcessSEH(DWORD exception_code); +#endif // GTEST_OS_WINDOWS + + // Returns true if "name" matches the ':' separated list of glob-style + // filters in "filter". + static bool MatchesFilter(const String& name, const char* filter); +}; + +// Returns the current application's name, removing directory path if that +// is present. Used by UnitTestOptions::GetOutputFile. +GTEST_API_ FilePath GetCurrentExecutableName(); + +// The role interface for getting the OS stack trace as a string. +class OsStackTraceGetterInterface { + public: + OsStackTraceGetterInterface() {} + virtual ~OsStackTraceGetterInterface() {} + + // Returns the current OS stack trace as a String. Parameters: + // + // max_depth - the maximum number of stack frames to be included + // in the trace. + // skip_count - the number of top frames to be skipped; doesn't count + // against max_depth. + virtual String CurrentStackTrace(int max_depth, int skip_count) = 0; + + // UponLeavingGTest() should be called immediately before Google Test calls + // user code. It saves some information about the current stack that + // CurrentStackTrace() will use to find and hide Google Test stack frames. + virtual void UponLeavingGTest() = 0; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetterInterface); +}; + +// A working implementation of the OsStackTraceGetterInterface interface. +class OsStackTraceGetter : public OsStackTraceGetterInterface { + public: + OsStackTraceGetter() : caller_frame_(NULL) {} + virtual String CurrentStackTrace(int max_depth, int skip_count); + virtual void UponLeavingGTest(); + + // This string is inserted in place of stack frames that are part of + // Google Test's implementation. + static const char* const kElidedFramesMarker; + + private: + Mutex mutex_; // protects all internal state + + // We save the stack frame below the frame that calls user code. + // We do this because the address of the frame immediately below + // the user code changes between the call to UponLeavingGTest() + // and any calls to CurrentStackTrace() from within the user code. + void* caller_frame_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(OsStackTraceGetter); +}; + +// Information about a Google Test trace point. +struct TraceInfo { + const char* file; + int line; + String message; +}; + +// This is the default global test part result reporter used in UnitTestImpl. +// This class should only be used by UnitTestImpl. +class DefaultGlobalTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultGlobalTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. Reports the test part + // result in the current test. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultGlobalTestPartResultReporter); +}; + +// This is the default per thread test part result reporter used in +// UnitTestImpl. This class should only be used by UnitTestImpl. +class DefaultPerThreadTestPartResultReporter + : public TestPartResultReporterInterface { + public: + explicit DefaultPerThreadTestPartResultReporter(UnitTestImpl* unit_test); + // Implements the TestPartResultReporterInterface. The implementation just + // delegates to the current global test part result reporter of *unit_test_. + virtual void ReportTestPartResult(const TestPartResult& result); + + private: + UnitTestImpl* const unit_test_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DefaultPerThreadTestPartResultReporter); +}; + +// The private implementation of the UnitTest class. We don't protect +// the methods under a mutex, as this class is not accessible by a +// user and the UnitTest class that delegates work to this class does +// proper locking. +class GTEST_API_ UnitTestImpl { + public: + explicit UnitTestImpl(UnitTest* parent); + virtual ~UnitTestImpl(); + + // There are two different ways to register your own TestPartResultReporter. + // You can register your own repoter to listen either only for test results + // from the current thread or for results from all threads. + // By default, each per-thread test result repoter just passes a new + // TestPartResult to the global test result reporter, which registers the + // test part result for the currently running test. + + // Returns the global test part result reporter. + TestPartResultReporterInterface* GetGlobalTestPartResultReporter(); + + // Sets the global test part result reporter. + void SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter); + + // Returns the test part result reporter for the current thread. + TestPartResultReporterInterface* GetTestPartResultReporterForCurrentThread(); + + // Sets the test part result reporter for the current thread. + void SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter); + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const { return !Failed(); } + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const { + return failed_test_case_count() > 0 || ad_hoc_test_result()->Failed(); + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[i]; + } + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i) { + const int index = GetElementOr(test_case_indices_, i, -1); + return index < 0 ? NULL : test_cases_[index]; + } + + // Provides access to the event listener list. + TestEventListeners* listeners() { return &listeners_; } + + // Returns the TestResult for the test that's currently running, or + // the TestResult for the ad hoc test if no test is running. + TestResult* current_test_result(); + + // Returns the TestResult for the ad hoc test. + const TestResult* ad_hoc_test_result() const { return &ad_hoc_test_result_; } + + // Sets the OS stack trace getter. + // + // Does nothing if the input and the current OS stack trace getter + // are the same; otherwise, deletes the old getter and makes the + // input the current getter. + void set_os_stack_trace_getter(OsStackTraceGetterInterface* getter); + + // Returns the current OS stack trace getter if it is not NULL; + // otherwise, creates an OsStackTraceGetter, makes it the current + // getter, and returns it. + OsStackTraceGetterInterface* os_stack_trace_getter(); + + // Returns the current OS stack trace as a String. + // + // The maximum number of stack frames to be included is specified by + // the gtest_stack_trace_depth flag. The skip_count parameter + // specifies the number of top frames to be skipped, which doesn't + // count against the number of frames to be included. + // + // For example, if Foo() calls Bar(), which in turn calls + // CurrentOsStackTraceExceptTop(1), Foo() will be included in the + // trace but Bar() and CurrentOsStackTraceExceptTop() won't. + String CurrentOsStackTraceExceptTop(int skip_count); + + // Finds and returns a TestCase with the given name. If one doesn't + // exist, creates one and returns it. + // + // Arguments: + // + // test_case_name: name of the test case + // type_param: the name of the test's type parameter, or NULL if + // this is not a typed or a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase* GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Adds a TestInfo to the unit test. + // + // Arguments: + // + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + // test_info: the TestInfo object + void AddTestInfo(Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + TestInfo* test_info) { + // In order to support thread-safe death tests, we need to + // remember the original working directory when the test program + // was first invoked. We cannot do this in RUN_ALL_TESTS(), as + // the user may have changed the current directory before calling + // RUN_ALL_TESTS(). Therefore we capture the current directory in + // AddTestInfo(), which is called to register a TEST or TEST_F + // before main() is reached. + if (original_working_dir_.IsEmpty()) { + original_working_dir_.Set(FilePath::GetCurrentDir()); + GTEST_CHECK_(!original_working_dir_.IsEmpty()) + << "Failed to get the current working directory."; + } + + GetTestCase(test_info->test_case_name(), + test_info->type_param(), + set_up_tc, + tear_down_tc)->AddTestInfo(test_info); + } + +#if GTEST_HAS_PARAM_TEST + // Returns ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() { + return parameterized_test_registry_; + } +#endif // GTEST_HAS_PARAM_TEST + + // Sets the TestCase object for the test that's currently running. + void set_current_test_case(TestCase* a_current_test_case) { + current_test_case_ = a_current_test_case; + } + + // Sets the TestInfo object for the test that's currently running. If + // current_test_info is NULL, the assertion results will be stored in + // ad_hoc_test_result_. + void set_current_test_info(TestInfo* a_current_test_info) { + current_test_info_ = a_current_test_info; + } + + // Registers all parameterized tests defined using TEST_P and + // INSTANTIATE_TEST_CASE_P, creating regular tests for each test/parameter + // combination. This method can be called more then once; it has guards + // protecting from registering the tests more then once. If + // value-parameterized tests are disabled, RegisterParameterizedTests is + // present but does nothing. + void RegisterParameterizedTests(); + + // Runs all tests in this UnitTest object, prints the result, and + // returns true if all tests are successful. If any exception is + // thrown during a test, this test is considered to be failed, but + // the rest of the tests will still be run. + bool RunAllTests(); + + // Clears the results of all tests, except the ad hoc tests. + void ClearNonAdHocTestResult() { + ForEach(test_cases_, TestCase::ClearTestCaseResult); + } + + // Clears the results of ad-hoc test assertions. + void ClearAdHocTestResult() { + ad_hoc_test_result_.Clear(); + } + + enum ReactionToSharding { + HONOR_SHARDING_PROTOCOL, + IGNORE_SHARDING_PROTOCOL + }; + + // Matches the full name of each test against the user-specified + // filter to decide whether the test should run, then records the + // result in each TestCase and TestInfo object. + // If shard_tests == HONOR_SHARDING_PROTOCOL, further filters tests + // based on sharding variables in the environment. + // Returns the number of tests that should run. + int FilterTests(ReactionToSharding shard_tests); + + // Prints the names of the tests matching the user-specified filter flag. + void ListTestsMatchingFilter(); + + const TestCase* current_test_case() const { return current_test_case_; } + TestInfo* current_test_info() { return current_test_info_; } + const TestInfo* current_test_info() const { return current_test_info_; } + + // Returns the vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector& environments() { return environments_; } + + // Getters for the per-thread Google Test trace stack. + std::vector& gtest_trace_stack() { + return *(gtest_trace_stack_.pointer()); + } + const std::vector& gtest_trace_stack() const { + return gtest_trace_stack_.get(); + } + +#if GTEST_HAS_DEATH_TEST + void InitDeathTestSubprocessControlInfo() { + internal_run_death_test_flag_.reset(ParseInternalRunDeathTestFlag()); + } + // Returns a pointer to the parsed --gtest_internal_run_death_test + // flag, or NULL if that flag was not specified. + // This information is useful only in a death test child process. + // Must not be called before a call to InitGoogleTest. + const InternalRunDeathTestFlag* internal_run_death_test_flag() const { + return internal_run_death_test_flag_.get(); + } + + // Returns a pointer to the current death test factory. + internal::DeathTestFactory* death_test_factory() { + return death_test_factory_.get(); + } + + void SuppressTestEventsIfInSubprocess(); + + friend class ReplaceDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + + // Initializes the event listener performing XML output as specified by + // UnitTestOptions. Must not be called before InitGoogleTest. + void ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Initializes the event listener for streaming test results to a socket. + // Must not be called before InitGoogleTest. + void ConfigureStreamingOutput(); +#endif + + // Performs initialization dependent upon flag values obtained in + // ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to + // ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest + // this function is also called from RunAllTests. Since this function can be + // called more than once, it has to be idempotent. + void PostFlagParsingInit(); + + // Gets the random seed used at the start of the current test iteration. + int random_seed() const { return random_seed_; } + + // Gets the random number generator. + internal::Random* random() { return &random_; } + + // Shuffles all test cases, and the tests within each test case, + // making sure that death tests are still run first. + void ShuffleTests(); + + // Restores the test cases and tests to their order before the first shuffle. + void UnshuffleTests(); + + // Returns the value of GTEST_FLAG(catch_exceptions) at the moment + // UnitTest::Run() starts. + bool catch_exceptions() const { return catch_exceptions_; } + + private: + friend class ::testing::UnitTest; + + // Used by UnitTest::Run() to capture the state of + // GTEST_FLAG(catch_exceptions) at the moment it starts. + void set_catch_exceptions(bool value) { catch_exceptions_ = value; } + + // The UnitTest object that owns this implementation object. + UnitTest* const parent_; + + // The working directory when the first TEST() or TEST_F() was + // executed. + internal::FilePath original_working_dir_; + + // The default test part result reporters. + DefaultGlobalTestPartResultReporter default_global_test_part_result_reporter_; + DefaultPerThreadTestPartResultReporter + default_per_thread_test_part_result_reporter_; + + // Points to (but doesn't own) the global test part result reporter. + TestPartResultReporterInterface* global_test_part_result_repoter_; + + // Protects read and write access to global_test_part_result_reporter_. + internal::Mutex global_test_part_result_reporter_mutex_; + + // Points to (but doesn't own) the per-thread test part result reporter. + internal::ThreadLocal + per_thread_test_part_result_reporter_; + + // The vector of environments that need to be set-up/torn-down + // before/after the tests are run. + std::vector environments_; + + // The vector of TestCases in their original order. It owns the + // elements in the vector. + std::vector test_cases_; + + // Provides a level of indirection for the test case list to allow + // easy shuffling and restoring the test case order. The i-th + // element of this vector is the index of the i-th test case in the + // shuffled order. + std::vector test_case_indices_; + +#if GTEST_HAS_PARAM_TEST + // ParameterizedTestRegistry object used to register value-parameterized + // tests. + internal::ParameterizedTestCaseRegistry parameterized_test_registry_; + + // Indicates whether RegisterParameterizedTests() has been called already. + bool parameterized_tests_registered_; +#endif // GTEST_HAS_PARAM_TEST + + // Index of the last death test case registered. Initially -1. + int last_death_test_case_; + + // This points to the TestCase for the currently running test. It + // changes as Google Test goes through one test case after another. + // When no test is running, this is set to NULL and Google Test + // stores assertion results in ad_hoc_test_result_. Initially NULL. + TestCase* current_test_case_; + + // This points to the TestInfo for the currently running test. It + // changes as Google Test goes through one test after another. When + // no test is running, this is set to NULL and Google Test stores + // assertion results in ad_hoc_test_result_. Initially NULL. + TestInfo* current_test_info_; + + // Normally, a user only writes assertions inside a TEST or TEST_F, + // or inside a function called by a TEST or TEST_F. Since Google + // Test keeps track of which test is current running, it can + // associate such an assertion with the test it belongs to. + // + // If an assertion is encountered when no TEST or TEST_F is running, + // Google Test attributes the assertion result to an imaginary "ad hoc" + // test, and records the result in ad_hoc_test_result_. + TestResult ad_hoc_test_result_; + + // The list of event listeners that can be used to track events inside + // Google Test. + TestEventListeners listeners_; + + // The OS stack trace getter. Will be deleted when the UnitTest + // object is destructed. By default, an OsStackTraceGetter is used, + // but the user can set this field to use a custom getter if that is + // desired. + OsStackTraceGetterInterface* os_stack_trace_getter_; + + // True iff PostFlagParsingInit() has been called. + bool post_flag_parse_init_performed_; + + // The random number seed used at the beginning of the test run. + int random_seed_; + + // Our random number generator. + internal::Random random_; + + // How long the test took to run, in milliseconds. + TimeInMillis elapsed_time_; + +#if GTEST_HAS_DEATH_TEST + // The decomposed components of the gtest_internal_run_death_test flag, + // parsed when RUN_ALL_TESTS is called. + internal::scoped_ptr internal_run_death_test_flag_; + internal::scoped_ptr death_test_factory_; +#endif // GTEST_HAS_DEATH_TEST + + // A per-thread stack of traces created by the SCOPED_TRACE() macro. + internal::ThreadLocal > gtest_trace_stack_; + + // The value of GTEST_FLAG(catch_exceptions) at the moment RunAllTests() + // starts. + bool catch_exceptions_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTestImpl); +}; // class UnitTestImpl + +// Convenience function for accessing the global UnitTest +// implementation object. +inline UnitTestImpl* GetUnitTestImpl() { + return UnitTest::GetInstance()->impl(); +} + +#if GTEST_USES_SIMPLE_RE + +// Internal helper functions for implementing the simple regular +// expression matcher. +GTEST_API_ bool IsInSet(char ch, const char* str); +GTEST_API_ bool IsAsciiDigit(char ch); +GTEST_API_ bool IsAsciiPunct(char ch); +GTEST_API_ bool IsRepeat(char ch); +GTEST_API_ bool IsAsciiWhiteSpace(char ch); +GTEST_API_ bool IsAsciiWordChar(char ch); +GTEST_API_ bool IsValidEscape(char ch); +GTEST_API_ bool AtomMatchesChar(bool escaped, char pattern, char ch); +GTEST_API_ bool ValidateRegex(const char* regex); +GTEST_API_ bool MatchRegexAtHead(const char* regex, const char* str); +GTEST_API_ bool MatchRepetitionAndRegexAtHead( + bool escaped, char ch, char repeat, const char* regex, const char* str); +GTEST_API_ bool MatchRegexAnywhere(const char* regex, const char* str); + +#endif // GTEST_USES_SIMPLE_RE + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, char** argv); +GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv); + +#if GTEST_HAS_DEATH_TEST + +// Returns the message describing the last system error, regardless of the +// platform. +GTEST_API_ String GetLastErrnoDescription(); + +# if GTEST_OS_WINDOWS +// Provides leak-safe Windows kernel handle ownership. +class AutoHandle { + public: + AutoHandle() : handle_(INVALID_HANDLE_VALUE) {} + explicit AutoHandle(HANDLE handle) : handle_(handle) {} + + ~AutoHandle() { Reset(); } + + HANDLE Get() const { return handle_; } + void Reset() { Reset(INVALID_HANDLE_VALUE); } + void Reset(HANDLE handle) { + if (handle != handle_) { + if (handle_ != INVALID_HANDLE_VALUE) + ::CloseHandle(handle_); + handle_ = handle; + } + } + + private: + HANDLE handle_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle); +}; +# endif // GTEST_OS_WINDOWS + +// Attempts to parse a string into a positive integer pointed to by the +// number parameter. Returns true if that is possible. +// GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we can use +// it here. +template +bool ParseNaturalNumber(const ::std::string& str, Integer* number) { + // Fail fast if the given string does not begin with a digit; + // this bypasses strtoXXX's "optional leading whitespace and plus + // or minus sign" semantics, which are undesirable here. + if (str.empty() || !IsDigit(str[0])) { + return false; + } + errno = 0; + + char* end; + // BiggestConvertible is the largest integer type that system-provided + // string-to-number conversion routines can return. + +# if GTEST_OS_WINDOWS && !defined(__GNUC__) + + // MSVC and C++ Builder define __int64 instead of the standard long long. + typedef unsigned __int64 BiggestConvertible; + const BiggestConvertible parsed = _strtoui64(str.c_str(), &end, 10); + +# else + + typedef unsigned long long BiggestConvertible; // NOLINT + const BiggestConvertible parsed = strtoull(str.c_str(), &end, 10); + +# endif // GTEST_OS_WINDOWS && !defined(__GNUC__) + + const bool parse_success = *end == '\0' && errno == 0; + + // TODO(vladl@google.com): Convert this to compile time assertion when it is + // available. + GTEST_CHECK_(sizeof(Integer) <= sizeof(parsed)); + + const Integer result = static_cast(parsed); + if (parse_success && static_cast(result) == parsed) { + *number = result; + return true; + } + return false; +} +#endif // GTEST_HAS_DEATH_TEST + +// TestResult contains some private methods that should be hidden from +// Google Test user but are required for testing. This class allow our tests +// to access them. +// +// This class is supplied only for the purpose of testing Google Test's own +// constructs. Do not use it in user tests, either directly or indirectly. +class TestResultAccessor { + public: + static void RecordProperty(TestResult* test_result, + const TestProperty& property) { + test_result->RecordProperty(property); + } + + static void ClearTestPartResults(TestResult* test_result) { + test_result->ClearTestPartResults(); + } + + static const std::vector& test_part_results( + const TestResult& test_result) { + return test_result.test_part_results(); + } +}; + +} // namespace internal +} // namespace testing + +#endif // GTEST_SRC_GTEST_INTERNAL_INL_H_ +#undef GTEST_IMPLEMENTATION_ + +#if GTEST_OS_WINDOWS +# define vsnprintf _vsnprintf +#endif // GTEST_OS_WINDOWS + +namespace testing { + +using internal::CountIf; +using internal::ForEach; +using internal::GetElementOr; +using internal::Shuffle; + +// Constants. + +// A test whose test case name or test name matches this filter is +// disabled and not run. +static const char kDisableTestFilter[] = "DISABLED_*:*/DISABLED_*"; + +// A test case whose name matches this filter is considered a death +// test case and will be run before test cases whose name doesn't +// match this filter. +static const char kDeathTestCaseFilter[] = "*DeathTest:*DeathTest/*"; + +// A test filter that matches everything. +static const char kUniversalFilter[] = "*"; + +// The default output file for XML output. +static const char kDefaultOutputFile[] = "test_detail.xml"; + +// The environment variable name for the test shard index. +static const char kTestShardIndex[] = "GTEST_SHARD_INDEX"; +// The environment variable name for the total number of test shards. +static const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS"; +// The environment variable name for the test shard status file. +static const char kTestShardStatusFile[] = "GTEST_SHARD_STATUS_FILE"; + +namespace internal { + +// The text used in failure messages to indicate the start of the +// stack trace. +const char kStackTraceMarker[] = "\nStack trace:\n"; + +// g_help_flag is true iff the --help flag or an equivalent form is +// specified on the command line. +bool g_help_flag = false; + +} // namespace internal + +GTEST_DEFINE_bool_( + also_run_disabled_tests, + internal::BoolFromGTestEnv("also_run_disabled_tests", false), + "Run disabled tests too, in addition to the tests normally being run."); + +GTEST_DEFINE_bool_( + break_on_failure, + internal::BoolFromGTestEnv("break_on_failure", false), + "True iff a failed assertion should be a debugger break-point."); + +GTEST_DEFINE_bool_( + catch_exceptions, + internal::BoolFromGTestEnv("catch_exceptions", true), + "True iff " GTEST_NAME_ + " should catch exceptions and treat them as test failures."); + +GTEST_DEFINE_string_( + color, + internal::StringFromGTestEnv("color", "auto"), + "Whether to use colors in the output. Valid values: yes, no, " + "and auto. 'auto' means to use colors if the output is " + "being sent to a terminal and the TERM environment variable " + "is set to xterm, xterm-color, xterm-256color, linux or cygwin."); + +GTEST_DEFINE_string_( + filter, + internal::StringFromGTestEnv("filter", kUniversalFilter), + "A colon-separated list of glob (not regex) patterns " + "for filtering the tests to run, optionally followed by a " + "'-' and a : separated list of negative patterns (tests to " + "exclude). A test is run if it matches one of the positive " + "patterns and does not match any of the negative patterns."); + +GTEST_DEFINE_bool_(list_tests, false, + "List all tests without running them."); + +GTEST_DEFINE_string_( + output, + internal::StringFromGTestEnv("output", ""), + "A format (currently must be \"xml\"), optionally followed " + "by a colon and an output file name or directory. A directory " + "is indicated by a trailing pathname separator. " + "Examples: \"xml:filename.xml\", \"xml::directoryname/\". " + "If a directory is specified, output files will be created " + "within that directory, with file-names based on the test " + "executable's name and, if necessary, made unique by adding " + "digits."); + +GTEST_DEFINE_bool_( + print_time, + internal::BoolFromGTestEnv("print_time", true), + "True iff " GTEST_NAME_ + " should display elapsed time in text output."); + +GTEST_DEFINE_int32_( + random_seed, + internal::Int32FromGTestEnv("random_seed", 0), + "Random number seed to use when shuffling test orders. Must be in range " + "[1, 99999], or 0 to use a seed based on the current time."); + +GTEST_DEFINE_int32_( + repeat, + internal::Int32FromGTestEnv("repeat", 1), + "How many times to repeat each test. Specify a negative number " + "for repeating forever. Useful for shaking out flaky tests."); + +GTEST_DEFINE_bool_( + show_internal_stack_frames, false, + "True iff " GTEST_NAME_ " should include internal stack frames when " + "printing test failure stack traces."); + +GTEST_DEFINE_bool_( + shuffle, + internal::BoolFromGTestEnv("shuffle", false), + "True iff " GTEST_NAME_ + " should randomize tests' order on every run."); + +GTEST_DEFINE_int32_( + stack_trace_depth, + internal::Int32FromGTestEnv("stack_trace_depth", kMaxStackTraceDepth), + "The maximum number of stack frames to print when an " + "assertion fails. The valid range is 0 through 100, inclusive."); + +GTEST_DEFINE_string_( + stream_result_to, + internal::StringFromGTestEnv("stream_result_to", ""), + "This flag specifies the host name and the port number on which to stream " + "test results. Example: \"localhost:555\". The flag is effective only on " + "Linux."); + +GTEST_DEFINE_bool_( + throw_on_failure, + internal::BoolFromGTestEnv("throw_on_failure", false), + "When this flag is specified, a failed assertion will throw an exception " + "if exceptions are enabled or exit the program with a non-zero code " + "otherwise."); + +namespace internal { + +// Generates a random number from [0, range), using a Linear +// Congruential Generator (LCG). Crashes if 'range' is 0 or greater +// than kMaxRange. +UInt32 Random::Generate(UInt32 range) { + // These constants are the same as are used in glibc's rand(3). + state_ = (1103515245U*state_ + 12345U) % kMaxRange; + + GTEST_CHECK_(range > 0) + << "Cannot generate a number in the range [0, 0)."; + GTEST_CHECK_(range <= kMaxRange) + << "Generation of a number in [0, " << range << ") was requested, " + << "but this can only generate numbers in [0, " << kMaxRange << ")."; + + // Converting via modulus introduces a bit of downward bias, but + // it's simple, and a linear congruential generator isn't too good + // to begin with. + return state_ % range; +} + +// GTestIsInitialized() returns true iff the user has initialized +// Google Test. Useful for catching the user mistake of not initializing +// Google Test before calling RUN_ALL_TESTS(). +// +// A user must call testing::InitGoogleTest() to initialize Google +// Test. g_init_gtest_count is set to the number of times +// InitGoogleTest() has been called. We don't protect this variable +// under a mutex as it is only accessed in the main thread. +int g_init_gtest_count = 0; +static bool GTestIsInitialized() { return g_init_gtest_count != 0; } + +// Iterates over a vector of TestCases, keeping a running sum of the +// results of calling a given int-returning method on each. +// Returns the sum. +static int SumOverTestCaseList(const std::vector& case_list, + int (TestCase::*method)() const) { + int sum = 0; + for (size_t i = 0; i < case_list.size(); i++) { + sum += (case_list[i]->*method)(); + } + return sum; +} + +// Returns true iff the test case passed. +static bool TestCasePassed(const TestCase* test_case) { + return test_case->should_run() && test_case->Passed(); +} + +// Returns true iff the test case failed. +static bool TestCaseFailed(const TestCase* test_case) { + return test_case->should_run() && test_case->Failed(); +} + +// Returns true iff test_case contains at least one test that should +// run. +static bool ShouldRunTestCase(const TestCase* test_case) { + return test_case->should_run(); +} + +// AssertHelper constructor. +AssertHelper::AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message) + : data_(new AssertHelperData(type, file, line, message)) { +} + +AssertHelper::~AssertHelper() { + delete data_; +} + +// Message assignment, for assertion streaming support. +void AssertHelper::operator=(const Message& message) const { + UnitTest::GetInstance()-> + AddTestPartResult(data_->type, data_->file, data_->line, + AppendUserMessage(data_->message, message), + UnitTest::GetInstance()->impl() + ->CurrentOsStackTraceExceptTop(1) + // Skips the stack frame for this function itself. + ); // NOLINT +} + +// Mutex for linked pointers. +GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// Application pathname gotten in InitGoogleTest. +String g_executable_path; + +// Returns the current application's name, removing directory path if that +// is present. +FilePath GetCurrentExecutableName() { + FilePath result; + +#if GTEST_OS_WINDOWS + result.Set(FilePath(g_executable_path).RemoveExtension("exe")); +#else + result.Set(FilePath(g_executable_path)); +#endif // GTEST_OS_WINDOWS + + return result.RemoveDirectoryName(); +} + +// Functions for processing the gtest_output flag. + +// Returns the output format, or "" for normal printed output. +String UnitTestOptions::GetOutputFormat() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + if (gtest_output_flag == NULL) return String(""); + + const char* const colon = strchr(gtest_output_flag, ':'); + return (colon == NULL) ? + String(gtest_output_flag) : + String(gtest_output_flag, colon - gtest_output_flag); +} + +// Returns the name of the requested output file, or the default if none +// was explicitly specified. +String UnitTestOptions::GetAbsolutePathToOutputFile() { + const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); + if (gtest_output_flag == NULL) + return String(""); + + const char* const colon = strchr(gtest_output_flag, ':'); + if (colon == NULL) + return String(internal::FilePath::ConcatPaths( + internal::FilePath( + UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(kDefaultOutputFile)).ToString() ); + + internal::FilePath output_name(colon + 1); + if (!output_name.IsAbsolutePath()) + // TODO(wan@google.com): on Windows \some\path is not an absolute + // path (as its meaning depends on the current drive), yet the + // following logic for turning it into an absolute path is wrong. + // Fix it. + output_name = internal::FilePath::ConcatPaths( + internal::FilePath(UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(colon + 1)); + + if (!output_name.IsDirectory()) + return output_name.ToString(); + + internal::FilePath result(internal::FilePath::GenerateUniqueFileName( + output_name, internal::GetCurrentExecutableName(), + GetOutputFormat().c_str())); + return result.ToString(); +} + +// Returns true iff the wildcard pattern matches the string. The +// first ':' or '\0' character in pattern marks the end of it. +// +// This recursive algorithm isn't very efficient, but is clear and +// works well enough for matching test names, which are short. +bool UnitTestOptions::PatternMatchesString(const char *pattern, + const char *str) { + switch (*pattern) { + case '\0': + case ':': // Either ':' or '\0' marks the end of the pattern. + return *str == '\0'; + case '?': // Matches any single character. + return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); + case '*': // Matches any string (possibly empty) of characters. + return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || + PatternMatchesString(pattern + 1, str); + default: // Non-special character. Matches itself. + return *pattern == *str && + PatternMatchesString(pattern + 1, str + 1); + } +} + +bool UnitTestOptions::MatchesFilter(const String& name, const char* filter) { + const char *cur_pattern = filter; + for (;;) { + if (PatternMatchesString(cur_pattern, name.c_str())) { + return true; + } + + // Finds the next pattern in the filter. + cur_pattern = strchr(cur_pattern, ':'); + + // Returns if no more pattern can be found. + if (cur_pattern == NULL) { + return false; + } + + // Skips the pattern separater (the ':' character). + cur_pattern++; + } +} + +// TODO(keithray): move String function implementations to gtest-string.cc. + +// Returns true iff the user-specified filter matches the test case +// name and the test name. +bool UnitTestOptions::FilterMatchesTest(const String &test_case_name, + const String &test_name) { + const String& full_name = String::Format("%s.%s", + test_case_name.c_str(), + test_name.c_str()); + + // Split --gtest_filter at '-', if there is one, to separate into + // positive filter and negative filter portions + const char* const p = GTEST_FLAG(filter).c_str(); + const char* const dash = strchr(p, '-'); + String positive; + String negative; + if (dash == NULL) { + positive = GTEST_FLAG(filter).c_str(); // Whole string is a positive filter + negative = String(""); + } else { + positive = String(p, dash - p); // Everything up to the dash + negative = String(dash+1); // Everything after the dash + if (positive.empty()) { + // Treat '-test1' as the same as '*-test1' + positive = kUniversalFilter; + } + } + + // A filter is a colon-separated list of patterns. It matches a + // test if any pattern in it matches the test. + return (MatchesFilter(full_name, positive.c_str()) && + !MatchesFilter(full_name, negative.c_str())); +} + +#if GTEST_HAS_SEH +// Returns EXCEPTION_EXECUTE_HANDLER if Google Test should handle the +// given SEH exception, or EXCEPTION_CONTINUE_SEARCH otherwise. +// This function is useful as an __except condition. +int UnitTestOptions::GTestShouldProcessSEH(DWORD exception_code) { + // Google Test should handle a SEH exception if: + // 1. the user wants it to, AND + // 2. this is not a breakpoint exception, AND + // 3. this is not a C++ exception (VC++ implements them via SEH, + // apparently). + // + // SEH exception code for C++ exceptions. + // (see http://support.microsoft.com/kb/185294 for more information). + const DWORD kCxxExceptionCode = 0xe06d7363; + + bool should_handle = true; + + if (!GTEST_FLAG(catch_exceptions)) + should_handle = false; + else if (exception_code == EXCEPTION_BREAKPOINT) + should_handle = false; + else if (exception_code == kCxxExceptionCode) + should_handle = false; + + return should_handle ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH; +} +#endif // GTEST_HAS_SEH + +} // namespace internal + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. Intercepts only failures from the current thread. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + TestPartResultArray* result) + : intercept_mode_(INTERCEPT_ONLY_CURRENT_THREAD), + result_(result) { + Init(); +} + +// The c'tor sets this object as the test part result reporter used by +// Google Test. The 'result' parameter specifies where to report the +// results. +ScopedFakeTestPartResultReporter::ScopedFakeTestPartResultReporter( + InterceptMode intercept_mode, TestPartResultArray* result) + : intercept_mode_(intercept_mode), + result_(result) { + Init(); +} + +void ScopedFakeTestPartResultReporter::Init() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + old_reporter_ = impl->GetGlobalTestPartResultReporter(); + impl->SetGlobalTestPartResultReporter(this); + } else { + old_reporter_ = impl->GetTestPartResultReporterForCurrentThread(); + impl->SetTestPartResultReporterForCurrentThread(this); + } +} + +// The d'tor restores the test part result reporter used by Google Test +// before. +ScopedFakeTestPartResultReporter::~ScopedFakeTestPartResultReporter() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + if (intercept_mode_ == INTERCEPT_ALL_THREADS) { + impl->SetGlobalTestPartResultReporter(old_reporter_); + } else { + impl->SetTestPartResultReporterForCurrentThread(old_reporter_); + } +} + +// Increments the test part result count and remembers the result. +// This method is from the TestPartResultReporterInterface interface. +void ScopedFakeTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + result_->Append(result); +} + +namespace internal { + +// Returns the type ID of ::testing::Test. We should always call this +// instead of GetTypeId< ::testing::Test>() to get the type ID of +// testing::Test. This is to work around a suspected linker bug when +// using Google Test as a framework on Mac OS X. The bug causes +// GetTypeId< ::testing::Test>() to return different values depending +// on whether the call is from the Google Test framework itself or +// from user test code. GetTestTypeId() is guaranteed to always +// return the same value, as it always calls GetTypeId<>() from the +// gtest.cc, which is within the Google Test framework. +TypeId GetTestTypeId() { + return GetTypeId(); +} + +// The value of GetTestTypeId() as seen from within the Google Test +// library. This is solely for testing GetTestTypeId(). +extern const TypeId kTestTypeIdInGoogleTest = GetTestTypeId(); + +// This predicate-formatter checks that 'results' contains a test part +// failure of the given type and that the failure message contains the +// given substring. +AssertionResult HasOneFailure(const char* /* results_expr */, + const char* /* type_expr */, + const char* /* substr_expr */, + const TestPartResultArray& results, + TestPartResult::Type type, + const string& substr) { + const String expected(type == TestPartResult::kFatalFailure ? + "1 fatal failure" : + "1 non-fatal failure"); + Message msg; + if (results.size() != 1) { + msg << "Expected: " << expected << "\n" + << " Actual: " << results.size() << " failures"; + for (int i = 0; i < results.size(); i++) { + msg << "\n" << results.GetTestPartResult(i); + } + return AssertionFailure() << msg; + } + + const TestPartResult& r = results.GetTestPartResult(0); + if (r.type() != type) { + return AssertionFailure() << "Expected: " << expected << "\n" + << " Actual:\n" + << r; + } + + if (strstr(r.message(), substr.c_str()) == NULL) { + return AssertionFailure() << "Expected: " << expected << " containing \"" + << substr << "\"\n" + << " Actual:\n" + << r; + } + + return AssertionSuccess(); +} + +// The constructor of SingleFailureChecker remembers where to look up +// test part results, what type of failure we expect, and what +// substring the failure message should contain. +SingleFailureChecker:: SingleFailureChecker( + const TestPartResultArray* results, + TestPartResult::Type type, + const string& substr) + : results_(results), + type_(type), + substr_(substr) {} + +// The destructor of SingleFailureChecker verifies that the given +// TestPartResultArray contains exactly one failure that has the given +// type and contains the given substring. If that's not the case, a +// non-fatal failure will be generated. +SingleFailureChecker::~SingleFailureChecker() { + EXPECT_PRED_FORMAT3(HasOneFailure, *results_, type_, substr_); +} + +DefaultGlobalTestPartResultReporter::DefaultGlobalTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultGlobalTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->current_test_result()->AddTestPartResult(result); + unit_test_->listeners()->repeater()->OnTestPartResult(result); +} + +DefaultPerThreadTestPartResultReporter::DefaultPerThreadTestPartResultReporter( + UnitTestImpl* unit_test) : unit_test_(unit_test) {} + +void DefaultPerThreadTestPartResultReporter::ReportTestPartResult( + const TestPartResult& result) { + unit_test_->GetGlobalTestPartResultReporter()->ReportTestPartResult(result); +} + +// Returns the global test part result reporter. +TestPartResultReporterInterface* +UnitTestImpl::GetGlobalTestPartResultReporter() { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + return global_test_part_result_repoter_; +} + +// Sets the global test part result reporter. +void UnitTestImpl::SetGlobalTestPartResultReporter( + TestPartResultReporterInterface* reporter) { + internal::MutexLock lock(&global_test_part_result_reporter_mutex_); + global_test_part_result_repoter_ = reporter; +} + +// Returns the test part result reporter for the current thread. +TestPartResultReporterInterface* +UnitTestImpl::GetTestPartResultReporterForCurrentThread() { + return per_thread_test_part_result_reporter_.get(); +} + +// Sets the test part result reporter for the current thread. +void UnitTestImpl::SetTestPartResultReporterForCurrentThread( + TestPartResultReporterInterface* reporter) { + per_thread_test_part_result_reporter_.set(reporter); +} + +// Gets the number of successful test cases. +int UnitTestImpl::successful_test_case_count() const { + return CountIf(test_cases_, TestCasePassed); +} + +// Gets the number of failed test cases. +int UnitTestImpl::failed_test_case_count() const { + return CountIf(test_cases_, TestCaseFailed); +} + +// Gets the number of all test cases. +int UnitTestImpl::total_test_case_count() const { + return static_cast(test_cases_.size()); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTestImpl::test_case_to_run_count() const { + return CountIf(test_cases_, ShouldRunTestCase); +} + +// Gets the number of successful tests. +int UnitTestImpl::successful_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::successful_test_count); +} + +// Gets the number of failed tests. +int UnitTestImpl::failed_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count); +} + +// Gets the number of disabled tests. +int UnitTestImpl::disabled_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count); +} + +// Gets the number of all tests. +int UnitTestImpl::total_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::total_test_count); +} + +// Gets the number of tests that should run. +int UnitTestImpl::test_to_run_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::test_to_run_count); +} + +// Returns the current OS stack trace as a String. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// CurrentOsStackTraceExceptTop(1), Foo() will be included in the +// trace but Bar() and CurrentOsStackTraceExceptTop() won't. +String UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) { + (void)skip_count; + return String(""); +} + +// Returns the current time in milliseconds. +TimeInMillis GetTimeInMillis() { +#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__) + // Difference between 1970-01-01 and 1601-01-01 in milliseconds. + // http://analogous.blogspot.com/2005/04/epoch.html + const TimeInMillis kJavaEpochToWinFileTimeDelta = + static_cast(116444736UL) * 100000UL; + const DWORD kTenthMicrosInMilliSecond = 10000; + + SYSTEMTIME now_systime; + FILETIME now_filetime; + ULARGE_INTEGER now_int64; + // TODO(kenton@google.com): Shouldn't this just use + // GetSystemTimeAsFileTime()? + GetSystemTime(&now_systime); + if (SystemTimeToFileTime(&now_systime, &now_filetime)) { + now_int64.LowPart = now_filetime.dwLowDateTime; + now_int64.HighPart = now_filetime.dwHighDateTime; + now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) - + kJavaEpochToWinFileTimeDelta; + return now_int64.QuadPart; + } + return 0; +#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ + __timeb64 now; + +# ifdef _MSC_VER + + // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 + // (deprecated function) there. + // TODO(kenton@google.com): Use GetTickCount()? Or use + // SystemTimeToFileTime() +# pragma warning(push) // Saves the current warning state. +# pragma warning(disable:4996) // Temporarily disables warning 4996. + _ftime64(&now); +# pragma warning(pop) // Restores the warning state. +# else + + _ftime64(&now); + +# endif // _MSC_VER + + return static_cast(now.time) * 1000 + now.millitm; +#elif GTEST_HAS_GETTIMEOFDAY_ + struct timeval now; + gettimeofday(&now, NULL); + return static_cast(now.tv_sec) * 1000 + now.tv_usec / 1000; +#else +# error "Don't know how to get the current time on your system." +#endif +} + +// Utilities + +// class String + +// Returns the input enclosed in double quotes if it's not NULL; +// otherwise returns "(null)". For example, "\"Hello\"" is returned +// for input "Hello". +// +// This is useful for printing a C string in the syntax of a literal. +// +// Known issue: escape sequences are not handled yet. +String String::ShowCStringQuoted(const char* c_str) { + return c_str ? String::Format("\"%s\"", c_str) : String("(null)"); +} + +// Copies at most length characters from str into a newly-allocated +// piece of memory of size length+1. The memory is allocated with new[]. +// A terminating null byte is written to the memory, and a pointer to it +// is returned. If str is NULL, NULL is returned. +static char* CloneString(const char* str, size_t length) { + if (str == NULL) { + return NULL; + } else { + char* const clone = new char[length + 1]; + posix::StrNCpy(clone, str, length); + clone[length] = '\0'; + return clone; + } +} + +// Clones a 0-terminated C string, allocating memory using new. The +// caller is responsible for deleting[] the return value. Returns the +// cloned string, or NULL if the input is NULL. +const char * String::CloneCString(const char* c_str) { + return (c_str == NULL) ? + NULL : CloneString(c_str, strlen(c_str)); +} + +#if GTEST_OS_WINDOWS_MOBILE +// Creates a UTF-16 wide string from the given ANSI string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the wide string, or NULL if the +// input is NULL. +LPCWSTR String::AnsiToUtf16(const char* ansi) { + if (!ansi) return NULL; + const int length = strlen(ansi); + const int unicode_length = + MultiByteToWideChar(CP_ACP, 0, ansi, length, + NULL, 0); + WCHAR* unicode = new WCHAR[unicode_length + 1]; + MultiByteToWideChar(CP_ACP, 0, ansi, length, + unicode, unicode_length); + unicode[unicode_length] = 0; + return unicode; +} + +// Creates an ANSI string from the given wide string, allocating +// memory using new. The caller is responsible for deleting the return +// value using delete[]. Returns the ANSI string, or NULL if the +// input is NULL. +const char* String::Utf16ToAnsi(LPCWSTR utf16_str) { + if (!utf16_str) return NULL; + const int ansi_length = + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + NULL, 0, NULL, NULL); + char* ansi = new char[ansi_length + 1]; + WideCharToMultiByte(CP_ACP, 0, utf16_str, -1, + ansi, ansi_length, NULL, NULL); + ansi[ansi_length] = 0; + return ansi; +} + +#endif // GTEST_OS_WINDOWS_MOBILE + +// Compares two C strings. Returns true iff they have the same content. +// +// Unlike strcmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CStringEquals(const char * lhs, const char * rhs) { + if ( lhs == NULL ) return rhs == NULL; + + if ( rhs == NULL ) return false; + + return strcmp(lhs, rhs) == 0; +} + +#if GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +// Converts an array of wide chars to a narrow string using the UTF-8 +// encoding, and streams the result to the given Message object. +static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length, + Message* msg) { + // TODO(wan): consider allowing a testing::String object to + // contain '\0'. This will make it behave more like std::string, + // and will allow ToUtf8String() to return the correct encoding + // for '\0' s.t. we can get rid of the conditional here (and in + // several other places). + for (size_t i = 0; i != length; ) { // NOLINT + if (wstr[i] != L'\0') { + *msg << WideStringToUtf8(wstr + i, static_cast(length - i)); + while (i != length && wstr[i] != L'\0') + i++; + } else { + *msg << '\0'; + i++; + } + } +} + +#endif // GTEST_HAS_STD_WSTRING || GTEST_HAS_GLOBAL_WSTRING + +} // namespace internal + +#if GTEST_HAS_STD_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::std::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING +// Converts the given wide string to a narrow string using the UTF-8 +// encoding, and streams the result to this Message object. +Message& Message::operator <<(const ::wstring& wstr) { + internal::StreamWideCharsToMessage(wstr.c_str(), wstr.length(), this); + return *this; +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +// AssertionResult constructors. +// Used in EXPECT_TRUE/FALSE(assertion_result). +AssertionResult::AssertionResult(const AssertionResult& other) + : success_(other.success_), + message_(other.message_.get() != NULL ? + new ::std::string(*other.message_) : + static_cast< ::std::string*>(NULL)) { +} + +// Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. +AssertionResult AssertionResult::operator!() const { + AssertionResult negation(!success_); + if (message_.get() != NULL) + negation << *message_; + return negation; +} + +// Makes a successful assertion result. +AssertionResult AssertionSuccess() { + return AssertionResult(true); +} + +// Makes a failed assertion result. +AssertionResult AssertionFailure() { + return AssertionResult(false); +} + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << message. +AssertionResult AssertionFailure(const Message& message) { + return AssertionFailure() << message; +} + +namespace internal { + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// expected_expression: "foo" +// actual_expression: "bar" +// expected_value: "5" +// actual_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string " (ignoring case)" will +// be inserted into the message. +AssertionResult EqFailure(const char* expected_expression, + const char* actual_expression, + const String& expected_value, + const String& actual_value, + bool ignoring_case) { + Message msg; + msg << "Value of: " << actual_expression; + if (actual_value != actual_expression) { + msg << "\n Actual: " << actual_value; + } + + msg << "\nExpected: " << expected_expression; + if (ignoring_case) { + msg << " (ignoring case)"; + } + if (expected_value != expected_expression) { + msg << "\nWhich is: " << expected_value; + } + + return AssertionFailure() << msg; +} + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +String GetBoolAssertionFailureMessage(const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value) { + const char* actual_message = assertion_result.message(); + Message msg; + msg << "Value of: " << expression_text + << "\n Actual: " << actual_predicate_value; + if (actual_message[0] != '\0') + msg << " (" << actual_message << ")"; + msg << "\nExpected: " << expected_predicate_value; + return msg.GetString(); +} + +// Helper function for implementing ASSERT_NEAR. +AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error) { + const double diff = fabs(val1 - val2); + if (diff <= abs_error) return AssertionSuccess(); + + // TODO(wan): do not print the value of an expression if it's + // already a literal. + return AssertionFailure() + << "The difference between " << expr1 << " and " << expr2 + << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n" + << expr1 << " evaluates to " << val1 << ",\n" + << expr2 << " evaluates to " << val2 << ", and\n" + << abs_error_expr << " evaluates to " << abs_error << "."; +} + + +// Helper template for implementing FloatLE() and DoubleLE(). +template +AssertionResult FloatingPointLE(const char* expr1, + const char* expr2, + RawType val1, + RawType val2) { + // Returns success if val1 is less than val2, + if (val1 < val2) { + return AssertionSuccess(); + } + + // or if val1 is almost equal to val2. + const FloatingPoint lhs(val1), rhs(val2); + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + // Note that the above two checks will both fail if either val1 or + // val2 is NaN, as the IEEE floating-point standard requires that + // any predicate involving a NaN must return false. + + ::std::stringstream val1_ss; + val1_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val1; + + ::std::stringstream val2_ss; + val2_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << val2; + + return AssertionFailure() + << "Expected: (" << expr1 << ") <= (" << expr2 << ")\n" + << " Actual: " << StringStreamToString(&val1_ss) << " vs " + << StringStreamToString(&val2_ss); +} + +} // namespace internal + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2) { + return internal::FloatingPointLE(expr1, expr2, val1, val2); +} + +namespace internal { + +// The helper function for {ASSERT|EXPECT}_EQ with int or enum +// arguments. +AssertionResult CmpHelperEQ(const char* expected_expression, + const char* actual_expression, + BiggestInt expected, + BiggestInt actual) { + if (expected == actual) { + return AssertionSuccess(); + } + + return EqFailure(expected_expression, + actual_expression, + FormatForComparisonFailureMessage(expected, actual), + FormatForComparisonFailureMessage(actual, expected), + false); +} + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_?? with integer or enum arguments. It is here +// just to avoid copy-and-paste of similar code. +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + BiggestInt val1, BiggestInt val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return AssertionFailure() \ + << "Expected: (" << expr1 << ") " #op " (" << expr2\ + << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ + << " vs " << FormatForComparisonFailureMessage(val2, val1);\ + }\ +} + +// Implements the helper function for {ASSERT|EXPECT}_NE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(NE, !=) +// Implements the helper function for {ASSERT|EXPECT}_LE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LE, <=) +// Implements the helper function for {ASSERT|EXPECT}_LT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(LT, < ) +// Implements the helper function for {ASSERT|EXPECT}_GE with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GE, >=) +// Implements the helper function for {ASSERT|EXPECT}_GT with int or +// enum arguments. +GTEST_IMPL_CMP_HELPER_(GT, > ) + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +AssertionResult CmpHelperSTREQ(const char* expected_expression, + const char* actual_expression, + const char* expected, + const char* actual) { + if (String::CStringEquals(expected, actual)) { + return AssertionSuccess(); + } + + return EqFailure(expected_expression, + actual_expression, + String::ShowCStringQuoted(expected), + String::ShowCStringQuoted(actual), + false); +} + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +AssertionResult CmpHelperSTRCASEEQ(const char* expected_expression, + const char* actual_expression, + const char* expected, + const char* actual) { + if (String::CaseInsensitiveCStringEquals(expected, actual)) { + return AssertionSuccess(); + } + + return EqFailure(expected_expression, + actual_expression, + String::ShowCStringQuoted(expected), + String::ShowCStringQuoted(actual), + true); +} + +// The helper function for {ASSERT|EXPECT}_STRNE. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2) { + if (!String::CaseInsensitiveCStringEquals(s1, s2)) { + return AssertionSuccess(); + } else { + return AssertionFailure() + << "Expected: (" << s1_expression << ") != (" + << s2_expression << ") (ignoring case), actual: \"" + << s1 << "\" vs \"" << s2 << "\""; + } +} + +} // namespace internal + +namespace { + +// Helper functions for implementing IsSubString() and IsNotSubstring(). + +// This group of overloaded functions return true iff needle is a +// substring of haystack. NULL is considered a substring of itself +// only. + +bool IsSubstringPred(const char* needle, const char* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return strstr(haystack, needle) != NULL; +} + +bool IsSubstringPred(const wchar_t* needle, const wchar_t* haystack) { + if (needle == NULL || haystack == NULL) + return needle == haystack; + + return wcsstr(haystack, needle) != NULL; +} + +// StringType here can be either ::std::string or ::std::wstring. +template +bool IsSubstringPred(const StringType& needle, + const StringType& haystack) { + return haystack.find(needle) != StringType::npos; +} + +// This function implements either IsSubstring() or IsNotSubstring(), +// depending on the value of the expected_to_be_substring parameter. +// StringType here can be const char*, const wchar_t*, ::std::string, +// or ::std::wstring. +template +AssertionResult IsSubstringImpl( + bool expected_to_be_substring, + const char* needle_expr, const char* haystack_expr, + const StringType& needle, const StringType& haystack) { + if (IsSubstringPred(needle, haystack) == expected_to_be_substring) + return AssertionSuccess(); + + const bool is_wide_string = sizeof(needle[0]) > 1; + const char* const begin_string_quote = is_wide_string ? "L\"" : "\""; + return AssertionFailure() + << "Value of: " << needle_expr << "\n" + << " Actual: " << begin_string_quote << needle << "\"\n" + << "Expected: " << (expected_to_be_substring ? "" : "not ") + << "a substring of " << haystack_expr << "\n" + << "Which is: " << begin_string_quote << haystack << "\""; +} + +} // namespace + +// IsSubstring() and IsNotSubstring() check whether needle is a +// substring of haystack (NULL is considered a substring of itself +// only), and return an appropriate error message when they fail. + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} + +#if GTEST_HAS_STD_WSTRING +AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(true, needle_expr, haystack_expr, needle, haystack); +} + +AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack) { + return IsSubstringImpl(false, needle_expr, haystack_expr, needle, haystack); +} +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +#if GTEST_OS_WINDOWS + +namespace { + +// Helper function for IsHRESULT{SuccessFailure} predicates +AssertionResult HRESULTFailureHelper(const char* expr, + const char* expected, + long hr) { // NOLINT +# if GTEST_OS_WINDOWS_MOBILE + + // Windows CE doesn't support FormatMessage. + const char error_text[] = ""; + +# else + + // Looks up the human-readable system message for the HRESULT code + // and since we're not passing any params to FormatMessage, we don't + // want inserts expanded. + const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD kBufSize = 4096; // String::Format can't exceed this length. + // Gets the system's human readable message string for this HRESULT. + char error_text[kBufSize] = { '\0' }; + DWORD message_length = ::FormatMessageA(kFlags, + 0, // no source, we're asking system + hr, // the error + 0, // no line width restrictions + error_text, // output buffer + kBufSize, // buf size + NULL); // no arguments for inserts + // Trims tailing white space (FormatMessage leaves a trailing cr-lf) + for (; message_length && IsSpace(error_text[message_length - 1]); + --message_length) { + error_text[message_length - 1] = '\0'; + } + +# endif // GTEST_OS_WINDOWS_MOBILE + + const String error_hex(String::Format("0x%08X ", hr)); + return ::testing::AssertionFailure() + << "Expected: " << expr << " " << expected << ".\n" + << " Actual: " << error_hex << error_text << "\n"; +} + +} // namespace + +AssertionResult IsHRESULTSuccess(const char* expr, long hr) { // NOLINT + if (SUCCEEDED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "succeeds", hr); +} + +AssertionResult IsHRESULTFailure(const char* expr, long hr) { // NOLINT + if (FAILED(hr)) { + return AssertionSuccess(); + } + return HRESULTFailureHelper(expr, "fails", hr); +} + +#endif // GTEST_OS_WINDOWS + +// Utility functions for encoding Unicode text (wide strings) in +// UTF-8. + +// A Unicode code-point can have upto 21 bits, and is encoded in UTF-8 +// like this: +// +// Code-point length Encoding +// 0 - 7 bits 0xxxxxxx +// 8 - 11 bits 110xxxxx 10xxxxxx +// 12 - 16 bits 1110xxxx 10xxxxxx 10xxxxxx +// 17 - 21 bits 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + +// The maximum code-point a one-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint1 = (static_cast(1) << 7) - 1; + +// The maximum code-point a two-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint2 = (static_cast(1) << (5 + 6)) - 1; + +// The maximum code-point a three-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint3 = (static_cast(1) << (4 + 2*6)) - 1; + +// The maximum code-point a four-byte UTF-8 sequence can represent. +const UInt32 kMaxCodePoint4 = (static_cast(1) << (3 + 3*6)) - 1; + +// Chops off the n lowest bits from a bit pattern. Returns the n +// lowest bits. As a side effect, the original bit pattern will be +// shifted to the right by n bits. +inline UInt32 ChopLowBits(UInt32* bits, int n) { + const UInt32 low_bits = *bits & ((static_cast(1) << n) - 1); + *bits >>= n; + return low_bits; +} + +// Converts a Unicode code point to a narrow string in UTF-8 encoding. +// code_point parameter is of type UInt32 because wchar_t may not be +// wide enough to contain a code point. +// The output buffer str must containt at least 32 characters. +// The function returns the address of the output buffer. +// If the code_point is not a valid Unicode code point +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. +char* CodePointToUtf8(UInt32 code_point, char* str) { + if (code_point <= kMaxCodePoint1) { + str[1] = '\0'; + str[0] = static_cast(code_point); // 0xxxxxxx + } else if (code_point <= kMaxCodePoint2) { + str[2] = '\0'; + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xC0 | code_point); // 110xxxxx + } else if (code_point <= kMaxCodePoint3) { + str[3] = '\0'; + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xE0 | code_point); // 1110xxxx + } else if (code_point <= kMaxCodePoint4) { + str[4] = '\0'; + str[3] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[2] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[1] = static_cast(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx + str[0] = static_cast(0xF0 | code_point); // 11110xxx + } else { + // The longest string String::Format can produce when invoked + // with these parameters is 28 character long (not including + // the terminating nul character). We are asking for 32 character + // buffer just in case. This is also enough for strncpy to + // null-terminate the destination string. + posix::StrNCpy( + str, String::Format("(Invalid Unicode 0x%X)", code_point).c_str(), 32); + str[31] = '\0'; // Makes sure no change in the format to strncpy leaves + // the result unterminated. + } + return str; +} + +// The following two functions only make sense if the the system +// uses UTF-16 for wide string encoding. All supported systems +// with 16 bit wchar_t (Windows, Cygwin, Symbian OS) do use UTF-16. + +// Determines if the arguments constitute UTF-16 surrogate pair +// and thus should be combined into a single Unicode code point +// using CreateCodePointFromUtf16SurrogatePair. +inline bool IsUtf16SurrogatePair(wchar_t first, wchar_t second) { + return sizeof(wchar_t) == 2 && + (first & 0xFC00) == 0xD800 && (second & 0xFC00) == 0xDC00; +} + +// Creates a Unicode code point from UTF16 surrogate pair. +inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first, + wchar_t second) { + const UInt32 mask = (1 << 10) - 1; + return (sizeof(wchar_t) == 2) ? + (((first & mask) << 10) | (second & mask)) + 0x10000 : + // This function should not be called when the condition is + // false, but we provide a sensible default in case it is. + static_cast(first); +} + +// Converts a wide string to a narrow string in UTF-8 encoding. +// The wide string is assumed to have the following encoding: +// UTF-16 if sizeof(wchar_t) == 2 (on Windows, Cygwin, Symbian OS) +// UTF-32 if sizeof(wchar_t) == 4 (on Linux) +// Parameter str points to a null-terminated wide string. +// Parameter num_chars may additionally limit the number +// of wchar_t characters processed. -1 is used when the entire string +// should be processed. +// If the string contains code points that are not valid Unicode code points +// (i.e. outside of Unicode range U+0 to U+10FFFF) they will be output +// as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding +// and contains invalid UTF-16 surrogate pairs, values in those pairs +// will be encoded as individual Unicode characters from Basic Normal Plane. +String WideStringToUtf8(const wchar_t* str, int num_chars) { + if (num_chars == -1) + num_chars = static_cast(wcslen(str)); + + ::std::stringstream stream; + for (int i = 0; i < num_chars; ++i) { + UInt32 unicode_code_point; + + if (str[i] == L'\0') { + break; + } else if (i + 1 < num_chars && IsUtf16SurrogatePair(str[i], str[i + 1])) { + unicode_code_point = CreateCodePointFromUtf16SurrogatePair(str[i], + str[i + 1]); + i++; + } else { + unicode_code_point = static_cast(str[i]); + } + + char buffer[32]; // CodePointToUtf8 requires a buffer this big. + stream << CodePointToUtf8(unicode_code_point, buffer); + } + return StringStreamToString(&stream); +} + +// Converts a wide C string to a String using the UTF-8 encoding. +// NULL will be converted to "(null)". +String String::ShowWideCString(const wchar_t * wide_c_str) { + if (wide_c_str == NULL) return String("(null)"); + + return String(internal::WideStringToUtf8(wide_c_str, -1).c_str()); +} + +// Similar to ShowWideCString(), except that this function encloses +// the converted string in double quotes. +String String::ShowWideCStringQuoted(const wchar_t* wide_c_str) { + if (wide_c_str == NULL) return String("(null)"); + + return String::Format("L\"%s\"", + String::ShowWideCString(wide_c_str).c_str()); +} + +// Compares two wide C strings. Returns true iff they have the same +// content. +// +// Unlike wcscmp(), this function can handle NULL argument(s). A NULL +// C string is considered different to any non-NULL C string, +// including the empty string. +bool String::WideCStringEquals(const wchar_t * lhs, const wchar_t * rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + + return wcscmp(lhs, rhs) == 0; +} + +// Helper function for *_STREQ on wide strings. +AssertionResult CmpHelperSTREQ(const char* expected_expression, + const char* actual_expression, + const wchar_t* expected, + const wchar_t* actual) { + if (String::WideCStringEquals(expected, actual)) { + return AssertionSuccess(); + } + + return EqFailure(expected_expression, + actual_expression, + String::ShowWideCStringQuoted(expected), + String::ShowWideCStringQuoted(actual), + false); +} + +// Helper function for *_STRNE on wide strings. +AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2) { + if (!String::WideCStringEquals(s1, s2)) { + return AssertionSuccess(); + } + + return AssertionFailure() << "Expected: (" << s1_expression << ") != (" + << s2_expression << "), actual: " + << String::ShowWideCStringQuoted(s1) + << " vs " << String::ShowWideCStringQuoted(s2); +} + +// Compares two C strings, ignoring case. Returns true iff they have +// the same content. +// +// Unlike strcasecmp(), this function can handle NULL argument(s). A +// NULL C string is considered different to any non-NULL C string, +// including the empty string. +bool String::CaseInsensitiveCStringEquals(const char * lhs, const char * rhs) { + if (lhs == NULL) + return rhs == NULL; + if (rhs == NULL) + return false; + return posix::StrCaseCmp(lhs, rhs) == 0; +} + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. +bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs) { + if (lhs == NULL) return rhs == NULL; + + if (rhs == NULL) return false; + +#if GTEST_OS_WINDOWS + return _wcsicmp(lhs, rhs) == 0; +#elif GTEST_OS_LINUX && !GTEST_OS_LINUX_ANDROID + return wcscasecmp(lhs, rhs) == 0; +#else + // Android, Mac OS X and Cygwin don't define wcscasecmp. + // Other unknown OSes may not define it either. + wint_t left, right; + do { + left = towlower(*lhs++); + right = towlower(*rhs++); + } while (left && left == right); + return left == right; +#endif // OS selector +} + +// Compares this with another String. +// Returns < 0 if this is less than rhs, 0 if this is equal to rhs, or > 0 +// if this is greater than rhs. +int String::Compare(const String & rhs) const { + const char* const lhs_c_str = c_str(); + const char* const rhs_c_str = rhs.c_str(); + + if (lhs_c_str == NULL) { + return rhs_c_str == NULL ? 0 : -1; // NULL < anything except NULL + } else if (rhs_c_str == NULL) { + return 1; + } + + const size_t shorter_str_len = + length() <= rhs.length() ? length() : rhs.length(); + for (size_t i = 0; i != shorter_str_len; i++) { + if (lhs_c_str[i] < rhs_c_str[i]) { + return -1; + } else if (lhs_c_str[i] > rhs_c_str[i]) { + return 1; + } + } + return (length() < rhs.length()) ? -1 : + (length() > rhs.length()) ? 1 : 0; +} + +// Returns true iff this String ends with the given suffix. *Any* +// String is considered to end with a NULL or empty suffix. +bool String::EndsWith(const char* suffix) const { + if (suffix == NULL || CStringEquals(suffix, "")) return true; + + if (c_str() == NULL) return false; + + const size_t this_len = strlen(c_str()); + const size_t suffix_len = strlen(suffix); + return (this_len >= suffix_len) && + CStringEquals(c_str() + this_len - suffix_len, suffix); +} + +// Returns true iff this String ends with the given suffix, ignoring case. +// Any String is considered to end with a NULL or empty suffix. +bool String::EndsWithCaseInsensitive(const char* suffix) const { + if (suffix == NULL || CStringEquals(suffix, "")) return true; + + if (c_str() == NULL) return false; + + const size_t this_len = strlen(c_str()); + const size_t suffix_len = strlen(suffix); + return (this_len >= suffix_len) && + CaseInsensitiveCStringEquals(c_str() + this_len - suffix_len, suffix); +} + +// Formats a list of arguments to a String, using the same format +// spec string as for printf. +// +// We do not use the StringPrintf class as it is not universally +// available. +// +// The result is limited to 4096 characters (including the tailing 0). +// If 4096 characters are not enough to format the input, or if +// there's an error, "" is +// returned. +String String::Format(const char * format, ...) { + va_list args; + va_start(args, format); + + char buffer[4096]; + const int kBufferSize = sizeof(buffer)/sizeof(buffer[0]); + + // MSVC 8 deprecates vsnprintf(), so we want to suppress warning + // 4996 (deprecated function) there. +#ifdef _MSC_VER // We are using MSVC. +# pragma warning(push) // Saves the current warning state. +# pragma warning(disable:4996) // Temporarily disables warning 4996. + + const int size = vsnprintf(buffer, kBufferSize, format, args); + +# pragma warning(pop) // Restores the warning state. +#else // We are not using MSVC. + const int size = vsnprintf(buffer, kBufferSize, format, args); +#endif // _MSC_VER + va_end(args); + + // vsnprintf()'s behavior is not portable. When the buffer is not + // big enough, it returns a negative value in MSVC, and returns the + // needed buffer size on Linux. When there is an output error, it + // always returns a negative value. For simplicity, we lump the two + // error cases together. + if (size < 0 || size >= kBufferSize) { + return String(""); + } else { + return String(buffer, size); + } +} + +// Converts the buffer in a stringstream to a String, converting NUL +// bytes to "\\0" along the way. +String StringStreamToString(::std::stringstream* ss) { + const ::std::string& str = ss->str(); + const char* const start = str.c_str(); + const char* const end = start + str.length(); + + // We need to use a helper stringstream to do this transformation + // because String doesn't support push_back(). + ::std::stringstream helper; + for (const char* ch = start; ch != end; ++ch) { + if (*ch == '\0') { + helper << "\\0"; // Replaces NUL with "\\0"; + } else { + helper.put(*ch); + } + } + + return String(helper.str().c_str()); +} + +// Appends the user-supplied message to the Google-Test-generated message. +String AppendUserMessage(const String& gtest_msg, + const Message& user_msg) { + // Appends the user message if it's non-empty. + const String user_msg_string = user_msg.GetString(); + if (user_msg_string.empty()) { + return gtest_msg; + } + + Message msg; + msg << gtest_msg << "\n" << user_msg_string; + + return msg.GetString(); +} + +} // namespace internal + +// class TestResult + +// Creates an empty TestResult. +TestResult::TestResult() + : death_test_count_(0), + elapsed_time_(0) { +} + +// D'tor. +TestResult::~TestResult() { +} + +// Returns the i-th test part result among all the results. i can +// range from 0 to total_part_count() - 1. If i is not in that range, +// aborts the program. +const TestPartResult& TestResult::GetTestPartResult(int i) const { + if (i < 0 || i >= total_part_count()) + internal::posix::Abort(); + return test_part_results_.at(i); +} + +// Returns the i-th test property. i can range from 0 to +// test_property_count() - 1. If i is not in that range, aborts the +// program. +const TestProperty& TestResult::GetTestProperty(int i) const { + if (i < 0 || i >= test_property_count()) + internal::posix::Abort(); + return test_properties_.at(i); +} + +// Clears the test part results. +void TestResult::ClearTestPartResults() { + test_part_results_.clear(); +} + +// Adds a test part result to the list. +void TestResult::AddTestPartResult(const TestPartResult& test_part_result) { + test_part_results_.push_back(test_part_result); +} + +// Adds a test property to the list. If a property with the same key as the +// supplied property is already represented, the value of this test_property +// replaces the old value for that key. +void TestResult::RecordProperty(const TestProperty& test_property) { + if (!ValidateTestProperty(test_property)) { + return; + } + internal::MutexLock lock(&test_properites_mutex_); + const std::vector::iterator property_with_matching_key = + std::find_if(test_properties_.begin(), test_properties_.end(), + internal::TestPropertyKeyIs(test_property.key())); + if (property_with_matching_key == test_properties_.end()) { + test_properties_.push_back(test_property); + return; + } + property_with_matching_key->SetValue(test_property.value()); +} + +// Adds a failure if the key is a reserved attribute of Google Test +// testcase tags. Returns true if the property is valid. +bool TestResult::ValidateTestProperty(const TestProperty& test_property) { + internal::String key(test_property.key()); + if (key == "name" || key == "status" || key == "time" || key == "classname") { + ADD_FAILURE() + << "Reserved key used in RecordProperty(): " + << key + << " ('name', 'status', 'time', and 'classname' are reserved by " + << GTEST_NAME_ << ")"; + return false; + } + return true; +} + +// Clears the object. +void TestResult::Clear() { + test_part_results_.clear(); + test_properties_.clear(); + death_test_count_ = 0; + elapsed_time_ = 0; +} + +// Returns true iff the test failed. +bool TestResult::Failed() const { + for (int i = 0; i < total_part_count(); ++i) { + if (GetTestPartResult(i).failed()) + return true; + } + return false; +} + +// Returns true iff the test part fatally failed. +static bool TestPartFatallyFailed(const TestPartResult& result) { + return result.fatally_failed(); +} + +// Returns true iff the test fatally failed. +bool TestResult::HasFatalFailure() const { + return CountIf(test_part_results_, TestPartFatallyFailed) > 0; +} + +// Returns true iff the test part non-fatally failed. +static bool TestPartNonfatallyFailed(const TestPartResult& result) { + return result.nonfatally_failed(); +} + +// Returns true iff the test has a non-fatal failure. +bool TestResult::HasNonfatalFailure() const { + return CountIf(test_part_results_, TestPartNonfatallyFailed) > 0; +} + +// Gets the number of all test parts. This is the sum of the number +// of successful test parts and the number of failed test parts. +int TestResult::total_part_count() const { + return static_cast(test_part_results_.size()); +} + +// Returns the number of the test properties. +int TestResult::test_property_count() const { + return static_cast(test_properties_.size()); +} + +// class Test + +// Creates a Test object. + +// The c'tor saves the values of all Google Test flags. +Test::Test() + : gtest_flag_saver_(new internal::GTestFlagSaver) { +} + +// The d'tor restores the values of all Google Test flags. +Test::~Test() { + delete gtest_flag_saver_; +} + +// Sets up the test fixture. +// +// A sub-class may override this. +void Test::SetUp() { +} + +// Tears down the test fixture. +// +// A sub-class may override this. +void Test::TearDown() { +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const char* key, const char* value) { + UnitTest::GetInstance()->RecordPropertyForCurrentTest(key, value); +} + +// Allows user supplied key value pairs to be recorded for later output. +void Test::RecordProperty(const char* key, int value) { + Message value_message; + value_message << value; + RecordProperty(key, value_message.GetString().c_str()); +} + +namespace internal { + +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const String& message) { + // This function is a friend of UnitTest and as such has access to + // AddTestPartResult. + UnitTest::GetInstance()->AddTestPartResult( + result_type, + NULL, // No info about the source file where the exception occurred. + -1, // We have no info on which line caused the exception. + message, + String()); // No stack trace, either. +} + +} // namespace internal + +// Google Test requires all tests in the same test case to use the same test +// fixture class. This function checks if the current test has the +// same fixture class as the first test in the current test case. If +// yes, it returns true; otherwise it generates a Google Test failure and +// returns false. +bool Test::HasSameFixtureClass() { + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + const TestCase* const test_case = impl->current_test_case(); + + // Info about the first test in the current test case. + const TestInfo* const first_test_info = test_case->test_info_list()[0]; + const internal::TypeId first_fixture_id = first_test_info->fixture_class_id_; + const char* const first_test_name = first_test_info->name(); + + // Info about the current test. + const TestInfo* const this_test_info = impl->current_test_info(); + const internal::TypeId this_fixture_id = this_test_info->fixture_class_id_; + const char* const this_test_name = this_test_info->name(); + + if (this_fixture_id != first_fixture_id) { + // Is the first test defined using TEST? + const bool first_is_TEST = first_fixture_id == internal::GetTestTypeId(); + // Is this test defined using TEST? + const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId(); + + if (first_is_TEST || this_is_TEST) { + // The user mixed TEST and TEST_F in this test case - we'll tell + // him/her how to fix it. + + // Gets the name of the TEST and the name of the TEST_F. Note + // that first_is_TEST and this_is_TEST cannot both be true, as + // the fixture IDs are different for the two tests. + const char* const TEST_name = + first_is_TEST ? first_test_name : this_test_name; + const char* const TEST_F_name = + first_is_TEST ? this_test_name : first_test_name; + + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class, so mixing TEST_F and TEST in the same test case is\n" + << "illegal. In test case " << this_test_info->test_case_name() + << ",\n" + << "test " << TEST_F_name << " is defined using TEST_F but\n" + << "test " << TEST_name << " is defined using TEST. You probably\n" + << "want to change the TEST to TEST_F or move it to another test\n" + << "case."; + } else { + // The user defined two fixture classes with the same name in + // two namespaces - we'll tell him/her how to fix it. + ADD_FAILURE() + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " + << this_test_info->test_case_name() << ",\n" + << "you defined test " << first_test_name + << " and test " << this_test_name << "\n" + << "using two different test fixture classes. This can happen if\n" + << "the two classes are from different namespaces or translation\n" + << "units and have the same name. You should probably rename one\n" + << "of the classes to put the tests into different test cases."; + } + return false; + } + + return true; +} + +#if GTEST_HAS_SEH + +// Adds an "exception thrown" fatal failure to the current test. This +// function returns its result via an output parameter pointer because VC++ +// prohibits creation of objects with destructors on stack in functions +// using __try (see error C2712). +static internal::String* FormatSehExceptionMessage(DWORD exception_code, + const char* location) { + Message message; + message << "SEH exception with code 0x" << std::setbase(16) << + exception_code << std::setbase(10) << " thrown in " << location << "."; + + return new internal::String(message.GetString()); +} + +#endif // GTEST_HAS_SEH + +#if GTEST_HAS_EXCEPTIONS + +// Adds an "exception thrown" fatal failure to the current test. +static internal::String FormatCxxExceptionMessage(const char* description, + const char* location) { + Message message; + if (description != NULL) { + message << "C++ exception with description \"" << description << "\""; + } else { + message << "Unknown C++ exception"; + } + message << " thrown in " << location << "."; + + return message.GetString(); +} + +static internal::String PrintTestPartResultToString( + const TestPartResult& test_part_result); + +// A failed Google Test assertion will throw an exception of this type when +// GTEST_FLAG(throw_on_failure) is true (if exceptions are enabled). We +// derive it from std::runtime_error, which is for errors presumably +// detectable only at run time. Since std::runtime_error inherits from +// std::exception, many testing frameworks know how to extract and print the +// message inside it. +class GoogleTestFailureException : public ::std::runtime_error { + public: + explicit GoogleTestFailureException(const TestPartResult& failure) + : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {} +}; +#endif // GTEST_HAS_EXCEPTIONS + +namespace internal { +// We put these helper functions in the internal namespace as IBM's xlC +// compiler rejects the code if they were declared static. + +// Runs the given method and handles SEH exceptions it throws, when +// SEH is supported; returns the 0-value for type Result in case of an +// SEH exception. (Microsoft compilers cannot handle SEH and C++ +// exceptions in the same function. Therefore, we provide a separate +// wrapper function for handling SEH exceptions.) +template +Result HandleSehExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { +#if GTEST_HAS_SEH + __try { + return (object->*method)(); + } __except (internal::UnitTestOptions::GTestShouldProcessSEH( // NOLINT + GetExceptionCode())) { + // We create the exception message on the heap because VC++ prohibits + // creation of objects with destructors on stack in functions using __try + // (see error C2712). + internal::String* exception_message = FormatSehExceptionMessage( + GetExceptionCode(), location); + internal::ReportFailureInUnknownLocation(TestPartResult::kFatalFailure, + *exception_message); + delete exception_message; + return static_cast(0); + } +#else + (void)location; + return (object->*method)(); +#endif // GTEST_HAS_SEH +} + +// Runs the given method and catches and reports C++ and/or SEH-style +// exceptions, if they are supported; returns the 0-value for type +// Result in case of an SEH exception. +template +Result HandleExceptionsInMethodIfSupported( + T* object, Result (T::*method)(), const char* location) { + // NOTE: The user code can affect the way in which Google Test handles + // exceptions by setting GTEST_FLAG(catch_exceptions), but only before + // RUN_ALL_TESTS() starts. It is technically possible to check the flag + // after the exception is caught and either report or re-throw the + // exception based on the flag's value: + // + // try { + // // Perform the test method. + // } catch (...) { + // if (GTEST_FLAG(catch_exceptions)) + // // Report the exception as failure. + // else + // throw; // Re-throws the original exception. + // } + // + // However, the purpose of this flag is to allow the program to drop into + // the debugger when the exception is thrown. On most platforms, once the + // control enters the catch block, the exception origin information is + // lost and the debugger will stop the program at the point of the + // re-throw in this function -- instead of at the point of the original + // throw statement in the code under test. For this reason, we perform + // the check early, sacrificing the ability to affect Google Test's + // exception handling in the method where the exception is thrown. + if (internal::GetUnitTestImpl()->catch_exceptions()) { +#if GTEST_HAS_EXCEPTIONS + try { + return HandleSehExceptionsInMethodIfSupported(object, method, location); + } catch (const GoogleTestFailureException&) { // NOLINT + // This exception doesn't originate in code under test. It makes no + // sense to report it as a test failure. + throw; + } catch (const std::exception& e) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(e.what(), location)); + } catch (...) { // NOLINT + internal::ReportFailureInUnknownLocation( + TestPartResult::kFatalFailure, + FormatCxxExceptionMessage(NULL, location)); + } + return static_cast(0); +#else + return HandleSehExceptionsInMethodIfSupported(object, method, location); +#endif // GTEST_HAS_EXCEPTIONS + } else { + return (object->*method)(); + } +} + +} // namespace internal + +// Runs the test and updates the test result. +void Test::Run() { + if (!HasSameFixtureClass()) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported(this, &Test::SetUp, "SetUp()"); + // We will run the test only if SetUp() was successful. + if (!HasFatalFailure()) { + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TestBody, "the test body"); + } + + // However, we want to clean up as much as possible. Hence we will + // always call TearDown(), even if SetUp() or the test body has + // failed. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &Test::TearDown, "TearDown()"); +} + +// Returns true iff the current test has a fatal failure. +bool Test::HasFatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()->HasFatalFailure(); +} + +// Returns true iff the current test has a non-fatal failure. +bool Test::HasNonfatalFailure() { + return internal::GetUnitTestImpl()->current_test_result()-> + HasNonfatalFailure(); +} + +// class TestInfo + +// Constructs a TestInfo object. It assumes ownership of the test factory +// object. +// TODO(vladl@google.com): Make a_test_case_name and a_name const string&'s +// to signify they cannot be NULLs. +TestInfo::TestInfo(const char* a_test_case_name, + const char* a_name, + const char* a_type_param, + const char* a_value_param, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory) + : test_case_name_(a_test_case_name), + name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + value_param_(a_value_param ? new std::string(a_value_param) : NULL), + fixture_class_id_(fixture_class_id), + should_run_(false), + is_disabled_(false), + matches_filter_(false), + factory_(factory), + result_() {} + +// Destructs a TestInfo object. +TestInfo::~TestInfo() { delete factory_; } + +namespace internal { + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param: the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param: text representation of the test's value parameter, +// or NULL if this is not a value-parameterized test. +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, const char* name, + const char* type_param, + const char* value_param, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory) { + TestInfo* const test_info = + new TestInfo(test_case_name, name, type_param, value_param, + fixture_class_id, factory); + GetUnitTestImpl()->AddTestInfo(set_up_tc, tear_down_tc, test_info); + return test_info; +} + +#if GTEST_HAS_PARAM_TEST +void ReportInvalidTestCaseType(const char* test_case_name, + const char* file, int line) { + Message errors; + errors + << "Attempted redefinition of test case " << test_case_name << ".\n" + << "All tests in the same test case must use the same test fixture\n" + << "class. However, in test case " << test_case_name << ", you tried\n" + << "to define a test using a fixture class different from the one\n" + << "used earlier. This can happen if the two fixture classes are\n" + << "from different namespaces and have the same name. You should\n" + << "probably rename one of the classes to put the tests into different\n" + << "test cases."; + + fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), + errors.GetString().c_str()); +} +#endif // GTEST_HAS_PARAM_TEST + +} // namespace internal + +namespace { + +// A predicate that checks the test name of a TestInfo against a known +// value. +// +// This is used for implementation of the TestCase class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestNameIs is copyable. +class TestNameIs { + public: + // Constructor. + // + // TestNameIs has NO default constructor. + explicit TestNameIs(const char* name) + : name_(name) {} + + // Returns true iff the test name of test_info matches name_. + bool operator()(const TestInfo * test_info) const { + return test_info && internal::String(test_info->name()).Compare(name_) == 0; + } + + private: + internal::String name_; +}; + +} // namespace + +namespace internal { + +// This method expands all parameterized tests registered with macros TEST_P +// and INSTANTIATE_TEST_CASE_P into regular tests and registers those. +// This will be done just once during the program runtime. +void UnitTestImpl::RegisterParameterizedTests() { +#if GTEST_HAS_PARAM_TEST + if (!parameterized_tests_registered_) { + parameterized_test_registry_.RegisterTests(); + parameterized_tests_registered_ = true; + } +#endif +} + +} // namespace internal + +// Creates the test object, runs it, records its result, and then +// deletes it. +void TestInfo::Run() { + if (!should_run_) return; + + // Tells UnitTest where to store test result. + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_info(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Notifies the unit test event listeners that a test is about to start. + repeater->OnTestStart(*this); + + const TimeInMillis start = internal::GetTimeInMillis(); + + impl->os_stack_trace_getter()->UponLeavingGTest(); + + // Creates the test object. + Test* const test = internal::HandleExceptionsInMethodIfSupported( + factory_, &internal::TestFactoryBase::CreateTest, + "the test fixture's constructor"); + + // Runs the test only if the test object was created and its + // constructor didn't generate a fatal failure. + if ((test != NULL) && !Test::HasFatalFailure()) { + // This doesn't throw as all user code that can throw are wrapped into + // exception handling code. + test->Run(); + } + + // Deletes the test object. + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + test, &Test::DeleteSelf_, "the test fixture's destructor"); + + result_.set_elapsed_time(internal::GetTimeInMillis() - start); + + // Notifies the unit test event listener that a test has just finished. + repeater->OnTestEnd(*this); + + // Tells UnitTest to stop associating assertion results to this + // test. + impl->set_current_test_info(NULL); +} + +// class TestCase + +// Gets the number of successful tests in this test case. +int TestCase::successful_test_count() const { + return CountIf(test_info_list_, TestPassed); +} + +// Gets the number of failed tests in this test case. +int TestCase::failed_test_count() const { + return CountIf(test_info_list_, TestFailed); +} + +int TestCase::disabled_test_count() const { + return CountIf(test_info_list_, TestDisabled); +} + +// Get the number of tests in this test case that should run. +int TestCase::test_to_run_count() const { + return CountIf(test_info_list_, ShouldRunTest); +} + +// Gets the number of all tests. +int TestCase::total_test_count() const { + return static_cast(test_info_list_.size()); +} + +// Creates a TestCase with the given name. +// +// Arguments: +// +// name: name of the test case +// a_type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase::TestCase(const char* a_name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) + : name_(a_name), + type_param_(a_type_param ? new std::string(a_type_param) : NULL), + set_up_tc_(set_up_tc), + tear_down_tc_(tear_down_tc), + should_run_(false), + elapsed_time_(0) { +} + +// Destructor of TestCase. +TestCase::~TestCase() { + // Deletes every Test in the collection. + ForEach(test_info_list_, internal::Delete); +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +const TestInfo* TestCase::GetTestInfo(int i) const { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Returns the i-th test among all the tests. i can range from 0 to +// total_test_count() - 1. If i is not in that range, returns NULL. +TestInfo* TestCase::GetMutableTestInfo(int i) { + const int index = GetElementOr(test_indices_, i, -1); + return index < 0 ? NULL : test_info_list_[index]; +} + +// Adds a test to this test case. Will delete the test upon +// destruction of the TestCase object. +void TestCase::AddTestInfo(TestInfo * test_info) { + test_info_list_.push_back(test_info); + test_indices_.push_back(static_cast(test_indices_.size())); +} + +// Runs every test in this TestCase. +void TestCase::Run() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_case(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + repeater->OnTestCaseStart(*this); + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunSetUpTestCase, "SetUpTestCase()"); + + const internal::TimeInMillis start = internal::GetTimeInMillis(); + for (int i = 0; i < total_test_count(); i++) { + GetMutableTestInfo(i)->Run(); + } + elapsed_time_ = internal::GetTimeInMillis() - start; + + impl->os_stack_trace_getter()->UponLeavingGTest(); + internal::HandleExceptionsInMethodIfSupported( + this, &TestCase::RunTearDownTestCase, "TearDownTestCase()"); + + repeater->OnTestCaseEnd(*this); + impl->set_current_test_case(NULL); +} + +// Clears the results of all tests in this test case. +void TestCase::ClearResult() { + ForEach(test_info_list_, TestInfo::ClearTestResult); +} + +// Shuffles the tests in this test case. +void TestCase::ShuffleTests(internal::Random* random) { + Shuffle(random, &test_indices_); +} + +// Restores the test order to before the first shuffle. +void TestCase::UnshuffleTests() { + for (size_t i = 0; i < test_indices_.size(); i++) { + test_indices_[i] = static_cast(i); + } +} + +// Formats a countable noun. Depending on its quantity, either the +// singular form or the plural form is used. e.g. +// +// FormatCountableNoun(1, "formula", "formuli") returns "1 formula". +// FormatCountableNoun(5, "book", "books") returns "5 books". +static internal::String FormatCountableNoun(int count, + const char * singular_form, + const char * plural_form) { + return internal::String::Format("%d %s", count, + count == 1 ? singular_form : plural_form); +} + +// Formats the count of tests. +static internal::String FormatTestCount(int test_count) { + return FormatCountableNoun(test_count, "test", "tests"); +} + +// Formats the count of test cases. +static internal::String FormatTestCaseCount(int test_case_count) { + return FormatCountableNoun(test_case_count, "test case", "test cases"); +} + +// Converts a TestPartResult::Type enum to human-friendly string +// representation. Both kNonFatalFailure and kFatalFailure are translated +// to "Failure", as the user usually doesn't care about the difference +// between the two when viewing the test result. +static const char * TestPartResultTypeToString(TestPartResult::Type type) { + switch (type) { + case TestPartResult::kSuccess: + return "Success"; + + case TestPartResult::kNonFatalFailure: + case TestPartResult::kFatalFailure: +#ifdef _MSC_VER + return "error: "; +#else + return "Failure\n"; +#endif + default: + return "Unknown result type"; + } +} + +// Prints a TestPartResult to a String. +static internal::String PrintTestPartResultToString( + const TestPartResult& test_part_result) { + return (Message() + << internal::FormatFileLocation(test_part_result.file_name(), + test_part_result.line_number()) + << " " << TestPartResultTypeToString(test_part_result.type()) + << test_part_result.message()).GetString(); +} + +// Prints a TestPartResult. +static void PrintTestPartResult(const TestPartResult& test_part_result) { + const internal::String& result = + PrintTestPartResultToString(test_part_result); + printf("%s\n", result.c_str()); + fflush(stdout); + // If the test program runs in Visual Studio or a debugger, the + // following statements add the test part result message to the Output + // window such that the user can double-click on it to jump to the + // corresponding source code location; otherwise they do nothing. +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + // We don't call OutputDebugString*() on Windows Mobile, as printing + // to stdout is done by OutputDebugString() there already - we don't + // want the same message printed twice. + ::OutputDebugStringA(result.c_str()); + ::OutputDebugStringA("\n"); +#endif +} + +// class PrettyUnitTestResultPrinter + +namespace internal { + +enum GTestColor { + COLOR_DEFAULT, + COLOR_RED, + COLOR_GREEN, + COLOR_YELLOW +}; + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + +// Returns the character attribute for the given color. +WORD GetColorAttribute(GTestColor color) { + switch (color) { + case COLOR_RED: return FOREGROUND_RED; + case COLOR_GREEN: return FOREGROUND_GREEN; + case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; + default: return 0; + } +} + +#else + +// Returns the ANSI color code for the given color. COLOR_DEFAULT is +// an invalid input. +const char* GetAnsiColorCode(GTestColor color) { + switch (color) { + case COLOR_RED: return "1"; + case COLOR_GREEN: return "2"; + case COLOR_YELLOW: return "3"; + default: return NULL; + }; +} + +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + +// Returns true iff Google Test should use colors in the output. +bool ShouldUseColor(bool stdout_is_tty) { + const char* const gtest_color = GTEST_FLAG(color).c_str(); + + if (String::CaseInsensitiveCStringEquals(gtest_color, "auto")) { +#if GTEST_OS_WINDOWS + // On Windows the TERM variable is usually not set, but the + // console there does support colors. + return stdout_is_tty; +#else + // On non-Windows platforms, we rely on the TERM variable. + const char* const term = posix::GetEnv("TERM"); + const bool term_supports_color = + String::CStringEquals(term, "xterm") || + String::CStringEquals(term, "xterm-color") || + String::CStringEquals(term, "xterm-256color") || + String::CStringEquals(term, "screen") || + String::CStringEquals(term, "linux") || + String::CStringEquals(term, "cygwin"); + return stdout_is_tty && term_supports_color; +#endif // GTEST_OS_WINDOWS + } + + return String::CaseInsensitiveCStringEquals(gtest_color, "yes") || + String::CaseInsensitiveCStringEquals(gtest_color, "true") || + String::CaseInsensitiveCStringEquals(gtest_color, "t") || + String::CStringEquals(gtest_color, "1"); + // We take "yes", "true", "t", and "1" as meaning "yes". If the + // value is neither one of these nor "auto", we treat it as "no" to + // be conservative. +} + +// Helpers for printing colored strings to stdout. Note that on Windows, we +// cannot simply emit special characters and have the terminal change colors. +// This routine must actually emit the characters rather than return a string +// that would be colored when printed, as can be done on Linux. +void ColoredPrintf(GTestColor color, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS + const bool use_color = false; +#else + static const bool in_color_mode = + ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); + const bool use_color = in_color_mode && (color != COLOR_DEFAULT); +#endif // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS + // The '!= 0' comparison is necessary to satisfy MSVC 7.1. + + if (!use_color) { + vprintf(fmt, args); + va_end(args); + return; + } + +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); + + // Gets the current text color. + CONSOLE_SCREEN_BUFFER_INFO buffer_info; + GetConsoleScreenBufferInfo(stdout_handle, &buffer_info); + const WORD old_color_attrs = buffer_info.wAttributes; + + // We need to flush the stream buffers into the console before each + // SetConsoleTextAttribute call lest it affect the text that is already + // printed but has not yet reached the console. + fflush(stdout); + SetConsoleTextAttribute(stdout_handle, + GetColorAttribute(color) | FOREGROUND_INTENSITY); + vprintf(fmt, args); + + fflush(stdout); + // Restores the text color. + SetConsoleTextAttribute(stdout_handle, old_color_attrs); +#else + printf("\033[0;3%sm", GetAnsiColorCode(color)); + vprintf(fmt, args); + printf("\033[m"); // Resets the terminal to default. +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE + va_end(args); +} + +void PrintFullTestCommentIfPresent(const TestInfo& test_info) { + const char* const type_param = test_info.type_param(); + const char* const value_param = test_info.value_param(); + + if (type_param != NULL || value_param != NULL) { + printf(", where "); + if (type_param != NULL) { + printf("TypeParam = %s", type_param); + if (value_param != NULL) + printf(" and "); + } + if (value_param != NULL) { + printf("GetParam() = %s", value_param); + } + } +} + +// This class implements the TestEventListener interface. +// +// Class PrettyUnitTestResultPrinter is copyable. +class PrettyUnitTestResultPrinter : public TestEventListener { + public: + PrettyUnitTestResultPrinter() {} + static void PrintTestName(const char * test_case, const char * test) { + printf("%s.%s", test_case, test); + } + + // The following methods override what's in the TestEventListener class. + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} + + private: + static void PrintFailedTests(const UnitTest& unit_test); + + internal::String test_case_name_; +}; + + // Fired before each iteration of tests starts. +void PrettyUnitTestResultPrinter::OnTestIterationStart( + const UnitTest& unit_test, int iteration) { + if (GTEST_FLAG(repeat) != 1) + printf("\nRepeating all tests (iteration %d) . . .\n\n", iteration + 1); + + const char* const filter = GTEST_FLAG(filter).c_str(); + + // Prints the filter if it's not *. This reminds the user that some + // tests may be skipped. + if (!internal::String::CStringEquals(filter, kUniversalFilter)) { + ColoredPrintf(COLOR_YELLOW, + "Note: %s filter = %s\n", GTEST_NAME_, filter); + } + + if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { + const Int32 shard_index = Int32FromEnvOrDie(kTestShardIndex, -1); + ColoredPrintf(COLOR_YELLOW, + "Note: This is test shard %d of %s.\n", + static_cast(shard_index) + 1, + internal::posix::GetEnv(kTestTotalShards)); + } + + if (GTEST_FLAG(shuffle)) { + ColoredPrintf(COLOR_YELLOW, + "Note: Randomizing tests' orders with a seed of %d .\n", + unit_test.random_seed()); + } + + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("Running %s from %s.\n", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment set-up.\n"); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { + test_case_name_ = test_case.name(); + const internal::String counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s", counts.c_str(), test_case_name_.c_str()); + if (test_case.type_param() == NULL) { + printf("\n"); + } else { + printf(", where TypeParam = %s\n", test_case.type_param()); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { + ColoredPrintf(COLOR_GREEN, "[ RUN ] "); + PrintTestName(test_case_name_.c_str(), test_info.name()); + printf("\n"); + fflush(stdout); +} + +// Called after an assertion failure. +void PrettyUnitTestResultPrinter::OnTestPartResult( + const TestPartResult& result) { + // If the test part succeeded, we don't need to do anything. + if (result.type() == TestPartResult::kSuccess) + return; + + // Print failure message from the assertion (e.g. expected this and got that). + PrintTestPartResult(result); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { + if (test_info.result()->Passed()) { + ColoredPrintf(COLOR_GREEN, "[ OK ] "); + } else { + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + } + PrintTestName(test_case_name_.c_str(), test_info.name()); + if (test_info.result()->Failed()) + PrintFullTestCommentIfPresent(test_info); + + if (GTEST_FLAG(print_time)) { + printf(" (%s ms)\n", internal::StreamableToString( + test_info.result()->elapsed_time()).c_str()); + } else { + printf("\n"); + } + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { + if (!GTEST_FLAG(print_time)) return; + + test_case_name_ = test_case.name(); + const internal::String counts = + FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("%s from %s (%s ms total)\n\n", + counts.c_str(), test_case_name_.c_str(), + internal::StreamableToString(test_case.elapsed_time()).c_str()); + fflush(stdout); +} + +void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart( + const UnitTest& /*unit_test*/) { + ColoredPrintf(COLOR_GREEN, "[----------] "); + printf("Global test environment tear-down\n"); + fflush(stdout); +} + +// Internal helper for printing the list of failed tests. +void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) { + const int failed_test_count = unit_test.failed_test_count(); + if (failed_test_count == 0) { + return; + } + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + const TestCase& test_case = *unit_test.GetTestCase(i); + if (!test_case.should_run() || (test_case.failed_test_count() == 0)) { + continue; + } + for (int j = 0; j < test_case.total_test_count(); ++j) { + const TestInfo& test_info = *test_case.GetTestInfo(j); + if (!test_info.should_run() || test_info.result()->Passed()) { + continue; + } + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s.%s", test_case.name(), test_info.name()); + PrintFullTestCommentIfPresent(test_info); + printf("\n"); + } + } +} + +void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + ColoredPrintf(COLOR_GREEN, "[==========] "); + printf("%s from %s ran.", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestCaseCount(unit_test.test_case_to_run_count()).c_str()); + if (GTEST_FLAG(print_time)) { + printf(" (%s ms total)", + internal::StreamableToString(unit_test.elapsed_time()).c_str()); + } + printf("\n"); + ColoredPrintf(COLOR_GREEN, "[ PASSED ] "); + printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); + + int num_failures = unit_test.failed_test_count(); + if (!unit_test.Passed()) { + const int failed_test_count = unit_test.failed_test_count(); + ColoredPrintf(COLOR_RED, "[ FAILED ] "); + printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str()); + PrintFailedTests(unit_test); + printf("\n%2d FAILED %s\n", num_failures, + num_failures == 1 ? "TEST" : "TESTS"); + } + + int num_disabled = unit_test.disabled_test_count(); + if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { + if (!num_failures) { + printf("\n"); // Add a spacer if no FAILURE banner is displayed. + } + ColoredPrintf(COLOR_YELLOW, + " YOU HAVE %d DISABLED %s\n\n", + num_disabled, + num_disabled == 1 ? "TEST" : "TESTS"); + } + // Ensure that Google Test output is printed before, e.g., heapchecker output. + fflush(stdout); +} + +// End PrettyUnitTestResultPrinter + +// class TestEventRepeater +// +// This class forwards events to other event listeners. +class TestEventRepeater : public TestEventListener { + public: + TestEventRepeater() : forwarding_enabled_(true) {} + virtual ~TestEventRepeater(); + void Append(TestEventListener *listener); + TestEventListener* Release(TestEventListener* listener); + + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled() const { return forwarding_enabled_; } + void set_forwarding_enabled(bool enable) { forwarding_enabled_ = enable; } + + virtual void OnTestProgramStart(const UnitTest& unit_test); + virtual void OnTestIterationStart(const UnitTest& unit_test, int iteration); + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test); + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test); + virtual void OnTestCaseStart(const TestCase& test_case); + virtual void OnTestStart(const TestInfo& test_info); + virtual void OnTestPartResult(const TestPartResult& result); + virtual void OnTestEnd(const TestInfo& test_info); + virtual void OnTestCaseEnd(const TestCase& test_case); + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test); + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test); + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + virtual void OnTestProgramEnd(const UnitTest& unit_test); + + private: + // Controls whether events will be forwarded to listeners_. Set to false + // in death test child processes. + bool forwarding_enabled_; + // The list of listeners that receive events. + std::vector listeners_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventRepeater); +}; + +TestEventRepeater::~TestEventRepeater() { + ForEach(listeners_, Delete); +} + +void TestEventRepeater::Append(TestEventListener *listener) { + listeners_.push_back(listener); +} + +// TODO(vladl@google.com): Factor the search functionality into Vector::Find. +TestEventListener* TestEventRepeater::Release(TestEventListener *listener) { + for (size_t i = 0; i < listeners_.size(); ++i) { + if (listeners_[i] == listener) { + listeners_.erase(listeners_.begin() + i); + return listener; + } + } + + return NULL; +} + +// Since most methods are very similar, use macros to reduce boilerplate. +// This defines a member that forwards the call to all listeners. +#define GTEST_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (size_t i = 0; i < listeners_.size(); i++) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} +// This defines a member that forwards the call to all listeners in reverse +// order. +#define GTEST_REVERSE_REPEATER_METHOD_(Name, Type) \ +void TestEventRepeater::Name(const Type& parameter) { \ + if (forwarding_enabled_) { \ + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { \ + listeners_[i]->Name(parameter); \ + } \ + } \ +} + +GTEST_REPEATER_METHOD_(OnTestProgramStart, UnitTest) +GTEST_REPEATER_METHOD_(OnEnvironmentsSetUpStart, UnitTest) +GTEST_REPEATER_METHOD_(OnTestCaseStart, TestCase) +GTEST_REPEATER_METHOD_(OnTestStart, TestInfo) +GTEST_REPEATER_METHOD_(OnTestPartResult, TestPartResult) +GTEST_REPEATER_METHOD_(OnEnvironmentsTearDownStart, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsSetUpEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnEnvironmentsTearDownEnd, UnitTest) +GTEST_REVERSE_REPEATER_METHOD_(OnTestEnd, TestInfo) +GTEST_REVERSE_REPEATER_METHOD_(OnTestCaseEnd, TestCase) +GTEST_REVERSE_REPEATER_METHOD_(OnTestProgramEnd, UnitTest) + +#undef GTEST_REPEATER_METHOD_ +#undef GTEST_REVERSE_REPEATER_METHOD_ + +void TestEventRepeater::OnTestIterationStart(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (size_t i = 0; i < listeners_.size(); i++) { + listeners_[i]->OnTestIterationStart(unit_test, iteration); + } + } +} + +void TestEventRepeater::OnTestIterationEnd(const UnitTest& unit_test, + int iteration) { + if (forwarding_enabled_) { + for (int i = static_cast(listeners_.size()) - 1; i >= 0; i--) { + listeners_[i]->OnTestIterationEnd(unit_test, iteration); + } + } +} + +// End TestEventRepeater + +// This class generates an XML output file. +class XmlUnitTestResultPrinter : public EmptyTestEventListener { + public: + explicit XmlUnitTestResultPrinter(const char* output_file); + + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + + private: + // Is c a whitespace character that is normalized to a space character + // when it appears in an XML attribute value? + static bool IsNormalizableWhitespace(char c) { + return c == 0x9 || c == 0xA || c == 0xD; + } + + // May c appear in a well-formed XML document? + static bool IsValidXmlCharacter(char c) { + return IsNormalizableWhitespace(c) || c >= 0x20; + } + + // Returns an XML-escaped copy of the input string str. If + // is_attribute is true, the text is meant to appear as an attribute + // value, and normalizable whitespace is preserved by replacing it + // with character references. + static String EscapeXml(const char* str, bool is_attribute); + + // Returns the given string with all characters invalid in XML removed. + static string RemoveInvalidXmlCharacters(const string& str); + + // Convenience wrapper around EscapeXml when str is an attribute value. + static String EscapeXmlAttribute(const char* str) { + return EscapeXml(str, true); + } + + // Convenience wrapper around EscapeXml when str is not an attribute value. + static String EscapeXmlText(const char* str) { return EscapeXml(str, false); } + + // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. + static void OutputXmlCDataSection(::std::ostream* stream, const char* data); + + // Streams an XML representation of a TestInfo object. + static void OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info); + + // Prints an XML representation of a TestCase object + static void PrintXmlTestCase(FILE* out, const TestCase& test_case); + + // Prints an XML summary of unit_test to output stream out. + static void PrintXmlUnitTest(FILE* out, const UnitTest& unit_test); + + // Produces a string representing the test properties in a result as space + // delimited XML attributes based on the property key="value" pairs. + // When the String is not empty, it includes a space at the beginning, + // to delimit this attribute from prior attributes. + static String TestPropertiesAsXmlAttributes(const TestResult& result); + + // The output file. + const String output_file_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(XmlUnitTestResultPrinter); +}; + +// Creates a new XmlUnitTestResultPrinter. +XmlUnitTestResultPrinter::XmlUnitTestResultPrinter(const char* output_file) + : output_file_(output_file) { + if (output_file_.c_str() == NULL || output_file_.empty()) { + fprintf(stderr, "XML output file may not be null\n"); + fflush(stderr); + exit(EXIT_FAILURE); + } +} + +// Called after the unit test ends. +void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + FILE* xmlout = NULL; + FilePath output_file(output_file_); + FilePath output_dir(output_file.RemoveFileName()); + + if (output_dir.CreateDirectoriesRecursively()) { + xmlout = posix::FOpen(output_file_.c_str(), "w"); + } + if (xmlout == NULL) { + // TODO(wan): report the reason of the failure. + // + // We don't do it for now as: + // + // 1. There is no urgent need for it. + // 2. It's a bit involved to make the errno variable thread-safe on + // all three operating systems (Linux, Windows, and Mac OS). + // 3. To interpret the meaning of errno in a thread-safe way, + // we need the strerror_r() function, which is not available on + // Windows. + fprintf(stderr, + "Unable to open file \"%s\"\n", + output_file_.c_str()); + fflush(stderr); + exit(EXIT_FAILURE); + } + PrintXmlUnitTest(xmlout, unit_test); + fclose(xmlout); +} + +// Returns an XML-escaped copy of the input string str. If is_attribute +// is true, the text is meant to appear as an attribute value, and +// normalizable whitespace is preserved by replacing it with character +// references. +// +// Invalid XML characters in str, if any, are stripped from the output. +// It is expected that most, if not all, of the text processed by this +// module will consist of ordinary English text. +// If this module is ever modified to produce version 1.1 XML output, +// most invalid characters can be retained using character references. +// TODO(wan): It might be nice to have a minimally invasive, human-readable +// escaping scheme for invalid characters, rather than dropping them. +String XmlUnitTestResultPrinter::EscapeXml(const char* str, bool is_attribute) { + Message m; + + if (str != NULL) { + for (const char* src = str; *src; ++src) { + switch (*src) { + case '<': + m << "<"; + break; + case '>': + m << ">"; + break; + case '&': + m << "&"; + break; + case '\'': + if (is_attribute) + m << "'"; + else + m << '\''; + break; + case '"': + if (is_attribute) + m << """; + else + m << '"'; + break; + default: + if (IsValidXmlCharacter(*src)) { + if (is_attribute && IsNormalizableWhitespace(*src)) + m << String::Format("&#x%02X;", unsigned(*src)); + else + m << *src; + } + break; + } + } + } + + return m.GetString(); +} + +// Returns the given string with all characters invalid in XML removed. +// Currently invalid characters are dropped from the string. An +// alternative is to replace them with certain characters such as . or ?. +string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters(const string& str) { + string output; + output.reserve(str.size()); + for (string::const_iterator it = str.begin(); it != str.end(); ++it) + if (IsValidXmlCharacter(*it)) + output.push_back(*it); + + return output; +} + +// The following routines generate an XML representation of a UnitTest +// object. +// +// This is how Google Test concepts map to the DTD: +// +// <-- corresponds to a UnitTest object +// <-- corresponds to a TestCase object +// <-- corresponds to a TestInfo object +// ... +// ... +// ... +// <-- individual assertion failures +// +// +// + +// Formats the given time in milliseconds as seconds. +std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { + ::std::stringstream ss; + ss << ms/1000.0; + return ss.str(); +} + +// Streams an XML CDATA section, escaping invalid CDATA sequences as needed. +void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, + const char* data) { + const char* segment = data; + *stream << ""); + if (next_segment != NULL) { + stream->write( + segment, static_cast(next_segment - segment)); + *stream << "]]>]]>"); + } else { + *stream << segment; + break; + } + } + *stream << "]]>"; +} + +// Prints an XML representation of a TestInfo object. +// TODO(wan): There is also value in printing properties with the plain printer. +void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info) { + const TestResult& result = *test_info.result(); + *stream << " \n"; + *stream << " "; + const string location = internal::FormatCompilerIndependentFileLocation( + part.file_name(), part.line_number()); + const string message = location + "\n" + part.message(); + OutputXmlCDataSection(stream, + RemoveInvalidXmlCharacters(message).c_str()); + *stream << "\n"; + } + } + + if (failures == 0) + *stream << " />\n"; + else + *stream << " \n"; +} + +// Prints an XML representation of a TestCase object +void XmlUnitTestResultPrinter::PrintXmlTestCase(FILE* out, + const TestCase& test_case) { + fprintf(out, + " \n", + FormatTimeInMillisAsSeconds(test_case.elapsed_time()).c_str()); + for (int i = 0; i < test_case.total_test_count(); ++i) { + ::std::stringstream stream; + OutputXmlTestInfo(&stream, test_case.name(), *test_case.GetTestInfo(i)); + fprintf(out, "%s", StringStreamToString(&stream).c_str()); + } + fprintf(out, " \n"); +} + +// Prints an XML summary of unit_test to output stream out. +void XmlUnitTestResultPrinter::PrintXmlUnitTest(FILE* out, + const UnitTest& unit_test) { + fprintf(out, "\n"); + fprintf(out, + "\n"); + for (int i = 0; i < unit_test.total_test_case_count(); ++i) + PrintXmlTestCase(out, *unit_test.GetTestCase(i)); + fprintf(out, "\n"); +} + +// Produces a string representing the test properties in a result as space +// delimited XML attributes based on the property key="value" pairs. +String XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( + const TestResult& result) { + Message attributes; + for (int i = 0; i < result.test_property_count(); ++i) { + const TestProperty& property = result.GetTestProperty(i); + attributes << " " << property.key() << "=" + << "\"" << EscapeXmlAttribute(property.value()) << "\""; + } + return attributes.GetString(); +} + +// End XmlUnitTestResultPrinter + +#if GTEST_CAN_STREAM_RESULTS_ + +// Streams test results to the given port on the given host machine. +class StreamingListener : public EmptyTestEventListener { + public: + // Escapes '=', '&', '%', and '\n' characters in str as "%xx". + static string UrlEncode(const char* str); + + StreamingListener(const string& host, const string& port) + : sockfd_(-1), host_name_(host), port_num_(port) { + MakeConnection(); + Send("gtest_streaming_protocol_version=1.0\n"); + } + + virtual ~StreamingListener() { + if (sockfd_ != -1) + CloseConnection(); + } + + void OnTestProgramStart(const UnitTest& /* unit_test */) { + Send("event=TestProgramStart\n"); + } + + void OnTestProgramEnd(const UnitTest& unit_test) { + // Note that Google Test current only report elapsed time for each + // test iteration, not for the entire test program. + Send(String::Format("event=TestProgramEnd&passed=%d\n", + unit_test.Passed())); + + // Notify the streaming server to stop. + CloseConnection(); + } + + void OnTestIterationStart(const UnitTest& /* unit_test */, int iteration) { + Send(String::Format("event=TestIterationStart&iteration=%d\n", + iteration)); + } + + void OnTestIterationEnd(const UnitTest& unit_test, int /* iteration */) { + Send(String::Format("event=TestIterationEnd&passed=%d&elapsed_time=%sms\n", + unit_test.Passed(), + StreamableToString(unit_test.elapsed_time()).c_str())); + } + + void OnTestCaseStart(const TestCase& test_case) { + Send(String::Format("event=TestCaseStart&name=%s\n", test_case.name())); + } + + void OnTestCaseEnd(const TestCase& test_case) { + Send(String::Format("event=TestCaseEnd&passed=%d&elapsed_time=%sms\n", + test_case.Passed(), + StreamableToString(test_case.elapsed_time()).c_str())); + } + + void OnTestStart(const TestInfo& test_info) { + Send(String::Format("event=TestStart&name=%s\n", test_info.name())); + } + + void OnTestEnd(const TestInfo& test_info) { + Send(String::Format( + "event=TestEnd&passed=%d&elapsed_time=%sms\n", + (test_info.result())->Passed(), + StreamableToString((test_info.result())->elapsed_time()).c_str())); + } + + void OnTestPartResult(const TestPartResult& test_part_result) { + const char* file_name = test_part_result.file_name(); + if (file_name == NULL) + file_name = ""; + Send(String::Format("event=TestPartResult&file=%s&line=%d&message=", + UrlEncode(file_name).c_str(), + test_part_result.line_number())); + Send(UrlEncode(test_part_result.message()) + "\n"); + } + + private: + // Creates a client socket and connects to the server. + void MakeConnection(); + + // Closes the socket. + void CloseConnection() { + GTEST_CHECK_(sockfd_ != -1) + << "CloseConnection() can be called only when there is a connection."; + + close(sockfd_); + sockfd_ = -1; + } + + // Sends a string to the socket. + void Send(const string& message) { + GTEST_CHECK_(sockfd_ != -1) + << "Send() can be called only when there is a connection."; + + const int len = static_cast(message.length()); + if (write(sockfd_, message.c_str(), len) != len) { + GTEST_LOG_(WARNING) + << "stream_result_to: failed to stream to " + << host_name_ << ":" << port_num_; + } + } + + int sockfd_; // socket file descriptor + const string host_name_; + const string port_num_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamingListener); +}; // class StreamingListener + +// Checks if str contains '=', '&', '%' or '\n' characters. If yes, +// replaces them by "%xx" where xx is their hexadecimal value. For +// example, replaces "=" with "%3D". This algorithm is O(strlen(str)) +// in both time and space -- important as the input str may contain an +// arbitrarily long test failure message and stack trace. +string StreamingListener::UrlEncode(const char* str) { + string result; + result.reserve(strlen(str) + 1); + for (char ch = *str; ch != '\0'; ch = *++str) { + switch (ch) { + case '%': + case '=': + case '&': + case '\n': + result.append(String::Format("%%%02x", static_cast(ch))); + break; + default: + result.push_back(ch); + break; + } + } + return result; +} + +void StreamingListener::MakeConnection() { + GTEST_CHECK_(sockfd_ == -1) + << "MakeConnection() can't be called when there is already a connection."; + + addrinfo hints; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; // To allow both IPv4 and IPv6 addresses. + hints.ai_socktype = SOCK_STREAM; + addrinfo* servinfo = NULL; + + // Use the getaddrinfo() to get a linked list of IP addresses for + // the given host name. + const int error_num = getaddrinfo( + host_name_.c_str(), port_num_.c_str(), &hints, &servinfo); + if (error_num != 0) { + GTEST_LOG_(WARNING) << "stream_result_to: getaddrinfo() failed: " + << gai_strerror(error_num); + } + + // Loop through all the results and connect to the first we can. + for (addrinfo* cur_addr = servinfo; sockfd_ == -1 && cur_addr != NULL; + cur_addr = cur_addr->ai_next) { + sockfd_ = socket( + cur_addr->ai_family, cur_addr->ai_socktype, cur_addr->ai_protocol); + if (sockfd_ != -1) { + // Connect the client socket to the server socket. + if (connect(sockfd_, cur_addr->ai_addr, cur_addr->ai_addrlen) == -1) { + close(sockfd_); + sockfd_ = -1; + } + } + } + + freeaddrinfo(servinfo); // all done with this structure + + if (sockfd_ == -1) { + GTEST_LOG_(WARNING) << "stream_result_to: failed to connect to " + << host_name_ << ":" << port_num_; + } +} + +// End of class Streaming Listener +#endif // GTEST_CAN_STREAM_RESULTS__ + +// Class ScopedTrace + +// Pushes the given source file location and message onto a per-thread +// trace stack maintained by Google Test. +// L < UnitTest::mutex_ +ScopedTrace::ScopedTrace(const char* file, int line, const Message& message) { + TraceInfo trace; + trace.file = file; + trace.line = line; + trace.message = message.GetString(); + + UnitTest::GetInstance()->PushGTestTrace(trace); +} + +// Pops the info pushed by the c'tor. +// L < UnitTest::mutex_ +ScopedTrace::~ScopedTrace() { + UnitTest::GetInstance()->PopGTestTrace(); +} + + +// class OsStackTraceGetter + +// Returns the current OS stack trace as a String. Parameters: +// +// max_depth - the maximum number of stack frames to be included +// in the trace. +// skip_count - the number of top frames to be skipped; doesn't count +// against max_depth. +// +// L < mutex_ +// We use "L < mutex_" to denote that the function may acquire mutex_. +String OsStackTraceGetter::CurrentStackTrace(int, int) { + return String(""); +} + +// L < mutex_ +void OsStackTraceGetter::UponLeavingGTest() { +} + +const char* const +OsStackTraceGetter::kElidedFramesMarker = + "... " GTEST_NAME_ " internal frames ..."; + +} // namespace internal + +// class TestEventListeners + +TestEventListeners::TestEventListeners() + : repeater_(new internal::TestEventRepeater()), + default_result_printer_(NULL), + default_xml_generator_(NULL) { +} + +TestEventListeners::~TestEventListeners() { delete repeater_; } + +// Returns the standard listener responsible for the default console +// output. Can be removed from the listeners list to shut down default +// console output. Note that removing this object from the listener list +// with Release transfers its ownership to the user. +void TestEventListeners::Append(TestEventListener* listener) { + repeater_->Append(listener); +} + +// Removes the given event listener from the list and returns it. It then +// becomes the caller's responsibility to delete the listener. Returns +// NULL if the listener is not found in the list. +TestEventListener* TestEventListeners::Release(TestEventListener* listener) { + if (listener == default_result_printer_) + default_result_printer_ = NULL; + else if (listener == default_xml_generator_) + default_xml_generator_ = NULL; + return repeater_->Release(listener); +} + +// Returns repeater that broadcasts the TestEventListener events to all +// subscribers. +TestEventListener* TestEventListeners::repeater() { return repeater_; } + +// Sets the default_result_printer attribute to the provided listener. +// The listener is also added to the listener list and previous +// default_result_printer is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultResultPrinter(TestEventListener* listener) { + if (default_result_printer_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_result_printer_); + default_result_printer_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Sets the default_xml_generator attribute to the provided listener. The +// listener is also added to the listener list and previous +// default_xml_generator is removed from it and deleted. The listener can +// also be NULL in which case it will not be added to the list. Does +// nothing if the previous and the current listener objects are the same. +void TestEventListeners::SetDefaultXmlGenerator(TestEventListener* listener) { + if (default_xml_generator_ != listener) { + // It is an error to pass this method a listener that is already in the + // list. + delete Release(default_xml_generator_); + default_xml_generator_ = listener; + if (listener != NULL) + Append(listener); + } +} + +// Controls whether events will be forwarded by the repeater to the +// listeners in the list. +bool TestEventListeners::EventForwardingEnabled() const { + return repeater_->forwarding_enabled(); +} + +void TestEventListeners::SuppressEventForwarding() { + repeater_->set_forwarding_enabled(false); +} + +// class UnitTest + +// Gets the singleton UnitTest object. The first time this method is +// called, a UnitTest object is constructed and returned. Consecutive +// calls will return the same object. +// +// We don't protect this under mutex_ as a user is not supposed to +// call this before main() starts, from which point on the return +// value will never change. +UnitTest * UnitTest::GetInstance() { + // When compiled with MSVC 7.1 in optimized mode, destroying the + // UnitTest object upon exiting the program messes up the exit code, + // causing successful tests to appear failed. We have to use a + // different implementation in this case to bypass the compiler bug. + // This implementation makes the compiler happy, at the cost of + // leaking the UnitTest object. + + // CodeGear C++Builder insists on a public destructor for the + // default implementation. Use this implementation to keep good OO + // design with private destructor. + +#if (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) + static UnitTest* const instance = new UnitTest; + return instance; +#else + static UnitTest instance; + return &instance; +#endif // (_MSC_VER == 1310 && !defined(_DEBUG)) || defined(__BORLANDC__) +} + +// Gets the number of successful test cases. +int UnitTest::successful_test_case_count() const { + return impl()->successful_test_case_count(); +} + +// Gets the number of failed test cases. +int UnitTest::failed_test_case_count() const { + return impl()->failed_test_case_count(); +} + +// Gets the number of all test cases. +int UnitTest::total_test_case_count() const { + return impl()->total_test_case_count(); +} + +// Gets the number of all test cases that contain at least one test +// that should run. +int UnitTest::test_case_to_run_count() const { + return impl()->test_case_to_run_count(); +} + +// Gets the number of successful tests. +int UnitTest::successful_test_count() const { + return impl()->successful_test_count(); +} + +// Gets the number of failed tests. +int UnitTest::failed_test_count() const { return impl()->failed_test_count(); } + +// Gets the number of disabled tests. +int UnitTest::disabled_test_count() const { + return impl()->disabled_test_count(); +} + +// Gets the number of all tests. +int UnitTest::total_test_count() const { return impl()->total_test_count(); } + +// Gets the number of tests that should run. +int UnitTest::test_to_run_count() const { return impl()->test_to_run_count(); } + +// Gets the elapsed time, in milliseconds. +internal::TimeInMillis UnitTest::elapsed_time() const { + return impl()->elapsed_time(); +} + +// Returns true iff the unit test passed (i.e. all test cases passed). +bool UnitTest::Passed() const { return impl()->Passed(); } + +// Returns true iff the unit test failed (i.e. some test case failed +// or something outside of all tests failed). +bool UnitTest::Failed() const { return impl()->Failed(); } + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +const TestCase* UnitTest::GetTestCase(int i) const { + return impl()->GetTestCase(i); +} + +// Gets the i-th test case among all the test cases. i can range from 0 to +// total_test_case_count() - 1. If i is not in that range, returns NULL. +TestCase* UnitTest::GetMutableTestCase(int i) { + return impl()->GetMutableTestCase(i); +} + +// Returns the list of event listeners that can be used to track events +// inside Google Test. +TestEventListeners& UnitTest::listeners() { + return *impl()->listeners(); +} + +// Registers and returns a global test environment. When a test +// program is run, all global test environments will be set-up in the +// order they were registered. After all tests in the program have +// finished, all global test environments will be torn-down in the +// *reverse* order they were registered. +// +// The UnitTest object takes ownership of the given environment. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +Environment* UnitTest::AddEnvironment(Environment* env) { + if (env == NULL) { + return NULL; + } + + impl_->environments().push_back(env); + return env; +} + +// Adds a TestPartResult to the current TestResult object. All Google Test +// assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) eventually call +// this to report their results. The user code should use the +// assertion macros instead of calling this directly. +// L < mutex_ +void UnitTest::AddTestPartResult(TestPartResult::Type result_type, + const char* file_name, + int line_number, + const internal::String& message, + const internal::String& os_stack_trace) { + Message msg; + msg << message; + + internal::MutexLock lock(&mutex_); + if (impl_->gtest_trace_stack().size() > 0) { + msg << "\n" << GTEST_NAME_ << " trace:"; + + for (int i = static_cast(impl_->gtest_trace_stack().size()); + i > 0; --i) { + const internal::TraceInfo& trace = impl_->gtest_trace_stack()[i - 1]; + msg << "\n" << internal::FormatFileLocation(trace.file, trace.line) + << " " << trace.message; + } + } + + if (os_stack_trace.c_str() != NULL && !os_stack_trace.empty()) { + msg << internal::kStackTraceMarker << os_stack_trace; + } + + const TestPartResult result = + TestPartResult(result_type, file_name, line_number, + msg.GetString().c_str()); + impl_->GetTestPartResultReporterForCurrentThread()-> + ReportTestPartResult(result); + + if (result_type != TestPartResult::kSuccess) { + // gtest_break_on_failure takes precedence over + // gtest_throw_on_failure. This allows a user to set the latter + // in the code (perhaps in order to use Google Test assertions + // with another testing framework) and specify the former on the + // command line for debugging. + if (GTEST_FLAG(break_on_failure)) { +#if GTEST_OS_WINDOWS + // Using DebugBreak on Windows allows gtest to still break into a debugger + // when a failure happens and both the --gtest_break_on_failure and + // the --gtest_catch_exceptions flags are specified. + DebugBreak(); +#else + // Dereference NULL through a volatile pointer to prevent the compiler + // from removing. We use this rather than abort() or __builtin_trap() for + // portability: Symbian doesn't implement abort() well, and some debuggers + // don't correctly trap abort(). + *static_cast(NULL) = 1; +#endif // GTEST_OS_WINDOWS + } else if (GTEST_FLAG(throw_on_failure)) { +#if GTEST_HAS_EXCEPTIONS + throw GoogleTestFailureException(result); +#else + // We cannot call abort() as it generates a pop-up in debug mode + // that cannot be suppressed in VC 7.1 or below. + exit(1); +#endif + } + } +} + +// Creates and adds a property to the current TestResult. If a property matching +// the supplied value already exists, updates its value instead. +void UnitTest::RecordPropertyForCurrentTest(const char* key, + const char* value) { + const TestProperty test_property(key, value); + impl_->current_test_result()->RecordProperty(test_property); +} + +// Runs all tests in this UnitTest object and prints the result. +// Returns 0 if successful, or 1 otherwise. +// +// We don't protect this under mutex_, as we only support calling it +// from the main thread. +int UnitTest::Run() { + // Captures the value of GTEST_FLAG(catch_exceptions). This value will be + // used for the duration of the program. + impl()->set_catch_exceptions(GTEST_FLAG(catch_exceptions)); + +#if GTEST_HAS_SEH + const bool in_death_test_child_process = + internal::GTEST_FLAG(internal_run_death_test).length() > 0; + + // Either the user wants Google Test to catch exceptions thrown by the + // tests or this is executing in the context of death test child + // process. In either case the user does not want to see pop-up dialogs + // about crashes - they are expected. + if (impl()->catch_exceptions() || in_death_test_child_process) { + +# if !GTEST_OS_WINDOWS_MOBILE + // SetErrorMode doesn't exist on CE. + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT | + SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); +# endif // !GTEST_OS_WINDOWS_MOBILE + +# if (defined(_MSC_VER) || GTEST_OS_WINDOWS_MINGW) && !GTEST_OS_WINDOWS_MOBILE + // Death test children can be terminated with _abort(). On Windows, + // _abort() can show a dialog with a warning message. This forces the + // abort message to go to stderr instead. + _set_error_mode(_OUT_TO_STDERR); +# endif + +# if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE + // In the debug version, Visual Studio pops up a separate dialog + // offering a choice to debug the aborted program. We need to suppress + // this dialog or it will pop up for every EXPECT/ASSERT_DEATH statement + // executed. Google Test will notify the user of any unexpected + // failure via stderr. + // + // VC++ doesn't define _set_abort_behavior() prior to the version 8.0. + // Users of prior VC versions shall suffer the agony and pain of + // clicking through the countless debug dialogs. + // TODO(vladl@google.com): find a way to suppress the abort dialog() in the + // debug mode when compiled with VC 7.1 or lower. + if (!GTEST_FLAG(break_on_failure)) + _set_abort_behavior( + 0x0, // Clear the following flags: + _WRITE_ABORT_MSG | _CALL_REPORTFAULT); // pop-up window, core dump. +# endif + +#if _MSC_VER >= 1310 && !GTEST_OS_WINDOWS_MOBILE + // Suppress the "Debug Assertion Failed" dialog in the debug mode. (As far + // as I know, these functions are available on Visual Studio .NET 2003 or + // later.) + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); + _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDERR); + _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG); + _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR); +#endif + + } +#endif // GTEST_HAS_SEH + + return internal::HandleExceptionsInMethodIfSupported( + impl(), + &internal::UnitTestImpl::RunAllTests, + "auxiliary test code (environments or event listeners)") ? 0 : 1; +} + +// Returns the working directory when the first TEST() or TEST_F() was +// executed. +const char* UnitTest::original_working_dir() const { + return impl_->original_working_dir_.c_str(); +} + +// Returns the TestCase object for the test that's currently running, +// or NULL if no test is running. +// L < mutex_ +const TestCase* UnitTest::current_test_case() const { + internal::MutexLock lock(&mutex_); + return impl_->current_test_case(); +} + +// Returns the TestInfo object for the test that's currently running, +// or NULL if no test is running. +// L < mutex_ +const TestInfo* UnitTest::current_test_info() const { + internal::MutexLock lock(&mutex_); + return impl_->current_test_info(); +} + +// Returns the random seed used at the start of the current test run. +int UnitTest::random_seed() const { return impl_->random_seed(); } + +#if GTEST_HAS_PARAM_TEST +// Returns ParameterizedTestCaseRegistry object used to keep track of +// value-parameterized tests and instantiate and register them. +// L < mutex_ +internal::ParameterizedTestCaseRegistry& + UnitTest::parameterized_test_registry() { + return impl_->parameterized_test_registry(); +} +#endif // GTEST_HAS_PARAM_TEST + +// Creates an empty UnitTest. +UnitTest::UnitTest() { + impl_ = new internal::UnitTestImpl(this); +} + +// Destructor of UnitTest. +UnitTest::~UnitTest() { + delete impl_; +} + +// Pushes a trace defined by SCOPED_TRACE() on to the per-thread +// Google Test trace stack. +// L < mutex_ +void UnitTest::PushGTestTrace(const internal::TraceInfo& trace) { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().push_back(trace); +} + +// Pops a trace from the per-thread Google Test trace stack. +// L < mutex_ +void UnitTest::PopGTestTrace() { + internal::MutexLock lock(&mutex_); + impl_->gtest_trace_stack().pop_back(); +} + +namespace internal { + +UnitTestImpl::UnitTestImpl(UnitTest* parent) + : parent_(parent), +#ifdef _MSC_VER +# pragma warning(push) // Saves the current warning state. +# pragma warning(disable:4355) // Temporarily disables warning 4355 + // (using this in initializer). + default_global_test_part_result_reporter_(this), + default_per_thread_test_part_result_reporter_(this), +# pragma warning(pop) // Restores the warning state again. +#else + default_global_test_part_result_reporter_(this), + default_per_thread_test_part_result_reporter_(this), +#endif // _MSC_VER + global_test_part_result_repoter_( + &default_global_test_part_result_reporter_), + per_thread_test_part_result_reporter_( + &default_per_thread_test_part_result_reporter_), +#if GTEST_HAS_PARAM_TEST + parameterized_test_registry_(), + parameterized_tests_registered_(false), +#endif // GTEST_HAS_PARAM_TEST + last_death_test_case_(-1), + current_test_case_(NULL), + current_test_info_(NULL), + ad_hoc_test_result_(), + os_stack_trace_getter_(NULL), + post_flag_parse_init_performed_(false), + random_seed_(0), // Will be overridden by the flag before first use. + random_(0), // Will be reseeded before first use. + elapsed_time_(0), +#if GTEST_HAS_DEATH_TEST + internal_run_death_test_flag_(NULL), + death_test_factory_(new DefaultDeathTestFactory), +#endif + // Will be overridden by the flag before first use. + catch_exceptions_(false) { + listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter); +} + +UnitTestImpl::~UnitTestImpl() { + // Deletes every TestCase. + ForEach(test_cases_, internal::Delete); + + // Deletes every Environment. + ForEach(environments_, internal::Delete); + + delete os_stack_trace_getter_; +} + +#if GTEST_HAS_DEATH_TEST +// Disables event forwarding if the control is currently in a death test +// subprocess. Must not be called before InitGoogleTest. +void UnitTestImpl::SuppressTestEventsIfInSubprocess() { + if (internal_run_death_test_flag_.get() != NULL) + listeners()->SuppressEventForwarding(); +} +#endif // GTEST_HAS_DEATH_TEST + +// Initializes event listeners performing XML output as specified by +// UnitTestOptions. Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureXmlOutput() { + const String& output_format = UnitTestOptions::GetOutputFormat(); + if (output_format == "xml") { + listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); + } else if (output_format != "") { + printf("WARNING: unrecognized output format \"%s\" ignored.\n", + output_format.c_str()); + fflush(stdout); + } +} + +#if GTEST_CAN_STREAM_RESULTS_ +// Initializes event listeners for streaming test results in String form. +// Must not be called before InitGoogleTest. +void UnitTestImpl::ConfigureStreamingOutput() { + const string& target = GTEST_FLAG(stream_result_to); + if (!target.empty()) { + const size_t pos = target.find(':'); + if (pos != string::npos) { + listeners()->Append(new StreamingListener(target.substr(0, pos), + target.substr(pos+1))); + } else { + printf("WARNING: unrecognized streaming target \"%s\" ignored.\n", + target.c_str()); + fflush(stdout); + } + } +} +#endif // GTEST_CAN_STREAM_RESULTS_ + +// Performs initialization dependent upon flag values obtained in +// ParseGoogleTestFlagsOnly. Is called from InitGoogleTest after the call to +// ParseGoogleTestFlagsOnly. In case a user neglects to call InitGoogleTest +// this function is also called from RunAllTests. Since this function can be +// called more than once, it has to be idempotent. +void UnitTestImpl::PostFlagParsingInit() { + // Ensures that this function does not execute more than once. + if (!post_flag_parse_init_performed_) { + post_flag_parse_init_performed_ = true; + +#if GTEST_HAS_DEATH_TEST + InitDeathTestSubprocessControlInfo(); + SuppressTestEventsIfInSubprocess(); +#endif // GTEST_HAS_DEATH_TEST + + // Registers parameterized tests. This makes parameterized tests + // available to the UnitTest reflection API without running + // RUN_ALL_TESTS. + RegisterParameterizedTests(); + + // Configures listeners for XML output. This makes it possible for users + // to shut down the default XML output before invoking RUN_ALL_TESTS. + ConfigureXmlOutput(); + +#if GTEST_CAN_STREAM_RESULTS_ + // Configures listeners for streaming test results to the specified server. + ConfigureStreamingOutput(); +#endif // GTEST_CAN_STREAM_RESULTS_ + } +} + +// A predicate that checks the name of a TestCase against a known +// value. +// +// This is used for implementation of the UnitTest class only. We put +// it in the anonymous namespace to prevent polluting the outer +// namespace. +// +// TestCaseNameIs is copyable. +class TestCaseNameIs { + public: + // Constructor. + explicit TestCaseNameIs(const String& name) + : name_(name) {} + + // Returns true iff the name of test_case matches name_. + bool operator()(const TestCase* test_case) const { + return test_case != NULL && strcmp(test_case->name(), name_.c_str()) == 0; + } + + private: + String name_; +}; + +// Finds and returns a TestCase with the given name. If one doesn't +// exist, creates one and returns it. It's the CALLER'S +// RESPONSIBILITY to ensure that this function is only called WHEN THE +// TESTS ARE NOT SHUFFLED. +// +// Arguments: +// +// test_case_name: name of the test case +// type_param: the name of the test case's type parameter, or NULL if +// this is not a typed or a type-parameterized test case. +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +TestCase* UnitTestImpl::GetTestCase(const char* test_case_name, + const char* type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc) { + // Can we find a TestCase with the given name? + const std::vector::const_iterator test_case = + std::find_if(test_cases_.begin(), test_cases_.end(), + TestCaseNameIs(test_case_name)); + + if (test_case != test_cases_.end()) + return *test_case; + + // No. Let's create one. + TestCase* const new_test_case = + new TestCase(test_case_name, type_param, set_up_tc, tear_down_tc); + + // Is this a death test case? + if (internal::UnitTestOptions::MatchesFilter(String(test_case_name), + kDeathTestCaseFilter)) { + // Yes. Inserts the test case after the last death test case + // defined so far. This only works when the test cases haven't + // been shuffled. Otherwise we may end up running a death test + // after a non-death test. + ++last_death_test_case_; + test_cases_.insert(test_cases_.begin() + last_death_test_case_, + new_test_case); + } else { + // No. Appends to the end of the list. + test_cases_.push_back(new_test_case); + } + + test_case_indices_.push_back(static_cast(test_case_indices_.size())); + return new_test_case; +} + +// Helpers for setting up / tearing down the given environment. They +// are for use in the ForEach() function. +static void SetUpEnvironment(Environment* env) { env->SetUp(); } +static void TearDownEnvironment(Environment* env) { env->TearDown(); } + +// Runs all tests in this UnitTest object, prints the result, and +// returns true if all tests are successful. If any exception is +// thrown during a test, the test is considered to be failed, but the +// rest of the tests will still be run. +// +// When parameterized tests are enabled, it expands and registers +// parameterized tests first in RegisterParameterizedTests(). +// All other functions called from RunAllTests() may safely assume that +// parameterized tests are ready to be counted and run. +bool UnitTestImpl::RunAllTests() { + // Makes sure InitGoogleTest() was called. + if (!GTestIsInitialized()) { + printf("%s", + "\nThis test program did NOT call ::testing::InitGoogleTest " + "before calling RUN_ALL_TESTS(). Please fix it.\n"); + return false; + } + + // Do not run any test if the --help flag was specified. + if (g_help_flag) + return true; + + // Repeats the call to the post-flag parsing initialization in case the + // user didn't call InitGoogleTest. + PostFlagParsingInit(); + + // Even if sharding is not on, test runners may want to use the + // GTEST_SHARD_STATUS_FILE to query whether the test supports the sharding + // protocol. + internal::WriteToShardStatusFileIfNeeded(); + + // True iff we are in a subprocess for running a thread-safe-style + // death test. + bool in_subprocess_for_death_test = false; + +#if GTEST_HAS_DEATH_TEST + in_subprocess_for_death_test = (internal_run_death_test_flag_.get() != NULL); +#endif // GTEST_HAS_DEATH_TEST + + const bool should_shard = ShouldShard(kTestTotalShards, kTestShardIndex, + in_subprocess_for_death_test); + + // Compares the full test names with the filter to decide which + // tests to run. + const bool has_tests_to_run = FilterTests(should_shard + ? HONOR_SHARDING_PROTOCOL + : IGNORE_SHARDING_PROTOCOL) > 0; + + // Lists the tests and exits if the --gtest_list_tests flag was specified. + if (GTEST_FLAG(list_tests)) { + // This must be called *after* FilterTests() has been called. + ListTestsMatchingFilter(); + return true; + } + + random_seed_ = GTEST_FLAG(shuffle) ? + GetRandomSeedFromFlag(GTEST_FLAG(random_seed)) : 0; + + // True iff at least one test has failed. + bool failed = false; + + TestEventListener* repeater = listeners()->repeater(); + + repeater->OnTestProgramStart(*parent_); + + // How many times to repeat the tests? We don't want to repeat them + // when we are inside the subprocess of a death test. + const int repeat = in_subprocess_for_death_test ? 1 : GTEST_FLAG(repeat); + // Repeats forever if the repeat count is negative. + const bool forever = repeat < 0; + for (int i = 0; forever || i != repeat; i++) { + // We want to preserve failures generated by ad-hoc test + // assertions executed before RUN_ALL_TESTS(). + ClearNonAdHocTestResult(); + + const TimeInMillis start = GetTimeInMillis(); + + // Shuffles test cases and tests if requested. + if (has_tests_to_run && GTEST_FLAG(shuffle)) { + random()->Reseed(random_seed_); + // This should be done before calling OnTestIterationStart(), + // such that a test event listener can see the actual test order + // in the event. + ShuffleTests(); + } + + // Tells the unit test event listeners that the tests are about to start. + repeater->OnTestIterationStart(*parent_, i); + + // Runs each test case if there is at least one test to run. + if (has_tests_to_run) { + // Sets up all environments beforehand. + repeater->OnEnvironmentsSetUpStart(*parent_); + ForEach(environments_, SetUpEnvironment); + repeater->OnEnvironmentsSetUpEnd(*parent_); + + // Runs the tests only if there was no fatal failure during global + // set-up. + if (!Test::HasFatalFailure()) { + for (int test_index = 0; test_index < total_test_case_count(); + test_index++) { + GetMutableTestCase(test_index)->Run(); + } + } + + // Tears down all environments in reverse order afterwards. + repeater->OnEnvironmentsTearDownStart(*parent_); + std::for_each(environments_.rbegin(), environments_.rend(), + TearDownEnvironment); + repeater->OnEnvironmentsTearDownEnd(*parent_); + } + + elapsed_time_ = GetTimeInMillis() - start; + + // Tells the unit test event listener that the tests have just finished. + repeater->OnTestIterationEnd(*parent_, i); + + // Gets the result and clears it. + if (!Passed()) { + failed = true; + } + + // Restores the original test order after the iteration. This + // allows the user to quickly repro a failure that happens in the + // N-th iteration without repeating the first (N - 1) iterations. + // This is not enclosed in "if (GTEST_FLAG(shuffle)) { ... }", in + // case the user somehow changes the value of the flag somewhere + // (it's always safe to unshuffle the tests). + UnshuffleTests(); + + if (GTEST_FLAG(shuffle)) { + // Picks a new random seed for each iteration. + random_seed_ = GetNextRandomSeed(random_seed_); + } + } + + repeater->OnTestProgramEnd(*parent_); + + return !failed; +} + +// Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file +// if the variable is present. If a file already exists at this location, this +// function will write over it. If the variable is present, but the file cannot +// be created, prints an error and exits. +void WriteToShardStatusFileIfNeeded() { + const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile); + if (test_shard_file != NULL) { + FILE* const file = posix::FOpen(test_shard_file, "w"); + if (file == NULL) { + ColoredPrintf(COLOR_RED, + "Could not write to the test shard status file \"%s\" " + "specified by the %s environment variable.\n", + test_shard_file, kTestShardStatusFile); + fflush(stdout); + exit(EXIT_FAILURE); + } + fclose(file); + } +} + +// Checks whether sharding is enabled by examining the relevant +// environment variable values. If the variables are present, +// but inconsistent (i.e., shard_index >= total_shards), prints +// an error and exits. If in_subprocess_for_death_test, sharding is +// disabled because it must only be applied to the original test +// process. Otherwise, we could filter out death tests we intended to execute. +bool ShouldShard(const char* total_shards_env, + const char* shard_index_env, + bool in_subprocess_for_death_test) { + if (in_subprocess_for_death_test) { + return false; + } + + const Int32 total_shards = Int32FromEnvOrDie(total_shards_env, -1); + const Int32 shard_index = Int32FromEnvOrDie(shard_index_env, -1); + + if (total_shards == -1 && shard_index == -1) { + return false; + } else if (total_shards == -1 && shard_index != -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestShardIndex << " = " << shard_index + << ", but have left " << kTestTotalShards << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (total_shards != -1 && shard_index == -1) { + const Message msg = Message() + << "Invalid environment variables: you have " + << kTestTotalShards << " = " << total_shards + << ", but have left " << kTestShardIndex << " unset.\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } else if (shard_index < 0 || shard_index >= total_shards) { + const Message msg = Message() + << "Invalid environment variables: we require 0 <= " + << kTestShardIndex << " < " << kTestTotalShards + << ", but you have " << kTestShardIndex << "=" << shard_index + << ", " << kTestTotalShards << "=" << total_shards << ".\n"; + ColoredPrintf(COLOR_RED, msg.GetString().c_str()); + fflush(stdout); + exit(EXIT_FAILURE); + } + + return total_shards > 1; +} + +// Parses the environment variable var as an Int32. If it is unset, +// returns default_val. If it is not an Int32, prints an error +// and aborts. +Int32 Int32FromEnvOrDie(const char* var, Int32 default_val) { + const char* str_val = posix::GetEnv(var); + if (str_val == NULL) { + return default_val; + } + + Int32 result; + if (!ParseInt32(Message() << "The value of environment variable " << var, + str_val, &result)) { + exit(EXIT_FAILURE); + } + return result; +} + +// Given the total number of shards, the shard index, and the test id, +// returns true iff the test should be run on this shard. The test id is +// some arbitrary but unique non-negative integer assigned to each test +// method. Assumes that 0 <= shard_index < total_shards. +bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) { + return (test_id % total_shards) == shard_index; +} + +// Compares the name of each test with the user-specified filter to +// decide whether the test should be run, then records the result in +// each TestCase and TestInfo object. +// If shard_tests == true, further filters tests based on sharding +// variables in the environment - see +// http://code.google.com/p/googletest/wiki/GoogleTestAdvancedGuide. +// Returns the number of tests that should run. +int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { + const Int32 total_shards = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestTotalShards, -1) : -1; + const Int32 shard_index = shard_tests == HONOR_SHARDING_PROTOCOL ? + Int32FromEnvOrDie(kTestShardIndex, -1) : -1; + + // num_runnable_tests are the number of tests that will + // run across all shards (i.e., match filter and are not disabled). + // num_selected_tests are the number of tests to be run on + // this shard. + int num_runnable_tests = 0; + int num_selected_tests = 0; + for (size_t i = 0; i < test_cases_.size(); i++) { + TestCase* const test_case = test_cases_[i]; + const String &test_case_name = test_case->name(); + test_case->set_should_run(false); + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + TestInfo* const test_info = test_case->test_info_list()[j]; + const String test_name(test_info->name()); + // A test is disabled if test case name or test name matches + // kDisableTestFilter. + const bool is_disabled = + internal::UnitTestOptions::MatchesFilter(test_case_name, + kDisableTestFilter) || + internal::UnitTestOptions::MatchesFilter(test_name, + kDisableTestFilter); + test_info->is_disabled_ = is_disabled; + + const bool matches_filter = + internal::UnitTestOptions::FilterMatchesTest(test_case_name, + test_name); + test_info->matches_filter_ = matches_filter; + + const bool is_runnable = + (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) && + matches_filter; + + const bool is_selected = is_runnable && + (shard_tests == IGNORE_SHARDING_PROTOCOL || + ShouldRunTestOnShard(total_shards, shard_index, + num_runnable_tests)); + + num_runnable_tests += is_runnable; + num_selected_tests += is_selected; + + test_info->should_run_ = is_selected; + test_case->set_should_run(test_case->should_run() || is_selected); + } + } + return num_selected_tests; +} + +// Prints the names of the tests matching the user-specified filter flag. +void UnitTestImpl::ListTestsMatchingFilter() { + for (size_t i = 0; i < test_cases_.size(); i++) { + const TestCase* const test_case = test_cases_[i]; + bool printed_test_case_name = false; + + for (size_t j = 0; j < test_case->test_info_list().size(); j++) { + const TestInfo* const test_info = + test_case->test_info_list()[j]; + if (test_info->matches_filter_) { + if (!printed_test_case_name) { + printed_test_case_name = true; + printf("%s.\n", test_case->name()); + } + printf(" %s\n", test_info->name()); + } + } + } + fflush(stdout); +} + +// Sets the OS stack trace getter. +// +// Does nothing if the input and the current OS stack trace getter are +// the same; otherwise, deletes the old getter and makes the input the +// current getter. +void UnitTestImpl::set_os_stack_trace_getter( + OsStackTraceGetterInterface* getter) { + if (os_stack_trace_getter_ != getter) { + delete os_stack_trace_getter_; + os_stack_trace_getter_ = getter; + } +} + +// Returns the current OS stack trace getter if it is not NULL; +// otherwise, creates an OsStackTraceGetter, makes it the current +// getter, and returns it. +OsStackTraceGetterInterface* UnitTestImpl::os_stack_trace_getter() { + if (os_stack_trace_getter_ == NULL) { + os_stack_trace_getter_ = new OsStackTraceGetter; + } + + return os_stack_trace_getter_; +} + +// Returns the TestResult for the test that's currently running, or +// the TestResult for the ad hoc test if no test is running. +TestResult* UnitTestImpl::current_test_result() { + return current_test_info_ ? + &(current_test_info_->result_) : &ad_hoc_test_result_; +} + +// Shuffles all test cases, and the tests within each test case, +// making sure that death tests are still run first. +void UnitTestImpl::ShuffleTests() { + // Shuffles the death test cases. + ShuffleRange(random(), 0, last_death_test_case_ + 1, &test_case_indices_); + + // Shuffles the non-death test cases. + ShuffleRange(random(), last_death_test_case_ + 1, + static_cast(test_cases_.size()), &test_case_indices_); + + // Shuffles the tests inside each test case. + for (size_t i = 0; i < test_cases_.size(); i++) { + test_cases_[i]->ShuffleTests(random()); + } +} + +// Restores the test cases and tests to their order before the first shuffle. +void UnitTestImpl::UnshuffleTests() { + for (size_t i = 0; i < test_cases_.size(); i++) { + // Unshuffles the tests in each test case. + test_cases_[i]->UnshuffleTests(); + // Resets the index of each test case. + test_case_indices_[i] = static_cast(i); + } +} + +// Returns the current OS stack trace as a String. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +String GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/, + int skip_count) { + // We pass skip_count + 1 to skip this wrapper function in addition + // to what the user really wants to skip. + return GetUnitTestImpl()->CurrentOsStackTraceExceptTop(skip_count + 1); +} + +// Used by the GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_ macro to +// suppress unreachable code warnings. +namespace { +class ClassUniqueToAlwaysTrue {}; +} + +bool IsTrue(bool condition) { return condition; } + +bool AlwaysTrue() { +#if GTEST_HAS_EXCEPTIONS + // This condition is always false so AlwaysTrue() never actually throws, + // but it makes the compiler think that it may throw. + if (IsTrue(false)) + throw ClassUniqueToAlwaysTrue(); +#endif // GTEST_HAS_EXCEPTIONS + return true; +} + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +bool SkipPrefix(const char* prefix, const char** pstr) { + const size_t prefix_len = strlen(prefix); + if (strncmp(*pstr, prefix, prefix_len) == 0) { + *pstr += prefix_len; + return true; + } + return false; +} + +// Parses a string as a command line flag. The string should have +// the format "--flag=value". When def_optional is true, the "=value" +// part can be omitted. +// +// Returns the value of the flag, or NULL if the parsing failed. +const char* ParseFlagValue(const char* str, + const char* flag, + bool def_optional) { + // str and flag must not be NULL. + if (str == NULL || flag == NULL) return NULL; + + // The flag must start with "--" followed by GTEST_FLAG_PREFIX_. + const String flag_str = String::Format("--%s%s", GTEST_FLAG_PREFIX_, flag); + const size_t flag_len = flag_str.length(); + if (strncmp(str, flag_str.c_str(), flag_len) != 0) return NULL; + + // Skips the flag name. + const char* flag_end = str + flag_len; + + // When def_optional is true, it's OK to not have a "=value" part. + if (def_optional && (flag_end[0] == '\0')) { + return flag_end; + } + + // If def_optional is true and there are more characters after the + // flag name, or if def_optional is false, there must be a '=' after + // the flag name. + if (flag_end[0] != '=') return NULL; + + // Returns the string after "=". + return flag_end + 1; +} + +// Parses a string for a bool flag, in the form of either +// "--flag=value" or "--flag". +// +// In the former case, the value is taken as true as long as it does +// not start with '0', 'f', or 'F'. +// +// In the latter case, the value is taken as true. +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseBoolFlag(const char* str, const char* flag, bool* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, true); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Converts the string value to a bool. + *value = !(*value_str == '0' || *value_str == 'f' || *value_str == 'F'); + return true; +} + +// Parses a string for an Int32 flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseInt32Flag(const char* str, const char* flag, Int32* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + return ParseInt32(Message() << "The value of flag --" << flag, + value_str, value); +} + +// Parses a string for a string flag, in the form of +// "--flag=value". +// +// On success, stores the value of the flag in *value, and returns +// true. On failure, returns false without changing *value. +bool ParseStringFlag(const char* str, const char* flag, String* value) { + // Gets the value of the flag as a string. + const char* const value_str = ParseFlagValue(str, flag, false); + + // Aborts if the parsing failed. + if (value_str == NULL) return false; + + // Sets *value to the value of the flag. + *value = value_str; + return true; +} + +// Determines whether a string has a prefix that Google Test uses for its +// flags, i.e., starts with GTEST_FLAG_PREFIX_ or GTEST_FLAG_PREFIX_DASH_. +// If Google Test detects that a command line flag has its prefix but is not +// recognized, it will print its help message. Flags starting with +// GTEST_INTERNAL_PREFIX_ followed by "internal_" are considered Google Test +// internal flags and do not trigger the help message. +static bool HasGoogleTestFlagPrefix(const char* str) { + return (SkipPrefix("--", &str) || + SkipPrefix("-", &str) || + SkipPrefix("/", &str)) && + !SkipPrefix(GTEST_FLAG_PREFIX_ "internal_", &str) && + (SkipPrefix(GTEST_FLAG_PREFIX_, &str) || + SkipPrefix(GTEST_FLAG_PREFIX_DASH_, &str)); +} + +// Prints a string containing code-encoded text. The following escape +// sequences can be used in the string to control the text color: +// +// @@ prints a single '@' character. +// @R changes the color to red. +// @G changes the color to green. +// @Y changes the color to yellow. +// @D changes to the default terminal text color. +// +// TODO(wan@google.com): Write tests for this once we add stdout +// capturing to Google Test. +static void PrintColorEncoded(const char* str) { + GTestColor color = COLOR_DEFAULT; // The current color. + + // Conceptually, we split the string into segments divided by escape + // sequences. Then we print one segment at a time. At the end of + // each iteration, the str pointer advances to the beginning of the + // next segment. + for (;;) { + const char* p = strchr(str, '@'); + if (p == NULL) { + ColoredPrintf(color, "%s", str); + return; + } + + ColoredPrintf(color, "%s", String(str, p - str).c_str()); + + const char ch = p[1]; + str = p + 2; + if (ch == '@') { + ColoredPrintf(color, "@"); + } else if (ch == 'D') { + color = COLOR_DEFAULT; + } else if (ch == 'R') { + color = COLOR_RED; + } else if (ch == 'G') { + color = COLOR_GREEN; + } else if (ch == 'Y') { + color = COLOR_YELLOW; + } else { + --str; + } + } +} + +static const char kColorEncodedHelpMessage[] = +"This program contains tests written using " GTEST_NAME_ ". You can use the\n" +"following command line flags to control its behavior:\n" +"\n" +"Test Selection:\n" +" @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n" +" List the names of all tests instead of running them. The name of\n" +" TEST(Foo, Bar) is \"Foo.Bar\".\n" +" @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS" + "[@G-@YNEGATIVE_PATTERNS]@D\n" +" Run only the tests whose name matches one of the positive patterns but\n" +" none of the negative patterns. '?' matches any single character; '*'\n" +" matches any substring; ':' separates two patterns.\n" +" @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n" +" Run all disabled tests too.\n" +"\n" +"Test Execution:\n" +" @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n" +" Run the tests repeatedly; use a negative count to repeat forever.\n" +" @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n" +" Randomize tests' orders on every iteration.\n" +" @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n" +" Random number seed to use for shuffling test orders (between 1 and\n" +" 99999, or 0 to use a seed based on the current time).\n" +"\n" +"Test Output:\n" +" @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" +" Enable/disable colored output. The default is @Gauto@D.\n" +" -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n" +" Don't print the elapsed time of each test.\n" +" @G--" GTEST_FLAG_PREFIX_ "output=xml@Y[@G:@YDIRECTORY_PATH@G" + GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n" +" Generate an XML report in the given directory or with the given file\n" +" name. @YFILE_PATH@D defaults to @Gtest_details.xml@D.\n" +#if GTEST_CAN_STREAM_RESULTS_ +" @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n" +" Stream test results to the given server.\n" +#endif // GTEST_CAN_STREAM_RESULTS_ +"\n" +"Assertion Behavior:\n" +#if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" +" Set the default death test style.\n" +#endif // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS +" @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n" +" Turn assertion failures into debugger break-points.\n" +" @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n" +" Turn assertion failures into C++ exceptions.\n" +" @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n" +" Do not report exceptions as test failures. Instead, allow them\n" +" to crash the program or throw a pop-up (on Windows).\n" +"\n" +"Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set " + "the corresponding\n" +"environment variable of a flag (all letters in upper-case). For example, to\n" +"disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_ + "color=no@D or set\n" +"the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n" +"\n" +"For more information, please read the " GTEST_NAME_ " documentation at\n" +"@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n" +"(not one in your own code or tests), please report it to\n" +"@G<" GTEST_DEV_EMAIL_ ">@D.\n"; + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. The type parameter CharType can be +// instantiated to either char or wchar_t. +template +void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) { + for (int i = 1; i < *argc; i++) { + const String arg_string = StreamableToString(argv[i]); + const char* const arg = arg_string.c_str(); + + using internal::ParseBoolFlag; + using internal::ParseInt32Flag; + using internal::ParseStringFlag; + + // Do we see a Google Test flag? + if (ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, + >EST_FLAG(also_run_disabled_tests)) || + ParseBoolFlag(arg, kBreakOnFailureFlag, + >EST_FLAG(break_on_failure)) || + ParseBoolFlag(arg, kCatchExceptionsFlag, + >EST_FLAG(catch_exceptions)) || + ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || + ParseStringFlag(arg, kDeathTestStyleFlag, + >EST_FLAG(death_test_style)) || + ParseBoolFlag(arg, kDeathTestUseFork, + >EST_FLAG(death_test_use_fork)) || + ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || + ParseStringFlag(arg, kInternalRunDeathTestFlag, + >EST_FLAG(internal_run_death_test)) || + ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || + ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || + ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || + ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || + ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || + ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || + ParseInt32Flag(arg, kStackTraceDepthFlag, + >EST_FLAG(stack_trace_depth)) || + ParseStringFlag(arg, kStreamResultToFlag, + >EST_FLAG(stream_result_to)) || + ParseBoolFlag(arg, kThrowOnFailureFlag, + >EST_FLAG(throw_on_failure)) + ) { + // Yes. Shift the remainder of the argv list left by one. Note + // that argv has (*argc + 1) elements, the last one always being + // NULL. The following loop moves the trailing NULL element as + // well. + for (int j = i; j != *argc; j++) { + argv[j] = argv[j + 1]; + } + + // Decrements the argument count. + (*argc)--; + + // We also need to decrement the iterator as we just removed + // an element. + i--; + } else if (arg_string == "--help" || arg_string == "-h" || + arg_string == "-?" || arg_string == "/?" || + HasGoogleTestFlagPrefix(arg)) { + // Both help flag and unrecognized Google Test flags (excluding + // internal ones) trigger help display. + g_help_flag = true; + } + } + + if (g_help_flag) { + // We print the help here instead of in RUN_ALL_TESTS(), as the + // latter may not be called at all if the user is using Google + // Test with another testing framework. + PrintColorEncoded(kColorEncodedHelpMessage); + } +} + +// Parses the command line for Google Test flags, without initializing +// other parts of Google Test. +void ParseGoogleTestFlagsOnly(int* argc, char** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); +} +void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv) { + ParseGoogleTestFlagsOnlyImpl(argc, argv); +} + +// The internal implementation of InitGoogleTest(). +// +// The type parameter CharType can be instantiated to either char or +// wchar_t. +template +void InitGoogleTestImpl(int* argc, CharType** argv) { + g_init_gtest_count++; + + // We don't want to run the initialization code twice. + if (g_init_gtest_count != 1) return; + + if (*argc <= 0) return; + + internal::g_executable_path = internal::StreamableToString(argv[0]); + +#if GTEST_HAS_DEATH_TEST + + g_argvs.clear(); + for (int i = 0; i != *argc; i++) { + g_argvs.push_back(StreamableToString(argv[i])); + } + +#endif // GTEST_HAS_DEATH_TEST + + ParseGoogleTestFlagsOnly(argc, argv); + GetUnitTestImpl()->PostFlagParsingInit(); +} + +} // namespace internal + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +void InitGoogleTest(int* argc, char** argv) { + internal::InitGoogleTestImpl(argc, argv); +} + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +void InitGoogleTest(int* argc, wchar_t** argv) { + internal::InitGoogleTestImpl(argc, argv); +} + +} // namespace testing +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan), vladl@google.com (Vlad Losev) +// +// This file implements death tests. + + +#if GTEST_HAS_DEATH_TEST + +# if GTEST_OS_MAC +# include +# endif // GTEST_OS_MAC + +# include +# include +# include +# include + +# if GTEST_OS_WINDOWS +# include +# else +# include +# include +# endif // GTEST_OS_WINDOWS + +#endif // GTEST_HAS_DEATH_TEST + + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick is to +// prevent a user from accidentally including gtest-internal-inl.h in +// his code. +#define GTEST_IMPLEMENTATION_ 1 +#undef GTEST_IMPLEMENTATION_ + +namespace testing { + +// Constants. + +// The default death test style. +static const char kDefaultDeathTestStyle[] = "fast"; + +GTEST_DEFINE_string_( + death_test_style, + internal::StringFromGTestEnv("death_test_style", kDefaultDeathTestStyle), + "Indicates how to run a death test in a forked child process: " + "\"threadsafe\" (child process re-executes the test binary " + "from the beginning, running only the specific death test) or " + "\"fast\" (child process runs the death test immediately " + "after forking)."); + +GTEST_DEFINE_bool_( + death_test_use_fork, + internal::BoolFromGTestEnv("death_test_use_fork", false), + "Instructs to use fork()/_exit() instead of clone() in death tests. " + "Ignored and always uses fork() on POSIX systems where clone() is not " + "implemented. Useful when running under valgrind or similar tools if " + "those do not support clone(). Valgrind 3.3.1 will just fail if " + "it sees an unsupported combination of clone() flags. " + "It is not recommended to use this flag w/o valgrind though it will " + "work in 99% of the cases. Once valgrind is fixed, this flag will " + "most likely be removed."); + +namespace internal { +GTEST_DEFINE_string_( + internal_run_death_test, "", + "Indicates the file, line number, temporal index of " + "the single death test to run, and a file descriptor to " + "which a success code may be sent, all separated by " + "colons. This flag is specified if and only if the current " + "process is a sub-process launched for running a thread-safe " + "death test. FOR INTERNAL USE ONLY."); +} // namespace internal + +#if GTEST_HAS_DEATH_TEST + +// ExitedWithCode constructor. +ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) { +} + +// ExitedWithCode function-call operator. +bool ExitedWithCode::operator()(int exit_status) const { +# if GTEST_OS_WINDOWS + + return exit_status == exit_code_; + +# else + + return WIFEXITED(exit_status) && WEXITSTATUS(exit_status) == exit_code_; + +# endif // GTEST_OS_WINDOWS +} + +# if !GTEST_OS_WINDOWS +// KilledBySignal constructor. +KilledBySignal::KilledBySignal(int signum) : signum_(signum) { +} + +// KilledBySignal function-call operator. +bool KilledBySignal::operator()(int exit_status) const { + return WIFSIGNALED(exit_status) && WTERMSIG(exit_status) == signum_; +} +# endif // !GTEST_OS_WINDOWS + +namespace internal { + +// Utilities needed for death tests. + +// Generates a textual description of a given exit code, in the format +// specified by wait(2). +static String ExitSummary(int exit_code) { + Message m; + +# if GTEST_OS_WINDOWS + + m << "Exited with exit status " << exit_code; + +# else + + if (WIFEXITED(exit_code)) { + m << "Exited with exit status " << WEXITSTATUS(exit_code); + } else if (WIFSIGNALED(exit_code)) { + m << "Terminated by signal " << WTERMSIG(exit_code); + } +# ifdef WCOREDUMP + if (WCOREDUMP(exit_code)) { + m << " (core dumped)"; + } +# endif +# endif // GTEST_OS_WINDOWS + + return m.GetString(); +} + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +bool ExitedUnsuccessfully(int exit_status) { + return !ExitedWithCode(0)(exit_status); +} + +# if !GTEST_OS_WINDOWS +// Generates a textual failure message when a death test finds more than +// one thread running, or cannot determine the number of threads, prior +// to executing the given statement. It is the responsibility of the +// caller not to pass a thread_count of 1. +static String DeathTestThreadWarning(size_t thread_count) { + Message msg; + msg << "Death tests use fork(), which is unsafe particularly" + << " in a threaded context. For this test, " << GTEST_NAME_ << " "; + if (thread_count == 0) + msg << "couldn't detect the number of threads."; + else + msg << "detected " << thread_count << " threads."; + return msg.GetString(); +} +# endif // !GTEST_OS_WINDOWS + +// Flag characters for reporting a death test that did not die. +static const char kDeathTestLived = 'L'; +static const char kDeathTestReturned = 'R'; +static const char kDeathTestThrew = 'T'; +static const char kDeathTestInternalError = 'I'; + +// An enumeration describing all of the possible ways that a death test can +// conclude. DIED means that the process died while executing the test +// code; LIVED means that process lived beyond the end of the test code; +// RETURNED means that the test statement attempted to execute a return +// statement, which is not allowed; THREW means that the test statement +// returned control by throwing an exception. IN_PROGRESS means the test +// has not yet concluded. +// TODO(vladl@google.com): Unify names and possibly values for +// AbortReason, DeathTestOutcome, and flag characters above. +enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW }; + +// Routine for aborting the program which is safe to call from an +// exec-style death test child process, in which case the error +// message is propagated back to the parent process. Otherwise, the +// message is simply printed to stderr. In either case, the program +// then exits with status 1. +void DeathTestAbort(const String& message) { + // On a POSIX system, this function may be called from a threadsafe-style + // death test child process, which operates on a very small stack. Use + // the heap for any additional non-minuscule memory requirements. + const InternalRunDeathTestFlag* const flag = + GetUnitTestImpl()->internal_run_death_test_flag(); + if (flag != NULL) { + FILE* parent = posix::FDOpen(flag->write_fd(), "w"); + fputc(kDeathTestInternalError, parent); + fprintf(parent, "%s", message.c_str()); + fflush(parent); + _exit(1); + } else { + fprintf(stderr, "%s", message.c_str()); + fflush(stderr); + posix::Abort(); + } +} + +// A replacement for CHECK that calls DeathTestAbort if the assertion +// fails. +# define GTEST_DEATH_TEST_CHECK_(expression) \ + do { \ + if (!::testing::internal::IsTrue(expression)) { \ + DeathTestAbort(::testing::internal::String::Format( \ + "CHECK failed: File %s, line %d: %s", \ + __FILE__, __LINE__, #expression)); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// This macro is similar to GTEST_DEATH_TEST_CHECK_, but it is meant for +// evaluating any system call that fulfills two conditions: it must return +// -1 on failure, and set errno to EINTR when it is interrupted and +// should be tried again. The macro expands to a loop that repeatedly +// evaluates the expression as long as it evaluates to -1 and sets +// errno to EINTR. If the expression evaluates to -1 but errno is +// something other than EINTR, DeathTestAbort is called. +# define GTEST_DEATH_TEST_CHECK_SYSCALL_(expression) \ + do { \ + int gtest_retval; \ + do { \ + gtest_retval = (expression); \ + } while (gtest_retval == -1 && errno == EINTR); \ + if (gtest_retval == -1) { \ + DeathTestAbort(::testing::internal::String::Format( \ + "CHECK failed: File %s, line %d: %s != -1", \ + __FILE__, __LINE__, #expression)); \ + } \ + } while (::testing::internal::AlwaysFalse()) + +// Returns the message describing the last system error in errno. +String GetLastErrnoDescription() { + return String(errno == 0 ? "" : posix::StrError(errno)); +} + +// This is called from a death test parent process to read a failure +// message from the death test child process and log it with the FATAL +// severity. On Windows, the message is read from a pipe handle. On other +// platforms, it is read from a file descriptor. +static void FailFromInternalError(int fd) { + Message error; + char buffer[256]; + int num_read; + + do { + while ((num_read = posix::Read(fd, buffer, 255)) > 0) { + buffer[num_read] = '\0'; + error << buffer; + } + } while (num_read == -1 && errno == EINTR); + + if (num_read == 0) { + GTEST_LOG_(FATAL) << error.GetString(); + } else { + const int last_error = errno; + GTEST_LOG_(FATAL) << "Error while reading death test internal: " + << GetLastErrnoDescription() << " [" << last_error << "]"; + } +} + +// Death test constructor. Increments the running death test count +// for the current test. +DeathTest::DeathTest() { + TestInfo* const info = GetUnitTestImpl()->current_test_info(); + if (info == NULL) { + DeathTestAbort("Cannot run a death test outside of a TEST or " + "TEST_F construct"); + } +} + +// Creates and returns a death test by dispatching to the current +// death test factory. +bool DeathTest::Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) { + return GetUnitTestImpl()->death_test_factory()->Create( + statement, regex, file, line, test); +} + +const char* DeathTest::LastMessage() { + return last_death_test_message_.c_str(); +} + +void DeathTest::set_last_death_test_message(const String& message) { + last_death_test_message_ = message; +} + +String DeathTest::last_death_test_message_; + +// Provides cross platform implementation for some death functionality. +class DeathTestImpl : public DeathTest { + protected: + DeathTestImpl(const char* a_statement, const RE* a_regex) + : statement_(a_statement), + regex_(a_regex), + spawned_(false), + status_(-1), + outcome_(IN_PROGRESS), + read_fd_(-1), + write_fd_(-1) {} + + // read_fd_ is expected to be closed and cleared by a derived class. + ~DeathTestImpl() { GTEST_DEATH_TEST_CHECK_(read_fd_ == -1); } + + void Abort(AbortReason reason); + virtual bool Passed(bool status_ok); + + const char* statement() const { return statement_; } + const RE* regex() const { return regex_; } + bool spawned() const { return spawned_; } + void set_spawned(bool is_spawned) { spawned_ = is_spawned; } + int status() const { return status_; } + void set_status(int a_status) { status_ = a_status; } + DeathTestOutcome outcome() const { return outcome_; } + void set_outcome(DeathTestOutcome an_outcome) { outcome_ = an_outcome; } + int read_fd() const { return read_fd_; } + void set_read_fd(int fd) { read_fd_ = fd; } + int write_fd() const { return write_fd_; } + void set_write_fd(int fd) { write_fd_ = fd; } + + // Called in the parent process only. Reads the result code of the death + // test child process via a pipe, interprets it to set the outcome_ + // member, and closes read_fd_. Outputs diagnostics and terminates in + // case of unexpected codes. + void ReadAndInterpretStatusByte(); + + private: + // The textual content of the code this object is testing. This class + // doesn't own this string and should not attempt to delete it. + const char* const statement_; + // The regular expression which test output must match. DeathTestImpl + // doesn't own this object and should not attempt to delete it. + const RE* const regex_; + // True if the death test child process has been successfully spawned. + bool spawned_; + // The exit status of the child process. + int status_; + // How the death test concluded. + DeathTestOutcome outcome_; + // Descriptor to the read end of the pipe to the child process. It is + // always -1 in the child process. The child keeps its write end of the + // pipe in write_fd_. + int read_fd_; + // Descriptor to the child's write end of the pipe to the parent process. + // It is always -1 in the parent process. The parent keeps its end of the + // pipe in read_fd_. + int write_fd_; +}; + +// Called in the parent process only. Reads the result code of the death +// test child process via a pipe, interprets it to set the outcome_ +// member, and closes read_fd_. Outputs diagnostics and terminates in +// case of unexpected codes. +void DeathTestImpl::ReadAndInterpretStatusByte() { + char flag; + int bytes_read; + + // The read() here blocks until data is available (signifying the + // failure of the death test) or until the pipe is closed (signifying + // its success), so it's okay to call this in the parent before + // the child process has exited. + do { + bytes_read = posix::Read(read_fd(), &flag, 1); + } while (bytes_read == -1 && errno == EINTR); + + if (bytes_read == 0) { + set_outcome(DIED); + } else if (bytes_read == 1) { + switch (flag) { + case kDeathTestReturned: + set_outcome(RETURNED); + break; + case kDeathTestThrew: + set_outcome(THREW); + break; + case kDeathTestLived: + set_outcome(LIVED); + break; + case kDeathTestInternalError: + FailFromInternalError(read_fd()); // Does not return. + break; + default: + GTEST_LOG_(FATAL) << "Death test child process reported " + << "unexpected status byte (" + << static_cast(flag) << ")"; + } + } else { + GTEST_LOG_(FATAL) << "Read from death test child process failed: " + << GetLastErrnoDescription(); + } + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Close(read_fd())); + set_read_fd(-1); +} + +// Signals that the death test code which should have exited, didn't. +// Should be called only in a death test child process. +// Writes a status byte to the child's status file descriptor, then +// calls _exit(1). +void DeathTestImpl::Abort(AbortReason reason) { + // The parent process considers the death test to be a failure if + // it finds any data in our pipe. So, here we write a single flag byte + // to the pipe, then exit. + const char status_ch = + reason == TEST_DID_NOT_DIE ? kDeathTestLived : + reason == TEST_THREW_EXCEPTION ? kDeathTestThrew : kDeathTestReturned; + + GTEST_DEATH_TEST_CHECK_SYSCALL_(posix::Write(write_fd(), &status_ch, 1)); + // We are leaking the descriptor here because on some platforms (i.e., + // when built as Windows DLL), destructors of global objects will still + // run after calling _exit(). On such systems, write_fd_ will be + // indirectly closed from the destructor of UnitTestImpl, causing double + // close if it is also closed here. On debug configurations, double close + // may assert. As there are no in-process buffers to flush here, we are + // relying on the OS to close the descriptor after the process terminates + // when the destructors are not run. + _exit(1); // Exits w/o any normal exit hooks (we were supposed to crash) +} + +// Returns an indented copy of stderr output for a death test. +// This makes distinguishing death test output lines from regular log lines +// much easier. +static ::std::string FormatDeathTestOutput(const ::std::string& output) { + ::std::string ret; + for (size_t at = 0; ; ) { + const size_t line_end = output.find('\n', at); + ret += "[ DEATH ] "; + if (line_end == ::std::string::npos) { + ret += output.substr(at); + break; + } + ret += output.substr(at, line_end + 1 - at); + at = line_end + 1; + } + return ret; +} + +// Assesses the success or failure of a death test, using both private +// members which have previously been set, and one argument: +// +// Private data members: +// outcome: An enumeration describing how the death test +// concluded: DIED, LIVED, THREW, or RETURNED. The death test +// fails in the latter three cases. +// status: The exit status of the child process. On *nix, it is in the +// in the format specified by wait(2). On Windows, this is the +// value supplied to the ExitProcess() API or a numeric code +// of the exception that terminated the program. +// regex: A regular expression object to be applied to +// the test's captured standard error output; the death test +// fails if it does not match. +// +// Argument: +// status_ok: true if exit_status is acceptable in the context of +// this particular death test, which fails if it is false +// +// Returns true iff all of the above conditions are met. Otherwise, the +// first failing condition, in the order given above, is the one that is +// reported. Also sets the last death test message string. +bool DeathTestImpl::Passed(bool status_ok) { + if (!spawned()) + return false; + + const String error_message = GetCapturedStderr(); + + bool success = false; + Message buffer; + + buffer << "Death test: " << statement() << "\n"; + switch (outcome()) { + case LIVED: + buffer << " Result: failed to die.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case THREW: + buffer << " Result: threw an exception.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case RETURNED: + buffer << " Result: illegal return in test statement.\n" + << " Error msg:\n" << FormatDeathTestOutput(error_message); + break; + case DIED: + if (status_ok) { + const bool matched = RE::PartialMatch(error_message.c_str(), *regex()); + if (matched) { + success = true; + } else { + buffer << " Result: died but not with expected error.\n" + << " Expected: " << regex()->pattern() << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + } else { + buffer << " Result: died but not with expected exit code:\n" + << " " << ExitSummary(status()) << "\n" + << "Actual msg:\n" << FormatDeathTestOutput(error_message); + } + break; + case IN_PROGRESS: + default: + GTEST_LOG_(FATAL) + << "DeathTest::Passed somehow called before conclusion of test"; + } + + DeathTest::set_last_death_test_message(buffer.GetString()); + return success; +} + +# if GTEST_OS_WINDOWS +// WindowsDeathTest implements death tests on Windows. Due to the +// specifics of starting new processes on Windows, death tests there are +// always threadsafe, and Google Test considers the +// --gtest_death_test_style=fast setting to be equivalent to +// --gtest_death_test_style=threadsafe there. +// +// A few implementation notes: Like the Linux version, the Windows +// implementation uses pipes for child-to-parent communication. But due to +// the specifics of pipes on Windows, some extra steps are required: +// +// 1. The parent creates a communication pipe and stores handles to both +// ends of it. +// 2. The parent starts the child and provides it with the information +// necessary to acquire the handle to the write end of the pipe. +// 3. The child acquires the write end of the pipe and signals the parent +// using a Windows event. +// 4. Now the parent can release the write end of the pipe on its side. If +// this is done before step 3, the object's reference count goes down to +// 0 and it is destroyed, preventing the child from acquiring it. The +// parent now has to release it, or read operations on the read end of +// the pipe will not return when the child terminates. +// 5. The parent reads child's output through the pipe (outcome code and +// any possible error messages) from the pipe, and its stderr and then +// determines whether to fail the test. +// +// Note: to distinguish Win32 API calls from the local method and function +// calls, the former are explicitly resolved in the global namespace. +// +class WindowsDeathTest : public DeathTestImpl { + public: + WindowsDeathTest(const char* a_statement, + const RE* a_regex, + const char* file, + int line) + : DeathTestImpl(a_statement, a_regex), file_(file), line_(line) {} + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + virtual TestRole AssumeRole(); + + private: + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; + // Handle to the write end of the pipe to the child process. + AutoHandle write_handle_; + // Child process handle. + AutoHandle child_handle_; + // Event the child process uses to signal the parent that it has + // acquired the handle to the write end of the pipe. After seeing this + // event the parent can release its own handles to make sure its + // ReadFile() calls return when the child terminates. + AutoHandle event_handle_; +}; + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int WindowsDeathTest::Wait() { + if (!spawned()) + return 0; + + // Wait until the child either signals that it has acquired the write end + // of the pipe or it dies. + const HANDLE wait_handles[2] = { child_handle_.Get(), event_handle_.Get() }; + switch (::WaitForMultipleObjects(2, + wait_handles, + FALSE, // Waits for any of the handles. + INFINITE)) { + case WAIT_OBJECT_0: + case WAIT_OBJECT_0 + 1: + break; + default: + GTEST_DEATH_TEST_CHECK_(false); // Should not get here. + } + + // The child has acquired the write end of the pipe or exited. + // We release the handle on our side and continue. + write_handle_.Reset(); + event_handle_.Reset(); + + ReadAndInterpretStatusByte(); + + // Waits for the child process to exit if it haven't already. This + // returns immediately if the child has already exited, regardless of + // whether previous calls to WaitForMultipleObjects synchronized on this + // handle or not. + GTEST_DEATH_TEST_CHECK_( + WAIT_OBJECT_0 == ::WaitForSingleObject(child_handle_.Get(), + INFINITE)); + DWORD status_code; + GTEST_DEATH_TEST_CHECK_( + ::GetExitCodeProcess(child_handle_.Get(), &status_code) != FALSE); + child_handle_.Reset(); + set_status(static_cast(status_code)); + return status(); +} + +// The AssumeRole process for a Windows death test. It creates a child +// process with the same executable as the current process to run the +// death test. The child process is given the --gtest_filter and +// --gtest_internal_run_death_test flags such that it knows to run the +// current death test only. +DeathTest::TestRole WindowsDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + // ParseInternalRunDeathTestFlag() has performed all the necessary + // processing. + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + // WindowsDeathTest uses an anonymous pipe to communicate results of + // a death test. + SECURITY_ATTRIBUTES handles_are_inheritable = { + sizeof(SECURITY_ATTRIBUTES), NULL, TRUE }; + HANDLE read_handle, write_handle; + GTEST_DEATH_TEST_CHECK_( + ::CreatePipe(&read_handle, &write_handle, &handles_are_inheritable, + 0) // Default buffer size. + != FALSE); + set_read_fd(::_open_osfhandle(reinterpret_cast(read_handle), + O_RDONLY)); + write_handle_.Reset(write_handle); + event_handle_.Reset(::CreateEvent( + &handles_are_inheritable, + TRUE, // The event will automatically reset to non-signaled state. + FALSE, // The initial state is non-signalled. + NULL)); // The even is unnamed. + GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != NULL); + const String filter_flag = String::Format("--%s%s=%s.%s", + GTEST_FLAG_PREFIX_, kFilterFlag, + info->test_case_name(), + info->name()); + const String internal_flag = String::Format( + "--%s%s=%s|%d|%d|%u|%Iu|%Iu", + GTEST_FLAG_PREFIX_, + kInternalRunDeathTestFlag, + file_, line_, + death_test_index, + static_cast(::GetCurrentProcessId()), + // size_t has the same with as pointers on both 32-bit and 64-bit + // Windows platforms. + // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx. + reinterpret_cast(write_handle), + reinterpret_cast(event_handle_.Get())); + + char executable_path[_MAX_PATH + 1]; // NOLINT + GTEST_DEATH_TEST_CHECK_( + _MAX_PATH + 1 != ::GetModuleFileNameA(NULL, + executable_path, + _MAX_PATH)); + + String command_line = String::Format("%s %s \"%s\"", + ::GetCommandLineA(), + filter_flag.c_str(), + internal_flag.c_str()); + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // Flush the log buffers since the log streams are shared with the child. + FlushInfoLog(); + + // The child process will share the standard handles with the parent. + STARTUPINFOA startup_info; + memset(&startup_info, 0, sizeof(STARTUPINFO)); + startup_info.dwFlags = STARTF_USESTDHANDLES; + startup_info.hStdInput = ::GetStdHandle(STD_INPUT_HANDLE); + startup_info.hStdOutput = ::GetStdHandle(STD_OUTPUT_HANDLE); + startup_info.hStdError = ::GetStdHandle(STD_ERROR_HANDLE); + + PROCESS_INFORMATION process_info; + GTEST_DEATH_TEST_CHECK_(::CreateProcessA( + executable_path, + const_cast(command_line.c_str()), + NULL, // Retuned process handle is not inheritable. + NULL, // Retuned thread handle is not inheritable. + TRUE, // Child inherits all inheritable handles (for write_handle_). + 0x0, // Default creation flags. + NULL, // Inherit the parent's environment. + UnitTest::GetInstance()->original_working_dir(), + &startup_info, + &process_info) != FALSE); + child_handle_.Reset(process_info.hProcess); + ::CloseHandle(process_info.hThread); + set_spawned(true); + return OVERSEE_TEST; +} +# else // We are not on Windows. + +// ForkingDeathTest provides implementations for most of the abstract +// methods of the DeathTest interface. Only the AssumeRole method is +// left undefined. +class ForkingDeathTest : public DeathTestImpl { + public: + ForkingDeathTest(const char* statement, const RE* regex); + + // All of these virtual functions are inherited from DeathTest. + virtual int Wait(); + + protected: + void set_child_pid(pid_t child_pid) { child_pid_ = child_pid; } + + private: + // PID of child process during death test; 0 in the child process itself. + pid_t child_pid_; +}; + +// Constructs a ForkingDeathTest. +ForkingDeathTest::ForkingDeathTest(const char* a_statement, const RE* a_regex) + : DeathTestImpl(a_statement, a_regex), + child_pid_(-1) {} + +// Waits for the child in a death test to exit, returning its exit +// status, or 0 if no child process exists. As a side effect, sets the +// outcome data member. +int ForkingDeathTest::Wait() { + if (!spawned()) + return 0; + + ReadAndInterpretStatusByte(); + + int status_value; + GTEST_DEATH_TEST_CHECK_SYSCALL_(waitpid(child_pid_, &status_value, 0)); + set_status(status_value); + return status_value; +} + +// A concrete death test class that forks, then immediately runs the test +// in the child process. +class NoExecDeathTest : public ForkingDeathTest { + public: + NoExecDeathTest(const char* a_statement, const RE* a_regex) : + ForkingDeathTest(a_statement, a_regex) { } + virtual TestRole AssumeRole(); +}; + +// The AssumeRole process for a fork-and-run death test. It implements a +// straightforward fork, with a simple pipe to transmit the status byte. +DeathTest::TestRole NoExecDeathTest::AssumeRole() { + const size_t thread_count = GetThreadCount(); + if (thread_count != 1) { + GTEST_LOG_(WARNING) << DeathTestThreadWarning(thread_count); + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + + DeathTest::set_last_death_test_message(""); + CaptureStderr(); + // When we fork the process below, the log file buffers are copied, but the + // file descriptors are shared. We flush all log files here so that closing + // the file descriptors in the child process doesn't throw off the + // synchronization between descriptors and buffers in the parent process. + // This is as close to the fork as possible to avoid a race condition in case + // there are multiple threads running before the death test, and another + // thread writes to the log file. + FlushInfoLog(); + + const pid_t child_pid = fork(); + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + set_child_pid(child_pid); + if (child_pid == 0) { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[0])); + set_write_fd(pipe_fd[1]); + // Redirects all logging to stderr in the child process to prevent + // concurrent writes to the log files. We capture stderr in the parent + // process and append the child process' output to a log. + LogToStderr(); + // Event forwarding to the listeners of event listener API mush be shut + // down in death test subprocesses. + GetUnitTestImpl()->listeners()->SuppressEventForwarding(); + return EXECUTE_TEST; + } else { + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; + } +} + +// A concrete death test class that forks and re-executes the main +// program from the beginning, with command-line flags set that cause +// only this specific death test to be run. +class ExecDeathTest : public ForkingDeathTest { + public: + ExecDeathTest(const char* a_statement, const RE* a_regex, + const char* file, int line) : + ForkingDeathTest(a_statement, a_regex), file_(file), line_(line) { } + virtual TestRole AssumeRole(); + private: + // The name of the file in which the death test is located. + const char* const file_; + // The line number on which the death test is located. + const int line_; +}; + +// Utility class for accumulating command-line arguments. +class Arguments { + public: + Arguments() { + args_.push_back(NULL); + } + + ~Arguments() { + for (std::vector::iterator i = args_.begin(); i != args_.end(); + ++i) { + free(*i); + } + } + void AddArgument(const char* argument) { + args_.insert(args_.end() - 1, posix::StrDup(argument)); + } + + template + void AddArguments(const ::std::vector& arguments) { + for (typename ::std::vector::const_iterator i = arguments.begin(); + i != arguments.end(); + ++i) { + args_.insert(args_.end() - 1, posix::StrDup(i->c_str())); + } + } + char* const* Argv() { + return &args_[0]; + } + private: + std::vector args_; +}; + +// A struct that encompasses the arguments to the child process of a +// threadsafe-style death test process. +struct ExecDeathTestArgs { + char* const* argv; // Command-line arguments for the child's call to exec + int close_fd; // File descriptor to close; the read end of a pipe +}; + +# if GTEST_OS_MAC +inline char** GetEnviron() { + // When Google Test is built as a framework on MacOS X, the environ variable + // is unavailable. Apple's documentation (man environ) recommends using + // _NSGetEnviron() instead. + return *_NSGetEnviron(); +} +# else +// Some POSIX platforms expect you to declare environ. extern "C" makes +// it reside in the global namespace. +extern "C" char** environ; +inline char** GetEnviron() { return environ; } +# endif // GTEST_OS_MAC + +// The main function for a threadsafe-style death test child process. +// This function is called in a clone()-ed process and thus must avoid +// any potentially unsafe operations like malloc or libc functions. +static int ExecDeathTestChildMain(void* child_arg) { + ExecDeathTestArgs* const args = static_cast(child_arg); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(args->close_fd)); + + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(String::Format("chdir(\"%s\") failed: %s", + original_dir, + GetLastErrnoDescription().c_str())); + return EXIT_FAILURE; + } + + // We can safely call execve() as it's a direct system call. We + // cannot use execvp() as it's a libc function and thus potentially + // unsafe. Since execve() doesn't search the PATH, the user must + // invoke the test program via a valid path that contains at least + // one path separator. + execve(args->argv[0], args->argv, GetEnviron()); + DeathTestAbort(String::Format("execve(%s, ...) in %s failed: %s", + args->argv[0], + original_dir, + GetLastErrnoDescription().c_str())); + return EXIT_FAILURE; +} + +// Two utility routines that together determine the direction the stack +// grows. +// This could be accomplished more elegantly by a single recursive +// function, but we want to guard against the unlikely possibility of +// a smart compiler optimizing the recursion away. +// +// GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining +// StackLowerThanAddress into StackGrowsDown, which then doesn't give +// correct answer. +bool StackLowerThanAddress(const void* ptr) GTEST_NO_INLINE_; +bool StackLowerThanAddress(const void* ptr) { + int dummy; + return &dummy < ptr; +} + +bool StackGrowsDown() { + int dummy; + return StackLowerThanAddress(&dummy); +} + +// A threadsafe implementation of fork(2) for threadsafe-style death tests +// that uses clone(2). It dies with an error message if anything goes +// wrong. +static pid_t ExecDeathTestFork(char* const* argv, int close_fd) { + ExecDeathTestArgs args = { argv, close_fd }; + pid_t child_pid = -1; + +# if GTEST_HAS_CLONE + const bool use_fork = GTEST_FLAG(death_test_use_fork); + + if (!use_fork) { + static const bool stack_grows_down = StackGrowsDown(); + const size_t stack_size = getpagesize(); + // MMAP_ANONYMOUS is not defined on Mac, so we use MAP_ANON instead. + void* const stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE, + MAP_ANON | MAP_PRIVATE, -1, 0); + GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED); + void* const stack_top = + static_cast(stack) + (stack_grows_down ? stack_size : 0); + + child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args); + + GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1); + } +# else + const bool use_fork = true; +# endif // GTEST_HAS_CLONE + + if (use_fork && (child_pid = fork()) == 0) { + ExecDeathTestChildMain(&args); + _exit(0); + } + + GTEST_DEATH_TEST_CHECK_(child_pid != -1); + return child_pid; +} + +// The AssumeRole process for a fork-and-exec death test. It re-executes the +// main program from the beginning, setting the --gtest_filter +// and --gtest_internal_run_death_test flags to cause only the current +// death test to be re-run. +DeathTest::TestRole ExecDeathTest::AssumeRole() { + const UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const TestInfo* const info = impl->current_test_info(); + const int death_test_index = info->result()->death_test_count(); + + if (flag != NULL) { + set_write_fd(flag->write_fd()); + return EXECUTE_TEST; + } + + int pipe_fd[2]; + GTEST_DEATH_TEST_CHECK_(pipe(pipe_fd) != -1); + // Clear the close-on-exec flag on the write end of the pipe, lest + // it be closed when the child process does an exec: + GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1); + + const String filter_flag = + String::Format("--%s%s=%s.%s", + GTEST_FLAG_PREFIX_, kFilterFlag, + info->test_case_name(), info->name()); + const String internal_flag = + String::Format("--%s%s=%s|%d|%d|%d", + GTEST_FLAG_PREFIX_, kInternalRunDeathTestFlag, + file_, line_, death_test_index, pipe_fd[1]); + Arguments args; + args.AddArguments(GetArgvs()); + args.AddArgument(filter_flag.c_str()); + args.AddArgument(internal_flag.c_str()); + + DeathTest::set_last_death_test_message(""); + + CaptureStderr(); + // See the comment in NoExecDeathTest::AssumeRole for why the next line + // is necessary. + FlushInfoLog(); + + const pid_t child_pid = ExecDeathTestFork(args.Argv(), pipe_fd[0]); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); + set_child_pid(child_pid); + set_read_fd(pipe_fd[0]); + set_spawned(true); + return OVERSEE_TEST; +} + +# endif // !GTEST_OS_WINDOWS + +// Creates a concrete DeathTest-derived class that depends on the +// --gtest_death_test_style flag, and sets the pointer pointed to +// by the "test" argument to its address. If the test should be +// skipped, sets that pointer to NULL. Returns true, unless the +// flag is set to an invalid value. +bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex, + const char* file, int line, + DeathTest** test) { + UnitTestImpl* const impl = GetUnitTestImpl(); + const InternalRunDeathTestFlag* const flag = + impl->internal_run_death_test_flag(); + const int death_test_index = impl->current_test_info() + ->increment_death_test_count(); + + if (flag != NULL) { + if (death_test_index > flag->index()) { + DeathTest::set_last_death_test_message(String::Format( + "Death test count (%d) somehow exceeded expected maximum (%d)", + death_test_index, flag->index())); + return false; + } + + if (!(flag->file() == file && flag->line() == line && + flag->index() == death_test_index)) { + *test = NULL; + return true; + } + } + +# if GTEST_OS_WINDOWS + + if (GTEST_FLAG(death_test_style) == "threadsafe" || + GTEST_FLAG(death_test_style) == "fast") { + *test = new WindowsDeathTest(statement, regex, file, line); + } + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") { + *test = new ExecDeathTest(statement, regex, file, line); + } else if (GTEST_FLAG(death_test_style) == "fast") { + *test = new NoExecDeathTest(statement, regex); + } + +# endif // GTEST_OS_WINDOWS + + else { // NOLINT - this is more readable than unbalanced brackets inside #if. + DeathTest::set_last_death_test_message(String::Format( + "Unknown death test style \"%s\" encountered", + GTEST_FLAG(death_test_style).c_str())); + return false; + } + + return true; +} + +// Splits a given string on a given delimiter, populating a given +// vector with the fields. GTEST_HAS_DEATH_TEST implies that we have +// ::std::string, so we can use it here. +static void SplitString(const ::std::string& str, char delimiter, + ::std::vector< ::std::string>* dest) { + ::std::vector< ::std::string> parsed; + ::std::string::size_type pos = 0; + while (::testing::internal::AlwaysTrue()) { + const ::std::string::size_type colon = str.find(delimiter, pos); + if (colon == ::std::string::npos) { + parsed.push_back(str.substr(pos)); + break; + } else { + parsed.push_back(str.substr(pos, colon - pos)); + pos = colon + 1; + } + } + dest->swap(parsed); +} + +# if GTEST_OS_WINDOWS +// Recreates the pipe and event handles from the provided parameters, +// signals the event, and returns a file descriptor wrapped around the pipe +// handle. This function is called in the child process only. +int GetStatusFileDescriptor(unsigned int parent_process_id, + size_t write_handle_as_size_t, + size_t event_handle_as_size_t) { + AutoHandle parent_process_handle(::OpenProcess(PROCESS_DUP_HANDLE, + FALSE, // Non-inheritable. + parent_process_id)); + if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) { + DeathTestAbort(String::Format("Unable to open parent process %u", + parent_process_id)); + } + + // TODO(vladl@google.com): Replace the following check with a + // compile-time assertion when available. + GTEST_CHECK_(sizeof(HANDLE) <= sizeof(size_t)); + + const HANDLE write_handle = + reinterpret_cast(write_handle_as_size_t); + HANDLE dup_write_handle; + + // The newly initialized handle is accessible only in in the parent + // process. To obtain one accessible within the child, we need to use + // DuplicateHandle. + if (!::DuplicateHandle(parent_process_handle.Get(), write_handle, + ::GetCurrentProcess(), &dup_write_handle, + 0x0, // Requested privileges ignored since + // DUPLICATE_SAME_ACCESS is used. + FALSE, // Request non-inheritable handler. + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort(String::Format( + "Unable to duplicate the pipe handle %Iu from the parent process %u", + write_handle_as_size_t, parent_process_id)); + } + + const HANDLE event_handle = reinterpret_cast(event_handle_as_size_t); + HANDLE dup_event_handle; + + if (!::DuplicateHandle(parent_process_handle.Get(), event_handle, + ::GetCurrentProcess(), &dup_event_handle, + 0x0, + FALSE, + DUPLICATE_SAME_ACCESS)) { + DeathTestAbort(String::Format( + "Unable to duplicate the event handle %Iu from the parent process %u", + event_handle_as_size_t, parent_process_id)); + } + + const int write_fd = + ::_open_osfhandle(reinterpret_cast(dup_write_handle), O_APPEND); + if (write_fd == -1) { + DeathTestAbort(String::Format( + "Unable to convert pipe handle %Iu to a file descriptor", + write_handle_as_size_t)); + } + + // Signals the parent that the write end of the pipe has been acquired + // so the parent can release its own write end. + ::SetEvent(dup_event_handle); + + return write_fd; +} +# endif // GTEST_OS_WINDOWS + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() { + if (GTEST_FLAG(internal_run_death_test) == "") return NULL; + + // GTEST_HAS_DEATH_TEST implies that we have ::std::string, so we + // can use it here. + int line = -1; + int index = -1; + ::std::vector< ::std::string> fields; + SplitString(GTEST_FLAG(internal_run_death_test).c_str(), '|', &fields); + int write_fd = -1; + +# if GTEST_OS_WINDOWS + + unsigned int parent_process_id = 0; + size_t write_handle_as_size_t = 0; + size_t event_handle_as_size_t = 0; + + if (fields.size() != 6 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &parent_process_id) + || !ParseNaturalNumber(fields[4], &write_handle_as_size_t) + || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) { + DeathTestAbort(String::Format( + "Bad --gtest_internal_run_death_test flag: %s", + GTEST_FLAG(internal_run_death_test).c_str())); + } + write_fd = GetStatusFileDescriptor(parent_process_id, + write_handle_as_size_t, + event_handle_as_size_t); +# else + + if (fields.size() != 4 + || !ParseNaturalNumber(fields[1], &line) + || !ParseNaturalNumber(fields[2], &index) + || !ParseNaturalNumber(fields[3], &write_fd)) { + DeathTestAbort(String::Format( + "Bad --gtest_internal_run_death_test flag: %s", + GTEST_FLAG(internal_run_death_test).c_str())); + } + +# endif // GTEST_OS_WINDOWS + + return new InternalRunDeathTestFlag(fields[0], line, index, write_fd); +} + +} // namespace internal + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace testing +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: keith.ray@gmail.com (Keith Ray) + + +#include + +#if GTEST_OS_WINDOWS_MOBILE +# include +#elif GTEST_OS_WINDOWS +# include +# include +#elif GTEST_OS_SYMBIAN || GTEST_OS_NACL +// Symbian OpenC and NaCl have PATH_MAX in sys/syslimits.h +# include +#else +# include +# include // Some Linux distributions define PATH_MAX here. +#endif // GTEST_OS_WINDOWS_MOBILE + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_MAX_ _MAX_PATH +#elif defined(PATH_MAX) +# define GTEST_PATH_MAX_ PATH_MAX +#elif defined(_XOPEN_PATH_MAX) +# define GTEST_PATH_MAX_ _XOPEN_PATH_MAX +#elif defined(_POSIX_PATH_MAX) +# define GTEST_PATH_MAX_ _POSIX_PATH_MAX +#else // conservately set to 256 +#define GTEST_PATH_MAX_ 256 +#endif // GTEST_OS_WINDOWS + + +namespace testing { +namespace internal { + +#if GTEST_OS_WINDOWS +// On Windows, '\\' is the standard path separator, but many tools and the +// Windows API also accept '/' as an alternate path separator. Unless otherwise +// noted, a file path can contain either kind of path separators, or a mixture +// of them. +const char kPathSeparator = '\\'; +const char kAlternatePathSeparator = '/'; +const char kPathSeparatorString[] = "\\"; +const char kAlternatePathSeparatorString[] = "/"; +# if GTEST_OS_WINDOWS_MOBILE +// Windows CE doesn't have a current directory. You should not use +// the current directory in tests on Windows CE, but this at least +// provides a reasonable fallback. +const char kCurrentDirectoryString[] = "\\"; +// Windows CE doesn't define INVALID_FILE_ATTRIBUTES +const DWORD kInvalidFileAttributes = 0xffffffff; +# else +const char kCurrentDirectoryString[] = ".\\"; +# endif // GTEST_OS_WINDOWS_MOBILE +#else +const char kPathSeparator = '/'; +const char kPathSeparatorString[] = "/"; +const char kCurrentDirectoryString[] = "./"; +#endif // GTEST_OS_WINDOWS + +// Returns whether the given character is a valid path separator. +static bool IsPathSeparator(char c) { +#if GTEST_HAS_ALT_PATH_SEP_ + return (c == kPathSeparator) || (c == kAlternatePathSeparator); +#else + return c == kPathSeparator; +#endif +} + +// Returns the current working directory, or "" if unsuccessful. +FilePath FilePath::GetCurrentDir() { +#if GTEST_OS_WINDOWS_MOBILE + // Windows CE doesn't have a current directory, so we just return + // something reasonable. + return FilePath(kCurrentDirectoryString); +#elif GTEST_OS_WINDOWS + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd); +#else + char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; + return FilePath(getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd); +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns a copy of the FilePath with the case-insensitive extension removed. +// Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns +// FilePath("dir/file"). If a case-insensitive extension is not +// found, returns a copy of the original FilePath. +FilePath FilePath::RemoveExtension(const char* extension) const { + String dot_extension(String::Format(".%s", extension)); + if (pathname_.EndsWithCaseInsensitive(dot_extension.c_str())) { + return FilePath(String(pathname_.c_str(), pathname_.length() - 4)); + } + return *this; +} + +// Returns a pointer to the last occurence of a valid path separator in +// the FilePath. On Windows, for example, both '/' and '\' are valid path +// separators. Returns NULL if no path separator was found. +const char* FilePath::FindLastPathSeparator() const { + const char* const last_sep = strrchr(c_str(), kPathSeparator); +#if GTEST_HAS_ALT_PATH_SEP_ + const char* const last_alt_sep = strrchr(c_str(), kAlternatePathSeparator); + // Comparing two pointers of which only one is NULL is undefined. + if (last_alt_sep != NULL && + (last_sep == NULL || last_alt_sep > last_sep)) { + return last_alt_sep; + } +#endif + return last_sep; +} + +// Returns a copy of the FilePath with the directory part removed. +// Example: FilePath("path/to/file").RemoveDirectoryName() returns +// FilePath("file"). If there is no directory part ("just_a_file"), it returns +// the FilePath unmodified. If there is no file part ("just_a_dir/") it +// returns an empty FilePath (""). +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveDirectoryName() const { + const char* const last_sep = FindLastPathSeparator(); + return last_sep ? FilePath(String(last_sep + 1)) : *this; +} + +// RemoveFileName returns the directory path with the filename removed. +// Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". +// If the FilePath is "a_file" or "/a_file", RemoveFileName returns +// FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does +// not have a file, like "just/a/dir/", it returns the FilePath unmodified. +// On Windows platform, '\' is the path separator, otherwise it is '/'. +FilePath FilePath::RemoveFileName() const { + const char* const last_sep = FindLastPathSeparator(); + String dir; + if (last_sep) { + dir = String(c_str(), last_sep + 1 - c_str()); + } else { + dir = kCurrentDirectoryString; + } + return FilePath(dir); +} + +// Helper functions for naming files in a directory for xml output. + +// Given directory = "dir", base_name = "test", number = 0, +// extension = "xml", returns "dir/test.xml". If number is greater +// than zero (e.g., 12), returns "dir/test_12.xml". +// On Windows platform, uses \ as the separator rather than /. +FilePath FilePath::MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension) { + String file; + if (number == 0) { + file = String::Format("%s.%s", base_name.c_str(), extension); + } else { + file = String::Format("%s_%d.%s", base_name.c_str(), number, extension); + } + return ConcatPaths(directory, FilePath(file)); +} + +// Given directory = "dir", relative_path = "test.xml", returns "dir/test.xml". +// On Windows, uses \ as the separator rather than /. +FilePath FilePath::ConcatPaths(const FilePath& directory, + const FilePath& relative_path) { + if (directory.IsEmpty()) + return relative_path; + const FilePath dir(directory.RemoveTrailingPathSeparator()); + return FilePath(String::Format("%s%c%s", dir.c_str(), kPathSeparator, + relative_path.c_str())); +} + +// Returns true if pathname describes something findable in the file-system, +// either a file, directory, or whatever. +bool FilePath::FileOrDirectoryExists() const { +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + return attributes != kInvalidFileAttributes; +#else + posix::StatStruct file_stat; + return posix::Stat(pathname_.c_str(), &file_stat) == 0; +#endif // GTEST_OS_WINDOWS_MOBILE +} + +// Returns true if pathname describes a directory in the file-system +// that exists. +bool FilePath::DirectoryExists() const { + bool result = false; +#if GTEST_OS_WINDOWS + // Don't strip off trailing separator if path is a root directory on + // Windows (like "C:\\"). + const FilePath& path(IsRootDirectory() ? *this : + RemoveTrailingPathSeparator()); +#else + const FilePath& path(*this); +#endif + +#if GTEST_OS_WINDOWS_MOBILE + LPCWSTR unicode = String::AnsiToUtf16(path.c_str()); + const DWORD attributes = GetFileAttributes(unicode); + delete [] unicode; + if ((attributes != kInvalidFileAttributes) && + (attributes & FILE_ATTRIBUTE_DIRECTORY)) { + result = true; + } +#else + posix::StatStruct file_stat; + result = posix::Stat(path.c_str(), &file_stat) == 0 && + posix::IsDir(file_stat); +#endif // GTEST_OS_WINDOWS_MOBILE + + return result; +} + +// Returns true if pathname describes a root directory. (Windows has one +// root directory per disk drive.) +bool FilePath::IsRootDirectory() const { +#if GTEST_OS_WINDOWS + // TODO(wan@google.com): on Windows a network share like + // \\server\share can be a root directory, although it cannot be the + // current directory. Handle this properly. + return pathname_.length() == 3 && IsAbsolutePath(); +#else + return pathname_.length() == 1 && IsPathSeparator(pathname_.c_str()[0]); +#endif +} + +// Returns true if pathname describes an absolute path. +bool FilePath::IsAbsolutePath() const { + const char* const name = pathname_.c_str(); +#if GTEST_OS_WINDOWS + return pathname_.length() >= 3 && + ((name[0] >= 'a' && name[0] <= 'z') || + (name[0] >= 'A' && name[0] <= 'Z')) && + name[1] == ':' && + IsPathSeparator(name[2]); +#else + return IsPathSeparator(name[0]); +#endif +} + +// Returns a pathname for a file that does not currently exist. The pathname +// will be directory/base_name.extension or +// directory/base_name_.extension if directory/base_name.extension +// already exists. The number will be incremented until a pathname is found +// that does not already exist. +// Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. +// There could be a race condition if two or more processes are calling this +// function at the same time -- they could both pick the same filename. +FilePath FilePath::GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension) { + FilePath full_pathname; + int number = 0; + do { + full_pathname.Set(MakeFileName(directory, base_name, number++, extension)); + } while (full_pathname.FileOrDirectoryExists()); + return full_pathname; +} + +// Returns true if FilePath ends with a path separator, which indicates that +// it is intended to represent a directory. Returns false otherwise. +// This does NOT check that a directory (or file) actually exists. +bool FilePath::IsDirectory() const { + return !pathname_.empty() && + IsPathSeparator(pathname_.c_str()[pathname_.length() - 1]); +} + +// Create directories so that path exists. Returns true if successful or if +// the directories already exist; returns false if unable to create directories +// for any reason. +bool FilePath::CreateDirectoriesRecursively() const { + if (!this->IsDirectory()) { + return false; + } + + if (pathname_.length() == 0 || this->DirectoryExists()) { + return true; + } + + const FilePath parent(this->RemoveTrailingPathSeparator().RemoveFileName()); + return parent.CreateDirectoriesRecursively() && this->CreateFolder(); +} + +// Create the directory so that path exists. Returns true if successful or +// if the directory already exists; returns false if unable to create the +// directory for any reason, including if the parent directory does not +// exist. Not named "CreateDirectory" because that's a macro on Windows. +bool FilePath::CreateFolder() const { +#if GTEST_OS_WINDOWS_MOBILE + FilePath removed_sep(this->RemoveTrailingPathSeparator()); + LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); + int result = CreateDirectory(unicode, NULL) ? 0 : -1; + delete [] unicode; +#elif GTEST_OS_WINDOWS + int result = _mkdir(pathname_.c_str()); +#else + int result = mkdir(pathname_.c_str(), 0777); +#endif // GTEST_OS_WINDOWS_MOBILE + + if (result == -1) { + return this->DirectoryExists(); // An error is OK if the directory exists. + } + return true; // No error. +} + +// If input name has a trailing separator character, remove it and return the +// name, otherwise return the name string unmodified. +// On Windows platform, uses \ as the separator, other platforms use /. +FilePath FilePath::RemoveTrailingPathSeparator() const { + return IsDirectory() + ? FilePath(String(pathname_.c_str(), pathname_.length() - 1)) + : *this; +} + +// Removes any redundant separators that might be in the pathname. +// For example, "bar///foo" becomes "bar/foo". Does not eliminate other +// redundancies that might be in a pathname involving "." or "..". +// TODO(wan@google.com): handle Windows network shares (e.g. \\server\share). +void FilePath::Normalize() { + if (pathname_.c_str() == NULL) { + pathname_ = ""; + return; + } + const char* src = pathname_.c_str(); + char* const dest = new char[pathname_.length() + 1]; + char* dest_ptr = dest; + memset(dest_ptr, 0, pathname_.length() + 1); + + while (*src != '\0') { + *dest_ptr = *src; + if (!IsPathSeparator(*src)) { + src++; + } else { +#if GTEST_HAS_ALT_PATH_SEP_ + if (*dest_ptr == kAlternatePathSeparator) { + *dest_ptr = kPathSeparator; + } +#endif + while (IsPathSeparator(*src)) + src++; + } + dest_ptr++; + } + *dest_ptr = '\0'; + pathname_ = dest; + delete[] dest; +} + +} // namespace internal +} // namespace testing +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + + +#include +#include +#include +#include + +#if GTEST_OS_WINDOWS_MOBILE +# include // For TerminateProcess() +#elif GTEST_OS_WINDOWS +# include +# include +#else +# include +#endif // GTEST_OS_WINDOWS_MOBILE + +#if GTEST_OS_MAC +# include +# include +# include +#endif // GTEST_OS_MAC + + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick is to +// prevent a user from accidentally including gtest-internal-inl.h in +// his code. +#define GTEST_IMPLEMENTATION_ 1 +#undef GTEST_IMPLEMENTATION_ + +namespace testing { +namespace internal { + +#if defined(_MSC_VER) || defined(__BORLANDC__) +// MSVC and C++Builder do not provide a definition of STDERR_FILENO. +const int kStdOutFileno = 1; +const int kStdErrFileno = 2; +#else +const int kStdOutFileno = STDOUT_FILENO; +const int kStdErrFileno = STDERR_FILENO; +#endif // _MSC_VER + +#if GTEST_OS_MAC + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +size_t GetThreadCount() { + const task_t task = mach_task_self(); + mach_msg_type_number_t thread_count; + thread_act_array_t thread_list; + const kern_return_t status = task_threads(task, &thread_list, &thread_count); + if (status == KERN_SUCCESS) { + // task_threads allocates resources in thread_list and we need to free them + // to avoid leaks. + vm_deallocate(task, + reinterpret_cast(thread_list), + sizeof(thread_t) * thread_count); + return static_cast(thread_count); + } else { + return 0; + } +} + +#else + +size_t GetThreadCount() { + // There's no portable way to detect the number of threads, so we just + // return 0 to indicate that we cannot detect it. + return 0; +} + +#endif // GTEST_OS_MAC + +#if GTEST_USES_POSIX_RE + +// Implements RE. Currently only needed for death tests. + +RE::~RE() { + if (is_valid_) { + // regfree'ing an invalid regex might crash because the content + // of the regex is undefined. Since the regex's are essentially + // the same, one cannot be valid (or invalid) without the other + // being so too. + regfree(&partial_regex_); + regfree(&full_regex_); + } + free(const_cast(pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.full_regex_, str, 1, &match, 0) == 0; +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + if (!re.is_valid_) return false; + + regmatch_t match; + return regexec(&re.partial_regex_, str, 1, &match, 0) == 0; +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = posix::StrDup(regex); + + // Reserves enough bytes to hold the regular expression used for a + // full match. + const size_t full_regex_len = strlen(regex) + 10; + char* const full_pattern = new char[full_regex_len]; + + snprintf(full_pattern, full_regex_len, "^(%s)$", regex); + is_valid_ = regcomp(&full_regex_, full_pattern, REG_EXTENDED) == 0; + // We want to call regcomp(&partial_regex_, ...) even if the + // previous expression returns false. Otherwise partial_regex_ may + // not be properly initialized can may cause trouble when it's + // freed. + // + // Some implementation of POSIX regex (e.g. on at least some + // versions of Cygwin) doesn't accept the empty string as a valid + // regex. We change it to an equivalent form "()" to be safe. + if (is_valid_) { + const char* const partial_regex = (*regex == '\0') ? "()" : regex; + is_valid_ = regcomp(&partial_regex_, partial_regex, REG_EXTENDED) == 0; + } + EXPECT_TRUE(is_valid_) + << "Regular expression \"" << regex + << "\" is not a valid POSIX Extended regular expression."; + + delete[] full_pattern; +} + +#elif GTEST_USES_SIMPLE_RE + +// Returns true iff ch appears anywhere in str (excluding the +// terminating '\0' character). +bool IsInSet(char ch, const char* str) { + return ch != '\0' && strchr(str, ch) != NULL; +} + +// Returns true iff ch belongs to the given classification. Unlike +// similar functions in , these aren't affected by the +// current locale. +bool IsAsciiDigit(char ch) { return '0' <= ch && ch <= '9'; } +bool IsAsciiPunct(char ch) { + return IsInSet(ch, "^-!\"#$%&'()*+,./:;<=>?@[\\]_`{|}~"); +} +bool IsRepeat(char ch) { return IsInSet(ch, "?*+"); } +bool IsAsciiWhiteSpace(char ch) { return IsInSet(ch, " \f\n\r\t\v"); } +bool IsAsciiWordChar(char ch) { + return ('a' <= ch && ch <= 'z') || ('A' <= ch && ch <= 'Z') || + ('0' <= ch && ch <= '9') || ch == '_'; +} + +// Returns true iff "\\c" is a supported escape sequence. +bool IsValidEscape(char c) { + return (IsAsciiPunct(c) || IsInSet(c, "dDfnrsStvwW")); +} + +// Returns true iff the given atom (specified by escaped and pattern) +// matches ch. The result is undefined if the atom is invalid. +bool AtomMatchesChar(bool escaped, char pattern_char, char ch) { + if (escaped) { // "\\p" where p is pattern_char. + switch (pattern_char) { + case 'd': return IsAsciiDigit(ch); + case 'D': return !IsAsciiDigit(ch); + case 'f': return ch == '\f'; + case 'n': return ch == '\n'; + case 'r': return ch == '\r'; + case 's': return IsAsciiWhiteSpace(ch); + case 'S': return !IsAsciiWhiteSpace(ch); + case 't': return ch == '\t'; + case 'v': return ch == '\v'; + case 'w': return IsAsciiWordChar(ch); + case 'W': return !IsAsciiWordChar(ch); + } + return IsAsciiPunct(pattern_char) && pattern_char == ch; + } + + return (pattern_char == '.' && ch != '\n') || pattern_char == ch; +} + +// Helper function used by ValidateRegex() to format error messages. +String FormatRegexSyntaxError(const char* regex, int index) { + return (Message() << "Syntax error at index " << index + << " in simple regular expression \"" << regex << "\": ").GetString(); +} + +// Generates non-fatal failures and returns false if regex is invalid; +// otherwise returns true. +bool ValidateRegex(const char* regex) { + if (regex == NULL) { + // TODO(wan@google.com): fix the source file location in the + // assertion failures to match where the regex is used in user + // code. + ADD_FAILURE() << "NULL is not a valid simple regular expression."; + return false; + } + + bool is_valid = true; + + // True iff ?, *, or + can follow the previous atom. + bool prev_repeatable = false; + for (int i = 0; regex[i]; i++) { + if (regex[i] == '\\') { // An escape sequence + i++; + if (regex[i] == '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "'\\' cannot appear at the end."; + return false; + } + + if (!IsValidEscape(regex[i])) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i - 1) + << "invalid escape sequence \"\\" << regex[i] << "\"."; + is_valid = false; + } + prev_repeatable = true; + } else { // Not an escape sequence. + const char ch = regex[i]; + + if (ch == '^' && i > 0) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'^' can only appear at the beginning."; + is_valid = false; + } else if (ch == '$' && regex[i + 1] != '\0') { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'$' can only appear at the end."; + is_valid = false; + } else if (IsInSet(ch, "()[]{}|")) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' is unsupported."; + is_valid = false; + } else if (IsRepeat(ch) && !prev_repeatable) { + ADD_FAILURE() << FormatRegexSyntaxError(regex, i) + << "'" << ch << "' can only follow a repeatable token."; + is_valid = false; + } + + prev_repeatable = !IsInSet(ch, "^$?*+"); + } + } + + return is_valid; +} + +// Matches a repeated regex atom followed by a valid simple regular +// expression. The regex atom is defined as c if escaped is false, +// or \c otherwise. repeat is the repetition meta character (?, *, +// or +). The behavior is undefined if str contains too many +// characters to be indexable by size_t, in which case the test will +// probably time out anyway. We are fine with this limitation as +// std::string has it too. +bool MatchRepetitionAndRegexAtHead( + bool escaped, char c, char repeat, const char* regex, + const char* str) { + const size_t min_count = (repeat == '+') ? 1 : 0; + const size_t max_count = (repeat == '?') ? 1 : + static_cast(-1) - 1; + // We cannot call numeric_limits::max() as it conflicts with the + // max() macro on Windows. + + for (size_t i = 0; i <= max_count; ++i) { + // We know that the atom matches each of the first i characters in str. + if (i >= min_count && MatchRegexAtHead(regex, str + i)) { + // We have enough matches at the head, and the tail matches too. + // Since we only care about *whether* the pattern matches str + // (as opposed to *how* it matches), there is no need to find a + // greedy match. + return true; + } + if (str[i] == '\0' || !AtomMatchesChar(escaped, c, str[i])) + return false; + } + return false; +} + +// Returns true iff regex matches a prefix of str. regex must be a +// valid simple regular expression and not start with "^", or the +// result is undefined. +bool MatchRegexAtHead(const char* regex, const char* str) { + if (*regex == '\0') // An empty regex matches a prefix of anything. + return true; + + // "$" only matches the end of a string. Note that regex being + // valid guarantees that there's nothing after "$" in it. + if (*regex == '$') + return *str == '\0'; + + // Is the first thing in regex an escape sequence? + const bool escaped = *regex == '\\'; + if (escaped) + ++regex; + if (IsRepeat(regex[1])) { + // MatchRepetitionAndRegexAtHead() calls MatchRegexAtHead(), so + // here's an indirect recursion. It terminates as the regex gets + // shorter in each recursion. + return MatchRepetitionAndRegexAtHead( + escaped, regex[0], regex[1], regex + 2, str); + } else { + // regex isn't empty, isn't "$", and doesn't start with a + // repetition. We match the first atom of regex with the first + // character of str and recurse. + return (*str != '\0') && AtomMatchesChar(escaped, *regex, *str) && + MatchRegexAtHead(regex + 1, str + 1); + } +} + +// Returns true iff regex matches any substring of str. regex must be +// a valid simple regular expression, or the result is undefined. +// +// The algorithm is recursive, but the recursion depth doesn't exceed +// the regex length, so we won't need to worry about running out of +// stack space normally. In rare cases the time complexity can be +// exponential with respect to the regex length + the string length, +// but usually it's must faster (often close to linear). +bool MatchRegexAnywhere(const char* regex, const char* str) { + if (regex == NULL || str == NULL) + return false; + + if (*regex == '^') + return MatchRegexAtHead(regex + 1, str); + + // A successful match can be anywhere in str. + do { + if (MatchRegexAtHead(regex, str)) + return true; + } while (*str++ != '\0'); + return false; +} + +// Implements the RE class. + +RE::~RE() { + free(const_cast(pattern_)); + free(const_cast(full_pattern_)); +} + +// Returns true iff regular expression re matches the entire str. +bool RE::FullMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.full_pattern_, str); +} + +// Returns true iff regular expression re matches a substring of str +// (including str itself). +bool RE::PartialMatch(const char* str, const RE& re) { + return re.is_valid_ && MatchRegexAnywhere(re.pattern_, str); +} + +// Initializes an RE from its string representation. +void RE::Init(const char* regex) { + pattern_ = full_pattern_ = NULL; + if (regex != NULL) { + pattern_ = posix::StrDup(regex); + } + + is_valid_ = ValidateRegex(regex); + if (!is_valid_) { + // No need to calculate the full pattern when the regex is invalid. + return; + } + + const size_t len = strlen(regex); + // Reserves enough bytes to hold the regular expression used for a + // full match: we need space to prepend a '^', append a '$', and + // terminate the string with '\0'. + char* buffer = static_cast(malloc(len + 3)); + full_pattern_ = buffer; + + if (*regex != '^') + *buffer++ = '^'; // Makes sure full_pattern_ starts with '^'. + + // We don't use snprintf or strncpy, as they trigger a warning when + // compiled with VC++ 8.0. + memcpy(buffer, regex, len); + buffer += len; + + if (len == 0 || regex[len - 1] != '$') + *buffer++ = '$'; // Makes sure full_pattern_ ends with '$'. + + *buffer = '\0'; +} + +#endif // GTEST_USES_POSIX_RE + +const char kUnknownFile[] = "unknown file"; + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) { + const char* const file_name = file == NULL ? kUnknownFile : file; + + if (line < 0) { + return String::Format("%s:", file_name).c_str(); + } +#ifdef _MSC_VER + return String::Format("%s(%d):", file_name, line).c_str(); +#else + return String::Format("%s:%d:", file_name, line).c_str(); +#endif // _MSC_VER +} + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +// Note that FormatCompilerIndependentFileLocation() does NOT append colon +// to the file location it produces, unlike FormatFileLocation(). +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation( + const char* file, int line) { + const char* const file_name = file == NULL ? kUnknownFile : file; + + if (line < 0) + return file_name; + else + return String::Format("%s:%d", file_name, line).c_str(); +} + + +GTestLog::GTestLog(GTestLogSeverity severity, const char* file, int line) + : severity_(severity) { + const char* const marker = + severity == GTEST_INFO ? "[ INFO ]" : + severity == GTEST_WARNING ? "[WARNING]" : + severity == GTEST_ERROR ? "[ ERROR ]" : "[ FATAL ]"; + GetStream() << ::std::endl << marker << " " + << FormatFileLocation(file, line).c_str() << ": "; +} + +// Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. +GTestLog::~GTestLog() { + GetStream() << ::std::endl; + if (severity_ == GTEST_FATAL) { + fflush(stderr); + posix::Abort(); + } +} +// Disable Microsoft deprecation warnings for POSIX functions called from +// this class (creat, dup, dup2, and close) +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable: 4996) +#endif // _MSC_VER + +#if GTEST_HAS_STREAM_REDIRECTION + +// Object that captures an output stream (stdout/stderr). +class CapturedStream { + public: + // The ctor redirects the stream to a temporary file. + CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) { + +# if GTEST_OS_WINDOWS + char temp_dir_path[MAX_PATH + 1] = { '\0' }; // NOLINT + char temp_file_path[MAX_PATH + 1] = { '\0' }; // NOLINT + + ::GetTempPathA(sizeof(temp_dir_path), temp_dir_path); + const UINT success = ::GetTempFileNameA(temp_dir_path, + "gtest_redir", + 0, // Generate unique file name. + temp_file_path); + GTEST_CHECK_(success != 0) + << "Unable to create a temporary file in " << temp_dir_path; + const int captured_fd = creat(temp_file_path, _S_IREAD | _S_IWRITE); + GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file " + << temp_file_path; + filename_ = temp_file_path; +# else + // There's no guarantee that a test has write access to the + // current directory, so we create the temporary file in the /tmp + // directory instead. + char name_template[] = "/tmp/captured_stream.XXXXXX"; + const int captured_fd = mkstemp(name_template); + filename_ = name_template; +# endif // GTEST_OS_WINDOWS + fflush(NULL); + dup2(captured_fd, fd_); + close(captured_fd); + } + + ~CapturedStream() { + remove(filename_.c_str()); + } + + String GetCapturedString() { + if (uncaptured_fd_ != -1) { + // Restores the original stream. + fflush(NULL); + dup2(uncaptured_fd_, fd_); + close(uncaptured_fd_); + uncaptured_fd_ = -1; + } + + FILE* const file = posix::FOpen(filename_.c_str(), "r"); + const String content = ReadEntireFile(file); + posix::FClose(file); + return content; + } + + private: + // Reads the entire content of a file as a String. + static String ReadEntireFile(FILE* file); + + // Returns the size (in bytes) of a file. + static size_t GetFileSize(FILE* file); + + const int fd_; // A stream to capture. + int uncaptured_fd_; + // Name of the temporary file holding the stderr output. + ::std::string filename_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(CapturedStream); +}; + +// Returns the size (in bytes) of a file. +size_t CapturedStream::GetFileSize(FILE* file) { + fseek(file, 0, SEEK_END); + return static_cast(ftell(file)); +} + +// Reads the entire content of a file as a string. +String CapturedStream::ReadEntireFile(FILE* file) { + const size_t file_size = GetFileSize(file); + char* const buffer = new char[file_size]; + + size_t bytes_last_read = 0; // # of bytes read in the last fread() + size_t bytes_read = 0; // # of bytes read so far + + fseek(file, 0, SEEK_SET); + + // Keeps reading the file until we cannot read further or the + // pre-determined file size is reached. + do { + bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file); + bytes_read += bytes_last_read; + } while (bytes_last_read > 0 && bytes_read < file_size); + + const String content(buffer, bytes_read); + delete[] buffer; + + return content; +} + +# ifdef _MSC_VER +# pragma warning(pop) +# endif // _MSC_VER + +static CapturedStream* g_captured_stderr = NULL; +static CapturedStream* g_captured_stdout = NULL; + +// Starts capturing an output stream (stdout/stderr). +void CaptureStream(int fd, const char* stream_name, CapturedStream** stream) { + if (*stream != NULL) { + GTEST_LOG_(FATAL) << "Only one " << stream_name + << " capturer can exist at a time."; + } + *stream = new CapturedStream(fd); +} + +// Stops capturing the output stream and returns the captured string. +String GetCapturedStream(CapturedStream** captured_stream) { + const String content = (*captured_stream)->GetCapturedString(); + + delete *captured_stream; + *captured_stream = NULL; + + return content; +} + +// Starts capturing stdout. +void CaptureStdout() { + CaptureStream(kStdOutFileno, "stdout", &g_captured_stdout); +} + +// Starts capturing stderr. +void CaptureStderr() { + CaptureStream(kStdErrFileno, "stderr", &g_captured_stderr); +} + +// Stops capturing stdout and returns the captured string. +String GetCapturedStdout() { return GetCapturedStream(&g_captured_stdout); } + +// Stops capturing stderr and returns the captured string. +String GetCapturedStderr() { return GetCapturedStream(&g_captured_stderr); } + +#endif // GTEST_HAS_STREAM_REDIRECTION + +#if GTEST_HAS_DEATH_TEST + +// A copy of all command line arguments. Set by InitGoogleTest(). +::std::vector g_argvs; + +// Returns the command line as a vector of strings. +const ::std::vector& GetArgvs() { return g_argvs; } + +#endif // GTEST_HAS_DEATH_TEST + +#if GTEST_OS_WINDOWS_MOBILE +namespace posix { +void Abort() { + DebugBreak(); + TerminateProcess(GetCurrentProcess(), 1); +} +} // namespace posix +#endif // GTEST_OS_WINDOWS_MOBILE + +// Returns the name of the environment variable corresponding to the +// given flag. For example, FlagToEnvVar("foo") will return +// "GTEST_FOO" in the open-source version. +static String FlagToEnvVar(const char* flag) { + const String full_flag = + (Message() << GTEST_FLAG_PREFIX_ << flag).GetString(); + + Message env_var; + for (size_t i = 0; i != full_flag.length(); i++) { + env_var << ToUpper(full_flag.c_str()[i]); + } + + return env_var.GetString(); +} + +// Parses 'str' for a 32-bit signed integer. If successful, writes +// the result to *value and returns true; otherwise leaves *value +// unchanged and returns false. +bool ParseInt32(const Message& src_text, const char* str, Int32* value) { + // Parses the environment variable as a decimal integer. + char* end = NULL; + const long long_value = strtol(str, &end, 10); // NOLINT + + // Has strtol() consumed all characters in the string? + if (*end != '\0') { + // No - an invalid character was encountered. + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value \"" << str << "\".\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + // Is the parsed value in the range of an Int32? + const Int32 result = static_cast(long_value); + if (long_value == LONG_MAX || long_value == LONG_MIN || + // The parsed value overflows as a long. (strtol() returns + // LONG_MAX or LONG_MIN when the input overflows.) + result != long_value + // The parsed value overflows as an Int32. + ) { + Message msg; + msg << "WARNING: " << src_text + << " is expected to be a 32-bit integer, but actually" + << " has value " << str << ", which overflows.\n"; + printf("%s", msg.GetString().c_str()); + fflush(stdout); + return false; + } + + *value = result; + return true; +} + +// Reads and returns the Boolean environment variable corresponding to +// the given flag; if it's not set, returns default_value. +// +// The value is considered true iff it's not "0". +bool BoolFromGTestEnv(const char* flag, bool default_value) { + const String env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + return string_value == NULL ? + default_value : strcmp(string_value, "0") != 0; +} + +// Reads and returns a 32-bit integer stored in the environment +// variable corresponding to the given flag; if it isn't set or +// doesn't represent a valid 32-bit integer, returns default_value. +Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) { + const String env_var = FlagToEnvVar(flag); + const char* const string_value = posix::GetEnv(env_var.c_str()); + if (string_value == NULL) { + // The environment variable is not set. + return default_value; + } + + Int32 result = default_value; + if (!ParseInt32(Message() << "Environment variable " << env_var, + string_value, &result)) { + printf("The default value %s is used.\n", + (Message() << default_value).GetString().c_str()); + fflush(stdout); + return default_value; + } + + return result; +} + +// Reads and returns the string environment variable corresponding to +// the given flag; if it's not set, returns default_value. +const char* StringFromGTestEnv(const char* flag, const char* default_value) { + const String env_var = FlagToEnvVar(flag); + const char* const value = posix::GetEnv(env_var.c_str()); + return value == NULL ? default_value : value; +} + +} // namespace internal +} // namespace testing +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Google Test - The Google C++ Testing Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// It uses the << operator when possible, and prints the bytes in the +// object otherwise. A user can override its behavior for a class +// type Foo by defining either operator<<(::std::ostream&, const Foo&) +// or void PrintTo(const Foo&, ::std::ostream*) in the namespace that +// defines Foo. + +#include +#include +#include // NOLINT +#include + +namespace testing { + +namespace { + +using ::std::ostream; + +#if GTEST_OS_WINDOWS_MOBILE // Windows CE does not define _snprintf_s. +# define snprintf _snprintf +#elif _MSC_VER >= 1400 // VC 8.0 and later deprecate snprintf and _snprintf. +# define snprintf _snprintf_s +#elif _MSC_VER +# define snprintf _snprintf +#endif // GTEST_OS_WINDOWS_MOBILE + +// Prints a segment of bytes in the given object. +void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start, + size_t count, ostream* os) { + char text[5] = ""; + for (size_t i = 0; i != count; i++) { + const size_t j = start + i; + if (i != 0) { + // Organizes the bytes into groups of 2 for easy parsing by + // human. + if ((j % 2) == 0) + *os << ' '; + else + *os << '-'; + } + snprintf(text, sizeof(text), "%02X", obj_bytes[j]); + *os << text; + } +} + +// Prints the bytes in the given value to the given ostream. +void PrintBytesInObjectToImpl(const unsigned char* obj_bytes, size_t count, + ostream* os) { + // Tells the user how big the object is. + *os << count << "-byte object <"; + + const size_t kThreshold = 132; + const size_t kChunkSize = 64; + // If the object size is bigger than kThreshold, we'll have to omit + // some details by printing only the first and the last kChunkSize + // bytes. + // TODO(wan): let the user control the threshold using a flag. + if (count < kThreshold) { + PrintByteSegmentInObjectTo(obj_bytes, 0, count, os); + } else { + PrintByteSegmentInObjectTo(obj_bytes, 0, kChunkSize, os); + *os << " ... "; + // Rounds up to 2-byte boundary. + const size_t resume_pos = (count - kChunkSize + 1)/2*2; + PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os); + } + *os << ">"; +} + +} // namespace + +namespace internal2 { + +// Delegates to PrintBytesInObjectToImpl() to print the bytes in the +// given object. The delegation simplifies the implementation, which +// uses the << operator and thus is easier done outside of the +// ::testing::internal namespace, which contains a << operator that +// sometimes conflicts with the one in STL. +void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, + ostream* os) { + PrintBytesInObjectToImpl(obj_bytes, count, os); +} + +} // namespace internal2 + +namespace internal { + +// Depending on the value of a char (or wchar_t), we print it in one +// of three formats: +// - as is if it's a printable ASCII (e.g. 'a', '2', ' '), +// - as a hexidecimal escape sequence (e.g. '\x7F'), or +// - as a special escape sequence (e.g. '\r', '\n'). +enum CharFormat { + kAsIs, + kHexEscape, + kSpecialEscape +}; + +// Returns true if c is a printable ASCII character. We test the +// value of c directly instead of calling isprint(), which is buggy on +// Windows Mobile. +inline bool IsPrintableAscii(wchar_t c) { + return 0x20 <= c && c <= 0x7E; +} + +// Prints a wide or narrow char c as a character literal without the +// quotes, escaping it when necessary; returns how c was formatted. +// The template argument UnsignedChar is the unsigned version of Char, +// which is the type of c. +template +static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { + switch (static_cast(c)) { + case L'\0': + *os << "\\0"; + break; + case L'\'': + *os << "\\'"; + break; + case L'\\': + *os << "\\\\"; + break; + case L'\a': + *os << "\\a"; + break; + case L'\b': + *os << "\\b"; + break; + case L'\f': + *os << "\\f"; + break; + case L'\n': + *os << "\\n"; + break; + case L'\r': + *os << "\\r"; + break; + case L'\t': + *os << "\\t"; + break; + case L'\v': + *os << "\\v"; + break; + default: + if (IsPrintableAscii(c)) { + *os << static_cast(c); + return kAsIs; + } else { + *os << String::Format("\\x%X", static_cast(c)); + return kHexEscape; + } + } + return kSpecialEscape; +} + +// Prints a char c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsWideStringLiteralTo(wchar_t c, ostream* os) { + switch (c) { + case L'\'': + *os << "'"; + return kAsIs; + case L'"': + *os << "\\\""; + return kSpecialEscape; + default: + return PrintAsCharLiteralTo(c, os); + } +} + +// Prints a char c as if it's part of a string literal, escaping it when +// necessary; returns how c was formatted. +static CharFormat PrintAsNarrowStringLiteralTo(char c, ostream* os) { + return PrintAsWideStringLiteralTo(static_cast(c), os); +} + +// Prints a wide or narrow character c and its code. '\0' is printed +// as "'\\0'", other unprintable characters are also properly escaped +// using the standard C++ escape sequence. The template argument +// UnsignedChar is the unsigned version of Char, which is the type of c. +template +void PrintCharAndCodeTo(Char c, ostream* os) { + // First, print c as a literal in the most readable form we can find. + *os << ((sizeof(c) > 1) ? "L'" : "'"); + const CharFormat format = PrintAsCharLiteralTo(c, os); + *os << "'"; + + // To aid user debugging, we also print c's code in decimal, unless + // it's 0 (in which case c was printed as '\\0', making the code + // obvious). + if (c == 0) + return; + *os << " (" << String::Format("%d", c).c_str(); + + // For more convenience, we print c's code again in hexidecimal, + // unless c was already printed in the form '\x##' or the code is in + // [1, 9]. + if (format == kHexEscape || (1 <= c && c <= 9)) { + // Do nothing. + } else { + *os << String::Format(", 0x%X", + static_cast(c)).c_str(); + } + *os << ")"; +} + +void PrintTo(unsigned char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} +void PrintTo(signed char c, ::std::ostream* os) { + PrintCharAndCodeTo(c, os); +} + +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its code. L'\0' is printed as "L'\\0'". +void PrintTo(wchar_t wc, ostream* os) { + PrintCharAndCodeTo(wc, os); +} + +// Prints the given array of characters to the ostream. +// The array starts at *begin, the length is len, it may include '\0' characters +// and may not be null-terminated. +static void PrintCharsAsStringTo(const char* begin, size_t len, ostream* os) { + *os << "\""; + bool is_previous_hex = false; + for (size_t index = 0; index < len; ++index) { + const char cur = begin[index]; + if (is_previous_hex && IsXDigit(cur)) { + // Previous character is of '\x..' form and this character can be + // interpreted as another hexadecimal digit in its number. Break string to + // disambiguate. + *os << "\" \""; + } + is_previous_hex = PrintAsNarrowStringLiteralTo(cur, os) == kHexEscape; + } + *os << "\""; +} + +// Prints a (const) char array of 'len' elements, starting at address 'begin'. +void UniversalPrintArray(const char* begin, size_t len, ostream* os) { + PrintCharsAsStringTo(begin, len, os); +} + +// Prints the given array of wide characters to the ostream. +// The array starts at *begin, the length is len, it may include L'\0' +// characters and may not be null-terminated. +static void PrintWideCharsAsStringTo(const wchar_t* begin, size_t len, + ostream* os) { + *os << "L\""; + bool is_previous_hex = false; + for (size_t index = 0; index < len; ++index) { + const wchar_t cur = begin[index]; + if (is_previous_hex && isascii(cur) && IsXDigit(static_cast(cur))) { + // Previous character is of '\x..' form and this character can be + // interpreted as another hexadecimal digit in its number. Break string to + // disambiguate. + *os << "\" L\""; + } + is_previous_hex = PrintAsWideStringLiteralTo(cur, os) == kHexEscape; + } + *os << "\""; +} + +// Prints the given C string to the ostream. +void PrintTo(const char* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintCharsAsStringTo(s, strlen(s), os); + } +} + +// MSVC compiler can be configured to define whar_t as a typedef +// of unsigned short. Defining an overload for const wchar_t* in that case +// would cause pointers to unsigned shorts be printed as wide strings, +// possibly accessing more memory than intended and causing invalid +// memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when +// wchar_t is implemented as a native type. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Prints the given wide C string to the ostream. +void PrintTo(const wchar_t* s, ostream* os) { + if (s == NULL) { + *os << "NULL"; + } else { + *os << ImplicitCast_(s) << " pointing to "; + PrintWideCharsAsStringTo(s, wcslen(s), os); + } +} +#endif // wchar_t is native + +// Prints a ::string object. +#if GTEST_HAS_GLOBAL_STRING +void PrintStringTo(const ::string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_GLOBAL_STRING + +void PrintStringTo(const ::std::string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} + +// Prints a ::wstring object. +#if GTEST_HAS_GLOBAL_WSTRING +void PrintWideStringTo(const ::wstring& s, ostream* os) { + PrintWideCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +void PrintWideStringTo(const ::std::wstring& s, ostream* os) { + PrintWideCharsAsStringTo(s.data(), s.size(), os); +} +#endif // GTEST_HAS_STD_WSTRING + +} // namespace internal + +} // namespace testing +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// +// The Google C++ Testing Framework (Google Test) + + +// Indicates that this translation unit is part of Google Test's +// implementation. It must come before gtest-internal-inl.h is +// included, or there will be a compiler error. This trick is to +// prevent a user from accidentally including gtest-internal-inl.h in +// his code. +#define GTEST_IMPLEMENTATION_ 1 +#undef GTEST_IMPLEMENTATION_ + +namespace testing { + +using internal::GetUnitTestImpl; + +// Gets the summary of the failure message by omitting the stack trace +// in it. +internal::String TestPartResult::ExtractSummary(const char* message) { + const char* const stack_trace = strstr(message, internal::kStackTraceMarker); + return stack_trace == NULL ? internal::String(message) : + internal::String(message, stack_trace - message); +} + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result) { + return os + << result.file_name() << ":" << result.line_number() << ": " + << (result.type() == TestPartResult::kSuccess ? "Success" : + result.type() == TestPartResult::kFatalFailure ? "Fatal failure" : + "Non-fatal failure") << ":\n" + << result.message() << std::endl; +} + +// Appends a TestPartResult to the array. +void TestPartResultArray::Append(const TestPartResult& result) { + array_.push_back(result); +} + +// Returns the TestPartResult at the given index (0-based). +const TestPartResult& TestPartResultArray::GetTestPartResult(int index) const { + if (index < 0 || index >= size()) { + printf("\nInvalid index (%d) into TestPartResultArray.\n", index); + internal::posix::Abort(); + } + + return array_[index]; +} + +// Returns the number of TestPartResult objects in the array. +int TestPartResultArray::size() const { + return static_cast(array_.size()); +} + +namespace internal { + +HasNewFatalFailureHelper::HasNewFatalFailureHelper() + : has_new_fatal_failure_(false), + original_reporter_(GetUnitTestImpl()-> + GetTestPartResultReporterForCurrentThread()) { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread(this); +} + +HasNewFatalFailureHelper::~HasNewFatalFailureHelper() { + GetUnitTestImpl()->SetTestPartResultReporterForCurrentThread( + original_reporter_); +} + +void HasNewFatalFailureHelper::ReportTestPartResult( + const TestPartResult& result) { + if (result.fatally_failed()) + has_new_fatal_failure_ = true; + original_reporter_->ReportTestPartResult(result); +} + +} // namespace internal + +} // namespace testing +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + + +namespace testing { +namespace internal { + +#if GTEST_HAS_TYPED_TEST_P + +// Skips to the first non-space char in str. Returns an empty string if str +// contains only whitespace characters. +static const char* SkipSpaces(const char* str) { + while (IsSpace(*str)) + str++; + return str; +} + +// Verifies that registered_tests match the test names in +// defined_test_names_; returns registered_tests if successful, or +// aborts the program otherwise. +const char* TypedTestCasePState::VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests) { + typedef ::std::set::const_iterator DefinedTestIter; + registered_ = true; + + // Skip initial whitespace in registered_tests since some + // preprocessors prefix stringizied literals with whitespace. + registered_tests = SkipSpaces(registered_tests); + + Message errors; + ::std::set tests; + for (const char* names = registered_tests; names != NULL; + names = SkipComma(names)) { + const String name = GetPrefixUntilComma(names); + if (tests.count(name) != 0) { + errors << "Test " << name << " is listed more than once.\n"; + continue; + } + + bool found = false; + for (DefinedTestIter it = defined_test_names_.begin(); + it != defined_test_names_.end(); + ++it) { + if (name == *it) { + found = true; + break; + } + } + + if (found) { + tests.insert(name); + } else { + errors << "No test named " << name + << " can be found in this test case.\n"; + } + } + + for (DefinedTestIter it = defined_test_names_.begin(); + it != defined_test_names_.end(); + ++it) { + if (tests.count(*it) == 0) { + errors << "You forgot to list test " << *it << ".\n"; + } + } + + const String& errors_str = errors.GetString(); + if (errors_str != "") { + fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), + errors_str.c_str()); + fflush(stderr); + posix::Abort(); + } + + return registered_tests; +} + +#endif // GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing diff --git a/tests/kfdtest/gtest-1.6.0/gtest/gtest.h b/tests/kfdtest/gtest-1.6.0/gtest/gtest.h new file mode 100644 index 0000000000..7b718dd02e --- /dev/null +++ b/tests/kfdtest/gtest-1.6.0/gtest/gtest.h @@ -0,0 +1,19640 @@ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the public API for Google Test. It should be +// included by any test program that uses Google Test. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! +// +// Acknowledgment: Google Test borrowed the idea of automatic test +// registration from Barthelemy Dagenais' (barthelemy@prologique.com) +// easyUnit framework. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_H_ + +#include +#include +#include + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file declares functions and macros used internally by +// Google Test. They are subject to change without notice. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan) +// +// Low-level types and utilities for porting Google Test to various +// platforms. They are subject to change without notice. DO NOT USE +// THEM IN USER CODE. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ + +// The user can define the following macros in the build script to +// control Google Test's behavior. If the user doesn't define a macro +// in this list, Google Test will define it. +// +// GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2) +// is/isn't available. +// GTEST_HAS_EXCEPTIONS - Define it to 1/0 to indicate that exceptions +// are enabled. +// GTEST_HAS_GLOBAL_STRING - Define it to 1/0 to indicate that ::string +// is/isn't available (some systems define +// ::string, which is different to std::string). +// GTEST_HAS_GLOBAL_WSTRING - Define it to 1/0 to indicate that ::string +// is/isn't available (some systems define +// ::wstring, which is different to std::wstring). +// GTEST_HAS_POSIX_RE - Define it to 1/0 to indicate that POSIX regular +// expressions are/aren't available. +// GTEST_HAS_PTHREAD - Define it to 1/0 to indicate that +// is/isn't available. +// GTEST_HAS_RTTI - Define it to 1/0 to indicate that RTTI is/isn't +// enabled. +// GTEST_HAS_STD_WSTRING - Define it to 1/0 to indicate that +// std::wstring does/doesn't work (Google Test can +// be used where std::wstring is unavailable). +// GTEST_HAS_TR1_TUPLE - Define it to 1/0 to indicate tr1::tuple +// is/isn't available. +// GTEST_HAS_SEH - Define it to 1/0 to indicate whether the +// compiler supports Microsoft's "Structured +// Exception Handling". +// GTEST_HAS_STREAM_REDIRECTION +// - Define it to 1/0 to indicate whether the +// platform supports I/O stream redirection using +// dup() and dup2(). +// GTEST_USE_OWN_TR1_TUPLE - Define it to 1/0 to indicate whether Google +// Test's own tr1 tuple implementation should be +// used. Unused when the user sets +// GTEST_HAS_TR1_TUPLE to 0. +// GTEST_LINKED_AS_SHARED_LIBRARY +// - Define to 1 when compiling tests that use +// Google Test as a shared library (known as +// DLL on Windows). +// GTEST_CREATE_SHARED_LIBRARY +// - Define to 1 when compiling Google Test itself +// as a shared library. + +// This header defines the following utilities: +// +// Macros indicating the current platform (defined to 1 if compiled on +// the given platform; otherwise undefined): +// GTEST_OS_AIX - IBM AIX +// GTEST_OS_CYGWIN - Cygwin +// GTEST_OS_HPUX - HP-UX +// GTEST_OS_LINUX - Linux +// GTEST_OS_LINUX_ANDROID - Google Android +// GTEST_OS_MAC - Mac OS X +// GTEST_OS_NACL - Google Native Client (NaCl) +// GTEST_OS_SOLARIS - Sun Solaris +// GTEST_OS_SYMBIAN - Symbian +// GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile) +// GTEST_OS_WINDOWS_DESKTOP - Windows Desktop +// GTEST_OS_WINDOWS_MINGW - MinGW +// GTEST_OS_WINDOWS_MOBILE - Windows Mobile +// GTEST_OS_ZOS - z/OS +// +// Among the platforms, Cygwin, Linux, Max OS X, and Windows have the +// most stable support. Since core members of the Google Test project +// don't have access to other platforms, support for them may be less +// stable. If you notice any problems on your platform, please notify +// googletestframework@googlegroups.com (patches for fixing them are +// even more welcome!). +// +// Note that it is possible that none of the GTEST_OS_* macros are defined. +// +// Macros indicating available Google Test features (defined to 1 if +// the corresponding feature is supported; otherwise undefined): +// GTEST_HAS_COMBINE - the Combine() function (for value-parameterized +// tests) +// GTEST_HAS_DEATH_TEST - death tests +// GTEST_HAS_PARAM_TEST - value-parameterized tests +// GTEST_HAS_TYPED_TEST - typed tests +// GTEST_HAS_TYPED_TEST_P - type-parameterized tests +// GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with +// GTEST_HAS_POSIX_RE (see above) which users can +// define themselves. +// GTEST_USES_SIMPLE_RE - our own simple regex is used; +// the above two are mutually exclusive. +// GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ(). +// +// Macros for basic C++ coding: +// GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning. +// GTEST_ATTRIBUTE_UNUSED_ - declares that a class' instances or a +// variable don't have to be used. +// GTEST_DISALLOW_ASSIGN_ - disables operator=. +// GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=. +// GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. +// +// Synchronization: +// Mutex, MutexLock, ThreadLocal, GetThreadCount() +// - synchronization primitives. +// GTEST_IS_THREADSAFE - defined to 1 to indicate that the above +// synchronization primitives have real implementations +// and Google Test is thread-safe; or 0 otherwise. +// +// Template meta programming: +// is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only. +// IteratorTraits - partial implementation of std::iterator_traits, which +// is not available in libCstd when compiled with Sun C++. +// +// Smart pointers: +// scoped_ptr - as in TR2. +// +// Regular expressions: +// RE - a simple regular expression class using the POSIX +// Extended Regular Expression syntax on UNIX-like +// platforms, or a reduced regular exception syntax on +// other platforms, including Windows. +// +// Logging: +// GTEST_LOG_() - logs messages at the specified severity level. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. +// +// Stdout and stderr capturing: +// CaptureStdout() - starts capturing stdout. +// GetCapturedStdout() - stops capturing stdout and returns the captured +// string. +// CaptureStderr() - starts capturing stderr. +// GetCapturedStderr() - stops capturing stderr and returns the captured +// string. +// +// Integer types: +// TypeWithSize - maps an integer to a int type. +// Int32, UInt32, Int64, UInt64, TimeInMillis +// - integers of known sizes. +// BiggestInt - the biggest signed integer type. +// +// Command-line utilities: +// GTEST_FLAG() - references a flag. +// GTEST_DECLARE_*() - declares a flag. +// GTEST_DEFINE_*() - defines a flag. +// GetArgvs() - returns the command line as a vector of strings. +// +// Environment variable utilities: +// GetEnv() - gets the value of an environment variable. +// BoolFromGTestEnv() - parses a bool environment variable. +// Int32FromGTestEnv() - parses an Int32 environment variable. +// StringFromGTestEnv() - parses a string environment variable. + +#include // for isspace, etc +#include // for ptrdiff_t +#include +#include +#include +#ifndef _WIN32_WCE +# include +# include +#endif // !_WIN32_WCE + +#include // NOLINT +#include // NOLINT +#include // NOLINT + +#define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com" +#define GTEST_FLAG_PREFIX_ "gtest_" +#define GTEST_FLAG_PREFIX_DASH_ "gtest-" +#define GTEST_FLAG_PREFIX_UPPER_ "GTEST_" +#define GTEST_NAME_ "Google Test" +#define GTEST_PROJECT_URL_ "http://code.google.com/p/googletest/" + +// Determines the version of gcc that is used to compile this. +#ifdef __GNUC__ +// 40302 means version 4.3.2. +# define GTEST_GCC_VER_ \ + (__GNUC__*10000 + __GNUC_MINOR__*100 + __GNUC_PATCHLEVEL__) +#endif // __GNUC__ + +// Determines the platform on which Google Test is compiled. +#ifdef __CYGWIN__ +# define GTEST_OS_CYGWIN 1 +#elif defined __SYMBIAN32__ +# define GTEST_OS_SYMBIAN 1 +#elif defined _WIN32 +# define GTEST_OS_WINDOWS 1 +# ifdef _WIN32_WCE +# define GTEST_OS_WINDOWS_MOBILE 1 +# elif defined(__MINGW__) || defined(__MINGW32__) +# define GTEST_OS_WINDOWS_MINGW 1 +# else +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif // _WIN32_WCE +#elif defined __APPLE__ +# define GTEST_OS_MAC 1 +#elif defined __linux__ +# define GTEST_OS_LINUX 1 +# ifdef ANDROID +# define GTEST_OS_LINUX_ANDROID 1 +# endif // ANDROID +#elif defined __MVS__ +# define GTEST_OS_ZOS 1 +#elif defined(__sun) && defined(__SVR4) +# define GTEST_OS_SOLARIS 1 +#elif defined(_AIX) +# define GTEST_OS_AIX 1 +#elif defined(__hpux) +# define GTEST_OS_HPUX 1 +#elif defined __native_client__ +# define GTEST_OS_NACL 1 +#endif // __CYGWIN__ + +// Brings in definitions for functions used in the testing::internal::posix +// namespace (read, write, close, chdir, isatty, stat). We do not currently +// use them on Windows Mobile. +#if !GTEST_OS_WINDOWS +// This assumes that non-Windows OSes provide unistd.h. For OSes where this +// is not the case, we need to include headers that provide the functions +// mentioned above. +# include +# if !GTEST_OS_NACL +// TODO(vladl@google.com): Remove this condition when Native Client SDK adds +// strings.h (tracked in +// http://code.google.com/p/nativeclient/issues/detail?id=1175). +# include // Native Client doesn't provide strings.h. +# endif +#elif !GTEST_OS_WINDOWS_MOBILE +# include +# include +#endif + +#if defined(_MSC_VER) +# include +#endif + +// Defines this to true iff Google Test can use POSIX regular expressions. +#ifndef GTEST_HAS_POSIX_RE +# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +#endif + +#if GTEST_HAS_POSIX_RE + +// On some platforms, needs someone to define size_t, and +// won't compile otherwise. We can #include it here as we already +// included , which is guaranteed to define size_t through +// . +# include // NOLINT + +# define GTEST_USES_POSIX_RE 1 + +#elif GTEST_OS_WINDOWS + +// is not available on Windows. Use our own simple regex +// implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#else + +// may not be available on this platform. Use our own +// simple regex implementation instead. +# define GTEST_USES_SIMPLE_RE 1 + +#endif // GTEST_HAS_POSIX_RE + +#ifndef GTEST_HAS_EXCEPTIONS +// The user didn't tell us whether exceptions are enabled, so we need +// to figure it out. +# if defined(_MSC_VER) || defined(__BORLANDC__) +// MSVC's and C++Builder's implementations of the STL use the _HAS_EXCEPTIONS +// macro to enable exceptions, so we'll do the same. +// Assumes that exceptions are enabled by default. +# ifndef _HAS_EXCEPTIONS +# define _HAS_EXCEPTIONS 1 +# endif // _HAS_EXCEPTIONS +# define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS +# elif defined(__GNUC__) && __EXCEPTIONS +// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__SUNPRO_CC) +// Sun Pro CC supports exceptions. However, there is no compile-time way of +// detecting whether they are enabled or not. Therefore, we assume that +// they are enabled unless the user tells us otherwise. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__IBMCPP__) && __EXCEPTIONS +// xlC defines __EXCEPTIONS to 1 iff exceptions are enabled. +# define GTEST_HAS_EXCEPTIONS 1 +# elif defined(__HP_aCC) +// Exception handling is in effect by default in HP aCC compiler. It has to +// be turned of by +noeh compiler option if desired. +# define GTEST_HAS_EXCEPTIONS 1 +# else +// For other compilers, we assume exceptions are disabled to be +// conservative. +# define GTEST_HAS_EXCEPTIONS 0 +# endif // defined(_MSC_VER) || defined(__BORLANDC__) +#endif // GTEST_HAS_EXCEPTIONS + +#if !defined(GTEST_HAS_STD_STRING) +// Even though we don't use this macro any longer, we keep it in case +// some clients still depend on it. +# define GTEST_HAS_STD_STRING 1 +#elif !GTEST_HAS_STD_STRING +// The user told us that ::std::string isn't available. +# error "Google Test cannot be used where ::std::string isn't available." +#endif // !defined(GTEST_HAS_STD_STRING) + +#ifndef GTEST_HAS_GLOBAL_STRING +// The user didn't tell us whether ::string is available, so we need +// to figure it out. + +# define GTEST_HAS_GLOBAL_STRING 0 + +#endif // GTEST_HAS_GLOBAL_STRING + +#ifndef GTEST_HAS_STD_WSTRING +// The user didn't tell us whether ::std::wstring is available, so we need +// to figure it out. +// TODO(wan@google.com): uses autoconf to detect whether ::std::wstring +// is available. + +// Cygwin 1.7 and below doesn't support ::std::wstring. +// Solaris' libc++ doesn't support it either. Android has +// no support for it at least as recent as Froyo (2.2). +# define GTEST_HAS_STD_WSTRING \ + (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS)) + +#endif // GTEST_HAS_STD_WSTRING + +#ifndef GTEST_HAS_GLOBAL_WSTRING +// The user didn't tell us whether ::wstring is available, so we need +// to figure it out. +# define GTEST_HAS_GLOBAL_WSTRING \ + (GTEST_HAS_STD_WSTRING && GTEST_HAS_GLOBAL_STRING) +#endif // GTEST_HAS_GLOBAL_WSTRING + +// Determines whether RTTI is available. +#ifndef GTEST_HAS_RTTI +// The user didn't tell us whether RTTI is enabled, so we need to +// figure it out. + +# ifdef _MSC_VER + +# ifdef _CPPRTTI // MSVC defines this macro iff RTTI is enabled. +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +// Starting with version 4.3.2, gcc defines __GXX_RTTI iff RTTI is enabled. +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40302) + +# ifdef __GXX_RTTI +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif // __GXX_RTTI + +// Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if +// both the typeid and dynamic_cast features are present. +# elif defined(__IBMCPP__) && (__IBMCPP__ >= 900) + +# ifdef __RTTI_ALL__ +# define GTEST_HAS_RTTI 1 +# else +# define GTEST_HAS_RTTI 0 +# endif + +# else + +// For all other compilers, we assume RTTI is enabled. +# define GTEST_HAS_RTTI 1 + +# endif // _MSC_VER + +#endif // GTEST_HAS_RTTI + +// It's this header's responsibility to #include when RTTI +// is enabled. +#if GTEST_HAS_RTTI +# include +#endif + +// Determines whether Google Test can use the pthreads library. +#ifndef GTEST_HAS_PTHREAD +// The user didn't tell us explicitly, so we assume pthreads support is +// available on Linux and Mac. +// +// To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0 +// to your compiler flags. +# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX) +#endif // GTEST_HAS_PTHREAD + +#if GTEST_HAS_PTHREAD +// gtest-port.h guarantees to #include when GTEST_HAS_PTHREAD is +// true. +# include // NOLINT + +// For timespec and nanosleep, used below. +# include // NOLINT +#endif + +// Determines whether Google Test can use tr1/tuple. You can define +// this macro to 0 to prevent Google Test from using tuple (any +// feature depending on tuple with be disabled in this mode). +#ifndef GTEST_HAS_TR1_TUPLE +// The user didn't tell us not to do it, so we assume it's OK. +# define GTEST_HAS_TR1_TUPLE 0 +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether Google Test's own tr1 tuple implementation +// should be used. +#ifndef GTEST_USE_OWN_TR1_TUPLE +// The user didn't tell us, so we need to figure it out. + +// We use our own TR1 tuple if we aren't sure the user has an +// implementation of it already. At this time, GCC 4.0.0+ and MSVC +// 2010 are the only mainstream compilers that come with a TR1 tuple +// implementation. NVIDIA's CUDA NVCC compiler pretends to be GCC by +// defining __GNUC__ and friends, but cannot compile GCC's tuple +// implementation. MSVC 2008 (9.0) provides TR1 tuple in a 323 MB +// Feature Pack download, which we cannot assume the user has. +# if (defined(__GNUC__) && !defined(__CUDACC__) && (GTEST_GCC_VER_ >= 40000)) \ + || _MSC_VER >= 1600 +# define GTEST_USE_OWN_TR1_TUPLE 0 +# else +# define GTEST_USE_OWN_TR1_TUPLE 1 +# endif + +#endif // GTEST_USE_OWN_TR1_TUPLE + +// To avoid conditional compilation everywhere, we make it +// gtest-port.h's responsibility to #include the header implementing +// tr1/tuple. +#if GTEST_HAS_TR1_TUPLE + +# if GTEST_USE_OWN_TR1_TUPLE +// This file was GENERATED by a script. DO NOT EDIT BY HAND!!! + +// Copyright 2009 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Implements a subset of TR1 tuple needed by Google Test and Google Mock. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ + +#include // For ::std::pair. + +// The compiler used in Symbian has a bug that prevents us from declaring the +// tuple template as a friend (it complains that tuple is redefined). This +// hack bypasses the bug by declaring the members that should otherwise be +// private as public. +// Sun Studio versions < 12 also have the above bug. +#if defined(__SYMBIAN32__) || (defined(__SUNPRO_CC) && __SUNPRO_CC < 0x590) +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ public: +#else +# define GTEST_DECLARE_TUPLE_AS_FRIEND_ \ + template friend class tuple; \ + private: +#endif + +// GTEST_n_TUPLE_(T) is the type of an n-tuple. +#define GTEST_0_TUPLE_(T) tuple<> +#define GTEST_1_TUPLE_(T) tuple +#define GTEST_2_TUPLE_(T) tuple +#define GTEST_3_TUPLE_(T) tuple +#define GTEST_4_TUPLE_(T) tuple +#define GTEST_5_TUPLE_(T) tuple +#define GTEST_6_TUPLE_(T) tuple +#define GTEST_7_TUPLE_(T) tuple +#define GTEST_8_TUPLE_(T) tuple +#define GTEST_9_TUPLE_(T) tuple +#define GTEST_10_TUPLE_(T) tuple + +// GTEST_n_TYPENAMES_(T) declares a list of n typenames. +#define GTEST_0_TYPENAMES_(T) +#define GTEST_1_TYPENAMES_(T) typename T##0 +#define GTEST_2_TYPENAMES_(T) typename T##0, typename T##1 +#define GTEST_3_TYPENAMES_(T) typename T##0, typename T##1, typename T##2 +#define GTEST_4_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3 +#define GTEST_5_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4 +#define GTEST_6_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5 +#define GTEST_7_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6 +#define GTEST_8_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, typename T##7 +#define GTEST_9_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8 +#define GTEST_10_TYPENAMES_(T) typename T##0, typename T##1, typename T##2, \ + typename T##3, typename T##4, typename T##5, typename T##6, \ + typename T##7, typename T##8, typename T##9 + +// In theory, defining stuff in the ::std namespace is undefined +// behavior. We can do this as we are playing the role of a standard +// library vendor. +namespace std { +namespace tr1 { + +template +class tuple; + +// Anything in namespace gtest_internal is Google Test's INTERNAL +// IMPLEMENTATION DETAIL and MUST NOT BE USED DIRECTLY in user code. +namespace gtest_internal { + +// ByRef::type is T if T is a reference; otherwise it's const T&. +template +struct ByRef { typedef const T& type; }; // NOLINT +template +struct ByRef { typedef T& type; }; // NOLINT + +// A handy wrapper for ByRef. +#define GTEST_BY_REF_(T) typename ::std::tr1::gtest_internal::ByRef::type + +// AddRef::type is T if T is a reference; otherwise it's T&. This +// is the same as tr1::add_reference::type. +template +struct AddRef { typedef T& type; }; // NOLINT +template +struct AddRef { typedef T& type; }; // NOLINT + +// A handy wrapper for AddRef. +#define GTEST_ADD_REF_(T) typename ::std::tr1::gtest_internal::AddRef::type + +// A helper for implementing get(). +template class Get; + +// A helper for implementing tuple_element. kIndexValid is true +// iff k < the number of fields in tuple type T. +template +struct TupleElement; + +template +struct TupleElement { typedef T0 type; }; + +template +struct TupleElement { typedef T1 type; }; + +template +struct TupleElement { typedef T2 type; }; + +template +struct TupleElement { typedef T3 type; }; + +template +struct TupleElement { typedef T4 type; }; + +template +struct TupleElement { typedef T5 type; }; + +template +struct TupleElement { typedef T6 type; }; + +template +struct TupleElement { typedef T7 type; }; + +template +struct TupleElement { typedef T8 type; }; + +template +struct TupleElement { typedef T9 type; }; + +} // namespace gtest_internal + +template <> +class tuple<> { + public: + tuple() {} + tuple(const tuple& /* t */) {} + tuple& operator=(const tuple& /* t */) { return *this; } +}; + +template +class GTEST_1_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0) : f0_(f0) {} + + tuple(const tuple& t) : f0_(t.f0_) {} + + template + tuple(const GTEST_1_TUPLE_(U)& t) : f0_(t.f0_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_1_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_1_TUPLE_(U)& t) { + f0_ = t.f0_; + return *this; + } + + T0 f0_; +}; + +template +class GTEST_2_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1) : f0_(f0), + f1_(f1) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_) {} + + template + tuple(const GTEST_2_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_) {} + template + tuple(const ::std::pair& p) : f0_(p.first), f1_(p.second) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_2_TUPLE_(U)& t) { + return CopyFrom(t); + } + template + tuple& operator=(const ::std::pair& p) { + f0_ = p.first; + f1_ = p.second; + return *this; + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_2_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + return *this; + } + + T0 f0_; + T1 f1_; +}; + +template +class GTEST_3_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2) : f0_(f0), f1_(f1), f2_(f2) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + template + tuple(const GTEST_3_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_3_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_3_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; +}; + +template +class GTEST_4_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_) {} + + template + tuple(const GTEST_4_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_4_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_4_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; +}; + +template +class GTEST_5_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, + GTEST_BY_REF_(T4) f4) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_) {} + + template + tuple(const GTEST_5_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_5_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_5_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; +}; + +template +class GTEST_6_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_) {} + + template + tuple(const GTEST_6_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_6_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_6_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; +}; + +template +class GTEST_7_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + template + tuple(const GTEST_7_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_7_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_7_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; +}; + +template +class GTEST_8_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, + GTEST_BY_REF_(T7) f7) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + template + tuple(const GTEST_8_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_8_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_8_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; +}; + +template +class GTEST_9_TUPLE_(T) { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8) : f0_(f0), f1_(f1), f2_(f2), f3_(f3), f4_(f4), + f5_(f5), f6_(f6), f7_(f7), f8_(f8) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + template + tuple(const GTEST_9_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_9_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_9_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; +}; + +template +class tuple { + public: + template friend class gtest_internal::Get; + + tuple() : f0_(), f1_(), f2_(), f3_(), f4_(), f5_(), f6_(), f7_(), f8_(), + f9_() {} + + explicit tuple(GTEST_BY_REF_(T0) f0, GTEST_BY_REF_(T1) f1, + GTEST_BY_REF_(T2) f2, GTEST_BY_REF_(T3) f3, GTEST_BY_REF_(T4) f4, + GTEST_BY_REF_(T5) f5, GTEST_BY_REF_(T6) f6, GTEST_BY_REF_(T7) f7, + GTEST_BY_REF_(T8) f8, GTEST_BY_REF_(T9) f9) : f0_(f0), f1_(f1), f2_(f2), + f3_(f3), f4_(f4), f5_(f5), f6_(f6), f7_(f7), f8_(f8), f9_(f9) {} + + tuple(const tuple& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), f3_(t.f3_), + f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), f9_(t.f9_) {} + + template + tuple(const GTEST_10_TUPLE_(U)& t) : f0_(t.f0_), f1_(t.f1_), f2_(t.f2_), + f3_(t.f3_), f4_(t.f4_), f5_(t.f5_), f6_(t.f6_), f7_(t.f7_), f8_(t.f8_), + f9_(t.f9_) {} + + tuple& operator=(const tuple& t) { return CopyFrom(t); } + + template + tuple& operator=(const GTEST_10_TUPLE_(U)& t) { + return CopyFrom(t); + } + + GTEST_DECLARE_TUPLE_AS_FRIEND_ + + template + tuple& CopyFrom(const GTEST_10_TUPLE_(U)& t) { + f0_ = t.f0_; + f1_ = t.f1_; + f2_ = t.f2_; + f3_ = t.f3_; + f4_ = t.f4_; + f5_ = t.f5_; + f6_ = t.f6_; + f7_ = t.f7_; + f8_ = t.f8_; + f9_ = t.f9_; + return *this; + } + + T0 f0_; + T1 f1_; + T2 f2_; + T3 f3_; + T4 f4_; + T5 f5_; + T6 f6_; + T7 f7_; + T8 f8_; + T9 f9_; +}; + +// 6.1.3.2 Tuple creation functions. + +// Known limitations: we don't support passing an +// std::tr1::reference_wrapper to make_tuple(). And we don't +// implement tie(). + +inline tuple<> make_tuple() { return tuple<>(); } + +template +inline GTEST_1_TUPLE_(T) make_tuple(const T0& f0) { + return GTEST_1_TUPLE_(T)(f0); +} + +template +inline GTEST_2_TUPLE_(T) make_tuple(const T0& f0, const T1& f1) { + return GTEST_2_TUPLE_(T)(f0, f1); +} + +template +inline GTEST_3_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2) { + return GTEST_3_TUPLE_(T)(f0, f1, f2); +} + +template +inline GTEST_4_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3) { + return GTEST_4_TUPLE_(T)(f0, f1, f2, f3); +} + +template +inline GTEST_5_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4) { + return GTEST_5_TUPLE_(T)(f0, f1, f2, f3, f4); +} + +template +inline GTEST_6_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5) { + return GTEST_6_TUPLE_(T)(f0, f1, f2, f3, f4, f5); +} + +template +inline GTEST_7_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6) { + return GTEST_7_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6); +} + +template +inline GTEST_8_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7) { + return GTEST_8_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7); +} + +template +inline GTEST_9_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8) { + return GTEST_9_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8); +} + +template +inline GTEST_10_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, + const T3& f3, const T4& f4, const T5& f5, const T6& f6, const T7& f7, + const T8& f8, const T9& f9) { + return GTEST_10_TUPLE_(T)(f0, f1, f2, f3, f4, f5, f6, f7, f8, f9); +} + +// 6.1.3.3 Tuple helper classes. + +template struct tuple_size; + +template +struct tuple_size { static const int value = 0; }; + +template +struct tuple_size { static const int value = 1; }; + +template +struct tuple_size { static const int value = 2; }; + +template +struct tuple_size { static const int value = 3; }; + +template +struct tuple_size { static const int value = 4; }; + +template +struct tuple_size { static const int value = 5; }; + +template +struct tuple_size { static const int value = 6; }; + +template +struct tuple_size { static const int value = 7; }; + +template +struct tuple_size { static const int value = 8; }; + +template +struct tuple_size { static const int value = 9; }; + +template +struct tuple_size { static const int value = 10; }; + +template +struct tuple_element { + typedef typename gtest_internal::TupleElement< + k < (tuple_size::value), k, Tuple>::type type; +}; + +#define GTEST_TUPLE_ELEMENT_(k, Tuple) typename tuple_element::type + +// 6.1.3.4 Element access. + +namespace gtest_internal { + +template <> +class Get<0> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + Field(Tuple& t) { return t.f0_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(0, Tuple)) + ConstField(const Tuple& t) { return t.f0_; } +}; + +template <> +class Get<1> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + Field(Tuple& t) { return t.f1_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(1, Tuple)) + ConstField(const Tuple& t) { return t.f1_; } +}; + +template <> +class Get<2> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + Field(Tuple& t) { return t.f2_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(2, Tuple)) + ConstField(const Tuple& t) { return t.f2_; } +}; + +template <> +class Get<3> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + Field(Tuple& t) { return t.f3_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(3, Tuple)) + ConstField(const Tuple& t) { return t.f3_; } +}; + +template <> +class Get<4> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + Field(Tuple& t) { return t.f4_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(4, Tuple)) + ConstField(const Tuple& t) { return t.f4_; } +}; + +template <> +class Get<5> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + Field(Tuple& t) { return t.f5_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(5, Tuple)) + ConstField(const Tuple& t) { return t.f5_; } +}; + +template <> +class Get<6> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + Field(Tuple& t) { return t.f6_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(6, Tuple)) + ConstField(const Tuple& t) { return t.f6_; } +}; + +template <> +class Get<7> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + Field(Tuple& t) { return t.f7_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(7, Tuple)) + ConstField(const Tuple& t) { return t.f7_; } +}; + +template <> +class Get<8> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + Field(Tuple& t) { return t.f8_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(8, Tuple)) + ConstField(const Tuple& t) { return t.f8_; } +}; + +template <> +class Get<9> { + public: + template + static GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + Field(Tuple& t) { return t.f9_; } // NOLINT + + template + static GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(9, Tuple)) + ConstField(const Tuple& t) { return t.f9_; } +}; + +} // namespace gtest_internal + +template +GTEST_ADD_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::Field(t); +} + +template +GTEST_BY_REF_(GTEST_TUPLE_ELEMENT_(k, GTEST_10_TUPLE_(T))) +get(const GTEST_10_TUPLE_(T)& t) { + return gtest_internal::Get::ConstField(t); +} + +// 6.1.3.5 Relational operators + +// We only implement == and !=, as we don't have a need for the rest yet. + +namespace gtest_internal { + +// SameSizeTuplePrefixComparator::Eq(t1, t2) returns true if the +// first k fields of t1 equals the first k fields of t2. +// SameSizeTuplePrefixComparator(k1, k2) would be a compiler error if +// k1 != k2. +template +struct SameSizeTuplePrefixComparator; + +template <> +struct SameSizeTuplePrefixComparator<0, 0> { + template + static bool Eq(const Tuple1& /* t1 */, const Tuple2& /* t2 */) { + return true; + } +}; + +template +struct SameSizeTuplePrefixComparator { + template + static bool Eq(const Tuple1& t1, const Tuple2& t2) { + return SameSizeTuplePrefixComparator::Eq(t1, t2) && + ::std::tr1::get(t1) == ::std::tr1::get(t2); + } +}; + +} // namespace gtest_internal + +template +inline bool operator==(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { + return gtest_internal::SameSizeTuplePrefixComparator< + tuple_size::value, + tuple_size::value>::Eq(t, u); +} + +template +inline bool operator!=(const GTEST_10_TUPLE_(T)& t, + const GTEST_10_TUPLE_(U)& u) { return !(t == u); } + +// 6.1.4 Pairs. +// Unimplemented. + +} // namespace tr1 +} // namespace std + +#undef GTEST_0_TUPLE_ +#undef GTEST_1_TUPLE_ +#undef GTEST_2_TUPLE_ +#undef GTEST_3_TUPLE_ +#undef GTEST_4_TUPLE_ +#undef GTEST_5_TUPLE_ +#undef GTEST_6_TUPLE_ +#undef GTEST_7_TUPLE_ +#undef GTEST_8_TUPLE_ +#undef GTEST_9_TUPLE_ +#undef GTEST_10_TUPLE_ + +#undef GTEST_0_TYPENAMES_ +#undef GTEST_1_TYPENAMES_ +#undef GTEST_2_TYPENAMES_ +#undef GTEST_3_TYPENAMES_ +#undef GTEST_4_TYPENAMES_ +#undef GTEST_5_TYPENAMES_ +#undef GTEST_6_TYPENAMES_ +#undef GTEST_7_TYPENAMES_ +#undef GTEST_8_TYPENAMES_ +#undef GTEST_9_TYPENAMES_ +#undef GTEST_10_TYPENAMES_ + +#undef GTEST_DECLARE_TUPLE_AS_FRIEND_ +#undef GTEST_BY_REF_ +#undef GTEST_ADD_REF_ +#undef GTEST_TUPLE_ELEMENT_ + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TUPLE_H_ +# elif GTEST_OS_SYMBIAN + +// On Symbian, BOOST_HAS_TR1_TUPLE causes Boost's TR1 tuple library to +// use STLport's tuple implementation, which unfortunately doesn't +// work as the copy of STLport distributed with Symbian is incomplete. +// By making sure BOOST_HAS_TR1_TUPLE is undefined, we force Boost to +// use its own tuple implementation. +# ifdef BOOST_HAS_TR1_TUPLE +# undef BOOST_HAS_TR1_TUPLE +# endif // BOOST_HAS_TR1_TUPLE + +// This prevents , which defines +// BOOST_HAS_TR1_TUPLE, from being #included by Boost's . +# define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED +# include + +# elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000) +// GCC 4.0+ implements tr1/tuple in the header. This does +// not conform to the TR1 spec, which requires the header to be . + +# if !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 +// Until version 4.3.2, gcc has a bug that causes , +// which is #included by , to not compile when RTTI is +// disabled. _TR1_FUNCTIONAL is the header guard for +// . Hence the following #define is a hack to prevent +// from being included. +# define _TR1_FUNCTIONAL 1 +# include +# undef _TR1_FUNCTIONAL // Allows the user to #include + // if he chooses to. +# else +# include // NOLINT +# endif // !GTEST_HAS_RTTI && GTEST_GCC_VER_ < 40302 + +# else +// If the compiler is not GCC 4.0+, we assume the user is using a +// spec-conforming TR1 implementation. +# include // NOLINT +# endif // GTEST_USE_OWN_TR1_TUPLE + +#endif // GTEST_HAS_TR1_TUPLE + +// Determines whether clone(2) is supported. +// Usually it will only be available on Linux, excluding +// Linux on the Itanium architecture. +// Also see http://linux.die.net/man/2/clone. +#ifndef GTEST_HAS_CLONE +// The user didn't tell us, so we need to figure it out. + +# if GTEST_OS_LINUX && !defined(__ia64__) +# define GTEST_HAS_CLONE 1 +# else +# define GTEST_HAS_CLONE 0 +# endif // GTEST_OS_LINUX && !defined(__ia64__) + +#endif // GTEST_HAS_CLONE + +// Determines whether to support stream redirection. This is used to test +// output correctness and to implement death tests. +#ifndef GTEST_HAS_STREAM_REDIRECTION +// By default, we assume that stream redirection is supported on all +// platforms except known mobile ones. +# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN +# define GTEST_HAS_STREAM_REDIRECTION 0 +# else +# define GTEST_HAS_STREAM_REDIRECTION 1 +# endif // !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_SYMBIAN +#endif // GTEST_HAS_STREAM_REDIRECTION + +// Determines whether to support death tests. +// Google Test does not support death tests for VC 7.1 and earlier as +// abort() in a VC 7.1 application compiled as GUI in debug config +// pops up a dialog window that cannot be suppressed programmatically. +#if (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ + (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \ + GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX) +# define GTEST_HAS_DEATH_TEST 1 +# include // NOLINT +#endif + +// We don't support MSVC 7.1 with exceptions disabled now. Therefore +// all the compilers we care about are adequate for supporting +// value-parameterized tests. +#define GTEST_HAS_PARAM_TEST 1 + +// Determines whether to support type-driven tests. + +// Typed tests need and variadic macros, which GCC, VC++ 8.0, +// Sun Pro CC, IBM Visual Age, and HP aCC support. +#if defined(__GNUC__) || (_MSC_VER >= 1400) || defined(__SUNPRO_CC) || \ + defined(__IBMCPP__) || defined(__HP_aCC) +# define GTEST_HAS_TYPED_TEST 1 +# define GTEST_HAS_TYPED_TEST_P 1 +#endif + +// Determines whether to support Combine(). This only makes sense when +// value-parameterized tests are enabled. The implementation doesn't +// work on Sun Studio since it doesn't understand templated conversion +// operators. +#if GTEST_HAS_PARAM_TEST && GTEST_HAS_TR1_TUPLE && !defined(__SUNPRO_CC) +# define GTEST_HAS_COMBINE 1 +#endif + +// Determines whether the system compiler uses UTF-16 for encoding wide strings. +#define GTEST_WIDE_STRING_USES_UTF16_ \ + (GTEST_OS_WINDOWS || GTEST_OS_CYGWIN || GTEST_OS_SYMBIAN || GTEST_OS_AIX) + +// Determines whether test results can be streamed to a socket. +#if GTEST_OS_LINUX +# define GTEST_CAN_STREAM_RESULTS_ 1 +#endif + +// Defines some utility macros. + +// The GNU compiler emits a warning if nested "if" statements are followed by +// an "else" statement and braces are not used to explicitly disambiguate the +// "else" binding. This leads to problems with code like: +// +// if (gate) +// ASSERT_*(condition) << "Some message"; +// +// The "switch (0) case 0:" idiom is used to suppress this. +#ifdef __INTEL_COMPILER +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ +#else +# define GTEST_AMBIGUOUS_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT +#endif + +// Use this annotation at the end of a struct/class definition to +// prevent the compiler from optimizing away instances that are never +// used. This is useful when all interesting logic happens inside the +// c'tor and / or d'tor. Example: +// +// struct Foo { +// Foo() { ... } +// } GTEST_ATTRIBUTE_UNUSED_; +// +// Also use it after a variable or parameter declaration to tell the +// compiler the variable/parameter does not have to be used. +#if defined(__GNUC__) && !defined(COMPILER_ICC) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +#else +# define GTEST_ATTRIBUTE_UNUSED_ +#endif + +// A macro to disallow operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_ASSIGN_(type)\ + void operator=(type const &) + +// A macro to disallow copy constructor and operator= +// This should be used in the private: declarations for a class. +#define GTEST_DISALLOW_COPY_AND_ASSIGN_(type)\ + type(type const &);\ + GTEST_DISALLOW_ASSIGN_(type) + +// Tell the compiler to warn about unused return values for functions declared +// with this macro. The macro should be used on function declarations +// following the argument list: +// +// Sprocket* AllocateSprocket() GTEST_MUST_USE_RESULT_; +#if defined(__GNUC__) && (GTEST_GCC_VER_ >= 30400) && !defined(COMPILER_ICC) +# define GTEST_MUST_USE_RESULT_ __attribute__ ((warn_unused_result)) +#else +# define GTEST_MUST_USE_RESULT_ +#endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC + +// Determine whether the compiler supports Microsoft's Structured Exception +// Handling. This is supported by several Windows compilers but generally +// does not exist on any other system. +#ifndef GTEST_HAS_SEH +// The user didn't tell us, so we need to figure it out. + +# if defined(_MSC_VER) || defined(__BORLANDC__) +// These two compilers are known to support SEH. +# define GTEST_HAS_SEH 1 +# else +// Assume no SEH. +# define GTEST_HAS_SEH 0 +# endif + +#endif // GTEST_HAS_SEH + +#ifdef _MSC_VER + +# if GTEST_LINKED_AS_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllimport) +# elif GTEST_CREATE_SHARED_LIBRARY +# define GTEST_API_ __declspec(dllexport) +# endif + +#endif // _MSC_VER + +#ifndef GTEST_API_ +# define GTEST_API_ +#endif + +#ifdef __GNUC__ +// Ask the compiler to never inline a given function. +# define GTEST_NO_INLINE_ __attribute__((noinline)) +#else +# define GTEST_NO_INLINE_ +#endif + +namespace testing { + +class Message; + +namespace internal { + +class String; + +// The GTEST_COMPILE_ASSERT_ macro can be used to verify that a compile time +// expression is true. For example, you could use it to verify the +// size of a static array: +// +// GTEST_COMPILE_ASSERT_(ARRAYSIZE(content_type_names) == CONTENT_NUM_TYPES, +// content_type_names_incorrect_size); +// +// or to make sure a struct is smaller than a certain size: +// +// GTEST_COMPILE_ASSERT_(sizeof(foo) < 128, foo_too_large); +// +// The second argument to the macro is the name of the variable. If +// the expression is false, most compilers will issue a warning/error +// containing the name of the variable. + +template +struct CompileAssert { +}; + +#define GTEST_COMPILE_ASSERT_(expr, msg) \ + typedef ::testing::internal::CompileAssert<(bool(expr))> \ + msg[bool(expr) ? 1 : -1] + +// Implementation details of GTEST_COMPILE_ASSERT_: +// +// - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1 +// elements (and thus is invalid) when the expression is false. +// +// - The simpler definition +// +// #define GTEST_COMPILE_ASSERT_(expr, msg) typedef char msg[(expr) ? 1 : -1] +// +// does not work, as gcc supports variable-length arrays whose sizes +// are determined at run-time (this is gcc's extension and not part +// of the C++ standard). As a result, gcc fails to reject the +// following code with the simple definition: +// +// int foo; +// GTEST_COMPILE_ASSERT_(foo, msg); // not supposed to compile as foo is +// // not a compile-time constant. +// +// - By using the type CompileAssert<(bool(expr))>, we ensures that +// expr is a compile-time constant. (Template arguments must be +// determined at compile-time.) +// +// - The outter parentheses in CompileAssert<(bool(expr))> are necessary +// to work around a bug in gcc 3.4.4 and 4.0.1. If we had written +// +// CompileAssert +// +// instead, these compilers will refuse to compile +// +// GTEST_COMPILE_ASSERT_(5 > 0, some_message); +// +// (They seem to think the ">" in "5 > 0" marks the end of the +// template argument list.) +// +// - The array size is (bool(expr) ? 1 : -1), instead of simply +// +// ((expr) ? 1 : -1). +// +// This is to avoid running into a bug in MS VC 7.1, which +// causes ((0.0) ? 1 : -1) to incorrectly evaluate to 1. + +// StaticAssertTypeEqHelper is used by StaticAssertTypeEq defined in gtest.h. +// +// This template is declared, but intentionally undefined. +template +struct StaticAssertTypeEqHelper; + +template +struct StaticAssertTypeEqHelper {}; + +#if GTEST_HAS_GLOBAL_STRING +typedef ::string string; +#else +typedef ::std::string string; +#endif // GTEST_HAS_GLOBAL_STRING + +#if GTEST_HAS_GLOBAL_WSTRING +typedef ::wstring wstring; +#elif GTEST_HAS_STD_WSTRING +typedef ::std::wstring wstring; +#endif // GTEST_HAS_GLOBAL_WSTRING + +// A helper for suppressing warnings on constant condition. It just +// returns 'condition'. +GTEST_API_ bool IsTrue(bool condition); + +// Defines scoped_ptr. + +// This implementation of scoped_ptr is PARTIAL - it only contains +// enough stuff to satisfy Google Test's need. +template +class scoped_ptr { + public: + typedef T element_type; + + explicit scoped_ptr(T* p = NULL) : ptr_(p) {} + ~scoped_ptr() { reset(); } + + T& operator*() const { return *ptr_; } + T* operator->() const { return ptr_; } + T* get() const { return ptr_; } + + T* release() { + T* const ptr = ptr_; + ptr_ = NULL; + return ptr; + } + + void reset(T* p = NULL) { + if (p != ptr_) { + if (IsTrue(sizeof(T) > 0)) { // Makes sure T is a complete type. + delete ptr_; + } + ptr_ = p; + } + } + private: + T* ptr_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(scoped_ptr); +}; + +// Defines RE. + +// A simple C++ wrapper for . It uses the POSIX Extended +// Regular Expression syntax. +class GTEST_API_ RE { + public: + // A copy constructor is required by the Standard to initialize object + // references from r-values. + RE(const RE& other) { Init(other.pattern()); } + + // Constructs an RE from a string. + RE(const ::std::string& regex) { Init(regex.c_str()); } // NOLINT + +#if GTEST_HAS_GLOBAL_STRING + + RE(const ::string& regex) { Init(regex.c_str()); } // NOLINT + +#endif // GTEST_HAS_GLOBAL_STRING + + RE(const char* regex) { Init(regex); } // NOLINT + ~RE(); + + // Returns the string representation of the regex. + const char* pattern() const { return pattern_; } + + // FullMatch(str, re) returns true iff regular expression re matches + // the entire str. + // PartialMatch(str, re) returns true iff regular expression re + // matches a substring of str (including str itself). + // + // TODO(wan@google.com): make FullMatch() and PartialMatch() work + // when str contains NUL characters. + static bool FullMatch(const ::std::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::std::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +#if GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const ::string& str, const RE& re) { + return FullMatch(str.c_str(), re); + } + static bool PartialMatch(const ::string& str, const RE& re) { + return PartialMatch(str.c_str(), re); + } + +#endif // GTEST_HAS_GLOBAL_STRING + + static bool FullMatch(const char* str, const RE& re); + static bool PartialMatch(const char* str, const RE& re); + + private: + void Init(const char* regex); + + // We use a const char* instead of a string, as Google Test may be used + // where string is not available. We also do not use Google Test's own + // String type here, in order to simplify dependencies between the + // files. + const char* pattern_; + bool is_valid_; + +#if GTEST_USES_POSIX_RE + + regex_t full_regex_; // For FullMatch(). + regex_t partial_regex_; // For PartialMatch(). + +#else // GTEST_USES_SIMPLE_RE + + const char* full_pattern_; // For FullMatch(); + +#endif + + GTEST_DISALLOW_ASSIGN_(RE); +}; + +// Formats a source file path and a line number as they would appear +// in an error message from the compiler used to compile this code. +GTEST_API_ ::std::string FormatFileLocation(const char* file, int line); + +// Formats a file location for compiler-independent XML output. +// Although this function is not platform dependent, we put it next to +// FormatFileLocation in order to contrast the two functions. +GTEST_API_ ::std::string FormatCompilerIndependentFileLocation(const char* file, + int line); + +// Defines logging utilities: +// GTEST_LOG_(severity) - logs messages at the specified severity level. The +// message itself is streamed into the macro. +// LogToStderr() - directs all log messages to stderr. +// FlushInfoLog() - flushes informational log messages. + +enum GTestLogSeverity { + GTEST_INFO, + GTEST_WARNING, + GTEST_ERROR, + GTEST_FATAL +}; + +// Formats log entry severity, provides a stream object for streaming the +// log message, and terminates the message with a newline when going out of +// scope. +class GTEST_API_ GTestLog { + public: + GTestLog(GTestLogSeverity severity, const char* file, int line); + + // Flushes the buffers and, if severity is GTEST_FATAL, aborts the program. + ~GTestLog(); + + ::std::ostream& GetStream() { return ::std::cerr; } + + private: + const GTestLogSeverity severity_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestLog); +}; + +#define GTEST_LOG_(severity) \ + ::testing::internal::GTestLog(::testing::internal::GTEST_##severity, \ + __FILE__, __LINE__).GetStream() + +inline void LogToStderr() {} +inline void FlushInfoLog() { fflush(NULL); } + +// INTERNAL IMPLEMENTATION - DO NOT USE. +// +// GTEST_CHECK_ is an all-mode assert. It aborts the program if the condition +// is not satisfied. +// Synopsys: +// GTEST_CHECK_(boolean_condition); +// or +// GTEST_CHECK_(boolean_condition) << "Additional message"; +// +// This checks the condition and if the condition is not satisfied +// it prints message about the condition violation, including the +// condition itself, plus additional message streamed into it, if any, +// and then it aborts the program. It aborts the program irrespective of +// whether it is built in the debug mode or not. +#define GTEST_CHECK_(condition) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::IsTrue(condition)) \ + ; \ + else \ + GTEST_LOG_(FATAL) << "Condition " #condition " failed. " + +// An all-mode assert to verify that the given POSIX-style function +// call returns 0 (indicating success). Known limitation: this +// doesn't expand to a balanced 'if' statement, so enclose the macro +// in {} if you need to use it as the only statement in an 'if' +// branch. +#define GTEST_CHECK_POSIX_SUCCESS_(posix_call) \ + if (const int gtest_error = (posix_call)) \ + GTEST_LOG_(FATAL) << #posix_call << "failed with error " \ + << gtest_error + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Use ImplicitCast_ as a safe version of static_cast for upcasting in +// the type hierarchy (e.g. casting a Foo* to a SuperclassOfFoo* or a +// const Foo*). When you use ImplicitCast_, the compiler checks that +// the cast is safe. Such explicit ImplicitCast_s are necessary in +// surprisingly many situations where C++ demands an exact type match +// instead of an argument type convertable to a target type. +// +// The syntax for using ImplicitCast_ is the same as for static_cast: +// +// ImplicitCast_(expr) +// +// ImplicitCast_ would have been part of the C++ standard library, +// but the proposal was submitted too late. It will probably make +// its way into the language in the future. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., implicit_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template +inline To ImplicitCast_(To x) { return x; } + +// When you upcast (that is, cast a pointer from type Foo to type +// SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts +// always succeed. When you downcast (that is, cast a pointer from +// type Foo to type SubclassOfFoo), static_cast<> isn't safe, because +// how do you know the pointer is really of type SubclassOfFoo? It +// could be a bare Foo, or of type DifferentSubclassOfFoo. Thus, +// when you downcast, you should use this macro. In debug mode, we +// use dynamic_cast<> to double-check the downcast is legal (we die +// if it's not). In normal mode, we do the efficient static_cast<> +// instead. Thus, it's important to test in debug mode to make sure +// the cast is legal! +// This is the only place in the code we should use dynamic_cast<>. +// In particular, you SHOULDN'T be using dynamic_cast<> in order to +// do RTTI (eg code like this: +// if (dynamic_cast(foo)) HandleASubclass1Object(foo); +// if (dynamic_cast(foo)) HandleASubclass2Object(foo); +// You should design the code some other way not to need this. +// +// This relatively ugly name is intentional. It prevents clashes with +// similar functions users may have (e.g., down_cast). The internal +// namespace alone is not enough because the function can be found by ADL. +template // use like this: DownCast_(foo); +inline To DownCast_(From* f) { // so we only accept pointers + // Ensures that To is a sub-type of From *. This test is here only + // for compile-time type checking, and has no overhead in an + // optimized build at run-time, as it will be optimized away + // completely. + if (false) { + const To to = NULL; + ::testing::internal::ImplicitCast_(to); + } + +#if GTEST_HAS_RTTI + // RTTI: debug mode only! + GTEST_CHECK_(f == NULL || dynamic_cast(f) != NULL); +#endif + return static_cast(f); +} + +// Downcasts the pointer of type Base to Derived. +// Derived must be a subclass of Base. The parameter MUST +// point to a class of type Derived, not any subclass of it. +// When RTTI is available, the function performs a runtime +// check to enforce this. +template +Derived* CheckedDowncastToActualType(Base* base) { +#if GTEST_HAS_RTTI + GTEST_CHECK_(typeid(*base) == typeid(Derived)); + return dynamic_cast(base); // NOLINT +#else + return static_cast(base); // Poor man's downcast. +#endif +} + +#if GTEST_HAS_STREAM_REDIRECTION + +// Defines the stderr capturer: +// CaptureStdout - starts capturing stdout. +// GetCapturedStdout - stops capturing stdout and returns the captured string. +// CaptureStderr - starts capturing stderr. +// GetCapturedStderr - stops capturing stderr and returns the captured string. +// +GTEST_API_ void CaptureStdout(); +GTEST_API_ String GetCapturedStdout(); +GTEST_API_ void CaptureStderr(); +GTEST_API_ String GetCapturedStderr(); + +#endif // GTEST_HAS_STREAM_REDIRECTION + + +#if GTEST_HAS_DEATH_TEST + +// A copy of all command line arguments. Set by InitGoogleTest(). +extern ::std::vector g_argvs; + +// GTEST_HAS_DEATH_TEST implies we have ::std::string. +const ::std::vector& GetArgvs(); + +#endif // GTEST_HAS_DEATH_TEST + +// Defines synchronization primitives. + +#if GTEST_HAS_PTHREAD + +// Sleeps for (roughly) n milli-seconds. This function is only for +// testing Google Test's own constructs. Don't use it in user tests, +// either directly or indirectly. +inline void SleepMilliseconds(int n) { + const timespec time = { + 0, // 0 seconds. + n * 1000L * 1000L, // And n ms. + }; + nanosleep(&time, NULL); +} + +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class Notification { + public: + Notification() : notified_(false) {} + + // Notifies all threads created with this notification to start. Must + // be called from the controller thread. + void Notify() { notified_ = true; } + + // Blocks until the controller thread notifies. Must be called from a test + // thread. + void WaitForNotification() { + while(!notified_) { + SleepMilliseconds(10); + } + } + + private: + volatile bool notified_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; + +// As a C-function, ThreadFuncWithCLinkage cannot be templated itself. +// Consequently, it cannot select a correct instantiation of ThreadWithParam +// in order to call its Run(). Introducing ThreadWithParamBase as a +// non-templated base class for ThreadWithParam allows us to bypass this +// problem. +class ThreadWithParamBase { + public: + virtual ~ThreadWithParamBase() {} + virtual void Run() = 0; +}; + +// pthread_create() accepts a pointer to a function type with the C linkage. +// According to the Standard (7.5/1), function types with different linkages +// are different even if they are otherwise identical. Some compilers (for +// example, SunStudio) treat them as different types. Since class methods +// cannot be defined with C-linkage we need to define a free C-function to +// pass into pthread_create(). +extern "C" inline void* ThreadFuncWithCLinkage(void* thread) { + static_cast(thread)->Run(); + return NULL; +} + +// Helper class for testing Google Test's multi-threading constructs. +// To use it, write: +// +// void ThreadFunc(int param) { /* Do things with param */ } +// Notification thread_can_start; +// ... +// // The thread_can_start parameter is optional; you can supply NULL. +// ThreadWithParam thread(&ThreadFunc, 5, &thread_can_start); +// thread_can_start.Notify(); +// +// These classes are only for testing Google Test's own constructs. Do +// not use them in user tests, either directly or indirectly. +template +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void (*UserThreadFunc)(T); + + ThreadWithParam( + UserThreadFunc func, T param, Notification* thread_can_start) + : func_(func), + param_(param), + thread_can_start_(thread_can_start), + finished_(false) { + ThreadWithParamBase* const base = this; + // The thread can be created only after all fields except thread_ + // have been initialized. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_create(&thread_, 0, &ThreadFuncWithCLinkage, base)); + } + ~ThreadWithParam() { Join(); } + + void Join() { + if (!finished_) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_join(thread_, 0)); + finished_ = true; + } + } + + virtual void Run() { + if (thread_can_start_ != NULL) + thread_can_start_->WaitForNotification(); + func_(param_); + } + + private: + const UserThreadFunc func_; // User-supplied thread function. + const T param_; // User-supplied parameter to the thread function. + // When non-NULL, used to block execution until the controller thread + // notifies. + Notification* const thread_can_start_; + bool finished_; // true iff we know that the thread function has finished. + pthread_t thread_; // The native thread object. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; + +// MutexBase and Mutex implement mutex on pthreads-based platforms. They +// are used in conjunction with class MutexLock: +// +// Mutex mutex; +// ... +// MutexLock lock(&mutex); // Acquires the mutex and releases it at the end +// // of the current scope. +// +// MutexBase implements behavior for both statically and dynamically +// allocated mutexes. Do not use MutexBase directly. Instead, write +// the following to define a static mutex: +// +// GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex); +// +// You can forward declare a static mutex like this: +// +// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex); +// +// To create a dynamic mutex, just define an object of type Mutex. +class MutexBase { + public: + // Acquires this mutex. + void Lock() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_)); + owner_ = pthread_self(); + } + + // Releases this mutex. + void Unlock() { + // We don't protect writing to owner_ here, as it's the caller's + // responsibility to ensure that the current thread holds the + // mutex when this is called. + owner_ = 0; + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_)); + } + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld() const { + GTEST_CHECK_(owner_ == pthread_self()) + << "The current thread is not holding the mutex @" << this; + } + + // A static mutex may be used before main() is entered. It may even + // be used before the dynamic initialization stage. Therefore we + // must be able to initialize a static mutex object at link time. + // This means MutexBase has to be a POD and its member variables + // have to be public. + public: + pthread_mutex_t mutex_; // The underlying pthread mutex. + pthread_t owner_; // The thread holding the mutex; 0 means no one holds it. +}; + +// Forward-declares a static mutex. +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::MutexBase mutex + +// Defines and statically (i.e. at link time) initializes a static mutex. +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, 0 } + +// The Mutex class can only be used for mutexes created at runtime. It +// shares its API with MutexBase otherwise. +class Mutex : public MutexBase { + public: + Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + owner_ = 0; + } + ~Mutex() { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_)); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +// We cannot name this class MutexLock as the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(MutexBase* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + MutexBase* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Helpers for ThreadLocal. + +// pthread_key_create() requires DeleteThreadLocalValue() to have +// C-linkage. Therefore it cannot be templatized to access +// ThreadLocal. Hence the need for class +// ThreadLocalValueHolderBase. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Called by pthread to delete thread-local data stored by +// pthread_setspecific(). +extern "C" inline void DeleteThreadLocalValue(void* value_holder) { + delete static_cast(value_holder); +} + +// Implements thread-local storage on pthreads-based systems. +// +// // Thread 1 +// ThreadLocal tl(100); // 100 is the default value for each thread. +// +// // Thread 2 +// tl.set(150); // Changes the value for thread 2 only. +// EXPECT_EQ(150, tl.get()); +// +// // Thread 1 +// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value. +// tl.set(200); +// EXPECT_EQ(200, tl.get()); +// +// The template type argument T must have a public copy constructor. +// In addition, the default ThreadLocal constructor requires T to have +// a public default constructor. +// +// An object managed for a thread by a ThreadLocal instance is deleted +// when the thread exits. Or, if the ThreadLocal instance dies in +// that thread, when the ThreadLocal dies. It's the user's +// responsibility to ensure that all other threads using a ThreadLocal +// have exited when it dies, or the per-thread objects for those +// threads will not be deleted. +// +// Google Test only uses global ThreadLocal objects. That means they +// will die after main() has returned. Therefore, no per-thread +// object managed by Google Test will be leaked as long as all threads +// using Google Test have exited when main() returns. +template +class ThreadLocal { + public: + ThreadLocal() : key_(CreateKey()), + default_() {} + explicit ThreadLocal(const T& value) : key_(CreateKey()), + default_(value) {} + + ~ThreadLocal() { + // Destroys the managed object for the current thread, if any. + DeleteThreadLocalValue(pthread_getspecific(key_)); + + // Releases resources associated with the key. This will *not* + // delete managed objects for other threads. + GTEST_CHECK_POSIX_SUCCESS_(pthread_key_delete(key_)); + } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of type T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + static pthread_key_t CreateKey() { + pthread_key_t key; + // When a thread exits, DeleteThreadLocalValue() will be called on + // the object managed for that thread. + GTEST_CHECK_POSIX_SUCCESS_( + pthread_key_create(&key, &DeleteThreadLocalValue)); + return key; + } + + T* GetOrCreateValue() const { + ThreadLocalValueHolderBase* const holder = + static_cast(pthread_getspecific(key_)); + if (holder != NULL) { + return CheckedDowncastToActualType(holder)->pointer(); + } + + ValueHolder* const new_holder = new ValueHolder(default_); + ThreadLocalValueHolderBase* const holder_base = new_holder; + GTEST_CHECK_POSIX_SUCCESS_(pthread_setspecific(key_, holder_base)); + return new_holder->pointer(); + } + + // A key pthreads uses for looking up per-thread values. + const pthread_key_t key_; + const T default_; // The default value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# define GTEST_IS_THREADSAFE 1 + +#else // GTEST_HAS_PTHREAD + +// A dummy implementation of synchronization primitives (mutex, lock, +// and thread-local variable). Necessary for compiling Google Test where +// mutex is not supported - using Google Test in multiple threads is not +// supported on such platforms. + +class Mutex { + public: + Mutex():owner_(0), handle_() + { + ::InitializeCriticalSection(&handle_); + } + + ~Mutex() + { + ::DeleteCriticalSection(&handle_); + } + + void Lock() + { + ::EnterCriticalSection(&handle_); + owner_ = ::GetCurrentThreadId(); + } + + void Unlock() + { + ::LeaveCriticalSection(&handle_); + owner_ = 0; + } + + // Does nothing if the current thread holds the mutex. Otherwise, crashes +// with high probability. + void AssertHeld() const { + GTEST_CHECK_(owner_ == ::GetCurrentThreadId()) + << "The current thread is not holding the mutex @" << this; + } + + private: + DWORD owner_; + CRITICAL_SECTION handle_; +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + static ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) + +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex* inMutex) : mutex_(inMutex) { + mutex_->Lock(); + } + + ~GTestMutexLock() { + mutex_->Unlock(); + } + private: + Mutex* mutex_; +}; + +typedef GTestMutexLock MutexLock; + +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +extern "C" inline void DeleteThreadLocalValue(void* value_holder) { + delete static_cast(value_holder); +} + +// Implements thread-local storage on windows system. +template +class ThreadLocal { + public: + ThreadLocal() : key_(CreateKey()), + default_() {} + explicit ThreadLocal(const T& value) : key_(CreateKey()), + default_(value) {} + + ~ThreadLocal() { + // Destroys the managed object for the current thread, if any. + DeleteThreadLocalValue(TlsGetValue(key_)); + + // Releases resources associated with the key. This will *not* + // delete managed objects for other threads. + GTEST_CHECK_(TlsFree(key_) > 0); + } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of type T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + static DWORD CreateKey() { + DWORD key; + // When a thread exits, DeleteThreadLocalValue() will be called on + // the object managed for that thread. + GTEST_CHECK_((key = TlsAlloc()) != TLS_OUT_OF_INDEXES); + return key; + } + + T* GetOrCreateValue() const { + ThreadLocalValueHolderBase* const holder = + static_cast(TlsGetValue(key_)); + if (holder != NULL) { + return CheckedDowncastToActualType(holder)->pointer(); + } + + ValueHolder* const new_holder = new ValueHolder(default_); + ThreadLocalValueHolderBase* const holder_base = new_holder; + GTEST_CHECK_(TlsSetValue(key_, holder_base) != 0); + return new_holder->pointer(); + } + + // A key pthreads uses for looking up per-thread values. + const DWORD key_; + const T default_; // The default value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +// The above synchronization primitives have dummy implementations. +// Therefore Google Test is not thread-safe. +# define GTEST_IS_THREADSAFE 0 + +#endif // GTEST_HAS_PTHREAD + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +GTEST_API_ size_t GetThreadCount(); + +// Passing non-POD classes through ellipsis (...) crashes the ARM +// compiler and generates a warning in Sun Studio. The Nokia Symbian +// and the IBM XL C/C++ compiler try to instantiate a copy constructor +// for objects passed through ellipsis (...), failing for uncopyable +// objects. We define this to ensure that only POD is passed through +// ellipsis on these systems. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) || defined(__SUNPRO_CC) +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_ELLIPSIS_NEEDS_POD_ 1 +#else +# define GTEST_CAN_COMPARE_NULL 1 +#endif + +// The Nokia Symbian and IBM XL C/C++ compilers cannot decide between +// const T& and const T* in a function template. These compilers +// _can_ decide between class template specializations for T and T*, +// so a tr1::type_traits-like is_pointer works. +#if defined(__SYMBIAN32__) || defined(__IBMCPP__) +# define GTEST_NEEDS_IS_POINTER_ 1 +#endif + +template +struct bool_constant { + typedef bool_constant type; + static const bool value = bool_value; +}; +template const bool bool_constant::value; + +typedef bool_constant false_type; +typedef bool_constant true_type; + +template +struct is_pointer : public false_type {}; + +template +struct is_pointer : public true_type {}; + +template +struct IteratorTraits { + typedef typename Iterator::value_type value_type; +}; + +template +struct IteratorTraits { + typedef T value_type; +}; + +template +struct IteratorTraits { + typedef T value_type; +}; + +#if GTEST_OS_WINDOWS +# define GTEST_PATH_SEP_ "\\" +# define GTEST_HAS_ALT_PATH_SEP_ 1 +// The biggest signed integer type the compiler supports. +typedef __int64 BiggestInt; +#else +# define GTEST_PATH_SEP_ "/" +# define GTEST_HAS_ALT_PATH_SEP_ 0 +typedef long long BiggestInt; // NOLINT +#endif // GTEST_OS_WINDOWS + +// Utilities for char. + +// isspace(int ch) and friends accept an unsigned char or EOF. char +// may be signed, depending on the compiler (or compiler flags). +// Therefore we need to cast a char to unsigned char before calling +// isspace(), etc. + +inline bool IsAlpha(char ch) { + return isalpha(static_cast(ch)) != 0; +} +inline bool IsAlNum(char ch) { + return isalnum(static_cast(ch)) != 0; +} +inline bool IsDigit(char ch) { + return isdigit(static_cast(ch)) != 0; +} +inline bool IsLower(char ch) { + return islower(static_cast(ch)) != 0; +} +inline bool IsSpace(char ch) { + return isspace(static_cast(ch)) != 0; +} +inline bool IsUpper(char ch) { + return isupper(static_cast(ch)) != 0; +} +inline bool IsXDigit(char ch) { + return isxdigit(static_cast(ch)) != 0; +} + +inline char ToLower(char ch) { + return static_cast(tolower(static_cast(ch))); +} +inline char ToUpper(char ch) { + return static_cast(toupper(static_cast(ch))); +} + +// The testing::internal::posix namespace holds wrappers for common +// POSIX functions. These wrappers hide the differences between +// Windows/MSVC and POSIX systems. Since some compilers define these +// standard functions as macros, the wrapper cannot have the same name +// as the wrapped function. + +namespace posix { + +// Functions with a different name on Windows. + +#if GTEST_OS_WINDOWS + +typedef struct _stat StatStruct; + +# ifdef __BORLANDC__ +inline int IsATTY(int fd) { return isatty(fd); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +# else // !__BORLANDC__ +# if GTEST_OS_WINDOWS_MOBILE +inline int IsATTY(int /* fd */) { return 0; } +# else +inline int IsATTY(int fd) { return _isatty(fd); } +# endif // GTEST_OS_WINDOWS_MOBILE +inline int StrCaseCmp(const char* s1, const char* s2) { + return _stricmp(s1, s2); +} +inline char* StrDup(const char* src) { return _strdup(src); } +# endif // __BORLANDC__ + +# if GTEST_OS_WINDOWS_MOBILE +inline int FileNo(FILE* file) { return reinterpret_cast(_fileno(file)); } +// Stat(), RmDir(), and IsDir() are not needed on Windows CE at this +// time and thus not defined there. +# else +inline int FileNo(FILE* file) { return _fileno(file); } +inline int Stat(const char* path, StatStruct* buf) { return _stat(path, buf); } +inline int RmDir(const char* dir) { return _rmdir(dir); } +inline bool IsDir(const StatStruct& st) { + return (_S_IFDIR & st.st_mode) != 0; +} +# endif // GTEST_OS_WINDOWS_MOBILE + +#else + +typedef struct stat StatStruct; + +inline int FileNo(FILE* file) { return fileno(file); } +inline int IsATTY(int fd) { return isatty(fd); } +inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); } +inline int StrCaseCmp(const char* s1, const char* s2) { + return strcasecmp(s1, s2); +} +inline char* StrDup(const char* src) { return strdup(src); } +inline int RmDir(const char* dir) { return rmdir(dir); } +inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } + +#endif // GTEST_OS_WINDOWS + +// Functions deprecated by MSVC 8.0. + +#ifdef _MSC_VER +// Temporarily disable warning 4996 (deprecated function). +# pragma warning(push) +# pragma warning(disable:4996) +#endif + +inline const char* StrNCpy(char* dest, const char* src, size_t n) { + return strncpy(dest, src, n); +} + +// ChDir(), FReopen(), FDOpen(), Read(), Write(), Close(), and +// StrError() aren't needed on Windows CE at this time and thus not +// defined there. + +#if !GTEST_OS_WINDOWS_MOBILE +inline int ChDir(const char* dir) { return chdir(dir); } +#endif +inline FILE* FOpen(const char* path, const char* mode) { + return fopen(path, mode); +} +#if !GTEST_OS_WINDOWS_MOBILE +inline FILE *FReopen(const char* path, const char* mode, FILE* stream) { + return freopen(path, mode, stream); +} +inline FILE* FDOpen(int fd, const char* mode) { return fdopen(fd, mode); } +#endif +inline int FClose(FILE* fp) { return fclose(fp); } +#if !GTEST_OS_WINDOWS_MOBILE +inline int Read(int fd, void* buf, unsigned int count) { + return static_cast(read(fd, buf, count)); +} +inline int Write(int fd, const void* buf, unsigned int count) { + return static_cast(write(fd, buf, count)); +} +inline int Close(int fd) { return close(fd); } +inline const char* StrError(int errnum) { return strerror(errnum); } +#endif +inline const char* GetEnv(const char* name) { +#if GTEST_OS_WINDOWS_MOBILE + // We are on Windows CE, which has no environment variables. + return NULL; +#elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) + // Environment variables which we programmatically clear will be set to the + // empty string rather than unset (NULL). Handle that case. + const char* const env = getenv(name); + return (env != NULL && env[0] != '\0') ? env : NULL; +#else + return getenv(name); +#endif +} + +#ifdef _MSC_VER +# pragma warning(pop) // Restores the warning state. +#endif + +#if GTEST_OS_WINDOWS_MOBILE +// Windows CE has no C library. The abort() function is used in +// several places in Google Test. This implementation provides a reasonable +// imitation of standard behaviour. +void Abort(); +#else +inline void Abort() { abort(); } +#endif // GTEST_OS_WINDOWS_MOBILE + +} // namespace posix + +// The maximum number a BiggestInt can represent. This definition +// works no matter BiggestInt is represented in one's complement or +// two's complement. +// +// We cannot rely on numeric_limits in STL, as __int64 and long long +// are not part of standard C++ and numeric_limits doesn't need to be +// defined for them. +const BiggestInt kMaxBiggestInt = + ~(static_cast(1) << (8*sizeof(BiggestInt) - 1)); + +// This template class serves as a compile-time function from size to +// type. It maps a size in bytes to a primitive type with that +// size. e.g. +// +// TypeWithSize<4>::UInt +// +// is typedef-ed to be unsigned int (unsigned integer made up of 4 +// bytes). +// +// Such functionality should belong to STL, but I cannot find it +// there. +// +// Google Test uses this class in the implementation of floating-point +// comparison. +// +// For now it only handles UInt (unsigned int) as that's all Google Test +// needs. Other types can be easily added in the future if need +// arises. +template +class TypeWithSize { + public: + // This prevents the user from using TypeWithSize with incorrect + // values of N. + typedef void UInt; +}; + +// The specialization for size 4. +template <> +class TypeWithSize<4> { + public: + // unsigned int has size 4 in both gcc and MSVC. + // + // As base/basictypes.h doesn't compile on Windows, we cannot use + // uint32, uint64, and etc here. + typedef int Int; + typedef unsigned int UInt; +}; + +// The specialization for size 8. +template <> +class TypeWithSize<8> { + public: + +#if GTEST_OS_WINDOWS + typedef __int64 Int; + typedef unsigned __int64 UInt; +#else + typedef long long Int; // NOLINT + typedef unsigned long long UInt; // NOLINT +#endif // GTEST_OS_WINDOWS +}; + +// Integer types of known sizes. +typedef TypeWithSize<4>::Int Int32; +typedef TypeWithSize<4>::UInt UInt32; +typedef TypeWithSize<8>::Int Int64; +typedef TypeWithSize<8>::UInt UInt64; +typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds. + +// Utilities for command line flags and environment variables. + +// Macro for referencing flags. +#define GTEST_FLAG(name) FLAGS_gtest_##name + +// Macros for declaring flags. +#define GTEST_DECLARE_bool_(name) GTEST_API_ extern bool GTEST_FLAG(name) +#define GTEST_DECLARE_int32_(name) \ + GTEST_API_ extern ::testing::internal::Int32 GTEST_FLAG(name) +#define GTEST_DECLARE_string_(name) \ + GTEST_API_ extern ::testing::internal::String GTEST_FLAG(name) + +// Macros for defining flags. +#define GTEST_DEFINE_bool_(name, default_val, doc) \ + GTEST_API_ bool GTEST_FLAG(name) = (default_val) +#define GTEST_DEFINE_int32_(name, default_val, doc) \ + GTEST_API_ ::testing::internal::Int32 GTEST_FLAG(name) = (default_val) +#define GTEST_DEFINE_string_(name, default_val, doc) \ + GTEST_API_ ::testing::internal::String GTEST_FLAG(name) = (default_val) + +// Parses 'str' for a 32-bit signed integer. If successful, writes the result +// to *value and returns true; otherwise leaves *value unchanged and returns +// false. +// TODO(chandlerc): Find a better way to refactor flag and environment parsing +// out of both gtest-port.cc and gtest.cc to avoid exporting this utility +// function. +bool ParseInt32(const Message& src_text, const char* str, Int32* value); + +// Parses a bool/Int32/string from the environment variable +// corresponding to the given Google Test flag. +bool BoolFromGTestEnv(const char* flag, bool default_val); +GTEST_API_ Int32 Int32FromGTestEnv(const char* flag, Int32 default_val); +const char* StringFromGTestEnv(const char* flag, const char* default_val); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ + +#if GTEST_OS_LINUX +# include +# include +# include +# include +#endif // GTEST_OS_LINUX + +#include +#include +#include +#include +#include + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file declares the String class and functions used internally by +// Google Test. They are subject to change without notice. They should not used +// by code external to Google Test. +// +// This header file is #included by . +// It should not be #included by other files. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ + +#ifdef __BORLANDC__ +// string.h is not guaranteed to provide strcpy on C++ Builder. +# include +#endif + +#include + +#include + +namespace testing { +namespace internal { + +// String - a UTF-8 string class. +// +// For historic reasons, we don't use std::string. +// +// TODO(wan@google.com): replace this class with std::string or +// implement it in terms of the latter. +// +// Note that String can represent both NULL and the empty string, +// while std::string cannot represent NULL. +// +// NULL and the empty string are considered different. NULL is less +// than anything (including the empty string) except itself. +// +// This class only provides minimum functionality necessary for +// implementing Google Test. We do not intend to implement a full-fledged +// string class here. +// +// Since the purpose of this class is to provide a substitute for +// std::string on platforms where it cannot be used, we define a copy +// constructor and assignment operators such that we don't need +// conditional compilation in a lot of places. +// +// In order to make the representation efficient, the d'tor of String +// is not virtual. Therefore DO NOT INHERIT FROM String. +class GTEST_API_ String { + public: + // Static utility methods + + // Returns the input enclosed in double quotes if it's not NULL; + // otherwise returns "(null)". For example, "\"Hello\"" is returned + // for input "Hello". + // + // This is useful for printing a C string in the syntax of a literal. + // + // Known issue: escape sequences are not handled yet. + static String ShowCStringQuoted(const char* c_str); + + // Clones a 0-terminated C string, allocating memory using new. The + // caller is responsible for deleting the return value using + // delete[]. Returns the cloned string, or NULL if the input is + // NULL. + // + // This is different from strdup() in string.h, which allocates + // memory using malloc(). + static const char* CloneCString(const char* c_str); + +#if GTEST_OS_WINDOWS_MOBILE + // Windows CE does not have the 'ANSI' versions of Win32 APIs. To be + // able to pass strings to Win32 APIs on CE we need to convert them + // to 'Unicode', UTF-16. + + // Creates a UTF-16 wide string from the given ANSI string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the wide string, or NULL if the + // input is NULL. + // + // The wide string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static LPCWSTR AnsiToUtf16(const char* c_str); + + // Creates an ANSI string from the given wide string, allocating + // memory using new. The caller is responsible for deleting the return + // value using delete[]. Returns the ANSI string, or NULL if the + // input is NULL. + // + // The returned string is created using the ANSI codepage (CP_ACP) to + // match the behaviour of the ANSI versions of Win32 calls and the + // C runtime. + static const char* Utf16ToAnsi(LPCWSTR utf16_str); +#endif + + // Compares two C strings. Returns true iff they have the same content. + // + // Unlike strcmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CStringEquals(const char* lhs, const char* rhs); + + // Converts a wide C string to a String using the UTF-8 encoding. + // NULL will be converted to "(null)". If an error occurred during + // the conversion, "(failed to convert from wide string)" is + // returned. + static String ShowWideCString(const wchar_t* wide_c_str); + + // Similar to ShowWideCString(), except that this function encloses + // the converted string in double quotes. + static String ShowWideCStringQuoted(const wchar_t* wide_c_str); + + // Compares two wide C strings. Returns true iff they have the same + // content. + // + // Unlike wcscmp(), this function can handle NULL argument(s). A + // NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool WideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); + + // Compares two C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike strcasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL C string, + // including the empty string. + static bool CaseInsensitiveCStringEquals(const char* lhs, + const char* rhs); + + // Compares two wide C strings, ignoring case. Returns true iff they + // have the same content. + // + // Unlike wcscasecmp(), this function can handle NULL argument(s). + // A NULL C string is considered different to any non-NULL wide C string, + // including the empty string. + // NB: The implementations on different platforms slightly differ. + // On windows, this method uses _wcsicmp which compares according to LC_CTYPE + // environment variable. On GNU platform this method uses wcscasecmp + // which compares according to LC_CTYPE category of the current locale. + // On MacOS X, it uses towlower, which also uses LC_CTYPE category of the + // current locale. + static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs, + const wchar_t* rhs); + + // Formats a list of arguments to a String, using the same format + // spec string as for printf. + // + // We do not use the StringPrintf class as it is not universally + // available. + // + // The result is limited to 4096 characters (including the tailing + // 0). If 4096 characters are not enough to format the input, + // "" is returned. + static String Format(const char* format, ...); + + // C'tors + + // The default c'tor constructs a NULL string. + String() : c_str_(NULL), length_(0) {} + + // Constructs a String by cloning a 0-terminated C string. + String(const char* a_c_str) { // NOLINT + if (a_c_str == NULL) { + c_str_ = NULL; + length_ = 0; + } else { + ConstructNonNull(a_c_str, strlen(a_c_str)); + } + } + + // Constructs a String by copying a given number of chars from a + // buffer. E.g. String("hello", 3) creates the string "hel", + // String("a\0bcd", 4) creates "a\0bc", String(NULL, 0) creates "", + // and String(NULL, 1) results in access violation. + String(const char* buffer, size_t a_length) { + ConstructNonNull(buffer, a_length); + } + + // The copy c'tor creates a new copy of the string. The two + // String objects do not share content. + String(const String& str) : c_str_(NULL), length_(0) { *this = str; } + + // D'tor. String is intended to be a final class, so the d'tor + // doesn't need to be virtual. + ~String() { delete[] c_str_; } + + // Allows a String to be implicitly converted to an ::std::string or + // ::string, and vice versa. Converting a String containing a NULL + // pointer to ::std::string or ::string is undefined behavior. + // Converting a ::std::string or ::string containing an embedded NUL + // character to a String will result in the prefix up to the first + // NUL character. + String(const ::std::string& str) { + ConstructNonNull(str.c_str(), str.length()); + } + + operator ::std::string() const { return ::std::string(c_str(), length()); } + +#if GTEST_HAS_GLOBAL_STRING + String(const ::string& str) { + ConstructNonNull(str.c_str(), str.length()); + } + + operator ::string() const { return ::string(c_str(), length()); } +#endif // GTEST_HAS_GLOBAL_STRING + + // Returns true iff this is an empty string (i.e. ""). + bool empty() const { return (c_str() != NULL) && (length() == 0); } + + // Compares this with another String. + // Returns < 0 if this is less than rhs, 0 if this is equal to rhs, or > 0 + // if this is greater than rhs. + int Compare(const String& rhs) const; + + // Returns true iff this String equals the given C string. A NULL + // string and a non-NULL string are considered not equal. + bool operator==(const char* a_c_str) const { return Compare(a_c_str) == 0; } + + // Returns true iff this String is less than the given String. A + // NULL string is considered less than "". + bool operator<(const String& rhs) const { return Compare(rhs) < 0; } + + // Returns true iff this String doesn't equal the given C string. A NULL + // string and a non-NULL string are considered not equal. + bool operator!=(const char* a_c_str) const { return !(*this == a_c_str); } + + // Returns true iff this String ends with the given suffix. *Any* + // String is considered to end with a NULL or empty suffix. + bool EndsWith(const char* suffix) const; + + // Returns true iff this String ends with the given suffix, not considering + // case. Any String is considered to end with a NULL or empty suffix. + bool EndsWithCaseInsensitive(const char* suffix) const; + + // Returns the length of the encapsulated string, or 0 if the + // string is NULL. + size_t length() const { return length_; } + + // Gets the 0-terminated C string this String object represents. + // The String object still owns the string. Therefore the caller + // should NOT delete the return value. + const char* c_str() const { return c_str_; } + + // Assigns a C string to this object. Self-assignment works. + const String& operator=(const char* a_c_str) { + return *this = String(a_c_str); + } + + // Assigns a String object to this object. Self-assignment works. + const String& operator=(const String& rhs) { + if (this != &rhs) { + delete[] c_str_; + if (rhs.c_str() == NULL) { + c_str_ = NULL; + length_ = 0; + } else { + ConstructNonNull(rhs.c_str(), rhs.length()); + } + } + + return *this; + } + + private: + // Constructs a non-NULL String from the given content. This + // function can only be called when c_str_ has not been allocated. + // ConstructNonNull(NULL, 0) results in an empty string (""). + // ConstructNonNull(NULL, non_zero) is undefined behavior. + void ConstructNonNull(const char* buffer, size_t a_length) { + char* const str = new char[a_length + 1]; + memcpy(str, buffer, a_length); + str[a_length] = '\0'; + c_str_ = str; + length_ = a_length; + } + + const char* c_str_; + size_t length_; +}; // class String + +// Streams a String to an ostream. Each '\0' character in the String +// is replaced with "\\0". +inline ::std::ostream& operator<<(::std::ostream& os, const String& str) { + if (str.c_str() == NULL) { + os << "(null)"; + } else { + const char* const c_str = str.c_str(); + for (size_t i = 0; i != str.length(); i++) { + if (c_str[i] == '\0') { + os << "\\0"; + } else { + os << c_str[i]; + } + } + } + return os; +} + +// Gets the content of the stringstream's buffer as a String. Each '\0' +// character in the buffer is replaced with "\\0". +GTEST_API_ String StringStreamToString(::std::stringstream* stream); + +// Converts a streamable value to a String. A NULL pointer is +// converted to "(null)". When the input value is a ::string, +// ::std::string, ::wstring, or ::std::wstring object, each NUL +// character in it is replaced with "\\0". + +// Declared here but defined in gtest.h, so that it has access +// to the definition of the Message class, required by the ARM +// compiler. +template +String StreamableToString(const T& streamable); + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: keith.ray@gmail.com (Keith Ray) +// +// Google Test filepath utilities +// +// This header file declares classes and functions used internally by +// Google Test. They are subject to change without notice. +// +// This file is #included in . +// Do not include this header file separately! + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ + + +namespace testing { +namespace internal { + +// FilePath - a class for file and directory pathname manipulation which +// handles platform-specific conventions (like the pathname separator). +// Used for helper functions for naming files in a directory for xml output. +// Except for Set methods, all methods are const or static, which provides an +// "immutable value object" -- useful for peace of mind. +// A FilePath with a value ending in a path separator ("like/this/") represents +// a directory, otherwise it is assumed to represent a file. In either case, +// it may or may not represent an actual file or directory in the file system. +// Names are NOT checked for syntax correctness -- no checking for illegal +// characters, malformed paths, etc. + +class GTEST_API_ FilePath { + public: + FilePath() : pathname_("") { } + FilePath(const FilePath& rhs) : pathname_(rhs.pathname_) { } + + explicit FilePath(const char* pathname) : pathname_(pathname) { + Normalize(); + } + + explicit FilePath(const String& pathname) : pathname_(pathname) { + Normalize(); + } + + FilePath& operator=(const FilePath& rhs) { + Set(rhs); + return *this; + } + + void Set(const FilePath& rhs) { + pathname_ = rhs.pathname_; + } + + String ToString() const { return pathname_; } + const char* c_str() const { return pathname_.c_str(); } + + // Returns the current working directory, or "" if unsuccessful. + static FilePath GetCurrentDir(); + + // Given directory = "dir", base_name = "test", number = 0, + // extension = "xml", returns "dir/test.xml". If number is greater + // than zero (e.g., 12), returns "dir/test_12.xml". + // On Windows platform, uses \ as the separator rather than /. + static FilePath MakeFileName(const FilePath& directory, + const FilePath& base_name, + int number, + const char* extension); + + // Given directory = "dir", relative_path = "test.xml", + // returns "dir/test.xml". + // On Windows, uses \ as the separator rather than /. + static FilePath ConcatPaths(const FilePath& directory, + const FilePath& relative_path); + + // Returns a pathname for a file that does not currently exist. The pathname + // will be directory/base_name.extension or + // directory/base_name_.extension if directory/base_name.extension + // already exists. The number will be incremented until a pathname is found + // that does not already exist. + // Examples: 'dir/foo_test.xml' or 'dir/foo_test_1.xml'. + // There could be a race condition if two or more processes are calling this + // function at the same time -- they could both pick the same filename. + static FilePath GenerateUniqueFileName(const FilePath& directory, + const FilePath& base_name, + const char* extension); + + // Returns true iff the path is NULL or "". + bool IsEmpty() const { return c_str() == NULL || *c_str() == '\0'; } + + // If input name has a trailing separator character, removes it and returns + // the name, otherwise return the name string unmodified. + // On Windows platform, uses \ as the separator, other platforms use /. + FilePath RemoveTrailingPathSeparator() const; + + // Returns a copy of the FilePath with the directory part removed. + // Example: FilePath("path/to/file").RemoveDirectoryName() returns + // FilePath("file"). If there is no directory part ("just_a_file"), it returns + // the FilePath unmodified. If there is no file part ("just_a_dir/") it + // returns an empty FilePath (""). + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveDirectoryName() const; + + // RemoveFileName returns the directory path with the filename removed. + // Example: FilePath("path/to/file").RemoveFileName() returns "path/to/". + // If the FilePath is "a_file" or "/a_file", RemoveFileName returns + // FilePath("./") or, on Windows, FilePath(".\\"). If the filepath does + // not have a file, like "just/a/dir/", it returns the FilePath unmodified. + // On Windows platform, '\' is the path separator, otherwise it is '/'. + FilePath RemoveFileName() const; + + // Returns a copy of the FilePath with the case-insensitive extension removed. + // Example: FilePath("dir/file.exe").RemoveExtension("EXE") returns + // FilePath("dir/file"). If a case-insensitive extension is not + // found, returns a copy of the original FilePath. + FilePath RemoveExtension(const char* extension) const; + + // Creates directories so that path exists. Returns true if successful or if + // the directories already exist; returns false if unable to create + // directories for any reason. Will also return false if the FilePath does + // not represent a directory (that is, it doesn't end with a path separator). + bool CreateDirectoriesRecursively() const; + + // Create the directory so that path exists. Returns true if successful or + // if the directory already exists; returns false if unable to create the + // directory for any reason, including if the parent directory does not + // exist. Not named "CreateDirectory" because that's a macro on Windows. + bool CreateFolder() const; + + // Returns true if FilePath describes something in the file-system, + // either a file, directory, or whatever, and that something exists. + bool FileOrDirectoryExists() const; + + // Returns true if pathname describes a directory in the file-system + // that exists. + bool DirectoryExists() const; + + // Returns true if FilePath ends with a path separator, which indicates that + // it is intended to represent a directory. Returns false otherwise. + // This does NOT check that a directory (or file) actually exists. + bool IsDirectory() const; + + // Returns true if pathname describes a root directory. (Windows has one + // root directory per disk drive.) + bool IsRootDirectory() const; + + // Returns true if pathname describes an absolute path. + bool IsAbsolutePath() const; + + private: + // Replaces multiple consecutive separators with a single separator. + // For example, "bar///foo" becomes "bar/foo". Does not eliminate other + // redundancies that might be in a pathname involving "." or "..". + // + // A pathname with multiple consecutive separators may occur either through + // user error or as a result of some scripts or APIs that generate a pathname + // with a trailing separator. On other platforms the same API or script + // may NOT generate a pathname with a trailing "/". Then elsewhere that + // pathname may have another "/" and pathname components added to it, + // without checking for the separator already being there. + // The script language and operating system may allow paths like "foo//bar" + // but some of the functions in FilePath will not handle that correctly. In + // particular, RemoveTrailingPathSeparator() only removes one separator, and + // it is called in CreateDirectoriesRecursively() assuming that it will change + // a pathname from directory syntax (trailing separator) to filename syntax. + // + // On Windows this method also replaces the alternate path separator '/' with + // the primary path separator '\\', so that for example "bar\\/\\foo" becomes + // "bar\\foo". + + void Normalize(); + + // Returns a pointer to the last occurence of a valid path separator in + // the FilePath. On Windows, for example, both '/' and '\' are valid path + // separators. Returns NULL if no path separator was found. + const char* FindLastPathSeparator() const; + + String pathname_; +}; // class FilePath + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +// This file was GENERATED by command: +// pump.py gtest-type-util.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Type utilities needed for implementing typed and type-parameterized +// tests. This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently we support at most 50 types in a list, and at most 50 +// type-parameterized tests in one type-parameterized test case. +// Please contact googletestframework@googlegroups.com if you need +// more. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + + +// #ifdef __GNUC__ is too general here. It is possible to use gcc without using +// libstdc++ (which is where cxxabi.h comes from). +# ifdef __GLIBCXX__ +# include +# elif defined(__HP_aCC) +# include +# endif // __GLIBCXX__ + +namespace testing { +namespace internal { + +// GetTypeName() returns a human-readable name of type T. +// NB: This function is also used in Google Mock, so don't move it inside of +// the typed-test-only section below. +template +String GetTypeName() { +# if GTEST_HAS_RTTI + + const char* const name = typeid(T).name(); +# if defined(__GLIBCXX__) || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +# ifdef __GLIBCXX__ + using abi::__cxa_demangle; +# endif // __GLIBCXX__ + char* const readable_name = __cxa_demangle(name, 0, 0, &status); + const String name_str(status == 0 ? readable_name : name); + free(readable_name); + return name_str; +# else + return name; +# endif // __GLIBCXX__ || __HP_aCC + +# else + + return ""; + +# endif // GTEST_HAS_RTTI +} + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// AssertyTypeEq::type is defined iff T1 and T2 are the same +// type. This can be used as a compile-time assertion to ensure that +// two types are equal. + +template +struct AssertTypeEq; + +template +struct AssertTypeEq { + typedef bool type; +}; + +// A unique type used as the default value for the arguments of class +// template Types. This allows us to simulate variadic templates +// (e.g. Types, Type, and etc), which C++ doesn't +// support directly. +struct None {}; + +// The following family of struct and struct templates are used to +// represent type lists. In particular, TypesN +// represents a type list with N types (T1, T2, ..., and TN) in it. +// Except for Types0, every struct in the family has two member types: +// Head for the first type in the list, and Tail for the rest of the +// list. + +// The empty type list. +struct Types0 {}; + +// Type lists of length 1, 2, 3, and so on. + +template +struct Types1 { + typedef T1 Head; + typedef Types0 Tail; +}; +template +struct Types2 { + typedef T1 Head; + typedef Types1 Tail; +}; + +template +struct Types3 { + typedef T1 Head; + typedef Types2 Tail; +}; + +template +struct Types4 { + typedef T1 Head; + typedef Types3 Tail; +}; + +template +struct Types5 { + typedef T1 Head; + typedef Types4 Tail; +}; + +template +struct Types6 { + typedef T1 Head; + typedef Types5 Tail; +}; + +template +struct Types7 { + typedef T1 Head; + typedef Types6 Tail; +}; + +template +struct Types8 { + typedef T1 Head; + typedef Types7 Tail; +}; + +template +struct Types9 { + typedef T1 Head; + typedef Types8 Tail; +}; + +template +struct Types10 { + typedef T1 Head; + typedef Types9 Tail; +}; + +template +struct Types11 { + typedef T1 Head; + typedef Types10 Tail; +}; + +template +struct Types12 { + typedef T1 Head; + typedef Types11 Tail; +}; + +template +struct Types13 { + typedef T1 Head; + typedef Types12 Tail; +}; + +template +struct Types14 { + typedef T1 Head; + typedef Types13 Tail; +}; + +template +struct Types15 { + typedef T1 Head; + typedef Types14 Tail; +}; + +template +struct Types16 { + typedef T1 Head; + typedef Types15 Tail; +}; + +template +struct Types17 { + typedef T1 Head; + typedef Types16 Tail; +}; + +template +struct Types18 { + typedef T1 Head; + typedef Types17 Tail; +}; + +template +struct Types19 { + typedef T1 Head; + typedef Types18 Tail; +}; + +template +struct Types20 { + typedef T1 Head; + typedef Types19 Tail; +}; + +template +struct Types21 { + typedef T1 Head; + typedef Types20 Tail; +}; + +template +struct Types22 { + typedef T1 Head; + typedef Types21 Tail; +}; + +template +struct Types23 { + typedef T1 Head; + typedef Types22 Tail; +}; + +template +struct Types24 { + typedef T1 Head; + typedef Types23 Tail; +}; + +template +struct Types25 { + typedef T1 Head; + typedef Types24 Tail; +}; + +template +struct Types26 { + typedef T1 Head; + typedef Types25 Tail; +}; + +template +struct Types27 { + typedef T1 Head; + typedef Types26 Tail; +}; + +template +struct Types28 { + typedef T1 Head; + typedef Types27 Tail; +}; + +template +struct Types29 { + typedef T1 Head; + typedef Types28 Tail; +}; + +template +struct Types30 { + typedef T1 Head; + typedef Types29 Tail; +}; + +template +struct Types31 { + typedef T1 Head; + typedef Types30 Tail; +}; + +template +struct Types32 { + typedef T1 Head; + typedef Types31 Tail; +}; + +template +struct Types33 { + typedef T1 Head; + typedef Types32 Tail; +}; + +template +struct Types34 { + typedef T1 Head; + typedef Types33 Tail; +}; + +template +struct Types35 { + typedef T1 Head; + typedef Types34 Tail; +}; + +template +struct Types36 { + typedef T1 Head; + typedef Types35 Tail; +}; + +template +struct Types37 { + typedef T1 Head; + typedef Types36 Tail; +}; + +template +struct Types38 { + typedef T1 Head; + typedef Types37 Tail; +}; + +template +struct Types39 { + typedef T1 Head; + typedef Types38 Tail; +}; + +template +struct Types40 { + typedef T1 Head; + typedef Types39 Tail; +}; + +template +struct Types41 { + typedef T1 Head; + typedef Types40 Tail; +}; + +template +struct Types42 { + typedef T1 Head; + typedef Types41 Tail; +}; + +template +struct Types43 { + typedef T1 Head; + typedef Types42 Tail; +}; + +template +struct Types44 { + typedef T1 Head; + typedef Types43 Tail; +}; + +template +struct Types45 { + typedef T1 Head; + typedef Types44 Tail; +}; + +template +struct Types46 { + typedef T1 Head; + typedef Types45 Tail; +}; + +template +struct Types47 { + typedef T1 Head; + typedef Types46 Tail; +}; + +template +struct Types48 { + typedef T1 Head; + typedef Types47 Tail; +}; + +template +struct Types49 { + typedef T1 Head; + typedef Types48 Tail; +}; + +template +struct Types50 { + typedef T1 Head; + typedef Types49 Tail; +}; + + +} // namespace internal + +// We don't want to require the users to write TypesN<...> directly, +// as that would require them to count the length. Types<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Types +// will appear as Types in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Types, and Google Test will translate +// that to TypesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Types template. +template +struct Types { + typedef internal::Types50 type; +}; + +template <> +struct Types { + typedef internal::Types0 type; +}; +template +struct Types { + typedef internal::Types1 type; +}; +template +struct Types { + typedef internal::Types2 type; +}; +template +struct Types { + typedef internal::Types3 type; +}; +template +struct Types { + typedef internal::Types4 type; +}; +template +struct Types { + typedef internal::Types5 type; +}; +template +struct Types { + typedef internal::Types6 type; +}; +template +struct Types { + typedef internal::Types7 type; +}; +template +struct Types { + typedef internal::Types8 type; +}; +template +struct Types { + typedef internal::Types9 type; +}; +template +struct Types { + typedef internal::Types10 type; +}; +template +struct Types { + typedef internal::Types11 type; +}; +template +struct Types { + typedef internal::Types12 type; +}; +template +struct Types { + typedef internal::Types13 type; +}; +template +struct Types { + typedef internal::Types14 type; +}; +template +struct Types { + typedef internal::Types15 type; +}; +template +struct Types { + typedef internal::Types16 type; +}; +template +struct Types { + typedef internal::Types17 type; +}; +template +struct Types { + typedef internal::Types18 type; +}; +template +struct Types { + typedef internal::Types19 type; +}; +template +struct Types { + typedef internal::Types20 type; +}; +template +struct Types { + typedef internal::Types21 type; +}; +template +struct Types { + typedef internal::Types22 type; +}; +template +struct Types { + typedef internal::Types23 type; +}; +template +struct Types { + typedef internal::Types24 type; +}; +template +struct Types { + typedef internal::Types25 type; +}; +template +struct Types { + typedef internal::Types26 type; +}; +template +struct Types { + typedef internal::Types27 type; +}; +template +struct Types { + typedef internal::Types28 type; +}; +template +struct Types { + typedef internal::Types29 type; +}; +template +struct Types { + typedef internal::Types30 type; +}; +template +struct Types { + typedef internal::Types31 type; +}; +template +struct Types { + typedef internal::Types32 type; +}; +template +struct Types { + typedef internal::Types33 type; +}; +template +struct Types { + typedef internal::Types34 type; +}; +template +struct Types { + typedef internal::Types35 type; +}; +template +struct Types { + typedef internal::Types36 type; +}; +template +struct Types { + typedef internal::Types37 type; +}; +template +struct Types { + typedef internal::Types38 type; +}; +template +struct Types { + typedef internal::Types39 type; +}; +template +struct Types { + typedef internal::Types40 type; +}; +template +struct Types { + typedef internal::Types41 type; +}; +template +struct Types { + typedef internal::Types42 type; +}; +template +struct Types { + typedef internal::Types43 type; +}; +template +struct Types { + typedef internal::Types44 type; +}; +template +struct Types { + typedef internal::Types45 type; +}; +template +struct Types { + typedef internal::Types46 type; +}; +template +struct Types { + typedef internal::Types47 type; +}; +template +struct Types { + typedef internal::Types48 type; +}; +template +struct Types { + typedef internal::Types49 type; +}; + +namespace internal { + +# define GTEST_TEMPLATE_ template class + +// The template "selector" struct TemplateSel is used to +// represent Tmpl, which must be a class template with one type +// parameter, as a type. TemplateSel::Bind::type is defined +// as the type Tmpl. This allows us to actually instantiate the +// template "selected" by TemplateSel. +// +// This trick is necessary for simulating typedef for class templates, +// which C++ doesn't support directly. +template +struct TemplateSel { + template + struct Bind { + typedef Tmpl type; + }; +}; + +# define GTEST_BIND_(TmplSel, T) \ + TmplSel::template Bind::type + +// A unique struct template used as the default value for the +// arguments of class template Templates. This allows us to simulate +// variadic templates (e.g. Templates, Templates, +// and etc), which C++ doesn't support directly. +template +struct NoneT {}; + +// The following family of struct and struct templates are used to +// represent template lists. In particular, TemplatesN represents a list of N templates (T1, T2, ..., and TN). Except +// for Templates0, every struct in the family has two member types: +// Head for the selector of the first template in the list, and Tail +// for the rest of the list. + +// The empty template list. +struct Templates0 {}; + +// Template lists of length 1, 2, 3, and so on. + +template +struct Templates1 { + typedef TemplateSel Head; + typedef Templates0 Tail; +}; +template +struct Templates2 { + typedef TemplateSel Head; + typedef Templates1 Tail; +}; + +template +struct Templates3 { + typedef TemplateSel Head; + typedef Templates2 Tail; +}; + +template +struct Templates4 { + typedef TemplateSel Head; + typedef Templates3 Tail; +}; + +template +struct Templates5 { + typedef TemplateSel Head; + typedef Templates4 Tail; +}; + +template +struct Templates6 { + typedef TemplateSel Head; + typedef Templates5 Tail; +}; + +template +struct Templates7 { + typedef TemplateSel Head; + typedef Templates6 Tail; +}; + +template +struct Templates8 { + typedef TemplateSel Head; + typedef Templates7 Tail; +}; + +template +struct Templates9 { + typedef TemplateSel Head; + typedef Templates8 Tail; +}; + +template +struct Templates10 { + typedef TemplateSel Head; + typedef Templates9 Tail; +}; + +template +struct Templates11 { + typedef TemplateSel Head; + typedef Templates10 Tail; +}; + +template +struct Templates12 { + typedef TemplateSel Head; + typedef Templates11 Tail; +}; + +template +struct Templates13 { + typedef TemplateSel Head; + typedef Templates12 Tail; +}; + +template +struct Templates14 { + typedef TemplateSel Head; + typedef Templates13 Tail; +}; + +template +struct Templates15 { + typedef TemplateSel Head; + typedef Templates14 Tail; +}; + +template +struct Templates16 { + typedef TemplateSel Head; + typedef Templates15 Tail; +}; + +template +struct Templates17 { + typedef TemplateSel Head; + typedef Templates16 Tail; +}; + +template +struct Templates18 { + typedef TemplateSel Head; + typedef Templates17 Tail; +}; + +template +struct Templates19 { + typedef TemplateSel Head; + typedef Templates18 Tail; +}; + +template +struct Templates20 { + typedef TemplateSel Head; + typedef Templates19 Tail; +}; + +template +struct Templates21 { + typedef TemplateSel Head; + typedef Templates20 Tail; +}; + +template +struct Templates22 { + typedef TemplateSel Head; + typedef Templates21 Tail; +}; + +template +struct Templates23 { + typedef TemplateSel Head; + typedef Templates22 Tail; +}; + +template +struct Templates24 { + typedef TemplateSel Head; + typedef Templates23 Tail; +}; + +template +struct Templates25 { + typedef TemplateSel Head; + typedef Templates24 Tail; +}; + +template +struct Templates26 { + typedef TemplateSel Head; + typedef Templates25 Tail; +}; + +template +struct Templates27 { + typedef TemplateSel Head; + typedef Templates26 Tail; +}; + +template +struct Templates28 { + typedef TemplateSel Head; + typedef Templates27 Tail; +}; + +template +struct Templates29 { + typedef TemplateSel Head; + typedef Templates28 Tail; +}; + +template +struct Templates30 { + typedef TemplateSel Head; + typedef Templates29 Tail; +}; + +template +struct Templates31 { + typedef TemplateSel Head; + typedef Templates30 Tail; +}; + +template +struct Templates32 { + typedef TemplateSel Head; + typedef Templates31 Tail; +}; + +template +struct Templates33 { + typedef TemplateSel Head; + typedef Templates32 Tail; +}; + +template +struct Templates34 { + typedef TemplateSel Head; + typedef Templates33 Tail; +}; + +template +struct Templates35 { + typedef TemplateSel Head; + typedef Templates34 Tail; +}; + +template +struct Templates36 { + typedef TemplateSel Head; + typedef Templates35 Tail; +}; + +template +struct Templates37 { + typedef TemplateSel Head; + typedef Templates36 Tail; +}; + +template +struct Templates38 { + typedef TemplateSel Head; + typedef Templates37 Tail; +}; + +template +struct Templates39 { + typedef TemplateSel Head; + typedef Templates38 Tail; +}; + +template +struct Templates40 { + typedef TemplateSel Head; + typedef Templates39 Tail; +}; + +template +struct Templates41 { + typedef TemplateSel Head; + typedef Templates40 Tail; +}; + +template +struct Templates42 { + typedef TemplateSel Head; + typedef Templates41 Tail; +}; + +template +struct Templates43 { + typedef TemplateSel Head; + typedef Templates42 Tail; +}; + +template +struct Templates44 { + typedef TemplateSel Head; + typedef Templates43 Tail; +}; + +template +struct Templates45 { + typedef TemplateSel Head; + typedef Templates44 Tail; +}; + +template +struct Templates46 { + typedef TemplateSel Head; + typedef Templates45 Tail; +}; + +template +struct Templates47 { + typedef TemplateSel Head; + typedef Templates46 Tail; +}; + +template +struct Templates48 { + typedef TemplateSel Head; + typedef Templates47 Tail; +}; + +template +struct Templates49 { + typedef TemplateSel Head; + typedef Templates48 Tail; +}; + +template +struct Templates50 { + typedef TemplateSel Head; + typedef Templates49 Tail; +}; + + +// We don't want to require the users to write TemplatesN<...> directly, +// as that would require them to count the length. Templates<...> is much +// easier to write, but generates horrible messages when there is a +// compiler error, as gcc insists on printing out each template +// argument, even if it has the default value (this means Templates +// will appear as Templates in the compiler +// errors). +// +// Our solution is to combine the best part of the two approaches: a +// user would write Templates, and Google Test will translate +// that to TemplatesN internally to make error messages +// readable. The translation is done by the 'type' member of the +// Templates template. +template +struct Templates { + typedef Templates50 type; +}; + +template <> +struct Templates { + typedef Templates0 type; +}; +template +struct Templates { + typedef Templates1 type; +}; +template +struct Templates { + typedef Templates2 type; +}; +template +struct Templates { + typedef Templates3 type; +}; +template +struct Templates { + typedef Templates4 type; +}; +template +struct Templates { + typedef Templates5 type; +}; +template +struct Templates { + typedef Templates6 type; +}; +template +struct Templates { + typedef Templates7 type; +}; +template +struct Templates { + typedef Templates8 type; +}; +template +struct Templates { + typedef Templates9 type; +}; +template +struct Templates { + typedef Templates10 type; +}; +template +struct Templates { + typedef Templates11 type; +}; +template +struct Templates { + typedef Templates12 type; +}; +template +struct Templates { + typedef Templates13 type; +}; +template +struct Templates { + typedef Templates14 type; +}; +template +struct Templates { + typedef Templates15 type; +}; +template +struct Templates { + typedef Templates16 type; +}; +template +struct Templates { + typedef Templates17 type; +}; +template +struct Templates { + typedef Templates18 type; +}; +template +struct Templates { + typedef Templates19 type; +}; +template +struct Templates { + typedef Templates20 type; +}; +template +struct Templates { + typedef Templates21 type; +}; +template +struct Templates { + typedef Templates22 type; +}; +template +struct Templates { + typedef Templates23 type; +}; +template +struct Templates { + typedef Templates24 type; +}; +template +struct Templates { + typedef Templates25 type; +}; +template +struct Templates { + typedef Templates26 type; +}; +template +struct Templates { + typedef Templates27 type; +}; +template +struct Templates { + typedef Templates28 type; +}; +template +struct Templates { + typedef Templates29 type; +}; +template +struct Templates { + typedef Templates30 type; +}; +template +struct Templates { + typedef Templates31 type; +}; +template +struct Templates { + typedef Templates32 type; +}; +template +struct Templates { + typedef Templates33 type; +}; +template +struct Templates { + typedef Templates34 type; +}; +template +struct Templates { + typedef Templates35 type; +}; +template +struct Templates { + typedef Templates36 type; +}; +template +struct Templates { + typedef Templates37 type; +}; +template +struct Templates { + typedef Templates38 type; +}; +template +struct Templates { + typedef Templates39 type; +}; +template +struct Templates { + typedef Templates40 type; +}; +template +struct Templates { + typedef Templates41 type; +}; +template +struct Templates { + typedef Templates42 type; +}; +template +struct Templates { + typedef Templates43 type; +}; +template +struct Templates { + typedef Templates44 type; +}; +template +struct Templates { + typedef Templates45 type; +}; +template +struct Templates { + typedef Templates46 type; +}; +template +struct Templates { + typedef Templates47 type; +}; +template +struct Templates { + typedef Templates48 type; +}; +template +struct Templates { + typedef Templates49 type; +}; + +// The TypeList template makes it possible to use either a single type +// or a Types<...> list in TYPED_TEST_CASE() and +// INSTANTIATE_TYPED_TEST_CASE_P(). + +template +struct TypeList { typedef Types1 type; }; + +template +struct TypeList > { + typedef typename Types::type type; +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ + +// Due to C++ preprocessor weirdness, we need double indirection to +// concatenate two tokens when one of them is __LINE__. Writing +// +// foo ## __LINE__ +// +// will result in the token foo__LINE__, instead of foo followed by +// the current line number. For more details, see +// http://www.parashift.com/c++-faq-lite/misc-technical-issues.html#faq-39.6 +#define GTEST_CONCAT_TOKEN_(foo, bar) GTEST_CONCAT_TOKEN_IMPL_(foo, bar) +#define GTEST_CONCAT_TOKEN_IMPL_(foo, bar) foo ## bar + +// Google Test defines the testing::Message class to allow construction of +// test messages via the << operator. The idea is that anything +// streamable to std::ostream can be streamed to a testing::Message. +// This allows a user to use his own types in Google Test assertions by +// overloading the << operator. +// +// util/gtl/stl_logging-inl.h overloads << for STL containers. These +// overloads cannot be defined in the std namespace, as that will be +// undefined behavior. Therefore, they are defined in the global +// namespace instead. +// +// C++'s symbol lookup rule (i.e. Koenig lookup) says that these +// overloads are visible in either the std namespace or the global +// namespace, but not other namespaces, including the testing +// namespace which Google Test's Message class is in. +// +// To allow STL containers (and other types that has a << operator +// defined in the global namespace) to be used in Google Test assertions, +// testing::Message must access the custom << operator from the global +// namespace. Hence this helper function. +// +// Note: Jeffrey Yasskin suggested an alternative fix by "using +// ::operator<<;" in the definition of Message's operator<<. That fix +// doesn't require a helper function, but unfortunately doesn't +// compile with MSVC. +template +inline void GTestStreamToHelper(std::ostream* os, const T& val) { + *os << val; +} + +class ProtocolMessage; +namespace proto2 { class Message; } + +namespace testing { + +// Forward declarations. + +class AssertionResult; // Result of an assertion. +class Message; // Represents a failure message. +class Test; // Represents a test. +class TestInfo; // Information about a test. +class TestPartResult; // Result of a test part. +class UnitTest; // A collection of test cases. + +template +::std::string PrintToString(const T& value); + +namespace internal { + +struct TraceInfo; // Information about a trace point. +class ScopedTrace; // Implements scoped trace. +class TestInfoImpl; // Opaque implementation of TestInfo +class UnitTestImpl; // Opaque implementation of UnitTest + +// How many times InitGoogleTest() has been called. +extern int g_init_gtest_count; + +// The text used in failure messages to indicate the start of the +// stack trace. +GTEST_API_ extern const char kStackTraceMarker[]; + +// A secret type that Google Test users don't know about. It has no +// definition on purpose. Therefore it's impossible to create a +// Secret object, which is what we want. +class Secret; + +// Two overloaded helpers for checking at compile time whether an +// expression is a null pointer literal (i.e. NULL or any 0-valued +// compile-time integral constant). Their return values have +// different sizes, so we can use sizeof() to test which version is +// picked by the compiler. These helpers have no implementations, as +// we only need their signatures. +// +// Given IsNullLiteralHelper(x), the compiler will pick the first +// version if x can be implicitly converted to Secret*, and pick the +// second version otherwise. Since Secret is a secret and incomplete +// type, the only expression a user can write that has type Secret* is +// a null pointer literal. Therefore, we know that x is a null +// pointer literal if and only if the first version is picked by the +// compiler. +char IsNullLiteralHelper(Secret* p); +char (&IsNullLiteralHelper(...))[2]; // NOLINT + +// A compile-time bool constant that is true if and only if x is a +// null pointer literal (i.e. NULL or any 0-valued compile-time +// integral constant). +#ifdef GTEST_ELLIPSIS_NEEDS_POD_ +// We lose support for NULL detection where the compiler doesn't like +// passing non-POD classes through ellipsis (...). +# define GTEST_IS_NULL_LITERAL_(x) false +#else +# define GTEST_IS_NULL_LITERAL_(x) \ + (sizeof(::testing::internal::IsNullLiteralHelper(x)) == 1) +#endif // GTEST_ELLIPSIS_NEEDS_POD_ + +// Appends the user-supplied message to the Google-Test-generated message. +GTEST_API_ String AppendUserMessage(const String& gtest_msg, + const Message& user_msg); + +// A helper class for creating scoped traces in user programs. +class GTEST_API_ ScopedTrace { + public: + // The c'tor pushes the given source file location and message onto + // a trace stack maintained by Google Test. + ScopedTrace(const char* file, int line, const Message& message); + + // The d'tor pops the info pushed by the c'tor. + // + // Note that the d'tor is not virtual in order to be efficient. + // Don't inherit from ScopedTrace! + ~ScopedTrace(); + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedTrace); +} GTEST_ATTRIBUTE_UNUSED_; // A ScopedTrace object does its job in its + // c'tor and d'tor. Therefore it doesn't + // need to be used otherwise. + +// Converts a streamable value to a String. A NULL pointer is +// converted to "(null)". When the input value is a ::string, +// ::std::string, ::wstring, or ::std::wstring object, each NUL +// character in it is replaced with "\\0". +// Declared here but defined in gtest.h, so that it has access +// to the definition of the Message class, required by the ARM +// compiler. +template +String StreamableToString(const T& streamable); + +// The Symbian compiler has a bug that prevents it from selecting the +// correct overload of FormatForComparisonFailureMessage (see below) +// unless we pass the first argument by reference. If we do that, +// however, Visual Age C++ 10.1 generates a compiler error. Therefore +// we only apply the work-around for Symbian. +#if defined(__SYMBIAN32__) +# define GTEST_CREF_WORKAROUND_ const& +#else +# define GTEST_CREF_WORKAROUND_ +#endif + +// When this operand is a const char* or char*, if the other operand +// is a ::std::string or ::string, we print this operand as a C string +// rather than a pointer (we do the same for wide strings); otherwise +// we print it as a pointer to be safe. + +// This internal macro is used to avoid duplicated code. +#define GTEST_FORMAT_IMPL_(operand2_type, operand1_printer)\ +inline String FormatForComparisonFailureMessage(\ + operand2_type::value_type* GTEST_CREF_WORKAROUND_ str, \ + const operand2_type& /*operand2*/) {\ + return operand1_printer(str);\ +}\ +inline String FormatForComparisonFailureMessage(\ + const operand2_type::value_type* GTEST_CREF_WORKAROUND_ str, \ + const operand2_type& /*operand2*/) {\ + return operand1_printer(str);\ +} + +GTEST_FORMAT_IMPL_(::std::string, String::ShowCStringQuoted) +#if GTEST_HAS_STD_WSTRING +GTEST_FORMAT_IMPL_(::std::wstring, String::ShowWideCStringQuoted) +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_STRING +GTEST_FORMAT_IMPL_(::string, String::ShowCStringQuoted) +#endif // GTEST_HAS_GLOBAL_STRING +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_FORMAT_IMPL_(::wstring, String::ShowWideCStringQuoted) +#endif // GTEST_HAS_GLOBAL_WSTRING + +#undef GTEST_FORMAT_IMPL_ + +// The next four overloads handle the case where the operand being +// printed is a char/wchar_t pointer and the other operand is not a +// string/wstring object. In such cases, we just print the operand as +// a pointer to be safe. +#define GTEST_FORMAT_CHAR_PTR_IMPL_(CharType) \ + template \ + String FormatForComparisonFailureMessage(CharType* GTEST_CREF_WORKAROUND_ p, \ + const T&) { \ + return PrintToString(static_cast(p)); \ + } + +GTEST_FORMAT_CHAR_PTR_IMPL_(char) +GTEST_FORMAT_CHAR_PTR_IMPL_(const char) +GTEST_FORMAT_CHAR_PTR_IMPL_(wchar_t) +GTEST_FORMAT_CHAR_PTR_IMPL_(const wchar_t) + +#undef GTEST_FORMAT_CHAR_PTR_IMPL_ + +// Constructs and returns the message for an equality assertion +// (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. +// +// The first four parameters are the expressions used in the assertion +// and their values, as strings. For example, for ASSERT_EQ(foo, bar) +// where foo is 5 and bar is 6, we have: +// +// expected_expression: "foo" +// actual_expression: "bar" +// expected_value: "5" +// actual_value: "6" +// +// The ignoring_case parameter is true iff the assertion is a +// *_STRCASEEQ*. When it's true, the string " (ignoring case)" will +// be inserted into the message. +GTEST_API_ AssertionResult EqFailure(const char* expected_expression, + const char* actual_expression, + const String& expected_value, + const String& actual_value, + bool ignoring_case); + +// Constructs a failure message for Boolean assertions such as EXPECT_TRUE. +GTEST_API_ String GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value); + +// This template class represents an IEEE floating-point number +// (either single-precision or double-precision, depending on the +// template parameters). +// +// The purpose of this class is to do more sophisticated number +// comparison. (Due to round-off error, etc, it's very unlikely that +// two floating-points will be equal exactly. Hence a naive +// comparison by the == operation often doesn't work.) +// +// Format of IEEE floating-point: +// +// The most-significant bit being the leftmost, an IEEE +// floating-point looks like +// +// sign_bit exponent_bits fraction_bits +// +// Here, sign_bit is a single bit that designates the sign of the +// number. +// +// For float, there are 8 exponent bits and 23 fraction bits. +// +// For double, there are 11 exponent bits and 52 fraction bits. +// +// More details can be found at +// http://en.wikipedia.org/wiki/IEEE_floating-point_standard. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +template +class FloatingPoint { + public: + // Defines the unsigned integer type that has the same size as the + // floating point number. + typedef typename TypeWithSize::UInt Bits; + + // Constants. + + // # of bits in a number. + static const size_t kBitCount = 8*sizeof(RawType); + + // # of fraction bits in a number. + static const size_t kFractionBitCount = + std::numeric_limits::digits - 1; + + // # of exponent bits in a number. + static const size_t kExponentBitCount = kBitCount - 1 - kFractionBitCount; + + // The mask for the sign bit. + static const Bits kSignBitMask = static_cast(1) << (kBitCount - 1); + + // The mask for the fraction bits. + static const Bits kFractionBitMask = + ~static_cast(0) >> (kExponentBitCount + 1); + + // The mask for the exponent bits. + static const Bits kExponentBitMask = ~(kSignBitMask | kFractionBitMask); + + // How many ULP's (Units in the Last Place) we want to tolerate when + // comparing two numbers. The larger the value, the more error we + // allow. A 0 value means that two numbers must be exactly the same + // to be considered equal. + // + // The maximum error of a single floating-point operation is 0.5 + // units in the last place. On Intel CPU's, all floating-point + // calculations are done with 80-bit precision, while double has 64 + // bits. Therefore, 4 should be enough for ordinary use. + // + // See the following article for more details on ULP: + // http://www.cygnus-software.com/papers/comparingfloats/comparingfloats.htm. + static const size_t kMaxUlps = 4; + + // Constructs a FloatingPoint from a raw floating-point number. + // + // On an Intel CPU, passing a non-normalized NAN (Not a Number) + // around may change its bits, although the new value is guaranteed + // to be also a NAN. Therefore, don't expect this constructor to + // preserve the bits in x when x is a NAN. + explicit FloatingPoint(const RawType& x) { u_.value_ = x; } + + // Static methods + + // Reinterprets a bit pattern as a floating-point number. + // + // This function is needed to test the AlmostEquals() method. + static RawType ReinterpretBits(const Bits bits) { + FloatingPoint fp(0); + fp.u_.bits_ = bits; + return fp.u_.value_; + } + + // Returns the floating-point number that represent positive infinity. + static RawType Infinity() { + return ReinterpretBits(kExponentBitMask); + } + + // Non-static methods + + // Returns the bits that represents this number. + const Bits &bits() const { return u_.bits_; } + + // Returns the exponent bits of this number. + Bits exponent_bits() const { return kExponentBitMask & u_.bits_; } + + // Returns the fraction bits of this number. + Bits fraction_bits() const { return kFractionBitMask & u_.bits_; } + + // Returns the sign bit of this number. + Bits sign_bit() const { return kSignBitMask & u_.bits_; } + + // Returns true iff this is NAN (not a number). + bool is_nan() const { + // It's a NAN if the exponent bits are all ones and the fraction + // bits are not entirely zeros. + return (exponent_bits() == kExponentBitMask) && (fraction_bits() != 0); + } + + // Returns true iff this number is at most kMaxUlps ULP's away from + // rhs. In particular, this function: + // + // - returns false if either number is (or both are) NAN. + // - treats really large numbers as almost equal to infinity. + // - thinks +0.0 and -0.0 are 0 DLP's apart. + bool AlmostEquals(const FloatingPoint& rhs) const { + // The IEEE standard says that any comparison operation involving + // a NAN must return false. + if (is_nan() || rhs.is_nan()) return false; + + return DistanceBetweenSignAndMagnitudeNumbers(u_.bits_, rhs.u_.bits_) + <= kMaxUlps; + } + + private: + // The data type used to store the actual floating-point number. + union FloatingPointUnion { + RawType value_; // The raw floating-point number. + Bits bits_; // The bits that represent the number. + }; + + // Converts an integer from the sign-and-magnitude representation to + // the biased representation. More precisely, let N be 2 to the + // power of (kBitCount - 1), an integer x is represented by the + // unsigned number x + N. + // + // For instance, + // + // -N + 1 (the most negative number representable using + // sign-and-magnitude) is represented by 1; + // 0 is represented by N; and + // N - 1 (the biggest number representable using + // sign-and-magnitude) is represented by 2N - 1. + // + // Read http://en.wikipedia.org/wiki/Signed_number_representations + // for more details on signed number representations. + static Bits SignAndMagnitudeToBiased(const Bits &sam) { + if (kSignBitMask & sam) { + // sam represents a negative number. + return ~sam + 1; + } else { + // sam represents a positive number. + return kSignBitMask | sam; + } + } + + // Given two numbers in the sign-and-magnitude representation, + // returns the distance between them as an unsigned number. + static Bits DistanceBetweenSignAndMagnitudeNumbers(const Bits &sam1, + const Bits &sam2) { + const Bits biased1 = SignAndMagnitudeToBiased(sam1); + const Bits biased2 = SignAndMagnitudeToBiased(sam2); + return (biased1 >= biased2) ? (biased1 - biased2) : (biased2 - biased1); + } + + FloatingPointUnion u_; +}; + +// Typedefs the instances of the FloatingPoint template class that we +// care to use. +typedef FloatingPoint Float; +typedef FloatingPoint Double; + +// In order to catch the mistake of putting tests that use different +// test fixture classes in the same test case, we need to assign +// unique IDs to fixture classes and compare them. The TypeId type is +// used to hold such IDs. The user should treat TypeId as an opaque +// type: the only operation allowed on TypeId values is to compare +// them for equality using the == operator. +typedef const void* TypeId; + +template +class TypeIdHelper { + public: + // dummy_ must not have a const type. Otherwise an overly eager + // compiler (e.g. MSVC 7.1 & 8.0) may try to merge + // TypeIdHelper::dummy_ for different Ts as an "optimization". + static bool dummy_; +}; + +template +bool TypeIdHelper::dummy_ = false; + +// GetTypeId() returns the ID of type T. Different values will be +// returned for different types. Calling the function twice with the +// same type argument is guaranteed to return the same ID. +template +TypeId GetTypeId() { + // The compiler is required to allocate a different + // TypeIdHelper::dummy_ variable for each T used to instantiate + // the template. Therefore, the address of dummy_ is guaranteed to + // be unique. + return &(TypeIdHelper::dummy_); +} + +// Returns the type ID of ::testing::Test. Always call this instead +// of GetTypeId< ::testing::Test>() to get the type ID of +// ::testing::Test, as the latter may give the wrong result due to a +// suspected linker bug when compiling Google Test as a Mac OS X +// framework. +GTEST_API_ TypeId GetTestTypeId(); + +// Defines the abstract factory interface that creates instances +// of a Test object. +class TestFactoryBase { + public: + virtual ~TestFactoryBase() {} + + // Creates a test instance to run. The instance is both created and destroyed + // within TestInfoImpl::Run() + virtual Test* CreateTest() = 0; + + protected: + TestFactoryBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestFactoryBase); +}; + +// This class provides implementation of TeastFactoryBase interface. +// It is used in TEST and TEST_F macros. +template +class TestFactoryImpl : public TestFactoryBase { + public: + virtual Test* CreateTest() { return new TestClass; } +}; + +#if GTEST_OS_WINDOWS + +// Predicate-formatters for implementing the HRESULT checking macros +// {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED} +// We pass a long instead of HRESULT to avoid causing an +// include dependency for the HRESULT type. +GTEST_API_ AssertionResult IsHRESULTSuccess(const char* expr, + long hr); // NOLINT +GTEST_API_ AssertionResult IsHRESULTFailure(const char* expr, + long hr); // NOLINT + +#endif // GTEST_OS_WINDOWS + +// Types of SetUpTestCase() and TearDownTestCase() functions. +typedef void (*SetUpTestCaseFunc)(); +typedef void (*TearDownTestCaseFunc)(); + +// Creates a new TestInfo object and registers it with Google Test; +// returns the created object. +// +// Arguments: +// +// test_case_name: name of the test case +// name: name of the test +// type_param the name of the test's type parameter, or NULL if +// this is not a typed or a type-parameterized test. +// value_param text representation of the test's value parameter, +// or NULL if this is not a type-parameterized test. +// fixture_class_id: ID of the test fixture class +// set_up_tc: pointer to the function that sets up the test case +// tear_down_tc: pointer to the function that tears down the test case +// factory: pointer to the factory that creates a test object. +// The newly created TestInfo instance will assume +// ownership of the factory object. +GTEST_API_ TestInfo* MakeAndRegisterTestInfo( + const char* test_case_name, const char* name, + const char* type_param, + const char* value_param, + TypeId fixture_class_id, + SetUpTestCaseFunc set_up_tc, + TearDownTestCaseFunc tear_down_tc, + TestFactoryBase* factory); + +// If *pstr starts with the given prefix, modifies *pstr to be right +// past the prefix and returns true; otherwise leaves *pstr unchanged +// and returns false. None of pstr, *pstr, and prefix can be NULL. +GTEST_API_ bool SkipPrefix(const char* prefix, const char** pstr); + +#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// State of the definition of a type-parameterized test case. +class GTEST_API_ TypedTestCasePState { + public: + TypedTestCasePState() : registered_(false) {} + + // Adds the given test name to defined_test_names_ and return true + // if the test case hasn't been registered; otherwise aborts the + // program. + bool AddTestName(const char* file, int line, const char* case_name, + const char* test_name) { + if (registered_) { + fprintf(stderr, "%s Test %s must be defined before " + "REGISTER_TYPED_TEST_CASE_P(%s, ...).\n", + FormatFileLocation(file, line).c_str(), test_name, case_name); + fflush(stderr); + posix::Abort(); + } + defined_test_names_.insert(test_name); + return true; + } + + // Verifies that registered_tests match the test names in + // defined_test_names_; returns registered_tests if successful, or + // aborts the program otherwise. + const char* VerifyRegisteredTestNames( + const char* file, int line, const char* registered_tests); + + private: + bool registered_; + ::std::set defined_test_names_; +}; + +// Skips to the first non-space char after the first comma in 'str'; +// returns NULL if no comma is found in 'str'. +inline const char* SkipComma(const char* str) { + const char* comma = strchr(str, ','); + if (comma == NULL) { + return NULL; + } + while (IsSpace(*(++comma))) {} + return comma; +} + +// Returns the prefix of 'str' before the first comma in it; returns +// the entire string if it contains no comma. +inline String GetPrefixUntilComma(const char* str) { + const char* comma = strchr(str, ','); + return comma == NULL ? String(str) : String(str, comma - str); +} + +// TypeParameterizedTest::Register() +// registers a list of type-parameterized tests with Google Test. The +// return value is insignificant - we just need to return something +// such that we can call this function in a namespace scope. +// +// Implementation note: The GTEST_TEMPLATE_ macro declares a template +// template parameter. It's defined in gtest-type-util.h. +template +class TypeParameterizedTest { + public: + // 'index' is the index of the test in the type list 'Types' + // specified in INSTANTIATE_TYPED_TEST_CASE_P(Prefix, TestCase, + // Types). Valid values for 'index' are [0, N - 1] where N is the + // length of Types. + static bool Register(const char* prefix, const char* case_name, + const char* test_names, int index) { + typedef typename Types::Head Type; + typedef Fixture FixtureClass; + typedef typename GTEST_BIND_(TestSel, Type) TestClass; + + // First, registers the first type-parameterized test in the type + // list. + MakeAndRegisterTestInfo( + String::Format("%s%s%s/%d", prefix, prefix[0] == '\0' ? "" : "/", + case_name, index).c_str(), + GetPrefixUntilComma(test_names).c_str(), + GetTypeName().c_str(), + NULL, // No value parameter. + GetTypeId(), + TestClass::SetUpTestCase, + TestClass::TearDownTestCase, + new TestFactoryImpl); + + // Next, recurses (at compile time) with the tail of the type list. + return TypeParameterizedTest + ::Register(prefix, case_name, test_names, index + 1); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTest { + public: + static bool Register(const char* /*prefix*/, const char* /*case_name*/, + const char* /*test_names*/, int /*index*/) { + return true; + } +}; + +// TypeParameterizedTestCase::Register() +// registers *all combinations* of 'Tests' and 'Types' with Google +// Test. The return value is insignificant - we just need to return +// something such that we can call this function in a namespace scope. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* prefix, const char* case_name, + const char* test_names) { + typedef typename Tests::Head Head; + + // First, register the first test in 'Test' for each type in 'Types'. + TypeParameterizedTest::Register( + prefix, case_name, test_names, 0); + + // Next, recurses (at compile time) with the tail of the test list. + return TypeParameterizedTestCase + ::Register(prefix, case_name, SkipComma(test_names)); + } +}; + +// The base case for the compile time recursion. +template +class TypeParameterizedTestCase { + public: + static bool Register(const char* /*prefix*/, const char* /*case_name*/, + const char* /*test_names*/) { + return true; + } +}; + +#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P + +// Returns the current OS stack trace as a String. +// +// The maximum number of stack frames to be included is specified by +// the gtest_stack_trace_depth flag. The skip_count parameter +// specifies the number of top frames to be skipped, which doesn't +// count against the number of frames to be included. +// +// For example, if Foo() calls Bar(), which in turn calls +// GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in +// the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. +GTEST_API_ String GetCurrentOsStackTraceExceptTop(UnitTest* unit_test, + int skip_count); + +// Helpers for suppressing warnings on unreachable code or constant +// condition. + +// Always returns true. +GTEST_API_ bool AlwaysTrue(); + +// Always returns false. +inline bool AlwaysFalse() { return !AlwaysTrue(); } + +// Helper for suppressing false warning from Clang on a const char* +// variable declared in a conditional expression always being NULL in +// the else branch. +struct GTEST_API_ ConstCharPtr { + ConstCharPtr(const char* str) : value(str) {} + operator bool() const { return true; } + const char* value; +}; + +// A simple Linear Congruential Generator for generating random +// numbers with a uniform distribution. Unlike rand() and srand(), it +// doesn't use global state (and therefore can't interfere with user +// code). Unlike rand_r(), it's portable. An LCG isn't very random, +// but it's good enough for our purposes. +class GTEST_API_ Random { + public: + static const UInt32 kMaxRange = 1u << 31; + + explicit Random(UInt32 seed) : state_(seed) {} + + void Reseed(UInt32 seed) { state_ = seed; } + + // Generates a random number from [0, range). Crashes if 'range' is + // 0 or greater than kMaxRange. + UInt32 Generate(UInt32 range); + + private: + UInt32 state_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(Random); +}; + +// Defining a variable of type CompileAssertTypesEqual will cause a +// compiler error iff T1 and T2 are different types. +template +struct CompileAssertTypesEqual; + +template +struct CompileAssertTypesEqual { +}; + +// Removes the reference from a type if it is a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::remove_reference, which is not widely available yet. +template +struct RemoveReference { typedef T type; }; // NOLINT +template +struct RemoveReference { typedef T type; }; // NOLINT + +// A handy wrapper around RemoveReference that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_REFERENCE_(T) \ + typename ::testing::internal::RemoveReference::type + +// Removes const from a type if it is a const type, otherwise leaves +// it unchanged. This is the same as tr1::remove_const, which is not +// widely available yet. +template +struct RemoveConst { typedef T type; }; // NOLINT +template +struct RemoveConst { typedef T type; }; // NOLINT + +// MSVC 8.0, Sun C++, and IBM XL C++ have a bug which causes the above +// definition to fail to remove the const in 'const int[3]' and 'const +// char[3][4]'. The following specialization works around the bug. +// However, it causes trouble with GCC and thus needs to be +// conditionally compiled. +#if defined(_MSC_VER) || defined(__SUNPRO_CC) || defined(__IBMCPP__) +template +struct RemoveConst { + typedef typename RemoveConst::type type[N]; +}; +#endif + +// A handy wrapper around RemoveConst that works when the argument +// T depends on template parameters. +#define GTEST_REMOVE_CONST_(T) \ + typename ::testing::internal::RemoveConst::type + +// Turns const U&, U&, const U, and U all into U. +#define GTEST_REMOVE_REFERENCE_AND_CONST_(T) \ + GTEST_REMOVE_CONST_(GTEST_REMOVE_REFERENCE_(T)) + +// Adds reference to a type if it is not a reference type, +// otherwise leaves it unchanged. This is the same as +// tr1::add_reference, which is not widely available yet. +template +struct AddReference { typedef T& type; }; // NOLINT +template +struct AddReference { typedef T& type; }; // NOLINT + +// A handy wrapper around AddReference that works when the argument T +// depends on template parameters. +#define GTEST_ADD_REFERENCE_(T) \ + typename ::testing::internal::AddReference::type + +// Adds a reference to const on top of T as necessary. For example, +// it transforms +// +// char ==> const char& +// const char ==> const char& +// char& ==> const char& +// const char& ==> const char& +// +// The argument T must depend on some template parameters. +#define GTEST_REFERENCE_TO_CONST_(T) \ + GTEST_ADD_REFERENCE_(const GTEST_REMOVE_REFERENCE_(T)) + +// ImplicitlyConvertible::value is a compile-time bool +// constant that's true iff type From can be implicitly converted to +// type To. +template +class ImplicitlyConvertible { + private: + // We need the following helper functions only for their types. + // They have no implementations. + + // MakeFrom() is an expression whose type is From. We cannot simply + // use From(), as the type From may not have a public default + // constructor. + static From MakeFrom(); + + // These two functions are overloaded. Given an expression + // Helper(x), the compiler will pick the first version if x can be + // implicitly converted to type To; otherwise it will pick the + // second version. + // + // The first version returns a value of size 1, and the second + // version returns a value of size 2. Therefore, by checking the + // size of Helper(x), which can be done at compile time, we can tell + // which version of Helper() is used, and hence whether x can be + // implicitly converted to type To. + static char Helper(To); + static char (&Helper(...))[2]; // NOLINT + + // We have to put the 'public' section after the 'private' section, + // or MSVC refuses to compile the code. + public: + // MSVC warns about implicitly converting from double to int for + // possible loss of data, so we need to temporarily disable the + // warning. +#ifdef _MSC_VER +# pragma warning(push) // Saves the current warning state. +# pragma warning(disable:4244) // Temporarily disables warning 4244. + + static const bool value = + sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1; +# pragma warning(pop) // Restores the warning state. +#elif defined(__BORLANDC__) + // C++Builder cannot use member overload resolution during template + // instantiation. The simplest workaround is to use its C++0x type traits + // functions (C++Builder 2009 and above only). + static const bool value = __is_convertible(From, To); +#else + static const bool value = + sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1; +#endif // _MSV_VER +}; +template +const bool ImplicitlyConvertible::value; + +// IsAProtocolMessage::value is a compile-time bool constant that's +// true iff T is type ProtocolMessage, proto2::Message, or a subclass +// of those. +template +struct IsAProtocolMessage + : public bool_constant< + ImplicitlyConvertible::value || + ImplicitlyConvertible::value> { +}; + +// When the compiler sees expression IsContainerTest(0), if C is an +// STL-style container class, the first overload of IsContainerTest +// will be viable (since both C::iterator* and C::const_iterator* are +// valid types and NULL can be implicitly converted to them). It will +// be picked over the second overload as 'int' is a perfect match for +// the type of argument 0. If C::iterator or C::const_iterator is not +// a valid type, the first overload is not viable, and the second +// overload will be picked. Therefore, we can determine whether C is +// a container class by checking the type of IsContainerTest(0). +// The value of the expression is insignificant. +// +// Note that we look for both C::iterator and C::const_iterator. The +// reason is that C++ injects the name of a class as a member of the +// class itself (e.g. you can refer to class iterator as either +// 'iterator' or 'iterator::iterator'). If we look for C::iterator +// only, for example, we would mistakenly think that a class named +// iterator is an STL container. +// +// Also note that the simpler approach of overloading +// IsContainerTest(typename C::const_iterator*) and +// IsContainerTest(...) doesn't work with Visual Age C++ and Sun C++. +typedef int IsContainer; +template +IsContainer IsContainerTest(int /* dummy */, + typename C::iterator* /* it */ = NULL, + typename C::const_iterator* /* const_it */ = NULL) { + return 0; +} + +typedef char IsNotContainer; +template +IsNotContainer IsContainerTest(long /* dummy */) { return '\0'; } + +// EnableIf::type is void when 'Cond' is true, and +// undefined when 'Cond' is false. To use SFINAE to make a function +// overload only apply when a particular expression is true, add +// "typename EnableIf::type* = 0" as the last parameter. +template struct EnableIf; +template<> struct EnableIf { typedef void type; }; // NOLINT + +// Utilities for native arrays. + +// ArrayEq() compares two k-dimensional native arrays using the +// elements' operator==, where k can be any integer >= 0. When k is +// 0, ArrayEq() degenerates into comparing a single pair of values. + +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs); + +// This generic version is used when k is 0. +template +inline bool ArrayEq(const T& lhs, const U& rhs) { return lhs == rhs; } + +// This overload is used when k >= 1. +template +inline bool ArrayEq(const T(&lhs)[N], const U(&rhs)[N]) { + return internal::ArrayEq(lhs, N, rhs); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous ArrayEq() function, arrays with different sizes would +// lead to different copies of the template code. +template +bool ArrayEq(const T* lhs, size_t size, const U* rhs) { + for (size_t i = 0; i != size; i++) { + if (!internal::ArrayEq(lhs[i], rhs[i])) + return false; + } + return true; +} + +// Finds the first element in the iterator range [begin, end) that +// equals elem. Element may be a native array type itself. +template +Iter ArrayAwareFind(Iter begin, Iter end, const Element& elem) { + for (Iter it = begin; it != end; ++it) { + if (internal::ArrayEq(*it, elem)) + return it; + } + return end; +} + +// CopyArray() copies a k-dimensional native array using the elements' +// operator=, where k can be any integer >= 0. When k is 0, +// CopyArray() degenerates into copying a single value. + +template +void CopyArray(const T* from, size_t size, U* to); + +// This generic version is used when k is 0. +template +inline void CopyArray(const T& from, U* to) { *to = from; } + +// This overload is used when k >= 1. +template +inline void CopyArray(const T(&from)[N], U(*to)[N]) { + internal::CopyArray(from, N, *to); +} + +// This helper reduces code bloat. If we instead put its logic inside +// the previous CopyArray() function, arrays with different sizes +// would lead to different copies of the template code. +template +void CopyArray(const T* from, size_t size, U* to) { + for (size_t i = 0; i != size; i++) { + internal::CopyArray(from[i], to + i); + } +} + +// The relation between an NativeArray object (see below) and the +// native array it represents. +enum RelationToSource { + kReference, // The NativeArray references the native array. + kCopy // The NativeArray makes a copy of the native array and + // owns the copy. +}; + +// Adapts a native array to a read-only STL-style container. Instead +// of the complete STL container concept, this adaptor only implements +// members useful for Google Mock's container matchers. New members +// should be added as needed. To simplify the implementation, we only +// support Element being a raw type (i.e. having no top-level const or +// reference modifier). It's the client's responsibility to satisfy +// this requirement. Element can be an array type itself (hence +// multi-dimensional arrays are supported). +template +class NativeArray { + public: + // STL-style container typedefs. + typedef Element value_type; + typedef Element* iterator; + typedef const Element* const_iterator; + + // Constructs from a native array. + NativeArray(const Element* array, size_t count, RelationToSource relation) { + Init(array, count, relation); + } + + // Copy constructor. + NativeArray(const NativeArray& rhs) { + Init(rhs.array_, rhs.size_, rhs.relation_to_source_); + } + + ~NativeArray() { + // Ensures that the user doesn't instantiate NativeArray with a + // const or reference type. + static_cast(StaticAssertTypeEqHelper()); + if (relation_to_source_ == kCopy) + delete[] array_; + } + + // STL-style container methods. + size_t size() const { return size_; } + const_iterator begin() const { return array_; } + const_iterator end() const { return array_ + size_; } + bool operator==(const NativeArray& rhs) const { + return size() == rhs.size() && + ArrayEq(begin(), size(), rhs.begin()); + } + + private: + // Initializes this object; makes a copy of the input array if + // 'relation' is kCopy. + void Init(const Element* array, size_t a_size, RelationToSource relation) { + if (relation == kReference) { + array_ = array; + } else { + Element* const copy = new Element[a_size]; + CopyArray(array, a_size, copy); + array_ = copy; + } + size_ = a_size; + relation_to_source_ = relation; + } + + const Element* array_; + size_t size_; + RelationToSource relation_to_source_; + + GTEST_DISALLOW_ASSIGN_(NativeArray); +}; + +} // namespace internal +} // namespace testing + +#define GTEST_MESSAGE_AT_(file, line, message, result_type) \ + ::testing::internal::AssertHelper(result_type, file, line, message) \ + = ::testing::Message() + +#define GTEST_MESSAGE_(message, result_type) \ + GTEST_MESSAGE_AT_(__FILE__, __LINE__, message, result_type) + +#define GTEST_FATAL_FAILURE_(message) \ + return GTEST_MESSAGE_(message, ::testing::TestPartResult::kFatalFailure) + +#define GTEST_NONFATAL_FAILURE_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kNonFatalFailure) + +#define GTEST_SUCCESS_(message) \ + GTEST_MESSAGE_(message, ::testing::TestPartResult::kSuccess) + +// Suppresses MSVC warnings 4072 (unreachable code) for the code following +// statement if it returns or throws (or doesn't return or throw in some +// situations). +#define GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) \ + if (::testing::internal::AlwaysTrue()) { statement; } + +#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::ConstCharPtr gtest_msg = "") { \ + bool gtest_caught_expected = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (expected_exception const&) { \ + gtest_caught_expected = true; \ + } \ + catch (...) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws a different type."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + if (!gtest_caught_expected) { \ + gtest_msg.value = \ + "Expected: " #statement " throws an exception of type " \ + #expected_exception ".\n Actual: it throws nothing."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__): \ + fail(gtest_msg.value) + +#define GTEST_TEST_NO_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__): \ + fail("Expected: " #statement " doesn't throw an exception.\n" \ + " Actual: it throws.") + +#define GTEST_TEST_ANY_THROW_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + bool gtest_caught_any = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } \ + catch (...) { \ + gtest_caught_any = true; \ + } \ + if (!gtest_caught_any) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testanythrow_, __LINE__): \ + fail("Expected: " #statement " throws an exception.\n" \ + " Actual: it doesn't.") + + +// Implements Boolean test assertions such as EXPECT_TRUE. expression can be +// either a boolean expression or an AssertionResult. text is a textual +// represenation of expression as it was passed into the EXPECT_TRUE. +#define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar_ = \ + ::testing::AssertionResult(expression)) \ + ; \ + else \ + fail(::testing::internal::GetBoolAssertionFailureMessage(\ + gtest_ar_, text, #actual, #expected).c_str()) + +#define GTEST_TEST_NO_FATAL_FAILURE_(statement, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + ::testing::internal::HasNewFatalFailureHelper gtest_fatal_failure_checker; \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + if (gtest_fatal_failure_checker.has_new_fatal_failure()) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__); \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_testnofatal_, __LINE__): \ + fail("Expected: " #statement " doesn't generate new fatal " \ + "failures in the current thread.\n" \ + " Actual: it does.") + +// Expands to the name of the class that implements the given test. +#define GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + test_case_name##_##test_name##_Test + +// Helper macro for defining tests. +#define GTEST_TEST_(test_case_name, test_name, parent_class, parent_id)\ +class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) : public parent_class {\ + public:\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {}\ + private:\ + virtual void TestBody();\ + static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_;\ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name));\ +};\ +\ +::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_case_name, test_name)\ + ::test_info_ =\ + ::testing::internal::MakeAndRegisterTestInfo(\ + #test_case_name, #test_name, NULL, NULL, \ + (parent_id), \ + parent_class::SetUpTestCase, \ + parent_class::TearDownTestCase, \ + new ::testing::internal::TestFactoryImpl<\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>);\ +void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the public API for death tests. It is +// #included by gtest.h so a user doesn't need to include this +// directly. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ + +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: wan@google.com (Zhanyong Wan), eefacm@gmail.com (Sean Mcafee) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines internal utilities needed for implementing +// death tests. They are subject to change without notice. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ + + +#include + +namespace testing { +namespace internal { + +GTEST_DECLARE_string_(internal_run_death_test); + +// Names of the flags (needed for parsing Google Test flags). +const char kDeathTestStyleFlag[] = "death_test_style"; +const char kDeathTestUseFork[] = "death_test_use_fork"; +const char kInternalRunDeathTestFlag[] = "internal_run_death_test"; + +#if GTEST_HAS_DEATH_TEST + +// DeathTest is a class that hides much of the complexity of the +// GTEST_DEATH_TEST_ macro. It is abstract; its static Create method +// returns a concrete class that depends on the prevailing death test +// style, as defined by the --gtest_death_test_style and/or +// --gtest_internal_run_death_test flags. + +// In describing the results of death tests, these terms are used with +// the corresponding definitions: +// +// exit status: The integer exit information in the format specified +// by wait(2) +// exit code: The integer code passed to exit(3), _exit(2), or +// returned from main() +class GTEST_API_ DeathTest { + public: + // Create returns false if there was an error determining the + // appropriate action to take for the current death test; for example, + // if the gtest_death_test_style flag is set to an invalid value. + // The LastMessage method will return a more detailed message in that + // case. Otherwise, the DeathTest pointer pointed to by the "test" + // argument is set. If the death test should be skipped, the pointer + // is set to NULL; otherwise, it is set to the address of a new concrete + // DeathTest object that controls the execution of the current test. + static bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); + DeathTest(); + virtual ~DeathTest() { } + + // A helper class that aborts a death test when it's deleted. + class ReturnSentinel { + public: + explicit ReturnSentinel(DeathTest* test) : test_(test) { } + ~ReturnSentinel() { test_->Abort(TEST_ENCOUNTERED_RETURN_STATEMENT); } + private: + DeathTest* const test_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ReturnSentinel); + } GTEST_ATTRIBUTE_UNUSED_; + + // An enumeration of possible roles that may be taken when a death + // test is encountered. EXECUTE means that the death test logic should + // be executed immediately. OVERSEE means that the program should prepare + // the appropriate environment for a child process to execute the death + // test, then wait for it to complete. + enum TestRole { OVERSEE_TEST, EXECUTE_TEST }; + + // An enumeration of the three reasons that a test might be aborted. + enum AbortReason { + TEST_ENCOUNTERED_RETURN_STATEMENT, + TEST_THREW_EXCEPTION, + TEST_DID_NOT_DIE + }; + + // Assumes one of the above roles. + virtual TestRole AssumeRole() = 0; + + // Waits for the death test to finish and returns its status. + virtual int Wait() = 0; + + // Returns true if the death test passed; that is, the test process + // exited during the test, its exit status matches a user-supplied + // predicate, and its stderr output matches a user-supplied regular + // expression. + // The user-supplied predicate may be a macro expression rather + // than a function pointer or functor, or else Wait and Passed could + // be combined. + virtual bool Passed(bool exit_status_ok) = 0; + + // Signals that the death test did not die as expected. + virtual void Abort(AbortReason reason) = 0; + + // Returns a human-readable outcome message regarding the outcome of + // the last death test. + static const char* LastMessage(); + + static void set_last_death_test_message(const String& message); + + private: + // A string containing a description of the outcome of the last death test. + static String last_death_test_message_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(DeathTest); +}; + +// Factory interface for death tests. May be mocked out for testing. +class DeathTestFactory { + public: + virtual ~DeathTestFactory() { } + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test) = 0; +}; + +// A concrete DeathTestFactory implementation for normal use. +class DefaultDeathTestFactory : public DeathTestFactory { + public: + virtual bool Create(const char* statement, const RE* regex, + const char* file, int line, DeathTest** test); +}; + +// Returns true if exit_status describes a process that was terminated +// by a signal, or exited normally with a nonzero exit code. +GTEST_API_ bool ExitedUnsuccessfully(int exit_status); + +// Traps C++ exceptions escaping statement and reports them as test +// failures. Note that trapping SEH exceptions is not implemented here. +# if GTEST_HAS_EXCEPTIONS +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } catch (const ::std::exception& gtest_exception) { \ + fprintf(\ + stderr, \ + "\n%s: Caught std::exception-derived exception escaping the " \ + "death test statement. Exception message: %s\n", \ + ::testing::internal::FormatFileLocation(__FILE__, __LINE__).c_str(), \ + gtest_exception.what()); \ + fflush(stderr); \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } catch (...) { \ + death_test->Abort(::testing::internal::DeathTest::TEST_THREW_EXCEPTION); \ + } + +# else +# define GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, death_test) \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) + +# endif + +// This macro is for implementing ASSERT_DEATH*, EXPECT_DEATH*, +// ASSERT_EXIT*, and EXPECT_EXIT*. +# define GTEST_DEATH_TEST_(statement, predicate, regex, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + const ::testing::internal::RE& gtest_regex = (regex); \ + ::testing::internal::DeathTest* gtest_dt; \ + if (!::testing::internal::DeathTest::Create(#statement, >est_regex, \ + __FILE__, __LINE__, >est_dt)) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + if (gtest_dt != NULL) { \ + ::testing::internal::scoped_ptr< ::testing::internal::DeathTest> \ + gtest_dt_ptr(gtest_dt); \ + switch (gtest_dt->AssumeRole()) { \ + case ::testing::internal::DeathTest::OVERSEE_TEST: \ + if (!gtest_dt->Passed(predicate(gtest_dt->Wait()))) { \ + goto GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__); \ + } \ + break; \ + case ::testing::internal::DeathTest::EXECUTE_TEST: { \ + ::testing::internal::DeathTest::ReturnSentinel \ + gtest_sentinel(gtest_dt); \ + GTEST_EXECUTE_DEATH_TEST_STATEMENT_(statement, gtest_dt); \ + gtest_dt->Abort(::testing::internal::DeathTest::TEST_DID_NOT_DIE); \ + break; \ + } \ + default: \ + break; \ + } \ + } \ + } else \ + GTEST_CONCAT_TOKEN_(gtest_label_, __LINE__): \ + fail(::testing::internal::DeathTest::LastMessage()) +// The symbol "fail" here expands to something into which a message +// can be streamed. + +// A class representing the parsed contents of the +// --gtest_internal_run_death_test flag, as it existed when +// RUN_ALL_TESTS was called. +class InternalRunDeathTestFlag { + public: + InternalRunDeathTestFlag(const String& a_file, + int a_line, + int an_index, + int a_write_fd) + : file_(a_file), line_(a_line), index_(an_index), + write_fd_(a_write_fd) {} + + ~InternalRunDeathTestFlag() { + if (write_fd_ >= 0) + posix::Close(write_fd_); + } + + String file() const { return file_; } + int line() const { return line_; } + int index() const { return index_; } + int write_fd() const { return write_fd_; } + + private: + String file_; + int line_; + int index_; + int write_fd_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(InternalRunDeathTestFlag); +}; + +// Returns a newly created InternalRunDeathTestFlag object with fields +// initialized from the GTEST_FLAG(internal_run_death_test) flag if +// the flag is specified; otherwise returns NULL. +InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag(); + +#else // GTEST_HAS_DEATH_TEST + +// This macro is used for implementing macros such as +// EXPECT_DEATH_IF_SUPPORTED and ASSERT_DEATH_IF_SUPPORTED on systems where +// death tests are not supported. Those macros must compile on such systems +// iff EXPECT_DEATH and ASSERT_DEATH compile with the same parameters on +// systems that support death tests. This allows one to write such a macro +// on a system that does not support death tests and be sure that it will +// compile on a death-test supporting system. +// +// Parameters: +// statement - A statement that a macro such as EXPECT_DEATH would test +// for program termination. This macro has to make sure this +// statement is compiled but not executed, to ensure that +// EXPECT_DEATH_IF_SUPPORTED compiles with a certain +// parameter iff EXPECT_DEATH compiles with it. +// regex - A regex that a macro such as EXPECT_DEATH would use to test +// the output of statement. This parameter has to be +// compiled but not evaluated by this macro, to ensure that +// this macro only accepts expressions that a macro such as +// EXPECT_DEATH would accept. +// terminator - Must be an empty statement for EXPECT_DEATH_IF_SUPPORTED +// and a return statement for ASSERT_DEATH_IF_SUPPORTED. +// This ensures that ASSERT_DEATH_IF_SUPPORTED will not +// compile inside functions where ASSERT_DEATH doesn't +// compile. +// +// The branch that has an always false condition is used to ensure that +// statement and regex are compiled (and thus syntactically correct) but +// never executed. The unreachable code macro protects the terminator +// statement from generating an 'unreachable code' warning in case +// statement unconditionally returns or throws. The Message constructor at +// the end allows the syntax of streaming additional messages into the +// macro, for compilational compatibility with EXPECT_DEATH/ASSERT_DEATH. +# define GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, terminator) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_LOG_(WARNING) \ + << "Death tests are not supported on this platform.\n" \ + << "Statement '" #statement "' cannot be verified."; \ + } else if (::testing::internal::AlwaysFalse()) { \ + ::testing::internal::RE::PartialMatch(".*", (regex)); \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + terminator; \ + } else \ + ::testing::Message() + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ + +namespace testing { + +// This flag controls the style of death tests. Valid values are "threadsafe", +// meaning that the death test child process will re-execute the test binary +// from the start, running only a single death test, or "fast", +// meaning that the child process will execute the test logic immediately +// after forking. +GTEST_DECLARE_string_(death_test_style); + +#if GTEST_HAS_DEATH_TEST + +// The following macros are useful for writing death tests. + +// Here's what happens when an ASSERT_DEATH* or EXPECT_DEATH* is +// executed: +// +// 1. It generates a warning if there is more than one active +// thread. This is because it's safe to fork() or clone() only +// when there is a single thread. +// +// 2. The parent process clone()s a sub-process and runs the death +// test in it; the sub-process exits with code 0 at the end of the +// death test, if it hasn't exited already. +// +// 3. The parent process waits for the sub-process to terminate. +// +// 4. The parent process checks the exit code and error message of +// the sub-process. +// +// Examples: +// +// ASSERT_DEATH(server.SendMessage(56, "Hello"), "Invalid port number"); +// for (int i = 0; i < 5; i++) { +// EXPECT_DEATH(server.ProcessRequest(i), +// "Invalid request .* in ProcessRequest()") +// << "Failed to die on request " << i); +// } +// +// ASSERT_EXIT(server.ExitNow(), ::testing::ExitedWithCode(0), "Exiting"); +// +// bool KilledBySIGHUP(int exit_code) { +// return WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGHUP; +// } +// +// ASSERT_EXIT(client.HangUpServer(), KilledBySIGHUP, "Hanging up!"); +// +// On the regular expressions used in death tests: +// +// On POSIX-compliant systems (*nix), we use the library, +// which uses the POSIX extended regex syntax. +// +// On other platforms (e.g. Windows), we only support a simple regex +// syntax implemented as part of Google Test. This limited +// implementation should be enough most of the time when writing +// death tests; though it lacks many features you can find in PCRE +// or POSIX extended regex syntax. For example, we don't support +// union ("x|y"), grouping ("(xy)"), brackets ("[xy]"), and +// repetition count ("x{5,7}"), among others. +// +// Below is the syntax that we do support. We chose it to be a +// subset of both PCRE and POSIX extended regex, so it's easy to +// learn wherever you come from. In the following: 'A' denotes a +// literal character, period (.), or a single \\ escape sequence; +// 'x' and 'y' denote regular expressions; 'm' and 'n' are for +// natural numbers. +// +// c matches any literal character c +// \\d matches any decimal digit +// \\D matches any character that's not a decimal digit +// \\f matches \f +// \\n matches \n +// \\r matches \r +// \\s matches any ASCII whitespace, including \n +// \\S matches any character that's not a whitespace +// \\t matches \t +// \\v matches \v +// \\w matches any letter, _, or decimal digit +// \\W matches any character that \\w doesn't match +// \\c matches any literal character c, which must be a punctuation +// . matches any single character except \n +// A? matches 0 or 1 occurrences of A +// A* matches 0 or many occurrences of A +// A+ matches 1 or many occurrences of A +// ^ matches the beginning of a string (not that of each line) +// $ matches the end of a string (not that of each line) +// xy matches x followed by y +// +// If you accidentally use PCRE or POSIX extended regex features +// not implemented by us, you will get a run-time failure. In that +// case, please try to rewrite your regular expression within the +// above syntax. +// +// This implementation is *not* meant to be as highly tuned or robust +// as a compiled regex library, but should perform well enough for a +// death test, which already incurs significant overhead by launching +// a child process. +// +// Known caveats: +// +// A "threadsafe" style death test obtains the path to the test +// program from argv[0] and re-executes it in the sub-process. For +// simplicity, the current implementation doesn't search the PATH +// when launching the sub-process. This means that the user must +// invoke the test program via a path that contains at least one +// path separator (e.g. path/to/foo_test and +// /absolute/path/to/bar_test are fine, but foo_test is not). This +// is rarely a problem as people usually don't put the test binary +// directory in PATH. +// +// TODO(wan@google.com): make thread-safe death tests search the PATH. + +// Asserts that a given statement causes the program to exit, with an +// integer exit status that satisfies predicate, and emitting error output +// that matches regex. +# define ASSERT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_FATAL_FAILURE_) + +// Like ASSERT_EXIT, but continues on to successive tests in the +// test case, if any: +# define EXPECT_EXIT(statement, predicate, regex) \ + GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_NONFATAL_FAILURE_) + +// Asserts that a given statement causes the program to exit, either by +// explicitly exiting with a nonzero exit code or being killed by a +// signal, and emitting error output that matches regex. +# define ASSERT_DEATH(statement, regex) \ + ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Like ASSERT_DEATH, but continues on to successive tests in the +// test case, if any: +# define EXPECT_DEATH(statement, regex) \ + EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) + +// Two predicate classes that can be used in {ASSERT,EXPECT}_EXIT*: + +// Tests that an exit code describes a normal exit with a given exit code. +class GTEST_API_ ExitedWithCode { + public: + explicit ExitedWithCode(int exit_code); + bool operator()(int exit_status) const; + private: + // No implementation - assignment is unsupported. + void operator=(const ExitedWithCode& other); + + const int exit_code_; +}; + +# if !GTEST_OS_WINDOWS +// Tests that an exit code describes an exit due to termination by a +// given signal. +class GTEST_API_ KilledBySignal { + public: + explicit KilledBySignal(int signum); + bool operator()(int exit_status) const; + private: + const int signum_; +}; +# endif // !GTEST_OS_WINDOWS + +// EXPECT_DEBUG_DEATH asserts that the given statements die in debug mode. +// The death testing framework causes this to have interesting semantics, +// since the sideeffects of the call are only visible in opt mode, and not +// in debug mode. +// +// In practice, this can be used to test functions that utilize the +// LOG(DFATAL) macro using the following style: +// +// int DieInDebugOr12(int* sideeffect) { +// if (sideeffect) { +// *sideeffect = 12; +// } +// LOG(DFATAL) << "death"; +// return 12; +// } +// +// TEST(TestCase, TestDieOr12WorksInDgbAndOpt) { +// int sideeffect = 0; +// // Only asserts in dbg. +// EXPECT_DEBUG_DEATH(DieInDebugOr12(&sideeffect), "death"); +// +// #ifdef NDEBUG +// // opt-mode has sideeffect visible. +// EXPECT_EQ(12, sideeffect); +// #else +// // dbg-mode no visible sideeffect. +// EXPECT_EQ(0, sideeffect); +// #endif +// } +// +// This will assert that DieInDebugReturn12InOpt() crashes in debug +// mode, usually due to a DCHECK or LOG(DFATAL), but returns the +// appropriate fallback value (12 in this case) in opt mode. If you +// need to test that a function has appropriate side-effects in opt +// mode, include assertions against the side-effects. A general +// pattern for this is: +// +// EXPECT_DEBUG_DEATH({ +// // Side-effects here will have an effect after this statement in +// // opt mode, but none in debug mode. +// EXPECT_EQ(12, DieInDebugOr12(&sideeffect)); +// }, "death"); +// +# ifdef NDEBUG + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + do { statement; } while (::testing::internal::AlwaysFalse()) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + do { statement; } while (::testing::internal::AlwaysFalse()) + +# else + +# define EXPECT_DEBUG_DEATH(statement, regex) \ + EXPECT_DEATH(statement, regex) + +# define ASSERT_DEBUG_DEATH(statement, regex) \ + ASSERT_DEATH(statement, regex) + +# endif // NDEBUG for EXPECT_DEBUG_DEATH +#endif // GTEST_HAS_DEATH_TEST + +// EXPECT_DEATH_IF_SUPPORTED(statement, regex) and +// ASSERT_DEATH_IF_SUPPORTED(statement, regex) expand to real death tests if +// death tests are supported; otherwise they just issue a warning. This is +// useful when you are combining death test assertions with normal test +// assertions in one test. +#if GTEST_HAS_DEATH_TEST +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH(statement, regex) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + ASSERT_DEATH(statement, regex) +#else +# define EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, ) +# define ASSERT_DEATH_IF_SUPPORTED(statement, regex) \ + GTEST_UNSUPPORTED_DEATH_TEST_(statement, regex, return) +#endif + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +// Copyright 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// The Google C++ Testing Framework (Google Test) +// +// This header file defines the Message class. +// +// IMPORTANT NOTE: Due to limitation of the C++ language, we have to +// leave some internal implementation details in this header file. +// They are clearly marked by comments like this: +// +// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +// +// Such code is NOT meant to be used by a user directly, and is subject +// to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user +// program! + +#ifndef GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#define GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ + +#include + + +namespace testing { + +// The Message class works like an ostream repeater. +// +// Typical usage: +// +// 1. You stream a bunch of values to a Message object. +// It will remember the text in a stringstream. +// 2. Then you stream the Message object to an ostream. +// This causes the text in the Message to be streamed +// to the ostream. +// +// For example; +// +// testing::Message foo; +// foo << 1 << " != " << 2; +// std::cout << foo; +// +// will print "1 != 2". +// +// Message is not intended to be inherited from. In particular, its +// destructor is not virtual. +// +// Note that stringstream behaves differently in gcc and in MSVC. You +// can stream a NULL char pointer to it in the former, but not in the +// latter (it causes an access violation if you do). The Message +// class hides this difference by treating a NULL char pointer as +// "(null)". +class GTEST_API_ Message { + private: + // The type of basic IO manipulators (endl, ends, and flush) for + // narrow streams. + typedef std::ostream& (*BasicNarrowIoManip)(std::ostream&); + + public: + // Constructs an empty Message. + // We allocate the stringstream separately because otherwise each use of + // ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's + // stack frame leading to huge stack frames in some cases; gcc does not reuse + // the stack space. + Message() : ss_(new ::std::stringstream) { + // By default, we want there to be enough precision when printing + // a double to a Message. + *ss_ << std::setprecision(std::numeric_limits::digits10 + 2); + } + + // Copy constructor. + Message(const Message& msg) : ss_(new ::std::stringstream) { // NOLINT + *ss_ << msg.GetString(); + } + + // Constructs a Message from a C-string. + explicit Message(const char* str) : ss_(new ::std::stringstream) { + *ss_ << str; + } + +#if GTEST_OS_SYMBIAN + // Streams a value (either a pointer or not) to this object. + template + inline Message& operator <<(const T& value) { + StreamHelper(typename internal::is_pointer::type(), value); + return *this; + } +#else + // Streams a non-pointer value to this object. + template + inline Message& operator <<(const T& val) { + ::GTestStreamToHelper(ss_.get(), val); + return *this; + } + + // Streams a pointer value to this object. + // + // This function is an overload of the previous one. When you + // stream a pointer to a Message, this definition will be used as it + // is more specialized. (The C++ Standard, section + // [temp.func.order].) If you stream a non-pointer, then the + // previous definition will be used. + // + // The reason for this overload is that streaming a NULL pointer to + // ostream is undefined behavior. Depending on the compiler, you + // may get "0", "(nil)", "(null)", or an access violation. To + // ensure consistent result across compilers, we always treat NULL + // as "(null)". + template + inline Message& operator <<(T* const& pointer) { // NOLINT + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + ::GTestStreamToHelper(ss_.get(), pointer); + } + return *this; + } +#endif // GTEST_OS_SYMBIAN + + // Since the basic IO manipulators are overloaded for both narrow + // and wide streams, we have to provide this specialized definition + // of operator <<, even though its body is the same as the + // templatized version above. Without this definition, streaming + // endl or other basic IO manipulators to Message will confuse the + // compiler. + Message& operator <<(BasicNarrowIoManip val) { + *ss_ << val; + return *this; + } + + // Instead of 1/0, we want to see true/false for bool values. + Message& operator <<(bool b) { + return *this << (b ? "true" : "false"); + } + + // These two overloads allow streaming a wide C string to a Message + // using the UTF-8 encoding. + Message& operator <<(const wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); + } + Message& operator <<(wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); + } + +#if GTEST_HAS_STD_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::std::wstring& wstr); +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_GLOBAL_WSTRING + // Converts the given wide string to a narrow string using the UTF-8 + // encoding, and streams the result to this Message object. + Message& operator <<(const ::wstring& wstr); +#endif // GTEST_HAS_GLOBAL_WSTRING + + // Gets the text streamed to this object so far as a String. + // Each '\0' character in the buffer is replaced with "\\0". + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + internal::String GetString() const { + return internal::StringStreamToString(ss_.get()); + } + + private: + +#if GTEST_OS_SYMBIAN + // These are needed as the Nokia Symbian Compiler cannot decide between + // const T& and const T* in a function template. The Nokia compiler _can_ + // decide between class template specializations for T and T*, so a + // tr1::type_traits-like is_pointer works, and we can overload on that. + template + inline void StreamHelper(internal::true_type /*dummy*/, T* pointer) { + if (pointer == NULL) { + *ss_ << "(null)"; + } else { + ::GTestStreamToHelper(ss_.get(), pointer); + } + } + template + inline void StreamHelper(internal::false_type /*dummy*/, const T& value) { + ::GTestStreamToHelper(ss_.get(), value); + } +#endif // GTEST_OS_SYMBIAN + + // We'll hold the text streamed to this object here. + const internal::scoped_ptr< ::std::stringstream> ss_; + + // We declare (but don't implement) this to prevent the compiler + // from implementing the assignment operator. + void operator=(const Message&); +}; + +// Streams a Message to an ostream. +inline std::ostream& operator <<(std::ostream& os, const Message& sb) { + return os << sb.GetString(); +} + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +// This file was GENERATED by command: +// pump.py gtest-param-test.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: vladl@google.com (Vlad Losev) +// +// Macros and functions for implementing parameterized tests +// in Google C++ Testing Framework (Google Test) +// +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ + + +// Value-parameterized tests allow you to test your code with different +// parameters without writing multiple copies of the same test. +// +// Here is how you use value-parameterized tests: + +#if 0 + +// To write value-parameterized tests, first you should define a fixture +// class. It is usually derived from testing::TestWithParam (see below for +// another inheritance scheme that's sometimes useful in more complicated +// class hierarchies), where the type of your parameter values. +// TestWithParam is itself derived from testing::Test. T can be any +// copyable type. If it's a raw pointer, you are responsible for managing the +// lifespan of the pointed values. + +class FooTest : public ::testing::TestWithParam { + // You can implement all the usual class fixture members here. +}; + +// Then, use the TEST_P macro to define as many parameterized tests +// for this fixture as you want. The _P suffix is for "parameterized" +// or "pattern", whichever you prefer to think. + +TEST_P(FooTest, DoesBlah) { + // Inside a test, access the test parameter with the GetParam() method + // of the TestWithParam class: + EXPECT_TRUE(foo.Blah(GetParam())); + ... +} + +TEST_P(FooTest, HasBlahBlah) { + ... +} + +// Finally, you can use INSTANTIATE_TEST_CASE_P to instantiate the test +// case with any set of parameters you want. Google Test defines a number +// of functions for generating test parameters. They return what we call +// (surprise!) parameter generators. Here is a summary of them, which +// are all in the testing namespace: +// +// +// Range(begin, end [, step]) - Yields values {begin, begin+step, +// begin+step+step, ...}. The values do not +// include end. step defaults to 1. +// Values(v1, v2, ..., vN) - Yields values {v1, v2, ..., vN}. +// ValuesIn(container) - Yields values from a C-style array, an STL +// ValuesIn(begin,end) container, or an iterator range [begin, end). +// Bool() - Yields sequence {false, true}. +// Combine(g1, g2, ..., gN) - Yields all combinations (the Cartesian product +// for the math savvy) of the values generated +// by the N generators. +// +// For more details, see comments at the definitions of these functions below +// in this file. +// +// The following statement will instantiate tests from the FooTest test case +// each with parameter values "meeny", "miny", and "moe". + +INSTANTIATE_TEST_CASE_P(InstantiationName, + FooTest, + Values("meeny", "miny", "moe")); + +// To distinguish different instances of the pattern, (yes, you +// can instantiate it more then once) the first argument to the +// INSTANTIATE_TEST_CASE_P macro is a prefix that will be added to the +// actual test case name. Remember to pick unique prefixes for different +// instantiations. The tests from the instantiation above will have +// these names: +// +// * InstantiationName/FooTest.DoesBlah/0 for "meeny" +// * InstantiationName/FooTest.DoesBlah/1 for "miny" +// * InstantiationName/FooTest.DoesBlah/2 for "moe" +// * InstantiationName/FooTest.HasBlahBlah/0 for "meeny" +// * InstantiationName/FooTest.HasBlahBlah/1 for "miny" +// * InstantiationName/FooTest.HasBlahBlah/2 for "moe" +// +// You can use these names in --gtest_filter. +// +// This statement will instantiate all tests from FooTest again, each +// with parameter values "cat" and "dog": + +const char* pets[] = {"cat", "dog"}; +INSTANTIATE_TEST_CASE_P(AnotherInstantiationName, FooTest, ValuesIn(pets)); + +// The tests from the instantiation above will have these names: +// +// * AnotherInstantiationName/FooTest.DoesBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.DoesBlah/1 for "dog" +// * AnotherInstantiationName/FooTest.HasBlahBlah/0 for "cat" +// * AnotherInstantiationName/FooTest.HasBlahBlah/1 for "dog" +// +// Please note that INSTANTIATE_TEST_CASE_P will instantiate all tests +// in the given test case, whether their definitions come before or +// AFTER the INSTANTIATE_TEST_CASE_P statement. +// +// Please also note that generator expressions (including parameters to the +// generators) are evaluated in InitGoogleTest(), after main() has started. +// This allows the user on one hand, to adjust generator parameters in order +// to dynamically determine a set of tests to run and on the other hand, +// give the user a chance to inspect the generated tests with Google Test +// reflection API before RUN_ALL_TESTS() is executed. +// +// You can see samples/sample7_unittest.cc and samples/sample8_unittest.cc +// for more examples. +// +// In the future, we plan to publish the API for defining new parameter +// generators. But for now this interface remains part of the internal +// implementation and is subject to change. +// +// +// A parameterized test fixture must be derived from testing::Test and from +// testing::WithParamInterface, where T is the type of the parameter +// values. Inheriting from TestWithParam satisfies that requirement because +// TestWithParam inherits from both Test and WithParamInterface. In more +// complicated hierarchies, however, it is occasionally useful to inherit +// separately from Test and WithParamInterface. For example: + +class BaseTest : public ::testing::Test { + // You can inherit all the usual members for a non-parameterized test + // fixture here. +}; + +class DerivedTest : public BaseTest, public ::testing::WithParamInterface { + // The usual test fixture members go here too. +}; + +TEST_F(BaseTest, HasFoo) { + // This is an ordinary non-parameterized test. +} + +TEST_P(DerivedTest, DoesBlah) { + // GetParam works just the same here as if you inherit from TestWithParam. + EXPECT_TRUE(foo.Blah(GetParam())); +} + +#endif // 0 + + +#if !GTEST_OS_SYMBIAN +# include +#endif + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ + +#include +#include +#include + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. +// Copyright 2003 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Authors: Dan Egnor (egnor@google.com) +// +// A "smart" pointer type with reference tracking. Every pointer to a +// particular object is kept on a circular linked list. When the last pointer +// to an object is destroyed or reassigned, the object is deleted. +// +// Used properly, this deletes the object when the last reference goes away. +// There are several caveats: +// - Like all reference counting schemes, cycles lead to leaks. +// - Each smart pointer is actually two pointers (8 bytes instead of 4). +// - Every time a pointer is assigned, the entire list of pointers to that +// object is traversed. This class is therefore NOT SUITABLE when there +// will often be more than two or three pointers to a particular object. +// - References are only tracked as long as linked_ptr<> objects are copied. +// If a linked_ptr<> is converted to a raw pointer and back, BAD THINGS +// will happen (double deletion). +// +// A good use of this class is storing object references in STL containers. +// You can safely put linked_ptr<> in a vector<>. +// Other uses may not be as good. +// +// Note: If you use an incomplete type with linked_ptr<>, the class +// *containing* linked_ptr<> must have a constructor and destructor (even +// if they do nothing!). +// +// Bill Gibbons suggested we use something like this. +// +// Thread Safety: +// Unlike other linked_ptr implementations, in this implementation +// a linked_ptr object is thread-safe in the sense that: +// - it's safe to copy linked_ptr objects concurrently, +// - it's safe to copy *from* a linked_ptr and read its underlying +// raw pointer (e.g. via get()) concurrently, and +// - it's safe to write to two linked_ptrs that point to the same +// shared object concurrently. +// TODO(wan@google.com): rename this to safe_linked_ptr to avoid +// confusion with normal linked_ptr. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ + +#include +#include + + +namespace testing { +namespace internal { + +// Protects copying of all linked_ptr objects. +GTEST_API_ GTEST_DECLARE_STATIC_MUTEX_(g_linked_ptr_mutex); + +// This is used internally by all instances of linked_ptr<>. It needs to be +// a non-template class because different types of linked_ptr<> can refer to +// the same object (linked_ptr(obj) vs linked_ptr(obj)). +// So, it needs to be possible for different types of linked_ptr to participate +// in the same circular linked list, so we need a single class type here. +// +// DO NOT USE THIS CLASS DIRECTLY YOURSELF. Use linked_ptr. +class linked_ptr_internal { + public: + // Create a new circle that includes only this instance. + void join_new() { + next_ = this; + } + + // Many linked_ptr operations may change p.link_ for some linked_ptr + // variable p in the same circle as this object. Therefore we need + // to prevent two such operations from occurring concurrently. + // + // Note that different types of linked_ptr objects can coexist in a + // circle (e.g. linked_ptr, linked_ptr, and + // linked_ptr). Therefore we must use a single mutex to + // protect all linked_ptr objects. This can create serious + // contention in production code, but is acceptable in a testing + // framework. + + // Join an existing circle. + // L < g_linked_ptr_mutex + void join(linked_ptr_internal const* ptr) { + MutexLock lock(&g_linked_ptr_mutex); + + linked_ptr_internal const* p = ptr; + while (p->next_ != ptr) p = p->next_; + p->next_ = this; + next_ = ptr; + } + + // Leave whatever circle we're part of. Returns true if we were the + // last member of the circle. Once this is done, you can join() another. + // L < g_linked_ptr_mutex + bool depart() { + MutexLock lock(&g_linked_ptr_mutex); + + if (next_ == this) return true; + linked_ptr_internal const* p = next_; + while (p->next_ != this) p = p->next_; + p->next_ = next_; + return false; + } + + private: + mutable linked_ptr_internal const* next_; +}; + +template +class linked_ptr { + public: + typedef T element_type; + + // Take over ownership of a raw pointer. This should happen as soon as + // possible after the object is created. + explicit linked_ptr(T* ptr = NULL) { capture(ptr); } + ~linked_ptr() { depart(); } + + // Copy an existing linked_ptr<>, adding ourselves to the list of references. + template linked_ptr(linked_ptr const& ptr) { copy(&ptr); } + linked_ptr(linked_ptr const& ptr) { // NOLINT + assert(&ptr != this); + copy(&ptr); + } + + // Assignment releases the old value and acquires the new. + template linked_ptr& operator=(linked_ptr const& ptr) { + depart(); + copy(&ptr); + return *this; + } + + linked_ptr& operator=(linked_ptr const& ptr) { + if (&ptr != this) { + depart(); + copy(&ptr); + } + return *this; + } + + // Smart pointer members. + void reset(T* ptr = NULL) { + depart(); + capture(ptr); + } + T* get() const { return value_; } + T* operator->() const { return value_; } + T& operator*() const { return *value_; } + + bool operator==(T* p) const { return value_ == p; } + bool operator!=(T* p) const { return value_ != p; } + template + bool operator==(linked_ptr const& ptr) const { + return value_ == ptr.get(); + } + template + bool operator!=(linked_ptr const& ptr) const { + return value_ != ptr.get(); + } + + private: + template + friend class linked_ptr; + + T* value_; + linked_ptr_internal link_; + + void depart() { + if (link_.depart()) delete value_; + } + + void capture(T* ptr) { + value_ = ptr; + link_.join_new(); + } + + template void copy(linked_ptr const* ptr) { + value_ = ptr->get(); + if (value_) + link_.join(&ptr->link_); + else + link_.join_new(); + } +}; + +template inline +bool operator==(T* ptr, const linked_ptr& x) { + return ptr == x.get(); +} + +template inline +bool operator!=(T* ptr, const linked_ptr& x) { + return ptr != x.get(); +} + +// A function to convert T* into linked_ptr +// Doing e.g. make_linked_ptr(new FooBarBaz(arg)) is a shorter notation +// for linked_ptr >(new FooBarBaz(arg)) +template +linked_ptr make_linked_ptr(T* ptr) { + return linked_ptr(ptr); +} + +} // namespace internal +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_LINKED_PTR_H_ +// Copyright 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +// Google Test - The Google C++ Testing Framework +// +// This file implements a universal value printer that can print a +// value of any type T: +// +// void ::testing::internal::UniversalPrinter::Print(value, ostream_ptr); +// +// A user can teach this function how to print a class type T by +// defining either operator<<() or PrintTo() in the namespace that +// defines T. More specifically, the FIRST defined function in the +// following list will be used (assuming T is defined in namespace +// foo): +// +// 1. foo::PrintTo(const T&, ostream*) +// 2. operator<<(ostream&, const T&) defined in either foo or the +// global namespace. +// +// If none of the above is defined, it will print the debug string of +// the value if it is a protocol buffer, or print the raw bytes in the +// value otherwise. +// +// To aid debugging: when T is a reference type, the address of the +// value is also printed; when T is a (const) char pointer, both the +// pointer value and the NUL-terminated string it points to are +// printed. +// +// We also provide some convenient wrappers: +// +// // Prints a value to a string. For a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// std::string ::testing::PrintToString(const T& value); +// +// // Prints a value tersely: for a reference type, the referenced +// // value (but not the address) is printed; for a (const or not) char +// // pointer, the NUL-terminated string (but not the pointer) is +// // printed. +// void ::testing::internal::UniversalTersePrint(const T& value, ostream*); +// +// // Prints value using the type inferred by the compiler. The difference +// // from UniversalTersePrint() is that this function prints both the +// // pointer and the NUL-terminated string for a (const or not) char pointer. +// void ::testing::internal::UniversalPrint(const T& value, ostream*); +// +// // Prints the fields of a tuple tersely to a string vector, one +// // element for each field. Tuple support must be enabled in +// // gtest-port.h. +// std::vector UniversalTersePrintTupleFieldsToStrings( +// const Tuple& value); +// +// Known limitation: +// +// The print primitives print the elements of an STL-style container +// using the compiler-inferred type of *iter where iter is a +// const_iterator of the container. When const_iterator is an input +// iterator but not a forward iterator, this inferred type may not +// match value_type, and the print output may be incorrect. In +// practice, this is rarely a problem as for most containers +// const_iterator is a forward iterator. We'll fix this if there's an +// actual need for it. Note that this fix cannot rely on value_type +// being defined as many user-defined container types don't have +// value_type. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ + +#include // NOLINT +#include +#include +#include +#include + +namespace testing { + +// Definitions in the 'internal' and 'internal2' name spaces are +// subject to change without notice. DO NOT USE THEM IN USER CODE! +namespace internal2 { + +// Prints the given number of bytes in the given object to the given +// ostream. +GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, + size_t count, + ::std::ostream* os); + +// For selecting which printer to use when a given type has neither << +// nor PrintTo(). +enum TypeKind { + kProtobuf, // a protobuf type + kConvertibleToInteger, // a type implicitly convertible to BiggestInt + // (e.g. a named or unnamed enum type) + kOtherType // anything else +}; + +// TypeWithoutFormatter::PrintValue(value, os) is called +// by the universal printer to print a value of type T when neither +// operator<< nor PrintTo() is defined for T, where kTypeKind is the +// "kind" of T as defined by enum TypeKind. +template +class TypeWithoutFormatter { + public: + // This default version is called when kTypeKind is kOtherType. + static void PrintValue(const T& value, ::std::ostream* os) { + PrintBytesInObjectTo(reinterpret_cast(&value), + sizeof(value), os); + } +}; + +// We print a protobuf using its ShortDebugString() when the string +// doesn't exceed this many characters; otherwise we print it using +// DebugString() for better readability. +const size_t kProtobufOneLinerMaxLength = 50; + +template +class TypeWithoutFormatter { + public: + static void PrintValue(const T& value, ::std::ostream* os) { + const ::testing::internal::string short_str = value.ShortDebugString(); + const ::testing::internal::string pretty_str = + short_str.length() <= kProtobufOneLinerMaxLength ? + short_str : ("\n" + value.DebugString()); + *os << ("<" + pretty_str + ">"); + } +}; + +template +class TypeWithoutFormatter { + public: + // Since T has no << operator or PrintTo() but can be implicitly + // converted to BiggestInt, we print it as a BiggestInt. + // + // Most likely T is an enum type (either named or unnamed), in which + // case printing it as an integer is the desired behavior. In case + // T is not an enum, printing it as an integer is the best we can do + // given that it has no user-defined printer. + static void PrintValue(const T& value, ::std::ostream* os) { + const internal::BiggestInt kBigInt = value; + *os << kBigInt; + } +}; + +// Prints the given value to the given ostream. If the value is a +// protocol message, its debug string is printed; if it's an enum or +// of a type implicitly convertible to BiggestInt, it's printed as an +// integer; otherwise the bytes in the value are printed. This is +// what UniversalPrinter::Print() does when it knows nothing about +// type T and T has neither << operator nor PrintTo(). +// +// A user can override this behavior for a class type Foo by defining +// a << operator in the namespace where Foo is defined. +// +// We put this operator in namespace 'internal2' instead of 'internal' +// to simplify the implementation, as much code in 'internal' needs to +// use << in STL, which would conflict with our own << were it defined +// in 'internal'. +// +// Note that this operator<< takes a generic std::basic_ostream type instead of the more restricted std::ostream. If +// we define it to take an std::ostream instead, we'll get an +// "ambiguous overloads" compiler error when trying to print a type +// Foo that supports streaming to std::basic_ostream, as the compiler cannot tell whether +// operator<<(std::ostream&, const T&) or +// operator<<(std::basic_stream, const Foo&) is more +// specific. +template +::std::basic_ostream& operator<<( + ::std::basic_ostream& os, const T& x) { + TypeWithoutFormatter::value ? kProtobuf : + internal::ImplicitlyConvertible::value ? + kConvertibleToInteger : kOtherType)>::PrintValue(x, &os); + return os; +} + +} // namespace internal2 +} // namespace testing + +// This namespace MUST NOT BE NESTED IN ::testing, or the name look-up +// magic needed for implementing UniversalPrinter won't work. +namespace testing_internal { + +// Used to print a value that is not an STL-style container when the +// user doesn't define PrintTo() for it. +template +void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) { + // With the following statement, during unqualified name lookup, + // testing::internal2::operator<< appears as if it was declared in + // the nearest enclosing namespace that contains both + // ::testing_internal and ::testing::internal2, i.e. the global + // namespace. For more details, refer to the C++ Standard section + // 7.3.4-1 [namespace.udir]. This allows us to fall back onto + // testing::internal2::operator<< in case T doesn't come with a << + // operator. + // + // We cannot write 'using ::testing::internal2::operator<<;', which + // gcc 3.3 fails to compile due to a compiler bug. + using namespace ::testing::internal2; // NOLINT + + // Assuming T is defined in namespace foo, in the next statement, + // the compiler will consider all of: + // + // 1. foo::operator<< (thanks to Koenig look-up), + // 2. ::operator<< (as the current namespace is enclosed in ::), + // 3. testing::internal2::operator<< (thanks to the using statement above). + // + // The operator<< whose type matches T best will be picked. + // + // We deliberately allow #2 to be a candidate, as sometimes it's + // impossible to define #1 (e.g. when foo is ::std, defining + // anything in it is undefined behavior unless you are a compiler + // vendor.). + *os << value; +} + +} // namespace testing_internal + +namespace testing { +namespace internal { + +// UniversalPrinter::Print(value, ostream_ptr) prints the given +// value to the given ostream. The caller must ensure that +// 'ostream_ptr' is not NULL, or the behavior is undefined. +// +// We define UniversalPrinter as a class template (as opposed to a +// function template), as we need to partially specialize it for +// reference types, which cannot be done with function templates. +template +class UniversalPrinter; + +template +void UniversalPrint(const T& value, ::std::ostream* os); + +// Used to print an STL-style container when the user doesn't define +// a PrintTo() for it. +template +void DefaultPrintTo(IsContainer /* dummy */, + false_type /* is not a pointer */, + const C& container, ::std::ostream* os) { + const size_t kMaxCount = 32; // The maximum number of elements to print. + *os << '{'; + size_t count = 0; + for (typename C::const_iterator it = container.begin(); + it != container.end(); ++it, ++count) { + if (count > 0) { + *os << ','; + if (count == kMaxCount) { // Enough has been printed. + *os << " ..."; + break; + } + } + *os << ' '; + // We cannot call PrintTo(*it, os) here as PrintTo() doesn't + // handle *it being a native array. + internal::UniversalPrint(*it, os); + } + + if (count > 0) { + *os << ' '; + } + *os << '}'; +} + +// Used to print a pointer that is neither a char pointer nor a member +// pointer, when the user doesn't define PrintTo() for it. (A member +// variable pointer or member function pointer doesn't really point to +// a location in the address space. Their representation is +// implementation-defined. Therefore they will be printed as raw +// bytes.) +template +void DefaultPrintTo(IsNotContainer /* dummy */, + true_type /* is a pointer */, + T* p, ::std::ostream* os) { + if (p == NULL) { + *os << "NULL"; + } else { + // C++ doesn't allow casting from a function pointer to any object + // pointer. + // + // IsTrue() silences warnings: "Condition is always true", + // "unreachable code". + if (IsTrue(ImplicitlyConvertible::value)) { + // T is not a function type. We just call << to print p, + // relying on ADL to pick up user-defined << for their pointer + // types, if any. + *os << p; + } else { + // T is a function type, so '*os << p' doesn't do what we want + // (it just prints p as bool). We want to print p as a const + // void*. However, we cannot cast it to const void* directly, + // even using reinterpret_cast, as earlier versions of gcc + // (e.g. 3.4.5) cannot compile the cast when p is a function + // pointer. Casting to uintptr_t first solves the problem. + *os << reinterpret_cast( + reinterpret_cast(p)); + } + } +} + +// Used to print a non-container, non-pointer value when the user +// doesn't define PrintTo() for it. +template +void DefaultPrintTo(IsNotContainer /* dummy */, + false_type /* is not a pointer */, + const T& value, ::std::ostream* os) { + ::testing_internal::DefaultPrintNonContainerTo(value, os); +} + +// Prints the given value using the << operator if it has one; +// otherwise prints the bytes in it. This is what +// UniversalPrinter::Print() does when PrintTo() is not specialized +// or overloaded for type T. +// +// A user can override this behavior for a class type Foo by defining +// an overload of PrintTo() in the namespace where Foo is defined. We +// give the user this option as sometimes defining a << operator for +// Foo is not desirable (e.g. the coding style may prevent doing it, +// or there is already a << operator but it doesn't do what the user +// wants). +template +void PrintTo(const T& value, ::std::ostream* os) { + // DefaultPrintTo() is overloaded. The type of its first two + // arguments determine which version will be picked. If T is an + // STL-style container, the version for container will be called; if + // T is a pointer, the pointer version will be called; otherwise the + // generic version will be called. + // + // Note that we check for container types here, prior to we check + // for protocol message types in our operator<<. The rationale is: + // + // For protocol messages, we want to give people a chance to + // override Google Mock's format by defining a PrintTo() or + // operator<<. For STL containers, other formats can be + // incompatible with Google Mock's format for the container + // elements; therefore we check for container types here to ensure + // that our format is used. + // + // The second argument of DefaultPrintTo() is needed to bypass a bug + // in Symbian's C++ compiler that prevents it from picking the right + // overload between: + // + // PrintTo(const T& x, ...); + // PrintTo(T* x, ...); + DefaultPrintTo(IsContainerTest(0), is_pointer(), value, os); +} + +// The following list of PrintTo() overloads tells +// UniversalPrinter::Print() how to print standard types (built-in +// types, strings, plain arrays, and pointers). + +// Overloads for various char types. +GTEST_API_ void PrintTo(unsigned char c, ::std::ostream* os); +GTEST_API_ void PrintTo(signed char c, ::std::ostream* os); +inline void PrintTo(char c, ::std::ostream* os) { + // When printing a plain char, we always treat it as unsigned. This + // way, the output won't be affected by whether the compiler thinks + // char is signed or not. + PrintTo(static_cast(c), os); +} + +// Overloads for other simple built-in types. +inline void PrintTo(bool x, ::std::ostream* os) { + *os << (x ? "true" : "false"); +} + +// Overload for wchar_t type. +// Prints a wchar_t as a symbol if it is printable or as its internal +// code otherwise and also as its decimal code (except for L'\0'). +// The L'\0' char is printed as "L'\\0'". The decimal code is printed +// as signed integer when wchar_t is implemented by the compiler +// as a signed type and is printed as an unsigned integer when wchar_t +// is implemented as an unsigned type. +GTEST_API_ void PrintTo(wchar_t wc, ::std::ostream* os); + +// Overloads for C strings. +GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); +inline void PrintTo(char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// signed/unsigned char is often used for representing binary data, so +// we print pointers to it as void* to be safe. +inline void PrintTo(const signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(signed char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(const unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +inline void PrintTo(unsigned char* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} + +// MSVC can be configured to define wchar_t as a typedef of unsigned +// short. It defines _NATIVE_WCHAR_T_DEFINED when wchar_t is a native +// type. When wchar_t is a typedef, defining an overload for const +// wchar_t* would cause unsigned short* be printed as a wide string, +// possibly causing invalid memory accesses. +#if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) +// Overloads for wide C strings +GTEST_API_ void PrintTo(const wchar_t* s, ::std::ostream* os); +inline void PrintTo(wchar_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +#endif + +// Overload for C arrays. Multi-dimensional arrays are printed +// properly. + +// Prints the given number of elements in an array, without printing +// the curly braces. +template +void PrintRawArrayTo(const T a[], size_t count, ::std::ostream* os) { + UniversalPrint(a[0], os); + for (size_t i = 1; i != count; i++) { + *os << ", "; + UniversalPrint(a[i], os); + } +} + +// Overloads for ::string and ::std::string. +#if GTEST_HAS_GLOBAL_STRING +GTEST_API_ void PrintStringTo(const ::string&s, ::std::ostream* os); +inline void PrintTo(const ::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_STRING + +GTEST_API_ void PrintStringTo(const ::std::string&s, ::std::ostream* os); +inline void PrintTo(const ::std::string& s, ::std::ostream* os) { + PrintStringTo(s, os); +} + +// Overloads for ::wstring and ::std::wstring. +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_API_ void PrintWideStringTo(const ::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_GLOBAL_WSTRING + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ void PrintWideStringTo(const ::std::wstring&s, ::std::ostream* os); +inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { + PrintWideStringTo(s, os); +} +#endif // GTEST_HAS_STD_WSTRING + +#if GTEST_HAS_TR1_TUPLE +// Overload for ::std::tr1::tuple. Needed for printing function arguments, +// which are packed as tuples. + +// Helper function for printing a tuple. T must be instantiated with +// a tuple type. +template +void PrintTupleTo(const T& t, ::std::ostream* os); + +// Overloaded PrintTo() for tuples of various arities. We support +// tuples of up-to 10 fields. The following implementation works +// regardless of whether tr1::tuple is implemented using the +// non-standard variadic template feature or not. + +inline void PrintTo(const ::std::tr1::tuple<>& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo(const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} + +template +void PrintTo( + const ::std::tr1::tuple& t, + ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_TR1_TUPLE + +// Overload for std::pair. +template +void PrintTo(const ::std::pair& value, ::std::ostream* os) { + *os << '('; + // We cannot use UniversalPrint(value.first, os) here, as T1 may be + // a reference type. The same for printing value.second. + UniversalPrinter::Print(value.first, os); + *os << ", "; + UniversalPrinter::Print(value.second, os); + *os << ')'; +} + +// Implements printing a non-reference type T by letting the compiler +// pick the right overload of PrintTo() for T. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. +#ifdef _MSC_VER +# pragma warning(push) // Saves the current warning state. +# pragma warning(disable:4180) // Temporarily disables warning 4180. +#endif // _MSC_VER + + // Note: we deliberately don't call this PrintTo(), as that name + // conflicts with ::testing::internal::PrintTo in the body of the + // function. + static void Print(const T& value, ::std::ostream* os) { + // By default, ::testing::internal::PrintTo() is used for printing + // the value. + // + // Thanks to Koenig look-up, if T is a class and has its own + // PrintTo() function defined in its namespace, that function will + // be visible here. Since it is more specific than the generic ones + // in ::testing::internal, it will be picked by the compiler in the + // following statement - exactly what we want. + PrintTo(value, os); + } + +#ifdef _MSC_VER +# pragma warning(pop) // Restores the warning state. +#endif // _MSC_VER +}; + +// UniversalPrintArray(begin, len, os) prints an array of 'len' +// elements, starting at address 'begin'. +template +void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { + if (len == 0) { + *os << "{}"; + } else { + *os << "{ "; + const size_t kThreshold = 18; + const size_t kChunkSize = 8; + // If the array has more than kThreshold elements, we'll have to + // omit some details by printing only the first and the last + // kChunkSize elements. + // TODO(wan@google.com): let the user control the threshold using a flag. + if (len <= kThreshold) { + PrintRawArrayTo(begin, len, os); + } else { + PrintRawArrayTo(begin, kChunkSize, os); + *os << ", ..., "; + PrintRawArrayTo(begin + len - kChunkSize, kChunkSize, os); + } + *os << " }"; + } +} +// This overload prints a (const) char array compactly. +GTEST_API_ void UniversalPrintArray(const char* begin, + size_t len, + ::std::ostream* os); + +// Implements printing an array type T[N]. +template +class UniversalPrinter { + public: + // Prints the given array, omitting some elements when there are too + // many. + static void Print(const T (&a)[N], ::std::ostream* os) { + UniversalPrintArray(a, N, os); + } +}; + +// Implements printing a reference type T&. +template +class UniversalPrinter { + public: + // MSVC warns about adding const to a function type, so we want to + // disable the warning. +#ifdef _MSC_VER +# pragma warning(push) // Saves the current warning state. +# pragma warning(disable:4180) // Temporarily disables warning 4180. +#endif // _MSC_VER + + static void Print(const T& value, ::std::ostream* os) { + // Prints the address of the value. We use reinterpret_cast here + // as static_cast doesn't compile when T is a function type. + *os << "@" << reinterpret_cast(&value) << " "; + + // Then prints the value itself. + UniversalPrint(value, os); + } + +#ifdef _MSC_VER +# pragma warning(pop) // Restores the warning state. +#endif // _MSC_VER +}; + +// Prints a value tersely: for a reference type, the referenced value +// (but not the address) is printed; for a (const) char pointer, the +// NUL-terminated string (but not the pointer) is printed. +template +void UniversalTersePrint(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); +} +inline void UniversalTersePrint(const char* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(string(str), os); + } +} +inline void UniversalTersePrint(char* str, ::std::ostream* os) { + UniversalTersePrint(static_cast(str), os); +} + +// Prints a value using the type inferred by the compiler. The +// difference between this and UniversalTersePrint() is that for a +// (const) char pointer, this prints both the pointer and the +// NUL-terminated string. +template +void UniversalPrint(const T& value, ::std::ostream* os) { + UniversalPrinter::Print(value, os); +} + +#if GTEST_HAS_TR1_TUPLE +typedef ::std::vector Strings; + +// This helper template allows PrintTo() for tuples and +// UniversalTersePrintTupleFieldsToStrings() to be defined by +// induction on the number of tuple fields. The idea is that +// TuplePrefixPrinter::PrintPrefixTo(t, os) prints the first N +// fields in tuple t, and can be defined in terms of +// TuplePrefixPrinter. + +// The inductive case. +template +struct TuplePrefixPrinter { + // Prints the first N fields of a tuple. + template + static void PrintPrefixTo(const Tuple& t, ::std::ostream* os) { + TuplePrefixPrinter::PrintPrefixTo(t, os); + *os << ", "; + UniversalPrinter::type> + ::Print(::std::tr1::get(t), os); + } + + // Tersely prints the first N fields of a tuple to a string vector, + // one element for each field. + template + static void TersePrintPrefixToStrings(const Tuple& t, Strings* strings) { + TuplePrefixPrinter::TersePrintPrefixToStrings(t, strings); + ::std::stringstream ss; + UniversalTersePrint(::std::tr1::get(t), &ss); + strings->push_back(ss.str()); + } +}; + +// Base cases. +template <> +struct TuplePrefixPrinter<0> { + template + static void PrintPrefixTo(const Tuple&, ::std::ostream*) {} + + template + static void TersePrintPrefixToStrings(const Tuple&, Strings*) {} +}; +// We have to specialize the entire TuplePrefixPrinter<> class +// template here, even though the definition of +// TersePrintPrefixToStrings() is the same as the generic version, as +// Embarcadero (formerly CodeGear, formerly Borland) C++ doesn't +// support specializing a method template of a class template. +template <> +struct TuplePrefixPrinter<1> { + template + static void PrintPrefixTo(const Tuple& t, ::std::ostream* os) { + UniversalPrinter::type>:: + Print(::std::tr1::get<0>(t), os); + } + + template + static void TersePrintPrefixToStrings(const Tuple& t, Strings* strings) { + ::std::stringstream ss; + UniversalTersePrint(::std::tr1::get<0>(t), &ss); + strings->push_back(ss.str()); + } +}; + +// Helper function for printing a tuple. T must be instantiated with +// a tuple type. +template +void PrintTupleTo(const T& t, ::std::ostream* os) { + *os << "("; + TuplePrefixPrinter< ::std::tr1::tuple_size::value>:: + PrintPrefixTo(t, os); + *os << ")"; +} + +// Prints the fields of a tuple tersely to a string vector, one +// element for each field. See the comment before +// UniversalTersePrint() for how we define "tersely". +template +Strings UniversalTersePrintTupleFieldsToStrings(const Tuple& value) { + Strings result; + TuplePrefixPrinter< ::std::tr1::tuple_size::value>:: + TersePrintPrefixToStrings(value, &result); + return result; +} +#endif // GTEST_HAS_TR1_TUPLE + +} // namespace internal + +template +::std::string PrintToString(const T& value) { + ::std::stringstream ss; + internal::UniversalTersePrint(value, &ss); + return ss.str(); +} + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ + +#if GTEST_HAS_PARAM_TEST + +namespace testing { +namespace internal { + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Outputs a message explaining invalid registration of different +// fixture class for the same test case. This may happen when +// TEST_P macro is used to define two tests with the same name +// but in different namespaces. +GTEST_API_ void ReportInvalidTestCaseType(const char* test_case_name, + const char* file, int line); + +template class ParamGeneratorInterface; +template class ParamGenerator; + +// Interface for iterating over elements provided by an implementation +// of ParamGeneratorInterface. +template +class ParamIteratorInterface { + public: + virtual ~ParamIteratorInterface() {} + // A pointer to the base generator instance. + // Used only for the purposes of iterator comparison + // to make sure that two iterators belong to the same generator. + virtual const ParamGeneratorInterface* BaseGenerator() const = 0; + // Advances iterator to point to the next element + // provided by the generator. The caller is responsible + // for not calling Advance() on an iterator equal to + // BaseGenerator()->End(). + virtual void Advance() = 0; + // Clones the iterator object. Used for implementing copy semantics + // of ParamIterator. + virtual ParamIteratorInterface* Clone() const = 0; + // Dereferences the current iterator and provides (read-only) access + // to the pointed value. It is the caller's responsibility not to call + // Current() on an iterator equal to BaseGenerator()->End(). + // Used for implementing ParamGenerator::operator*(). + virtual const T* Current() const = 0; + // Determines whether the given iterator and other point to the same + // element in the sequence generated by the generator. + // Used for implementing ParamGenerator::operator==(). + virtual bool Equals(const ParamIteratorInterface& other) const = 0; +}; + +// Class iterating over elements provided by an implementation of +// ParamGeneratorInterface. It wraps ParamIteratorInterface +// and implements the const forward iterator concept. +template +class ParamIterator { + public: + typedef T value_type; + typedef const T& reference; + typedef ptrdiff_t difference_type; + + // ParamIterator assumes ownership of the impl_ pointer. + ParamIterator(const ParamIterator& other) : impl_(other.impl_->Clone()) {} + ParamIterator& operator=(const ParamIterator& other) { + if (this != &other) + impl_.reset(other.impl_->Clone()); + return *this; + } + + const T& operator*() const { return *impl_->Current(); } + const T* operator->() const { return impl_->Current(); } + // Prefix version of operator++. + ParamIterator& operator++() { + impl_->Advance(); + return *this; + } + // Postfix version of operator++. + ParamIterator operator++(int /*unused*/) { + ParamIteratorInterface* clone = impl_->Clone(); + impl_->Advance(); + return ParamIterator(clone); + } + bool operator==(const ParamIterator& other) const { + return impl_.get() == other.impl_.get() || impl_->Equals(*other.impl_); + } + bool operator!=(const ParamIterator& other) const { + return !(*this == other); + } + + private: + friend class ParamGenerator; + explicit ParamIterator(ParamIteratorInterface* impl) : impl_(impl) {} + scoped_ptr > impl_; +}; + +// ParamGeneratorInterface is the binary interface to access generators +// defined in other translation units. +template +class ParamGeneratorInterface { + public: + typedef T ParamType; + + virtual ~ParamGeneratorInterface() {} + + // Generator interface definition + virtual ParamIteratorInterface* Begin() const = 0; + virtual ParamIteratorInterface* End() const = 0; +}; + +// Wraps ParamGeneratorInterface and provides general generator syntax +// compatible with the STL Container concept. +// This class implements copy initialization semantics and the contained +// ParamGeneratorInterface instance is shared among all copies +// of the original object. This is possible because that instance is immutable. +template +class ParamGenerator { + public: + typedef ParamIterator iterator; + + explicit ParamGenerator(ParamGeneratorInterface* impl) : impl_(impl) {} + ParamGenerator(const ParamGenerator& other) : impl_(other.impl_) {} + + ParamGenerator& operator=(const ParamGenerator& other) { + impl_ = other.impl_; + return *this; + } + + iterator begin() const { return iterator(impl_->Begin()); } + iterator end() const { return iterator(impl_->End()); } + + private: + linked_ptr > impl_; +}; + +// Generates values from a range of two comparable values. Can be used to +// generate sequences of user-defined types that implement operator+() and +// operator<(). +// This class is used in the Range() function. +template +class RangeGenerator : public ParamGeneratorInterface { + public: + RangeGenerator(T begin, T end, IncrementT step) + : begin_(begin), end_(end), + step_(step), end_index_(CalculateEndIndex(begin, end, step)) {} + virtual ~RangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, begin_, 0, step_); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, end_, end_index_, step_); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, T value, int index, + IncrementT step) + : base_(base), value_(value), index_(index), step_(step) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + value_ = value_ + step_; + index_++; + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const T* Current() const { return &value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const int other_index = + CheckedDowncastToActualType(&other)->index_; + return index_ == other_index; + } + + private: + Iterator(const Iterator& other) + : ParamIteratorInterface(), + base_(other.base_), value_(other.value_), index_(other.index_), + step_(other.step_) {} + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + T value_; + int index_; + const IncrementT step_; + }; // class RangeGenerator::Iterator + + static int CalculateEndIndex(const T& begin, + const T& end, + const IncrementT& step) { + int end_index = 0; + for (T i = begin; i < end; i = i + step) + end_index++; + return end_index; + } + + // No implementation - assignment is unsupported. + void operator=(const RangeGenerator& other); + + const T begin_; + const T end_; + const IncrementT step_; + // The index for the end() iterator. All the elements in the generated + // sequence are indexed (0-based) to aid iterator comparison. + const int end_index_; +}; // class RangeGenerator + + +// Generates values from a pair of STL-style iterators. Used in the +// ValuesIn() function. The elements are copied from the source range +// since the source can be located on the stack, and the generator +// is likely to persist beyond that stack frame. +template +class ValuesInIteratorRangeGenerator : public ParamGeneratorInterface { + public: + template + ValuesInIteratorRangeGenerator(ForwardIterator begin, ForwardIterator end) + : container_(begin, end) {} + virtual ~ValuesInIteratorRangeGenerator() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, container_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, container_.end()); + } + + private: + typedef typename ::std::vector ContainerType; + + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + typename ContainerType::const_iterator iterator) + : base_(base), iterator_(iterator) {} + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + virtual void Advance() { + ++iterator_; + value_.reset(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + // We need to use cached value referenced by iterator_ because *iterator_ + // can return a temporary object (and of type other then T), so just + // having "return &*iterator_;" doesn't work. + // value_ is updated here and not in Advance() because Advance() + // can advance iterator_ beyond the end of the range, and we cannot + // detect that fact. The client code, on the other hand, is + // responsible for not calling Current() on an out-of-range iterator. + virtual const T* Current() const { + if (value_.get() == NULL) + value_.reset(new T(*iterator_)); + return value_.get(); + } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + return iterator_ == + CheckedDowncastToActualType(&other)->iterator_; + } + + private: + Iterator(const Iterator& other) + // The explicit constructor call suppresses a false warning + // emitted by gcc when supplied with the -Wextra option. + : ParamIteratorInterface(), + base_(other.base_), + iterator_(other.iterator_) {} + + const ParamGeneratorInterface* const base_; + typename ContainerType::const_iterator iterator_; + // A cached value of *iterator_. We keep it here to allow access by + // pointer in the wrapping iterator's operator->(). + // value_ needs to be mutable to be accessed in Current(). + // Use of scoped_ptr helps manage cached value's lifetime, + // which is bound by the lifespan of the iterator itself. + mutable scoped_ptr value_; + }; // class ValuesInIteratorRangeGenerator::Iterator + + // No implementation - assignment is unsupported. + void operator=(const ValuesInIteratorRangeGenerator& other); + + const ContainerType container_; +}; // class ValuesInIteratorRangeGenerator + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Stores a parameter value and later creates tests parameterized with that +// value. +template +class ParameterizedTestFactory : public TestFactoryBase { + public: + typedef typename TestClass::ParamType ParamType; + explicit ParameterizedTestFactory(ParamType parameter) : + parameter_(parameter) {} + virtual Test* CreateTest() { + TestClass::SetParam(¶meter_); + return new TestClass(); + } + + private: + const ParamType parameter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactoryBase is a base class for meta-factories that create +// test factories for passing into MakeAndRegisterTestInfo function. +template +class TestMetaFactoryBase { + public: + virtual ~TestMetaFactoryBase() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) = 0; +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// TestMetaFactory creates test factories for passing into +// MakeAndRegisterTestInfo function. Since MakeAndRegisterTestInfo receives +// ownership of test factory pointer, same factory object cannot be passed +// into that method twice. But ParameterizedTestCaseInfo is going to call +// it for each Test/Parameter value combination. Thus it needs meta factory +// creator class. +template +class TestMetaFactory + : public TestMetaFactoryBase { + public: + typedef typename TestCase::ParamType ParamType; + + TestMetaFactory() {} + + virtual TestFactoryBase* CreateTestFactory(ParamType parameter) { + return new ParameterizedTestFactory(parameter); + } + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestMetaFactory); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfoBase is a generic interface +// to ParameterizedTestCaseInfo classes. ParameterizedTestCaseInfoBase +// accumulates test information provided by TEST_P macro invocations +// and generators provided by INSTANTIATE_TEST_CASE_P macro invocations +// and uses that information to register all resulting test instances +// in RegisterTests method. The ParameterizeTestCaseRegistry class holds +// a collection of pointers to the ParameterizedTestCaseInfo objects +// and calls RegisterTests() on each of them when asked. +class ParameterizedTestCaseInfoBase { + public: + virtual ~ParameterizedTestCaseInfoBase() {} + + // Base part of test case name for display purposes. + virtual const string& GetTestCaseName() const = 0; + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const = 0; + // UnitTest class invokes this method to register tests in this + // test case right before running them in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + virtual void RegisterTests() = 0; + + protected: + ParameterizedTestCaseInfoBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfoBase); +}; + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseInfo accumulates tests obtained from TEST_P +// macro invocations for a particular test case and generators +// obtained from INSTANTIATE_TEST_CASE_P macro invocations for that +// test case. It registers tests with all values generated by all +// generators when asked. +template +class ParameterizedTestCaseInfo : public ParameterizedTestCaseInfoBase { + public: + // ParamType and GeneratorCreationFunc are private types but are required + // for declarations of public methods AddTestPattern() and + // AddTestCaseInstantiation(). + typedef typename TestCase::ParamType ParamType; + // A function that returns an instance of appropriate generator type. + typedef ParamGenerator(GeneratorCreationFunc)(); + + explicit ParameterizedTestCaseInfo(const char* name) + : test_case_name_(name) {} + + // Test case base name for display purposes. + virtual const string& GetTestCaseName() const { return test_case_name_; } + // Test case id to verify identity. + virtual TypeId GetTestCaseTypeId() const { return GetTypeId(); } + // TEST_P macro uses AddTestPattern() to record information + // about a single test in a LocalTestInfo structure. + // test_case_name is the base name of the test case (without invocation + // prefix). test_base_name is the name of an individual test without + // parameter index. For the test SequenceA/FooTest.DoBar/1 FooTest is + // test case base name and DoBar is test base name. + void AddTestPattern(const char* test_case_name, + const char* test_base_name, + TestMetaFactoryBase* meta_factory) { + tests_.push_back(linked_ptr(new TestInfo(test_case_name, + test_base_name, + meta_factory))); + } + // INSTANTIATE_TEST_CASE_P macro uses AddGenerator() to record information + // about a generator. + int AddTestCaseInstantiation(const string& instantiation_name, + GeneratorCreationFunc* func, + const char* /* file */, + int /* line */) { + instantiations_.push_back(::std::make_pair(instantiation_name, func)); + return 0; // Return value used only to run this method in namespace scope. + } + // UnitTest class invokes this method to register tests in this test case + // test cases right before running tests in RUN_ALL_TESTS macro. + // This method should not be called more then once on any single + // instance of a ParameterizedTestCaseInfoBase derived class. + // UnitTest has a guard to prevent from calling this method more then once. + virtual void RegisterTests() { + for (typename TestInfoContainer::iterator test_it = tests_.begin(); + test_it != tests_.end(); ++test_it) { + linked_ptr test_info = *test_it; + for (typename InstantiationContainer::iterator gen_it = + instantiations_.begin(); gen_it != instantiations_.end(); + ++gen_it) { + const string& instantiation_name = gen_it->first; + ParamGenerator generator((*gen_it->second)()); + + Message test_case_name_stream; + if ( !instantiation_name.empty() ) + test_case_name_stream << instantiation_name << "/"; + test_case_name_stream << test_info->test_case_base_name; + + int i = 0; + for (typename ParamGenerator::iterator param_it = + generator.begin(); + param_it != generator.end(); ++param_it, ++i) { + Message test_name_stream; + test_name_stream << test_info->test_base_name << "/" << i; + MakeAndRegisterTestInfo( + test_case_name_stream.GetString().c_str(), + test_name_stream.GetString().c_str(), + NULL, // No type parameter. + PrintToString(*param_it).c_str(), + GetTestCaseTypeId(), + TestCase::SetUpTestCase, + TestCase::TearDownTestCase, + test_info->test_meta_factory->CreateTestFactory(*param_it)); + } // for param_it + } // for gen_it + } // for test_it + } // RegisterTests + + private: + // LocalTestInfo structure keeps information about a single test registered + // with TEST_P macro. + struct TestInfo { + TestInfo(const char* a_test_case_base_name, + const char* a_test_base_name, + TestMetaFactoryBase* a_test_meta_factory) : + test_case_base_name(a_test_case_base_name), + test_base_name(a_test_base_name), + test_meta_factory(a_test_meta_factory) {} + + const string test_case_base_name; + const string test_base_name; + const scoped_ptr > test_meta_factory; + }; + typedef ::std::vector > TestInfoContainer; + // Keeps pairs of + // received from INSTANTIATE_TEST_CASE_P macros. + typedef ::std::vector > + InstantiationContainer; + + const string test_case_name_; + TestInfoContainer tests_; + InstantiationContainer instantiations_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseInfo); +}; // class ParameterizedTestCaseInfo + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// ParameterizedTestCaseRegistry contains a map of ParameterizedTestCaseInfoBase +// classes accessed by test case names. TEST_P and INSTANTIATE_TEST_CASE_P +// macros use it to locate their corresponding ParameterizedTestCaseInfo +// descriptors. +class ParameterizedTestCaseRegistry { + public: + ParameterizedTestCaseRegistry() {} + ~ParameterizedTestCaseRegistry() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + delete *it; + } + } + + // Looks up or creates and returns a structure containing information about + // tests and instantiations of a particular test case. + template + ParameterizedTestCaseInfo* GetTestCasePatternHolder( + const char* test_case_name, + const char* file, + int line) { + ParameterizedTestCaseInfo* typed_test_info = NULL; + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + if ((*it)->GetTestCaseName() == test_case_name) { + if ((*it)->GetTestCaseTypeId() != GetTypeId()) { + // Complain about incorrect usage of Google Test facilities + // and terminate the program since we cannot guaranty correct + // test case setup and tear-down in this case. + ReportInvalidTestCaseType(test_case_name, file, line); + posix::Abort(); + } else { + // At this point we are sure that the object we found is of the same + // type we are looking for, so we downcast it to that type + // without further checks. + typed_test_info = CheckedDowncastToActualType< + ParameterizedTestCaseInfo >(*it); + } + break; + } + } + if (typed_test_info == NULL) { + typed_test_info = new ParameterizedTestCaseInfo(test_case_name); + test_case_infos_.push_back(typed_test_info); + } + return typed_test_info; + } + void RegisterTests() { + for (TestCaseInfoContainer::iterator it = test_case_infos_.begin(); + it != test_case_infos_.end(); ++it) { + (*it)->RegisterTests(); + } + } + + private: + typedef ::std::vector TestCaseInfoContainer; + + TestCaseInfoContainer test_case_infos_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ParameterizedTestCaseRegistry); +}; + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +// This file was GENERATED by command: +// pump.py gtest-param-util-generated.h.pump +// DO NOT EDIT BY HAND!!! + +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: vladl@google.com (Vlad Losev) + +// Type and function utilities for implementing parameterized tests. +// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! +// +// Currently Google Test supports at most 50 arguments in Values, +// and at most 10 arguments in Combine. Please contact +// googletestframework@googlegroups.com if you need more. +// Please note that the number of arguments to Combine is limited +// by the maximum arity of the implementation of tr1::tuple which is +// currently set at 10. + +#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ +#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + +// scripts/fuse_gtest.py depends on gtest's own header being #included +// *unconditionally*. Therefore these #includes cannot be moved +// inside #if GTEST_HAS_PARAM_TEST. + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Forward declarations of ValuesIn(), which is implemented in +// include/gtest/gtest-param-test.h. +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end); + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]); + +template +internal::ParamGenerator ValuesIn( + const Container& container); + +namespace internal { + +// Used in the Values() function to provide polymorphic capabilities. +template +class ValueArray1 { + public: + explicit ValueArray1(T1 v1) : v1_(v1) {} + + template + operator ParamGenerator() const { return ValuesIn(&v1_, &v1_ + 1); } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray1& other); + + const T1 v1_; +}; + +template +class ValueArray2 { + public: + ValueArray2(T1 v1, T2 v2) : v1_(v1), v2_(v2) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray2& other); + + const T1 v1_; + const T2 v2_; +}; + +template +class ValueArray3 { + public: + ValueArray3(T1 v1, T2 v2, T3 v3) : v1_(v1), v2_(v2), v3_(v3) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray3& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; +}; + +template +class ValueArray4 { + public: + ValueArray4(T1 v1, T2 v2, T3 v3, T4 v4) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray4& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; +}; + +template +class ValueArray5 { + public: + ValueArray5(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray5& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; +}; + +template +class ValueArray6 { + public: + ValueArray6(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray6& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; +}; + +template +class ValueArray7 { + public: + ValueArray7(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray7& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; +}; + +template +class ValueArray8 { + public: + ValueArray8(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray8& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; +}; + +template +class ValueArray9 { + public: + ValueArray9(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray9& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; +}; + +template +class ValueArray10 { + public: + ValueArray10(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray10& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; +}; + +template +class ValueArray11 { + public: + ValueArray11(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray11& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; +}; + +template +class ValueArray12 { + public: + ValueArray12(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray12& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; +}; + +template +class ValueArray13 { + public: + ValueArray13(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray13& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; +}; + +template +class ValueArray14 { + public: + ValueArray14(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray14& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; +}; + +template +class ValueArray15 { + public: + ValueArray15(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray15& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; +}; + +template +class ValueArray16 { + public: + ValueArray16(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray16& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; +}; + +template +class ValueArray17 { + public: + ValueArray17(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray17& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; +}; + +template +class ValueArray18 { + public: + ValueArray18(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray18& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; +}; + +template +class ValueArray19 { + public: + ValueArray19(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray19& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; +}; + +template +class ValueArray20 { + public: + ValueArray20(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray20& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; +}; + +template +class ValueArray21 { + public: + ValueArray21(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray21& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; +}; + +template +class ValueArray22 { + public: + ValueArray22(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray22& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; +}; + +template +class ValueArray23 { + public: + ValueArray23(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, + v23_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray23& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; +}; + +template +class ValueArray24 { + public: + ValueArray24(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray24& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; +}; + +template +class ValueArray25 { + public: + ValueArray25(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray25& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; +}; + +template +class ValueArray26 { + public: + ValueArray26(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray26& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; +}; + +template +class ValueArray27 { + public: + ValueArray27(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray27& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; +}; + +template +class ValueArray28 { + public: + ValueArray28(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray28& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; +}; + +template +class ValueArray29 { + public: + ValueArray29(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray29& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; +}; + +template +class ValueArray30 { + public: + ValueArray30(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray30& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; +}; + +template +class ValueArray31 { + public: + ValueArray31(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray31& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; +}; + +template +class ValueArray32 { + public: + ValueArray32(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray32& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; +}; + +template +class ValueArray33 { + public: + ValueArray33(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray33& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; +}; + +template +class ValueArray34 { + public: + ValueArray34(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray34& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; +}; + +template +class ValueArray35 { + public: + ValueArray35(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, + v35_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray35& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; +}; + +template +class ValueArray36 { + public: + ValueArray36(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray36& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; +}; + +template +class ValueArray37 { + public: + ValueArray37(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray37& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; +}; + +template +class ValueArray38 { + public: + ValueArray38(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray38& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; +}; + +template +class ValueArray39 { + public: + ValueArray39(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray39& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; +}; + +template +class ValueArray40 { + public: + ValueArray40(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray40& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; +}; + +template +class ValueArray41 { + public: + ValueArray41(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray41& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; +}; + +template +class ValueArray42 { + public: + ValueArray42(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray42& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; +}; + +template +class ValueArray43 { + public: + ValueArray43(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), + v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), + v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), + v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), + v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), + v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), + v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray43& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; +}; + +template +class ValueArray44 { + public: + ValueArray44(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), + v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), + v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), v18_(v18), + v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), v24_(v24), + v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), v30_(v30), + v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), v36_(v36), + v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), v42_(v42), + v43_(v43), v44_(v44) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray44& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; +}; + +template +class ValueArray45 { + public: + ValueArray45(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), + v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), v11_(v11), + v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), v17_(v17), + v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), v23_(v23), + v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), v29_(v29), + v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), v35_(v35), + v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), v41_(v41), + v42_(v42), v43_(v43), v44_(v44), v45_(v45) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray45& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; +}; + +template +class ValueArray46 { + public: + ValueArray46(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) : v1_(v1), v2_(v2), v3_(v3), + v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray46& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; +}; + +template +class ValueArray47 { + public: + ValueArray47(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) : v1_(v1), v2_(v2), + v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), v10_(v10), + v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), v16_(v16), + v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), v22_(v22), + v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), v28_(v28), + v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), v34_(v34), + v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), v40_(v40), + v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), v46_(v46), + v47_(v47) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, + v47_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray47& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; +}; + +template +class ValueArray48 { + public: + ValueArray48(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48) : v1_(v1), + v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), v8_(v8), v9_(v9), + v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), v15_(v15), + v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), v21_(v21), + v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), v27_(v27), + v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), v33_(v33), + v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), v39_(v39), + v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), v45_(v45), + v46_(v46), v47_(v47), v48_(v48) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, v47_, + v48_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray48& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; +}; + +template +class ValueArray49 { + public: + ValueArray49(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, + T49 v49) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, v47_, + v48_, v49_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray49& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; +}; + +template +class ValueArray50 { + public: + ValueArray50(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, T48 v48, T49 v49, + T50 v50) : v1_(v1), v2_(v2), v3_(v3), v4_(v4), v5_(v5), v6_(v6), v7_(v7), + v8_(v8), v9_(v9), v10_(v10), v11_(v11), v12_(v12), v13_(v13), v14_(v14), + v15_(v15), v16_(v16), v17_(v17), v18_(v18), v19_(v19), v20_(v20), + v21_(v21), v22_(v22), v23_(v23), v24_(v24), v25_(v25), v26_(v26), + v27_(v27), v28_(v28), v29_(v29), v30_(v30), v31_(v31), v32_(v32), + v33_(v33), v34_(v34), v35_(v35), v36_(v36), v37_(v37), v38_(v38), + v39_(v39), v40_(v40), v41_(v41), v42_(v42), v43_(v43), v44_(v44), + v45_(v45), v46_(v46), v47_(v47), v48_(v48), v49_(v49), v50_(v50) {} + + template + operator ParamGenerator() const { + const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, + v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, + v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, + v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, v47_, + v48_, v49_, v50_}; + return ValuesIn(array); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const ValueArray50& other); + + const T1 v1_; + const T2 v2_; + const T3 v3_; + const T4 v4_; + const T5 v5_; + const T6 v6_; + const T7 v7_; + const T8 v8_; + const T9 v9_; + const T10 v10_; + const T11 v11_; + const T12 v12_; + const T13 v13_; + const T14 v14_; + const T15 v15_; + const T16 v16_; + const T17 v17_; + const T18 v18_; + const T19 v19_; + const T20 v20_; + const T21 v21_; + const T22 v22_; + const T23 v23_; + const T24 v24_; + const T25 v25_; + const T26 v26_; + const T27 v27_; + const T28 v28_; + const T29 v29_; + const T30 v30_; + const T31 v31_; + const T32 v32_; + const T33 v33_; + const T34 v34_; + const T35 v35_; + const T36 v36_; + const T37 v37_; + const T38 v38_; + const T39 v39_; + const T40 v40_; + const T41 v41_; + const T42 v42_; + const T43 v43_; + const T44 v44_; + const T45 v45_; + const T46 v46_; + const T47 v47_; + const T48 v48_; + const T49 v49_; + const T50 v50_; +}; + +# if GTEST_HAS_COMBINE +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Generates values from the Cartesian product of values produced +// by the argument generators. +// +template +class CartesianProductGenerator2 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator2(const ParamGenerator& g1, + const ParamGenerator& g2) + : g1_(g1), g2_(g2) {} + virtual ~CartesianProductGenerator2() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current2_; + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + ParamType current_value_; + }; // class CartesianProductGenerator2::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator2& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; +}; // class CartesianProductGenerator2 + + +template +class CartesianProductGenerator3 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator3(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + virtual ~CartesianProductGenerator3() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current3_; + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + ParamType current_value_; + }; // class CartesianProductGenerator3::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator3& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; +}; // class CartesianProductGenerator3 + + +template +class CartesianProductGenerator4 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator4(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + virtual ~CartesianProductGenerator4() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current4_; + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + ParamType current_value_; + }; // class CartesianProductGenerator4::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator4& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; +}; // class CartesianProductGenerator4 + + +template +class CartesianProductGenerator5 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator5(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + virtual ~CartesianProductGenerator5() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current5_; + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + ParamType current_value_; + }; // class CartesianProductGenerator5::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator5& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; +}; // class CartesianProductGenerator5 + + +template +class CartesianProductGenerator6 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator6(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + virtual ~CartesianProductGenerator6() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current6_; + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + ParamType current_value_; + }; // class CartesianProductGenerator6::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator6& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; +}; // class CartesianProductGenerator6 + + +template +class CartesianProductGenerator7 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator7(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + virtual ~CartesianProductGenerator7() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current7_; + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + ParamType current_value_; + }; // class CartesianProductGenerator7::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator7& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; +}; // class CartesianProductGenerator7 + + +template +class CartesianProductGenerator8 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator8(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + virtual ~CartesianProductGenerator8() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current8_; + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + ParamType current_value_; + }; // class CartesianProductGenerator8::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator8& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; +}; // class CartesianProductGenerator8 + + +template +class CartesianProductGenerator9 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator9(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + virtual ~CartesianProductGenerator9() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current9_; + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + ParamType current_value_; + }; // class CartesianProductGenerator9::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator9& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; +}; // class CartesianProductGenerator9 + + +template +class CartesianProductGenerator10 + : public ParamGeneratorInterface< ::std::tr1::tuple > { + public: + typedef ::std::tr1::tuple ParamType; + + CartesianProductGenerator10(const ParamGenerator& g1, + const ParamGenerator& g2, const ParamGenerator& g3, + const ParamGenerator& g4, const ParamGenerator& g5, + const ParamGenerator& g6, const ParamGenerator& g7, + const ParamGenerator& g8, const ParamGenerator& g9, + const ParamGenerator& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + virtual ~CartesianProductGenerator10() {} + + virtual ParamIteratorInterface* Begin() const { + return new Iterator(this, g1_, g1_.begin(), g2_, g2_.begin(), g3_, + g3_.begin(), g4_, g4_.begin(), g5_, g5_.begin(), g6_, g6_.begin(), g7_, + g7_.begin(), g8_, g8_.begin(), g9_, g9_.begin(), g10_, g10_.begin()); + } + virtual ParamIteratorInterface* End() const { + return new Iterator(this, g1_, g1_.end(), g2_, g2_.end(), g3_, g3_.end(), + g4_, g4_.end(), g5_, g5_.end(), g6_, g6_.end(), g7_, g7_.end(), g8_, + g8_.end(), g9_, g9_.end(), g10_, g10_.end()); + } + + private: + class Iterator : public ParamIteratorInterface { + public: + Iterator(const ParamGeneratorInterface* base, + const ParamGenerator& g1, + const typename ParamGenerator::iterator& current1, + const ParamGenerator& g2, + const typename ParamGenerator::iterator& current2, + const ParamGenerator& g3, + const typename ParamGenerator::iterator& current3, + const ParamGenerator& g4, + const typename ParamGenerator::iterator& current4, + const ParamGenerator& g5, + const typename ParamGenerator::iterator& current5, + const ParamGenerator& g6, + const typename ParamGenerator::iterator& current6, + const ParamGenerator& g7, + const typename ParamGenerator::iterator& current7, + const ParamGenerator& g8, + const typename ParamGenerator::iterator& current8, + const ParamGenerator& g9, + const typename ParamGenerator::iterator& current9, + const ParamGenerator& g10, + const typename ParamGenerator::iterator& current10) + : base_(base), + begin1_(g1.begin()), end1_(g1.end()), current1_(current1), + begin2_(g2.begin()), end2_(g2.end()), current2_(current2), + begin3_(g3.begin()), end3_(g3.end()), current3_(current3), + begin4_(g4.begin()), end4_(g4.end()), current4_(current4), + begin5_(g5.begin()), end5_(g5.end()), current5_(current5), + begin6_(g6.begin()), end6_(g6.end()), current6_(current6), + begin7_(g7.begin()), end7_(g7.end()), current7_(current7), + begin8_(g8.begin()), end8_(g8.end()), current8_(current8), + begin9_(g9.begin()), end9_(g9.end()), current9_(current9), + begin10_(g10.begin()), end10_(g10.end()), current10_(current10) { + ComputeCurrentValue(); + } + virtual ~Iterator() {} + + virtual const ParamGeneratorInterface* BaseGenerator() const { + return base_; + } + // Advance should not be called on beyond-of-range iterators + // so no component iterators must be beyond end of range, either. + virtual void Advance() { + assert(!AtEnd()); + ++current10_; + if (current10_ == end10_) { + current10_ = begin10_; + ++current9_; + } + if (current9_ == end9_) { + current9_ = begin9_; + ++current8_; + } + if (current8_ == end8_) { + current8_ = begin8_; + ++current7_; + } + if (current7_ == end7_) { + current7_ = begin7_; + ++current6_; + } + if (current6_ == end6_) { + current6_ = begin6_; + ++current5_; + } + if (current5_ == end5_) { + current5_ = begin5_; + ++current4_; + } + if (current4_ == end4_) { + current4_ = begin4_; + ++current3_; + } + if (current3_ == end3_) { + current3_ = begin3_; + ++current2_; + } + if (current2_ == end2_) { + current2_ = begin2_; + ++current1_; + } + ComputeCurrentValue(); + } + virtual ParamIteratorInterface* Clone() const { + return new Iterator(*this); + } + virtual const ParamType* Current() const { return ¤t_value_; } + virtual bool Equals(const ParamIteratorInterface& other) const { + // Having the same base generator guarantees that the other + // iterator is of the same type and we can downcast. + GTEST_CHECK_(BaseGenerator() == other.BaseGenerator()) + << "The program attempted to compare iterators " + << "from different generators." << std::endl; + const Iterator* typed_other = + CheckedDowncastToActualType(&other); + // We must report iterators equal if they both point beyond their + // respective ranges. That can happen in a variety of fashions, + // so we have to consult AtEnd(). + return (AtEnd() && typed_other->AtEnd()) || + ( + current1_ == typed_other->current1_ && + current2_ == typed_other->current2_ && + current3_ == typed_other->current3_ && + current4_ == typed_other->current4_ && + current5_ == typed_other->current5_ && + current6_ == typed_other->current6_ && + current7_ == typed_other->current7_ && + current8_ == typed_other->current8_ && + current9_ == typed_other->current9_ && + current10_ == typed_other->current10_); + } + + private: + Iterator(const Iterator& other) + : base_(other.base_), + begin1_(other.begin1_), + end1_(other.end1_), + current1_(other.current1_), + begin2_(other.begin2_), + end2_(other.end2_), + current2_(other.current2_), + begin3_(other.begin3_), + end3_(other.end3_), + current3_(other.current3_), + begin4_(other.begin4_), + end4_(other.end4_), + current4_(other.current4_), + begin5_(other.begin5_), + end5_(other.end5_), + current5_(other.current5_), + begin6_(other.begin6_), + end6_(other.end6_), + current6_(other.current6_), + begin7_(other.begin7_), + end7_(other.end7_), + current7_(other.current7_), + begin8_(other.begin8_), + end8_(other.end8_), + current8_(other.current8_), + begin9_(other.begin9_), + end9_(other.end9_), + current9_(other.current9_), + begin10_(other.begin10_), + end10_(other.end10_), + current10_(other.current10_) { + ComputeCurrentValue(); + } + + void ComputeCurrentValue() { + if (!AtEnd()) + current_value_ = ParamType(*current1_, *current2_, *current3_, + *current4_, *current5_, *current6_, *current7_, *current8_, + *current9_, *current10_); + } + bool AtEnd() const { + // We must report iterator past the end of the range when either of the + // component iterators has reached the end of its range. + return + current1_ == end1_ || + current2_ == end2_ || + current3_ == end3_ || + current4_ == end4_ || + current5_ == end5_ || + current6_ == end6_ || + current7_ == end7_ || + current8_ == end8_ || + current9_ == end9_ || + current10_ == end10_; + } + + // No implementation - assignment is unsupported. + void operator=(const Iterator& other); + + const ParamGeneratorInterface* const base_; + // begin[i]_ and end[i]_ define the i-th range that Iterator traverses. + // current[i]_ is the actual traversing iterator. + const typename ParamGenerator::iterator begin1_; + const typename ParamGenerator::iterator end1_; + typename ParamGenerator::iterator current1_; + const typename ParamGenerator::iterator begin2_; + const typename ParamGenerator::iterator end2_; + typename ParamGenerator::iterator current2_; + const typename ParamGenerator::iterator begin3_; + const typename ParamGenerator::iterator end3_; + typename ParamGenerator::iterator current3_; + const typename ParamGenerator::iterator begin4_; + const typename ParamGenerator::iterator end4_; + typename ParamGenerator::iterator current4_; + const typename ParamGenerator::iterator begin5_; + const typename ParamGenerator::iterator end5_; + typename ParamGenerator::iterator current5_; + const typename ParamGenerator::iterator begin6_; + const typename ParamGenerator::iterator end6_; + typename ParamGenerator::iterator current6_; + const typename ParamGenerator::iterator begin7_; + const typename ParamGenerator::iterator end7_; + typename ParamGenerator::iterator current7_; + const typename ParamGenerator::iterator begin8_; + const typename ParamGenerator::iterator end8_; + typename ParamGenerator::iterator current8_; + const typename ParamGenerator::iterator begin9_; + const typename ParamGenerator::iterator end9_; + typename ParamGenerator::iterator current9_; + const typename ParamGenerator::iterator begin10_; + const typename ParamGenerator::iterator end10_; + typename ParamGenerator::iterator current10_; + ParamType current_value_; + }; // class CartesianProductGenerator10::Iterator + + // No implementation - assignment is unsupported. + void operator=(const CartesianProductGenerator10& other); + + const ParamGenerator g1_; + const ParamGenerator g2_; + const ParamGenerator g3_; + const ParamGenerator g4_; + const ParamGenerator g5_; + const ParamGenerator g6_; + const ParamGenerator g7_; + const ParamGenerator g8_; + const ParamGenerator g9_; + const ParamGenerator g10_; +}; // class CartesianProductGenerator10 + + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Helper classes providing Combine() with polymorphic features. They allow +// casting CartesianProductGeneratorN to ParamGenerator if T is +// convertible to U. +// +template +class CartesianProductHolder2 { + public: +CartesianProductHolder2(const Generator1& g1, const Generator2& g2) + : g1_(g1), g2_(g2) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator2( + static_cast >(g1_), + static_cast >(g2_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder2& other); + + const Generator1 g1_; + const Generator2 g2_; +}; // class CartesianProductHolder2 + +template +class CartesianProductHolder3 { + public: +CartesianProductHolder3(const Generator1& g1, const Generator2& g2, + const Generator3& g3) + : g1_(g1), g2_(g2), g3_(g3) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator3( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder3& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; +}; // class CartesianProductHolder3 + +template +class CartesianProductHolder4 { + public: +CartesianProductHolder4(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator4( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder4& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; +}; // class CartesianProductHolder4 + +template +class CartesianProductHolder5 { + public: +CartesianProductHolder5(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator5( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder5& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; +}; // class CartesianProductHolder5 + +template +class CartesianProductHolder6 { + public: +CartesianProductHolder6(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator6( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder6& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; +}; // class CartesianProductHolder6 + +template +class CartesianProductHolder7 { + public: +CartesianProductHolder7(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator7( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder7& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; +}; // class CartesianProductHolder7 + +template +class CartesianProductHolder8 { + public: +CartesianProductHolder8(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), + g8_(g8) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator8( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder8& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; +}; // class CartesianProductHolder8 + +template +class CartesianProductHolder9 { + public: +CartesianProductHolder9(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator9( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder9& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; +}; // class CartesianProductHolder9 + +template +class CartesianProductHolder10 { + public: +CartesianProductHolder10(const Generator1& g1, const Generator2& g2, + const Generator3& g3, const Generator4& g4, const Generator5& g5, + const Generator6& g6, const Generator7& g7, const Generator8& g8, + const Generator9& g9, const Generator10& g10) + : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7), g8_(g8), + g9_(g9), g10_(g10) {} + template + operator ParamGenerator< ::std::tr1::tuple >() const { + return ParamGenerator< ::std::tr1::tuple >( + new CartesianProductGenerator10( + static_cast >(g1_), + static_cast >(g2_), + static_cast >(g3_), + static_cast >(g4_), + static_cast >(g5_), + static_cast >(g6_), + static_cast >(g7_), + static_cast >(g8_), + static_cast >(g9_), + static_cast >(g10_))); + } + + private: + // No implementation - assignment is unsupported. + void operator=(const CartesianProductHolder10& other); + + const Generator1 g1_; + const Generator2 g2_; + const Generator3 g3_; + const Generator4 g4_; + const Generator5 g5_; + const Generator6 g6_; + const Generator7 g7_; + const Generator8 g8_; + const Generator9 g9_; + const Generator10 g10_; +}; // class CartesianProductHolder10 + +# endif // GTEST_HAS_COMBINE + +} // namespace internal +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ + +#if GTEST_HAS_PARAM_TEST + +namespace testing { + +// Functions producing parameter generators. +// +// Google Test uses these generators to produce parameters for value- +// parameterized tests. When a parameterized test case is instantiated +// with a particular generator, Google Test creates and runs tests +// for each element in the sequence produced by the generator. +// +// In the following sample, tests from test case FooTest are instantiated +// each three times with parameter values 3, 5, and 8: +// +// class FooTest : public TestWithParam { ... }; +// +// TEST_P(FooTest, TestThis) { +// } +// TEST_P(FooTest, TestThat) { +// } +// INSTANTIATE_TEST_CASE_P(TestSequence, FooTest, Values(3, 5, 8)); +// + +// Range() returns generators providing sequences of values in a range. +// +// Synopsis: +// Range(start, end) +// - returns a generator producing a sequence of values {start, start+1, +// start+2, ..., }. +// Range(start, end, step) +// - returns a generator producing a sequence of values {start, start+step, +// start+step+step, ..., }. +// Notes: +// * The generated sequences never include end. For example, Range(1, 5) +// returns a generator producing a sequence {1, 2, 3, 4}. Range(1, 9, 2) +// returns a generator producing {1, 3, 5, 7}. +// * start and end must have the same type. That type may be any integral or +// floating-point type or a user defined type satisfying these conditions: +// * It must be assignable (have operator=() defined). +// * It must have operator+() (operator+(int-compatible type) for +// two-operand version). +// * It must have operator<() defined. +// Elements in the resulting sequences will also have that type. +// * Condition start < end must be satisfied in order for resulting sequences +// to contain any elements. +// +template +internal::ParamGenerator Range(T start, T end, IncrementT step) { + return internal::ParamGenerator( + new internal::RangeGenerator(start, end, step)); +} + +template +internal::ParamGenerator Range(T start, T end) { + return Range(start, end, 1); +} + +// ValuesIn() function allows generation of tests with parameters coming from +// a container. +// +// Synopsis: +// ValuesIn(const T (&array)[N]) +// - returns a generator producing sequences with elements from +// a C-style array. +// ValuesIn(const Container& container) +// - returns a generator producing sequences with elements from +// an STL-style container. +// ValuesIn(Iterator begin, Iterator end) +// - returns a generator producing sequences with elements from +// a range [begin, end) defined by a pair of STL-style iterators. These +// iterators can also be plain C pointers. +// +// Please note that ValuesIn copies the values from the containers +// passed in and keeps them to generate tests in RUN_ALL_TESTS(). +// +// Examples: +// +// This instantiates tests from test case StringTest +// each with C-string values of "foo", "bar", and "baz": +// +// const char* strings[] = {"foo", "bar", "baz"}; +// INSTANTIATE_TEST_CASE_P(StringSequence, SrtingTest, ValuesIn(strings)); +// +// This instantiates tests from test case StlStringTest +// each with STL strings with values "a" and "b": +// +// ::std::vector< ::std::string> GetParameterStrings() { +// ::std::vector< ::std::string> v; +// v.push_back("a"); +// v.push_back("b"); +// return v; +// } +// +// INSTANTIATE_TEST_CASE_P(CharSequence, +// StlStringTest, +// ValuesIn(GetParameterStrings())); +// +// +// This will also instantiate tests from CharTest +// each with parameter values 'a' and 'b': +// +// ::std::list GetParameterChars() { +// ::std::list list; +// list.push_back('a'); +// list.push_back('b'); +// return list; +// } +// ::std::list l = GetParameterChars(); +// INSTANTIATE_TEST_CASE_P(CharSequence2, +// CharTest, +// ValuesIn(l.begin(), l.end())); +// +template +internal::ParamGenerator< + typename ::testing::internal::IteratorTraits::value_type> +ValuesIn(ForwardIterator begin, ForwardIterator end) { + typedef typename ::testing::internal::IteratorTraits + ::value_type ParamType; + return internal::ParamGenerator( + new internal::ValuesInIteratorRangeGenerator(begin, end)); +} + +template +internal::ParamGenerator ValuesIn(const T (&array)[N]) { + return ValuesIn(array, array + N); +} + +template +internal::ParamGenerator ValuesIn( + const Container& container) { + return ValuesIn(container.begin(), container.end()); +} + +// Values() allows generating tests from explicitly specified list of +// parameters. +// +// Synopsis: +// Values(T v1, T v2, ..., T vN) +// - returns a generator producing sequences with elements v1, v2, ..., vN. +// +// For example, this instantiates tests from test case BarTest each +// with values "one", "two", and "three": +// +// INSTANTIATE_TEST_CASE_P(NumSequence, BarTest, Values("one", "two", "three")); +// +// This instantiates tests from test case BazTest each with values 1, 2, 3.5. +// The exact type of values will depend on the type of parameter in BazTest. +// +// INSTANTIATE_TEST_CASE_P(FloatingNumbers, BazTest, Values(1, 2, 3.5)); +// +// Currently, Values() supports from 1 to 50 parameters. +// +template +internal::ValueArray1 Values(T1 v1) { + return internal::ValueArray1(v1); +} + +template +internal::ValueArray2 Values(T1 v1, T2 v2) { + return internal::ValueArray2(v1, v2); +} + +template +internal::ValueArray3 Values(T1 v1, T2 v2, T3 v3) { + return internal::ValueArray3(v1, v2, v3); +} + +template +internal::ValueArray4 Values(T1 v1, T2 v2, T3 v3, T4 v4) { + return internal::ValueArray4(v1, v2, v3, v4); +} + +template +internal::ValueArray5 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5) { + return internal::ValueArray5(v1, v2, v3, v4, v5); +} + +template +internal::ValueArray6 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6) { + return internal::ValueArray6(v1, v2, v3, v4, v5, v6); +} + +template +internal::ValueArray7 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7) { + return internal::ValueArray7(v1, v2, v3, v4, v5, + v6, v7); +} + +template +internal::ValueArray8 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8) { + return internal::ValueArray8(v1, v2, v3, v4, + v5, v6, v7, v8); +} + +template +internal::ValueArray9 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9) { + return internal::ValueArray9(v1, v2, v3, + v4, v5, v6, v7, v8, v9); +} + +template +internal::ValueArray10 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10) { + return internal::ValueArray10(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10); +} + +template +internal::ValueArray11 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11) { + return internal::ValueArray11(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11); +} + +template +internal::ValueArray12 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12) { + return internal::ValueArray12(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12); +} + +template +internal::ValueArray13 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13) { + return internal::ValueArray13(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13); +} + +template +internal::ValueArray14 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14) { + return internal::ValueArray14(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14); +} + +template +internal::ValueArray15 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15) { + return internal::ValueArray15(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); +} + +template +internal::ValueArray16 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16) { + return internal::ValueArray16(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16); +} + +template +internal::ValueArray17 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17) { + return internal::ValueArray17(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17); +} + +template +internal::ValueArray18 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18) { + return internal::ValueArray18(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18); +} + +template +internal::ValueArray19 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19) { + return internal::ValueArray19(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19); +} + +template +internal::ValueArray20 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20) { + return internal::ValueArray20(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20); +} + +template +internal::ValueArray21 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21) { + return internal::ValueArray21(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21); +} + +template +internal::ValueArray22 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22) { + return internal::ValueArray22(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22); +} + +template +internal::ValueArray23 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23) { + return internal::ValueArray23(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23); +} + +template +internal::ValueArray24 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24) { + return internal::ValueArray24(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24); +} + +template +internal::ValueArray25 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25) { + return internal::ValueArray25(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25); +} + +template +internal::ValueArray26 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26) { + return internal::ValueArray26(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26); +} + +template +internal::ValueArray27 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27) { + return internal::ValueArray27(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27); +} + +template +internal::ValueArray28 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28) { + return internal::ValueArray28(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28); +} + +template +internal::ValueArray29 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29) { + return internal::ValueArray29(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29); +} + +template +internal::ValueArray30 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30) { + return internal::ValueArray30(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30); +} + +template +internal::ValueArray31 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31) { + return internal::ValueArray31(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31); +} + +template +internal::ValueArray32 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32) { + return internal::ValueArray32(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32); +} + +template +internal::ValueArray33 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33) { + return internal::ValueArray33(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33); +} + +template +internal::ValueArray34 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34) { + return internal::ValueArray34(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34); +} + +template +internal::ValueArray35 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35) { + return internal::ValueArray35(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35); +} + +template +internal::ValueArray36 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36) { + return internal::ValueArray36(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36); +} + +template +internal::ValueArray37 Values(T1 v1, T2 v2, T3 v3, + T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37) { + return internal::ValueArray37(v1, v2, v3, + v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37); +} + +template +internal::ValueArray38 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38) { + return internal::ValueArray38(v1, v2, + v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, + v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, + v33, v34, v35, v36, v37, v38); +} + +template +internal::ValueArray39 Values(T1 v1, T2 v2, + T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, + T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, + T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, + T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, + T37 v37, T38 v38, T39 v39) { + return internal::ValueArray39(v1, + v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, + v32, v33, v34, v35, v36, v37, v38, v39); +} + +template +internal::ValueArray40 Values(T1 v1, + T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, + T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, + T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, + T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, + T36 v36, T37 v37, T38 v38, T39 v39, T40 v40) { + return internal::ValueArray40(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, + v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40); +} + +template +internal::ValueArray41 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41) { + return internal::ValueArray41(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, + v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, + v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41); +} + +template +internal::ValueArray42 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42) { + return internal::ValueArray42(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, + v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, v41, + v42); +} + +template +internal::ValueArray43 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43) { + return internal::ValueArray43(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, + v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, v40, + v41, v42, v43); +} + +template +internal::ValueArray44 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, + T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, T17 v17, + T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, T25 v25, + T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, T33 v33, + T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, T41 v41, + T42 v42, T43 v43, T44 v44) { + return internal::ValueArray44(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, + v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, + v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, v39, + v40, v41, v42, v43, v44); +} + +template +internal::ValueArray45 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, T8 v8, + T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, T16 v16, + T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, T24 v24, + T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, T32 v32, + T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, T40 v40, + T41 v41, T42 v42, T43 v43, T44 v44, T45 v45) { + return internal::ValueArray45(v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, + v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, + v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, v38, + v39, v40, v41, v42, v43, v44, v45); +} + +template +internal::ValueArray46 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46) { + return internal::ValueArray46(v1, v2, v3, v4, v5, v6, v7, v8, v9, + v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46); +} + +template +internal::ValueArray47 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, T7 v7, + T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47) { + return internal::ValueArray47(v1, v2, v3, v4, v5, v6, v7, v8, + v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, v37, + v38, v39, v40, v41, v42, v43, v44, v45, v46, v47); +} + +template +internal::ValueArray48 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, T6 v6, + T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, T15 v15, + T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, T23 v23, + T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, T31 v31, + T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, T39 v39, + T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, T47 v47, + T48 v48) { + return internal::ValueArray48(v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, + v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, v36, + v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48); +} + +template +internal::ValueArray49 Values(T1 v1, T2 v2, T3 v3, T4 v4, T5 v5, + T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, T14 v14, + T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, T22 v22, + T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, T30 v30, + T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, T38 v38, + T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, T46 v46, + T47 v47, T48 v48, T49 v49) { + return internal::ValueArray49(v1, v2, v3, v4, v5, v6, + v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, + v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, v34, v35, + v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, v48, v49); +} + +template +internal::ValueArray50 Values(T1 v1, T2 v2, T3 v3, T4 v4, + T5 v5, T6 v6, T7 v7, T8 v8, T9 v9, T10 v10, T11 v11, T12 v12, T13 v13, + T14 v14, T15 v15, T16 v16, T17 v17, T18 v18, T19 v19, T20 v20, T21 v21, + T22 v22, T23 v23, T24 v24, T25 v25, T26 v26, T27 v27, T28 v28, T29 v29, + T30 v30, T31 v31, T32 v32, T33 v33, T34 v34, T35 v35, T36 v36, T37 v37, + T38 v38, T39 v39, T40 v40, T41 v41, T42 v42, T43 v43, T44 v44, T45 v45, + T46 v46, T47 v47, T48 v48, T49 v49, T50 v50) { + return internal::ValueArray50(v1, v2, v3, v4, + v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, v32, v33, + v34, v35, v36, v37, v38, v39, v40, v41, v42, v43, v44, v45, v46, v47, + v48, v49, v50); +} + +// Bool() allows generating tests with parameters in a set of (false, true). +// +// Synopsis: +// Bool() +// - returns a generator producing sequences with elements {false, true}. +// +// It is useful when testing code that depends on Boolean flags. Combinations +// of multiple flags can be tested when several Bool()'s are combined using +// Combine() function. +// +// In the following example all tests in the test case FlagDependentTest +// will be instantiated twice with parameters false and true. +// +// class FlagDependentTest : public testing::TestWithParam { +// virtual void SetUp() { +// external_flag = GetParam(); +// } +// } +// INSTANTIATE_TEST_CASE_P(BoolSequence, FlagDependentTest, Bool()); +// +inline internal::ParamGenerator Bool() { + return Values(false, true); +} + +# if GTEST_HAS_COMBINE +// Combine() allows the user to combine two or more sequences to produce +// values of a Cartesian product of those sequences' elements. +// +// Synopsis: +// Combine(gen1, gen2, ..., genN) +// - returns a generator producing sequences with elements coming from +// the Cartesian product of elements from the sequences generated by +// gen1, gen2, ..., genN. The sequence elements will have a type of +// tuple where T1, T2, ..., TN are the types +// of elements from sequences produces by gen1, gen2, ..., genN. +// +// Combine can have up to 10 arguments. This number is currently limited +// by the maximum number of elements in the tuple implementation used by Google +// Test. +// +// Example: +// +// This will instantiate tests in test case AnimalTest each one with +// the parameter values tuple("cat", BLACK), tuple("cat", WHITE), +// tuple("dog", BLACK), and tuple("dog", WHITE): +// +// enum Color { BLACK, GRAY, WHITE }; +// class AnimalTest +// : public testing::TestWithParam > {...}; +// +// TEST_P(AnimalTest, AnimalLooksNice) {...} +// +// INSTANTIATE_TEST_CASE_P(AnimalVariations, AnimalTest, +// Combine(Values("cat", "dog"), +// Values(BLACK, WHITE))); +// +// This will instantiate tests in FlagDependentTest with all variations of two +// Boolean flags: +// +// class FlagDependentTest +// : public testing::TestWithParam > { +// virtual void SetUp() { +// // Assigns external_flag_1 and external_flag_2 values from the tuple. +// tie(external_flag_1, external_flag_2) = GetParam(); +// } +// }; +// +// TEST_P(FlagDependentTest, TestFeature1) { +// // Test your code using external_flag_1 and external_flag_2 here. +// } +// INSTANTIATE_TEST_CASE_P(TwoBoolSequence, FlagDependentTest, +// Combine(Bool(), Bool())); +// +template +internal::CartesianProductHolder2 Combine( + const Generator1& g1, const Generator2& g2) { + return internal::CartesianProductHolder2( + g1, g2); +} + +template +internal::CartesianProductHolder3 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3) { + return internal::CartesianProductHolder3( + g1, g2, g3); +} + +template +internal::CartesianProductHolder4 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4) { + return internal::CartesianProductHolder4( + g1, g2, g3, g4); +} + +template +internal::CartesianProductHolder5 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5) { + return internal::CartesianProductHolder5( + g1, g2, g3, g4, g5); +} + +template +internal::CartesianProductHolder6 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6) { + return internal::CartesianProductHolder6( + g1, g2, g3, g4, g5, g6); +} + +template +internal::CartesianProductHolder7 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7) { + return internal::CartesianProductHolder7( + g1, g2, g3, g4, g5, g6, g7); +} + +template +internal::CartesianProductHolder8 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8) { + return internal::CartesianProductHolder8( + g1, g2, g3, g4, g5, g6, g7, g8); +} + +template +internal::CartesianProductHolder9 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9) { + return internal::CartesianProductHolder9( + g1, g2, g3, g4, g5, g6, g7, g8, g9); +} + +template +internal::CartesianProductHolder10 Combine( + const Generator1& g1, const Generator2& g2, const Generator3& g3, + const Generator4& g4, const Generator5& g5, const Generator6& g6, + const Generator7& g7, const Generator8& g8, const Generator9& g9, + const Generator10& g10) { + return internal::CartesianProductHolder10( + g1, g2, g3, g4, g5, g6, g7, g8, g9, g10); +} +# endif // GTEST_HAS_COMBINE + + + +# define TEST_P(test_case_name, test_name) \ + class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \ + : public test_case_name { \ + public: \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ + virtual void TestBody(); \ + private: \ + static int AddToRegistry() { \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, __FILE__, __LINE__)->AddTestPattern(\ + #test_case_name, \ + #test_name, \ + new ::testing::internal::TestMetaFactory< \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>()); \ + return 0; \ + } \ + static int gtest_registering_dummy_; \ + GTEST_DISALLOW_COPY_AND_ASSIGN_(\ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)); \ + }; \ + int GTEST_TEST_CLASS_NAME_(test_case_name, \ + test_name)::gtest_registering_dummy_ = \ + GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() + +# define INSTANTIATE_TEST_CASE_P(prefix, test_case_name, generator) \ + ::testing::internal::ParamGenerator \ + gtest_##prefix##test_case_name##_EvalGenerator_() { return generator; } \ + int gtest_##prefix##test_case_name##_dummy_ = \ + ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ + GetTestCasePatternHolder(\ + #test_case_name, __FILE__, __LINE__)->AddTestCaseInstantiation(\ + #prefix, \ + >est_##prefix##test_case_name##_EvalGenerator_, \ + __FILE__, __LINE__) + +} // namespace testing + +#endif // GTEST_HAS_PARAM_TEST + +#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) +// +// Google C++ Testing Framework definitions useful in production code. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ + +// When you need to test the private or protected members of a class, +// use the FRIEND_TEST macro to declare your tests as friends of the +// class. For example: +// +// class MyClass { +// private: +// void MyMethod(); +// FRIEND_TEST(MyClassTest, MyMethod); +// }; +// +// class MyClassTest : public testing::Test { +// // ... +// }; +// +// TEST_F(MyClassTest, MyMethod) { +// // Can call MyClass::MyMethod() here. +// } + +#define FRIEND_TEST(test_case_name, test_name)\ +friend class test_case_name##_##test_name##_Test + +#endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +// Copyright 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: mheule@google.com (Markus Heule) +// + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ + +#include +#include + +namespace testing { + +// A copyable object representing the result of a test part (i.e. an +// assertion or an explicit FAIL(), ADD_FAILURE(), or SUCCESS()). +// +// Don't inherit from TestPartResult as its destructor is not virtual. +class GTEST_API_ TestPartResult { + public: + // The possible outcomes of a test part (i.e. an assertion or an + // explicit SUCCEED(), FAIL(), or ADD_FAILURE()). + enum Type { + kSuccess, // Succeeded. + kNonFatalFailure, // Failed but the test can continue. + kFatalFailure // Failed and the test should be terminated. + }; + + // C'tor. TestPartResult does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestPartResult object. + TestPartResult(Type a_type, + const char* a_file_name, + int a_line_number, + const char* a_message) + : type_(a_type), + file_name_(a_file_name), + line_number_(a_line_number), + summary_(ExtractSummary(a_message)), + message_(a_message) { + } + + // Gets the outcome of the test part. + Type type() const { return type_; } + + // Gets the name of the source file where the test part took place, or + // NULL if it's unknown. + const char* file_name() const { return file_name_.c_str(); } + + // Gets the line in the source file where the test part took place, + // or -1 if it's unknown. + int line_number() const { return line_number_; } + + // Gets the summary of the failure message. + const char* summary() const { return summary_.c_str(); } + + // Gets the message associated with the test part. + const char* message() const { return message_.c_str(); } + + // Returns true iff the test part passed. + bool passed() const { return type_ == kSuccess; } + + // Returns true iff the test part failed. + bool failed() const { return type_ != kSuccess; } + + // Returns true iff the test part non-fatally failed. + bool nonfatally_failed() const { return type_ == kNonFatalFailure; } + + // Returns true iff the test part fatally failed. + bool fatally_failed() const { return type_ == kFatalFailure; } + private: + Type type_; + + // Gets the summary of the failure message by omitting the stack + // trace in it. + static internal::String ExtractSummary(const char* message); + + // The name of the source file where the test part took place, or + // NULL if the source file is unknown. + internal::String file_name_; + // The line in the source file where the test part took place, or -1 + // if the line number is unknown. + int line_number_; + internal::String summary_; // The test failure summary. + internal::String message_; // The test failure message. +}; + +// Prints a TestPartResult object. +std::ostream& operator<<(std::ostream& os, const TestPartResult& result); + +// An array of TestPartResult objects. +// +// Don't inherit from TestPartResultArray as its destructor is not +// virtual. +class GTEST_API_ TestPartResultArray { + public: + TestPartResultArray() {} + + // Appends the given TestPartResult to the array. + void Append(const TestPartResult& result); + + // Returns the TestPartResult at the given index (0-based). + const TestPartResult& GetTestPartResult(int index) const; + + // Returns the number of TestPartResult objects in the array. + int size() const; + + private: + std::vector array_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestPartResultArray); +}; + +// This interface knows how to report a test part result. +class TestPartResultReporterInterface { + public: + virtual ~TestPartResultReporterInterface() {} + + virtual void ReportTestPartResult(const TestPartResult& result) = 0; +}; + +namespace internal { + +// This helper class is used by {ASSERT|EXPECT}_NO_FATAL_FAILURE to check if a +// statement generates new fatal failures. To do so it registers itself as the +// current test part result reporter. Besides checking if fatal failures were +// reported, it only delegates the reporting to the former result reporter. +// The original result reporter is restored in the destructor. +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +class GTEST_API_ HasNewFatalFailureHelper + : public TestPartResultReporterInterface { + public: + HasNewFatalFailureHelper(); + virtual ~HasNewFatalFailureHelper(); + virtual void ReportTestPartResult(const TestPartResult& result); + bool has_new_fatal_failure() const { return has_new_fatal_failure_; } + private: + bool has_new_fatal_failure_; + TestPartResultReporterInterface* original_reporter_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(HasNewFatalFailureHelper); +}; + +} // namespace internal + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +// Copyright 2008 Google Inc. +// All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: wan@google.com (Zhanyong Wan) + +#ifndef GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#define GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ + +// This header implements typed tests and type-parameterized tests. + +// Typed (aka type-driven) tests repeat the same test for types in a +// list. You must know which types you want to test with when writing +// typed tests. Here's how you do it: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + public: + ... + typedef std::list List; + static T shared_; + T value_; +}; + +// Next, associate a list of types with the test case, which will be +// repeated for each type in the list. The typedef is necessary for +// the macro to parse correctly. +typedef testing::Types MyTypes; +TYPED_TEST_CASE(FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// TYPED_TEST_CASE(FooTest, int); + +// Then, use TYPED_TEST() instead of TEST_F() to define as many typed +// tests for this test case as you want. +TYPED_TEST(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + // Since we are inside a derived class template, C++ requires use to + // visit the members of FooTest via 'this'. + TypeParam n = this->value_; + + // To visit static members of the fixture, add the TestFixture:: + // prefix. + n += TestFixture::shared_; + + // To refer to typedefs in the fixture, add the "typename + // TestFixture::" prefix. + typename TestFixture::List values; + values.push_back(n); + ... +} + +TYPED_TEST(FooTest, HasPropertyA) { ... } + +#endif // 0 + +// Type-parameterized tests are abstract test patterns parameterized +// by a type. Compared with typed tests, type-parameterized tests +// allow you to define the test pattern without knowing what the type +// parameters are. The defined pattern can be instantiated with +// different types any number of times, in any number of translation +// units. +// +// If you are designing an interface or concept, you can define a +// suite of type-parameterized tests to verify properties that any +// valid implementation of the interface/concept should have. Then, +// each implementation can easily instantiate the test suite to verify +// that it conforms to the requirements, without having to write +// similar tests repeatedly. Here's an example: + +#if 0 + +// First, define a fixture class template. It should be parameterized +// by a type. Remember to derive it from testing::Test. +template +class FooTest : public testing::Test { + ... +}; + +// Next, declare that you will define a type-parameterized test case +// (the _P suffix is for "parameterized" or "pattern", whichever you +// prefer): +TYPED_TEST_CASE_P(FooTest); + +// Then, use TYPED_TEST_P() to define as many type-parameterized tests +// for this type-parameterized test case as you want. +TYPED_TEST_P(FooTest, DoesBlah) { + // Inside a test, refer to TypeParam to get the type parameter. + TypeParam n = 0; + ... +} + +TYPED_TEST_P(FooTest, HasPropertyA) { ... } + +// Now the tricky part: you need to register all test patterns before +// you can instantiate them. The first argument of the macro is the +// test case name; the rest are the names of the tests in this test +// case. +REGISTER_TYPED_TEST_CASE_P(FooTest, + DoesBlah, HasPropertyA); + +// Finally, you are free to instantiate the pattern with the types you +// want. If you put the above code in a header file, you can #include +// it in multiple C++ source files and instantiate it multiple times. +// +// To distinguish different instances of the pattern, the first +// argument to the INSTANTIATE_* macro is a prefix that will be added +// to the actual test case name. Remember to pick unique prefixes for +// different instances. +typedef testing::Types MyTypes; +INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, MyTypes); + +// If the type list contains only one type, you can write that type +// directly without Types<...>: +// INSTANTIATE_TYPED_TEST_CASE_P(My, FooTest, int); + +#endif // 0 + + +// Implements typed tests. + +#if GTEST_HAS_TYPED_TEST + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the typedef for the type parameters of the +// given test case. +# define GTEST_TYPE_PARAMS_(TestCaseName) gtest_type_params_##TestCaseName##_ + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define TYPED_TEST_CASE(CaseName, Types) \ + typedef ::testing::internal::TypeList< Types >::type \ + GTEST_TYPE_PARAMS_(CaseName) + +# define TYPED_TEST(CaseName, TestName) \ + template \ + class GTEST_TEST_CLASS_NAME_(CaseName, TestName) \ + : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + bool gtest_##CaseName##_##TestName##_registered_ GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTest< \ + CaseName, \ + ::testing::internal::TemplateSel< \ + GTEST_TEST_CLASS_NAME_(CaseName, TestName)>, \ + GTEST_TYPE_PARAMS_(CaseName)>::Register(\ + "", #CaseName, #TestName, 0); \ + template \ + void GTEST_TEST_CLASS_NAME_(CaseName, TestName)::TestBody() + +#endif // GTEST_HAS_TYPED_TEST + +// Implements type-parameterized tests. + +#if GTEST_HAS_TYPED_TEST_P + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the namespace name that the type-parameterized tests for +// the given type-parameterized test case are defined in. The exact +// name of the namespace is subject to change without notice. +# define GTEST_CASE_NAMESPACE_(TestCaseName) \ + gtest_case_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// +// Expands to the name of the variable used to remember the names of +// the defined tests in the given test case. +# define GTEST_TYPED_TEST_CASE_P_STATE_(TestCaseName) \ + gtest_typed_test_case_p_state_##TestCaseName##_ + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE DIRECTLY. +// +// Expands to the name of the variable used to remember the names of +// the registered tests in the given test case. +# define GTEST_REGISTERED_TEST_NAMES_(TestCaseName) \ + gtest_registered_test_names_##TestCaseName##_ + +// The variables defined in the type-parameterized test macros are +// static as typically these macros are used in a .h file that can be +// #included in multiple translation units linked together. +# define TYPED_TEST_CASE_P(CaseName) \ + static ::testing::internal::TypedTestCasePState \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName) + +# define TYPED_TEST_P(CaseName, TestName) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + template \ + class TestName : public CaseName { \ + private: \ + typedef CaseName TestFixture; \ + typedef gtest_TypeParam_ TypeParam; \ + virtual void TestBody(); \ + }; \ + static bool gtest_##TestName##_defined_ GTEST_ATTRIBUTE_UNUSED_ = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).AddTestName(\ + __FILE__, __LINE__, #CaseName, #TestName); \ + } \ + template \ + void GTEST_CASE_NAMESPACE_(CaseName)::TestName::TestBody() + +# define REGISTER_TYPED_TEST_CASE_P(CaseName, ...) \ + namespace GTEST_CASE_NAMESPACE_(CaseName) { \ + typedef ::testing::internal::Templates<__VA_ARGS__>::type gtest_AllTests_; \ + } \ + static const char* const GTEST_REGISTERED_TEST_NAMES_(CaseName) = \ + GTEST_TYPED_TEST_CASE_P_STATE_(CaseName).VerifyRegisteredTestNames(\ + __FILE__, __LINE__, #__VA_ARGS__) + +// The 'Types' template argument below must have spaces around it +// since some compilers may choke on '>>' when passing a template +// instance (e.g. Types) +# define INSTANTIATE_TYPED_TEST_CASE_P(Prefix, CaseName, Types) \ + bool gtest_##Prefix##_##CaseName GTEST_ATTRIBUTE_UNUSED_ = \ + ::testing::internal::TypeParameterizedTestCase::type>::Register(\ + #Prefix, #CaseName, GTEST_REGISTERED_TEST_NAMES_(CaseName)) + +#endif // GTEST_HAS_TYPED_TEST_P + +#endif // GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ + +// Depending on the platform, different string classes are available. +// On Linux, in addition to ::std::string, Google also makes use of +// class ::string, which has the same interface as ::std::string, but +// has a different implementation. +// +// The user can define GTEST_HAS_GLOBAL_STRING to 1 to indicate that +// ::string is available AND is a distinct type to ::std::string, or +// define it to 0 to indicate otherwise. +// +// If the user's ::std::string and ::string are the same class due to +// aliasing, he should define GTEST_HAS_GLOBAL_STRING to 0. +// +// If the user doesn't define GTEST_HAS_GLOBAL_STRING, it is defined +// heuristically. + +namespace testing { + +// Declares the flags. + +// This flag temporary enables the disabled tests. +GTEST_DECLARE_bool_(also_run_disabled_tests); + +// This flag brings the debugger on an assertion failure. +GTEST_DECLARE_bool_(break_on_failure); + +// This flag controls whether Google Test catches all test-thrown exceptions +// and logs them as failures. +GTEST_DECLARE_bool_(catch_exceptions); + +// This flag enables using colors in terminal output. Available values are +// "yes" to enable colors, "no" (disable colors), or "auto" (the default) +// to let Google Test decide. +GTEST_DECLARE_string_(color); + +// This flag sets up the filter to select by name using a glob pattern +// the tests to run. If the filter is not given all tests are executed. +GTEST_DECLARE_string_(filter); + +// This flag causes the Google Test to list tests. None of the tests listed +// are actually run if the flag is provided. +GTEST_DECLARE_bool_(list_tests); + +// This flag controls whether Google Test emits a detailed XML report to a file +// in addition to its normal textual output. +GTEST_DECLARE_string_(output); + +// This flags control whether Google Test prints the elapsed time for each +// test. +GTEST_DECLARE_bool_(print_time); + +// This flag specifies the random number seed. +GTEST_DECLARE_int32_(random_seed); + +// This flag sets how many times the tests are repeated. The default value +// is 1. If the value is -1 the tests are repeating forever. +GTEST_DECLARE_int32_(repeat); + +// This flag controls whether Google Test includes Google Test internal +// stack frames in failure stack traces. +GTEST_DECLARE_bool_(show_internal_stack_frames); + +// When this flag is specified, tests' order is randomized on every iteration. +GTEST_DECLARE_bool_(shuffle); + +// This flag specifies the maximum number of stack frames to be +// printed in a failure message. +GTEST_DECLARE_int32_(stack_trace_depth); + +// When this flag is specified, a failed assertion will throw an +// exception if exceptions are enabled, or exit the program with a +// non-zero code otherwise. +GTEST_DECLARE_bool_(throw_on_failure); + +// When this flag is set with a "host:port" string, on supported +// platforms test results are streamed to the specified port on +// the specified host machine. +GTEST_DECLARE_string_(stream_result_to); + +// The upper limit for valid stack trace depths. +const int kMaxStackTraceDepth = 100; + +namespace internal { + +class AssertHelper; +class DefaultGlobalTestPartResultReporter; +class ExecDeathTest; +class NoExecDeathTest; +class FinalSuccessChecker; +class GTestFlagSaver; +class TestResultAccessor; +class TestEventListenersAccessor; +class TestEventRepeater; +class WindowsDeathTest; +class UnitTestImpl* GetUnitTestImpl(); +void ReportFailureInUnknownLocation(TestPartResult::Type result_type, + const String& message); + +// Converts a streamable value to a String. A NULL pointer is +// converted to "(null)". When the input value is a ::string, +// ::std::string, ::wstring, or ::std::wstring object, each NUL +// character in it is replaced with "\\0". +// Declared in gtest-internal.h but defined here, so that it has access +// to the definition of the Message class, required by the ARM +// compiler. +template +String StreamableToString(const T& streamable) { + return (Message() << streamable).GetString(); +} + +} // namespace internal + +// The friend relationship of some of these classes is cyclic. +// If we don't forward declare them the compiler might confuse the classes +// in friendship clauses with same named classes on the scope. +class Test; +class TestCase; +class TestInfo; +class UnitTest; + +// A class for indicating whether an assertion was successful. When +// the assertion wasn't successful, the AssertionResult object +// remembers a non-empty message that describes how it failed. +// +// To create an instance of this class, use one of the factory functions +// (AssertionSuccess() and AssertionFailure()). +// +// This class is useful for two purposes: +// 1. Defining predicate functions to be used with Boolean test assertions +// EXPECT_TRUE/EXPECT_FALSE and their ASSERT_ counterparts +// 2. Defining predicate-format functions to be +// used with predicate assertions (ASSERT_PRED_FORMAT*, etc). +// +// For example, if you define IsEven predicate: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then the failed expectation EXPECT_TRUE(IsEven(Fib(5))) +// will print the message +// +// Value of: IsEven(Fib(5)) +// Actual: false (5 is odd) +// Expected: true +// +// instead of a more opaque +// +// Value of: IsEven(Fib(5)) +// Actual: false +// Expected: true +// +// in case IsEven is a simple Boolean predicate. +// +// If you expect your predicate to be reused and want to support informative +// messages in EXPECT_FALSE and ASSERT_FALSE (negative assertions show up +// about half as often as positive ones in our tests), supply messages for +// both success and failure cases: +// +// testing::AssertionResult IsEven(int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess() << n << " is even"; +// else +// return testing::AssertionFailure() << n << " is odd"; +// } +// +// Then a statement EXPECT_FALSE(IsEven(Fib(6))) will print +// +// Value of: IsEven(Fib(6)) +// Actual: true (8 is even) +// Expected: false +// +// NB: Predicates that support negative Boolean assertions have reduced +// performance in positive ones so be careful not to use them in tests +// that have lots (tens of thousands) of positive Boolean assertions. +// +// To use this class with EXPECT_PRED_FORMAT assertions such as: +// +// // Verifies that Foo() returns an even number. +// EXPECT_PRED_FORMAT1(IsEven, Foo()); +// +// you need to define: +// +// testing::AssertionResult IsEven(const char* expr, int n) { +// if ((n % 2) == 0) +// return testing::AssertionSuccess(); +// else +// return testing::AssertionFailure() +// << "Expected: " << expr << " is even\n Actual: it's " << n; +// } +// +// If Foo() returns 5, you will see the following message: +// +// Expected: Foo() is even +// Actual: it's 5 +// +class GTEST_API_ AssertionResult { + public: + // Copy constructor. + // Used in EXPECT_TRUE/FALSE(assertion_result). + AssertionResult(const AssertionResult& other); + // Used in the EXPECT_TRUE/FALSE(bool_expression). + explicit AssertionResult(bool success) : success_(success) {} + + // Returns true iff the assertion succeeded. + operator bool() const { return success_; } // NOLINT + + // Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. + AssertionResult operator!() const; + + // Returns the text streamed into this AssertionResult. Test assertions + // use it when they fail (i.e., the predicate's outcome doesn't match the + // assertion's expectation). When nothing has been streamed into the + // object, returns an empty string. + const char* message() const { + return message_.get() != NULL ? message_->c_str() : ""; + } + // TODO(vladl@google.com): Remove this after making sure no clients use it. + // Deprecated; please use message() instead. + const char* failure_message() const { return message(); } + + // Streams a custom failure message into this object. + template AssertionResult& operator<<(const T& value) { + AppendMessage(Message() << value); + return *this; + } + + // Allows streaming basic output manipulators such as endl or flush into + // this object. + AssertionResult& operator<<( + ::std::ostream& (*basic_manipulator)(::std::ostream& stream)) { + AppendMessage(Message() << basic_manipulator); + return *this; + } + + private: + // Appends the contents of message to message_. + void AppendMessage(const Message& a_message) { + if (message_.get() == NULL) + message_.reset(new ::std::string); + message_->append(a_message.GetString().c_str()); + } + + // Stores result of the assertion predicate. + bool success_; + // Stores the message describing the condition in case the expectation + // construct is not satisfied with the predicate's outcome. + // Referenced via a pointer to avoid taking too much stack frame space + // with test assertions. + internal::scoped_ptr< ::std::string> message_; + + GTEST_DISALLOW_ASSIGN_(AssertionResult); +}; + +// Makes a successful assertion result. +GTEST_API_ AssertionResult AssertionSuccess(); + +// Makes a failed assertion result. +GTEST_API_ AssertionResult AssertionFailure(); + +// Makes a failed assertion result with the given failure message. +// Deprecated; use AssertionFailure() << msg. +GTEST_API_ AssertionResult AssertionFailure(const Message& msg); + +// The abstract class that all tests inherit from. +// +// In Google Test, a unit test program contains one or many TestCases, and +// each TestCase contains one or many Tests. +// +// When you define a test using the TEST macro, you don't need to +// explicitly derive from Test - the TEST macro automatically does +// this for you. +// +// The only time you derive from Test is when defining a test fixture +// to be used a TEST_F. For example: +// +// class FooTest : public testing::Test { +// protected: +// virtual void SetUp() { ... } +// virtual void TearDown() { ... } +// ... +// }; +// +// TEST_F(FooTest, Bar) { ... } +// TEST_F(FooTest, Baz) { ... } +// +// Test is not copyable. +class GTEST_API_ Test { + public: + friend class TestInfo; + + // Defines types for pointers to functions that set up and tear down + // a test case. + typedef internal::SetUpTestCaseFunc SetUpTestCaseFunc; + typedef internal::TearDownTestCaseFunc TearDownTestCaseFunc; + + // The d'tor is virtual as we intend to inherit from Test. + virtual ~Test(); + + // Sets up the stuff shared by all tests in this test case. + // + // Google Test will call Foo::SetUpTestCase() before running the first + // test in test case Foo. Hence a sub-class can define its own + // SetUpTestCase() method to shadow the one defined in the super + // class. + static void SetUpTestCase() {} + + // Tears down the stuff shared by all tests in this test case. + // + // Google Test will call Foo::TearDownTestCase() after running the last + // test in test case Foo. Hence a sub-class can define its own + // TearDownTestCase() method to shadow the one defined in the super + // class. + static void TearDownTestCase() {} + + // Returns true iff the current test has a fatal failure. + static bool HasFatalFailure(); + + // Returns true iff the current test has a non-fatal failure. + static bool HasNonfatalFailure(); + + // Returns true iff the current test has a (either fatal or + // non-fatal) failure. + static bool HasFailure() { return HasFatalFailure() || HasNonfatalFailure(); } + + // Logs a property for the current test. Only the last value for a given + // key is remembered. + // These are public static so they can be called from utility functions + // that are not members of the test fixture. + // The arguments are const char* instead strings, as Google Test is used + // on platforms where string doesn't compile. + // + // Note that a driving consideration for these RecordProperty methods + // was to produce xml output suited to the Greenspan charting utility, + // which at present will only chart values that fit in a 32-bit int. It + // is the user's responsibility to restrict their values to 32-bit ints + // if they intend them to be used with Greenspan. + static void RecordProperty(const char* key, const char* value); + static void RecordProperty(const char* key, int value); + + protected: + // Creates a Test object. + Test(); + + // Sets up the test fixture. + virtual void SetUp(); + + // Tears down the test fixture. + virtual void TearDown(); + + private: + // Returns true iff the current test has the same fixture class as + // the first test in the current test case. + static bool HasSameFixtureClass(); + + // Runs the test after the test fixture has been set up. + // + // A sub-class must implement this to define the test logic. + // + // DO NOT OVERRIDE THIS FUNCTION DIRECTLY IN A USER PROGRAM. + // Instead, use the TEST or TEST_F macro. + virtual void TestBody() = 0; + + // Sets up, executes, and tears down the test. + void Run(); + + // Deletes self. We deliberately pick an unusual name for this + // internal method to avoid clashing with names used in user TESTs. + void DeleteSelf_() { delete this; } + + // Uses a GTestFlagSaver to save and restore all Google Test flags. + const internal::GTestFlagSaver* const gtest_flag_saver_; + + // Often a user mis-spells SetUp() as Setup() and spends a long time + // wondering why it is never called by Google Test. The declaration of + // the following method is solely for catching such an error at + // compile time: + // + // - The return type is deliberately chosen to be not void, so it + // will be a conflict if a user declares void Setup() in his test + // fixture. + // + // - This method is private, so it will be another compiler error + // if a user calls it from his test fixture. + // + // DO NOT OVERRIDE THIS FUNCTION. + // + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } + + // We disallow copying Tests. + GTEST_DISALLOW_COPY_AND_ASSIGN_(Test); +}; + +typedef internal::TimeInMillis TimeInMillis; + +// A copyable object representing a user specified test property which can be +// output as a key/value string pair. +// +// Don't inherit from TestProperty as its destructor is not virtual. +class TestProperty { + public: + // C'tor. TestProperty does NOT have a default constructor. + // Always use this constructor (with parameters) to create a + // TestProperty object. + TestProperty(const char* a_key, const char* a_value) : + key_(a_key), value_(a_value) { + } + + // Gets the user supplied key. + const char* key() const { + return key_.c_str(); + } + + // Gets the user supplied value. + const char* value() const { + return value_.c_str(); + } + + // Sets a new value, overriding the one supplied in the constructor. + void SetValue(const char* new_value) { + value_ = new_value; + } + + private: + // The key supplied by the user. + internal::String key_; + // The value supplied by the user. + internal::String value_; +}; + +// The result of a single Test. This includes a list of +// TestPartResults, a list of TestProperties, a count of how many +// death tests there are in the Test, and how much time it took to run +// the Test. +// +// TestResult is not copyable. +class GTEST_API_ TestResult { + public: + // Creates an empty TestResult. + TestResult(); + + // D'tor. Do not inherit from TestResult. + ~TestResult(); + + // Gets the number of all test parts. This is the sum of the number + // of successful test parts and the number of failed test parts. + int total_part_count() const; + + // Returns the number of the test properties. + int test_property_count() const; + + // Returns true iff the test passed (i.e. no test part failed). + bool Passed() const { return !Failed(); } + + // Returns true iff the test failed. + bool Failed() const; + + // Returns true iff the test fatally failed. + bool HasFatalFailure() const; + + // Returns true iff the test has a non-fatal failure. + bool HasNonfatalFailure() const; + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test part result among all the results. i can range + // from 0 to test_property_count() - 1. If i is not in that range, aborts + // the program. + const TestPartResult& GetTestPartResult(int i) const; + + // Returns the i-th test property. i can range from 0 to + // test_property_count() - 1. If i is not in that range, aborts the + // program. + const TestProperty& GetTestProperty(int i) const; + + private: + friend class TestInfo; + friend class UnitTest; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::ExecDeathTest; + friend class internal::TestResultAccessor; + friend class internal::UnitTestImpl; + friend class internal::WindowsDeathTest; + + // Gets the vector of TestPartResults. + const std::vector& test_part_results() const { + return test_part_results_; + } + + // Gets the vector of TestProperties. + const std::vector& test_properties() const { + return test_properties_; + } + + // Sets the elapsed time. + void set_elapsed_time(TimeInMillis elapsed) { elapsed_time_ = elapsed; } + + // Adds a test property to the list. The property is validated and may add + // a non-fatal failure if invalid (e.g., if it conflicts with reserved + // key names). If a property is already recorded for the same key, the + // value will be updated, rather than storing multiple values for the same + // key. + void RecordProperty(const TestProperty& test_property); + + // Adds a failure if the key is a reserved attribute of Google Test + // testcase tags. Returns true if the property is valid. + // TODO(russr): Validate attribute names are legal and human readable. + static bool ValidateTestProperty(const TestProperty& test_property); + + // Adds a test part result to the list. + void AddTestPartResult(const TestPartResult& test_part_result); + + // Returns the death test count. + int death_test_count() const { return death_test_count_; } + + // Increments the death test count, returning the new count. + int increment_death_test_count() { return ++death_test_count_; } + + // Clears the test part results. + void ClearTestPartResults(); + + // Clears the object. + void Clear(); + + // Protects mutable state of the property vector and of owned + // properties, whose values may be updated. + internal::Mutex test_properites_mutex_; + + // The vector of TestPartResults + std::vector test_part_results_; + // The vector of TestProperties + std::vector test_properties_; + // Running count of death tests. + int death_test_count_; + // The elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + + // We disallow copying TestResult. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestResult); +}; // class TestResult + +// A TestInfo object stores the following information about a test: +// +// Test case name +// Test name +// Whether the test should be run +// A function pointer that creates the test object when invoked +// Test result +// +// The constructor of TestInfo registers itself with the UnitTest +// singleton such that the RUN_ALL_TESTS() macro knows which tests to +// run. +class GTEST_API_ TestInfo { + public: + // Destructs a TestInfo object. This function is not virtual, so + // don't inherit from TestInfo. + ~TestInfo(); + + // Returns the test case name. + const char* test_case_name() const { return test_case_name_.c_str(); } + + // Returns the test name. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a typed + // or a type-parameterized test. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns the text representation of the value parameter, or NULL if this + // is not a value-parameterized test. + const char* value_param() const { + if (value_param_.get() != NULL) + return value_param_->c_str(); + return NULL; + } + + // Returns true if this test should run, that is if the test is not disabled + // (or it is disabled but the also_run_disabled_tests flag has been specified) + // and its full name matches the user-specified filter. + // + // Google Test allows the user to filter the tests by their full names. + // The full name of a test Bar in test case Foo is defined as + // "Foo.Bar". Only the tests that match the filter will run. + // + // A filter is a colon-separated list of glob (not regex) patterns, + // optionally followed by a '-' and a colon-separated list of + // negative patterns (tests to exclude). A test is run if it + // matches one of the positive patterns and does not match any of + // the negative patterns. + // + // For example, *A*:Foo.* is a filter that matches any string that + // contains the character 'A' or starts with "Foo.". + bool should_run() const { return should_run_; } + + // Returns the result of the test. + const TestResult* result() const { return &result_; } + + private: + +#if GTEST_HAS_DEATH_TEST + friend class internal::DefaultDeathTestFactory; +#endif // GTEST_HAS_DEATH_TEST + friend class Test; + friend class TestCase; + friend class internal::UnitTestImpl; + friend TestInfo* internal::MakeAndRegisterTestInfo( + const char* test_case_name, const char* name, + const char* type_param, + const char* value_param, + internal::TypeId fixture_class_id, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc, + internal::TestFactoryBase* factory); + + // Constructs a TestInfo object. The newly constructed instance assumes + // ownership of the factory object. + TestInfo(const char* test_case_name, const char* name, + const char* a_type_param, + const char* a_value_param, + internal::TypeId fixture_class_id, + internal::TestFactoryBase* factory); + + // Increments the number of death tests encountered in this test so + // far. + int increment_death_test_count() { + return result_.increment_death_test_count(); + } + + // Creates the test object, runs it, records its result, and then + // deletes it. + void Run(); + + static void ClearTestResult(TestInfo* test_info) { + test_info->result_.Clear(); + } + + // These fields are immutable properties of the test. + const std::string test_case_name_; // Test case name + const std::string name_; // Test name + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // Text representation of the value parameter, or NULL if this is not a + // value-parameterized test. + const internal::scoped_ptr value_param_; + const internal::TypeId fixture_class_id_; // ID of the test fixture class + bool should_run_; // True iff this test should run + bool is_disabled_; // True iff this test is disabled + bool matches_filter_; // True if this test matches the + // user-specified filter. + internal::TestFactoryBase* const factory_; // The factory that creates + // the test object + + // This field is mutable and needs to be reset before running the + // test for the second time. + TestResult result_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestInfo); +}; + +// A test case, which consists of a vector of TestInfos. +// +// TestCase is not copyable. +class GTEST_API_ TestCase { + public: + // Creates a TestCase with the given name. + // + // TestCase does NOT have a default constructor. Always use this + // constructor to create a TestCase object. + // + // Arguments: + // + // name: name of the test case + // a_type_param: the name of the test's type parameter, or NULL if + // this is not a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test case + // tear_down_tc: pointer to the function that tears down the test case + TestCase(const char* name, const char* a_type_param, + Test::SetUpTestCaseFunc set_up_tc, + Test::TearDownTestCaseFunc tear_down_tc); + + // Destructor of TestCase. + virtual ~TestCase(); + + // Gets the name of the TestCase. + const char* name() const { return name_.c_str(); } + + // Returns the name of the parameter type, or NULL if this is not a + // type-parameterized test case. + const char* type_param() const { + if (type_param_.get() != NULL) + return type_param_->c_str(); + return NULL; + } + + // Returns true if any test in this test case should run. + bool should_run() const { return should_run_; } + + // Gets the number of successful tests in this test case. + int successful_test_count() const; + + // Gets the number of failed tests in this test case. + int failed_test_count() const; + + // Gets the number of disabled tests in this test case. + int disabled_test_count() const; + + // Get the number of tests in this test case that should run. + int test_to_run_count() const; + + // Gets the number of all tests in this test case. + int total_test_count() const; + + // Returns true iff the test case passed. + bool Passed() const { return !Failed(); } + + // Returns true iff the test case failed. + bool Failed() const { return failed_test_count() > 0; } + + // Returns the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const { return elapsed_time_; } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + const TestInfo* GetTestInfo(int i) const; + + private: + friend class Test; + friend class internal::UnitTestImpl; + + // Gets the (mutable) vector of TestInfos in this TestCase. + std::vector& test_info_list() { return test_info_list_; } + + // Gets the (immutable) vector of TestInfos in this TestCase. + const std::vector& test_info_list() const { + return test_info_list_; + } + + // Returns the i-th test among all the tests. i can range from 0 to + // total_test_count() - 1. If i is not in that range, returns NULL. + TestInfo* GetMutableTestInfo(int i); + + // Sets the should_run member. + void set_should_run(bool should) { should_run_ = should; } + + // Adds a TestInfo to this test case. Will delete the TestInfo upon + // destruction of the TestCase object. + void AddTestInfo(TestInfo * test_info); + + // Clears the results of all tests in this test case. + void ClearResult(); + + // Clears the results of all tests in the given test case. + static void ClearTestCaseResult(TestCase* test_case) { + test_case->ClearResult(); + } + + // Runs every test in this TestCase. + void Run(); + + // Runs SetUpTestCase() for this TestCase. This wrapper is needed + // for catching exceptions thrown from SetUpTestCase(). + void RunSetUpTestCase() { (*set_up_tc_)(); } + + // Runs TearDownTestCase() for this TestCase. This wrapper is + // needed for catching exceptions thrown from TearDownTestCase(). + void RunTearDownTestCase() { (*tear_down_tc_)(); } + + // Returns true iff test passed. + static bool TestPassed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Passed(); + } + + // Returns true iff test failed. + static bool TestFailed(const TestInfo* test_info) { + return test_info->should_run() && test_info->result()->Failed(); + } + + // Returns true iff test is disabled. + static bool TestDisabled(const TestInfo* test_info) { + return test_info->is_disabled_; + } + + // Returns true if the given test should run. + static bool ShouldRunTest(const TestInfo* test_info) { + return test_info->should_run(); + } + + // Shuffles the tests in this test case. + void ShuffleTests(internal::Random* random); + + // Restores the test order to before the first shuffle. + void UnshuffleTests(); + + // Name of the test case. + internal::String name_; + // Name of the parameter type, or NULL if this is not a typed or a + // type-parameterized test. + const internal::scoped_ptr type_param_; + // The vector of TestInfos in their original order. It owns the + // elements in the vector. + std::vector test_info_list_; + // Provides a level of indirection for the test list to allow easy + // shuffling and restoring the test order. The i-th element in this + // vector is the index of the i-th test in the shuffled test list. + std::vector test_indices_; + // Pointer to the function that sets up the test case. + Test::SetUpTestCaseFunc set_up_tc_; + // Pointer to the function that tears down the test case. + Test::TearDownTestCaseFunc tear_down_tc_; + // True iff any test in this test case should run. + bool should_run_; + // Elapsed time, in milliseconds. + TimeInMillis elapsed_time_; + + // We disallow copying TestCases. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestCase); +}; + +// An Environment object is capable of setting up and tearing down an +// environment. The user should subclass this to define his own +// environment(s). +// +// An Environment object does the set-up and tear-down in virtual +// methods SetUp() and TearDown() instead of the constructor and the +// destructor, as: +// +// 1. You cannot safely throw from a destructor. This is a problem +// as in some cases Google Test is used where exceptions are enabled, and +// we may want to implement ASSERT_* using exceptions where they are +// available. +// 2. You cannot use ASSERT_* directly in a constructor or +// destructor. +class Environment { + public: + // The d'tor is virtual as we need to subclass Environment. + virtual ~Environment() {} + + // Override this to define how to set up the environment. + virtual void SetUp() {} + + // Override this to define how to tear down the environment. + virtual void TearDown() {} + private: + // If you see an error about overriding the following function or + // about it being private, you have mis-spelled SetUp() as Setup(). + struct Setup_should_be_spelled_SetUp {}; + virtual Setup_should_be_spelled_SetUp* Setup() { return NULL; } +}; + +// The interface for tracing execution of tests. The methods are organized in +// the order the corresponding events are fired. +class TestEventListener { + public: + virtual ~TestEventListener() {} + + // Fired before any test activity starts. + virtual void OnTestProgramStart(const UnitTest& unit_test) = 0; + + // Fired before each iteration of tests starts. There may be more than + // one iteration if GTEST_FLAG(repeat) is set. iteration is the iteration + // index, starting from 0. + virtual void OnTestIterationStart(const UnitTest& unit_test, + int iteration) = 0; + + // Fired before environment set-up for each iteration of tests starts. + virtual void OnEnvironmentsSetUpStart(const UnitTest& unit_test) = 0; + + // Fired after environment set-up for each iteration of tests ends. + virtual void OnEnvironmentsSetUpEnd(const UnitTest& unit_test) = 0; + + // Fired before the test case starts. + virtual void OnTestCaseStart(const TestCase& test_case) = 0; + + // Fired before the test starts. + virtual void OnTestStart(const TestInfo& test_info) = 0; + + // Fired after a failed assertion or a SUCCEED() invocation. + virtual void OnTestPartResult(const TestPartResult& test_part_result) = 0; + + // Fired after the test ends. + virtual void OnTestEnd(const TestInfo& test_info) = 0; + + // Fired after the test case ends. + virtual void OnTestCaseEnd(const TestCase& test_case) = 0; + + // Fired before environment tear-down for each iteration of tests starts. + virtual void OnEnvironmentsTearDownStart(const UnitTest& unit_test) = 0; + + // Fired after environment tear-down for each iteration of tests ends. + virtual void OnEnvironmentsTearDownEnd(const UnitTest& unit_test) = 0; + + // Fired after each iteration of tests finishes. + virtual void OnTestIterationEnd(const UnitTest& unit_test, + int iteration) = 0; + + // Fired after all test activities have ended. + virtual void OnTestProgramEnd(const UnitTest& unit_test) = 0; +}; + +// The convenience class for users who need to override just one or two +// methods and are not concerned that a possible change to a signature of +// the methods they override will not be caught during the build. For +// comments about each method please see the definition of TestEventListener +// above. +class EmptyTestEventListener : public TestEventListener { + public: + virtual void OnTestProgramStart(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationStart(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestCaseStart(const TestCase& /*test_case*/) {} + virtual void OnTestStart(const TestInfo& /*test_info*/) {} + virtual void OnTestPartResult(const TestPartResult& /*test_part_result*/) {} + virtual void OnTestEnd(const TestInfo& /*test_info*/) {} + virtual void OnTestCaseEnd(const TestCase& /*test_case*/) {} + virtual void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) {} + virtual void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) {} + virtual void OnTestIterationEnd(const UnitTest& /*unit_test*/, + int /*iteration*/) {} + virtual void OnTestProgramEnd(const UnitTest& /*unit_test*/) {} +}; + +// TestEventListeners lets users add listeners to track events in Google Test. +class GTEST_API_ TestEventListeners { + public: + TestEventListeners(); + ~TestEventListeners(); + + // Appends an event listener to the end of the list. Google Test assumes + // the ownership of the listener (i.e. it will delete the listener when + // the test program finishes). + void Append(TestEventListener* listener); + + // Removes the given event listener from the list and returns it. It then + // becomes the caller's responsibility to delete the listener. Returns + // NULL if the listener is not found in the list. + TestEventListener* Release(TestEventListener* listener); + + // Returns the standard listener responsible for the default console + // output. Can be removed from the listeners list to shut down default + // console output. Note that removing this object from the listener list + // with Release transfers its ownership to the caller and makes this + // function return NULL the next time. + TestEventListener* default_result_printer() const { + return default_result_printer_; + } + + // Returns the standard listener responsible for the default XML output + // controlled by the --gtest_output=xml flag. Can be removed from the + // listeners list by users who want to shut down the default XML output + // controlled by this flag and substitute it with custom one. Note that + // removing this object from the listener list with Release transfers its + // ownership to the caller and makes this function return NULL the next + // time. + TestEventListener* default_xml_generator() const { + return default_xml_generator_; + } + + private: + friend class TestCase; + friend class TestInfo; + friend class internal::DefaultGlobalTestPartResultReporter; + friend class internal::NoExecDeathTest; + friend class internal::TestEventListenersAccessor; + friend class internal::UnitTestImpl; + + // Returns repeater that broadcasts the TestEventListener events to all + // subscribers. + TestEventListener* repeater(); + + // Sets the default_result_printer attribute to the provided listener. + // The listener is also added to the listener list and previous + // default_result_printer is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultResultPrinter(TestEventListener* listener); + + // Sets the default_xml_generator attribute to the provided listener. The + // listener is also added to the listener list and previous + // default_xml_generator is removed from it and deleted. The listener can + // also be NULL in which case it will not be added to the list. Does + // nothing if the previous and the current listener objects are the same. + void SetDefaultXmlGenerator(TestEventListener* listener); + + // Controls whether events will be forwarded by the repeater to the + // listeners in the list. + bool EventForwardingEnabled() const; + void SuppressEventForwarding(); + + // The actual list of listeners. + internal::TestEventRepeater* repeater_; + // Listener responsible for the standard result output. + TestEventListener* default_result_printer_; + // Listener responsible for the creation of the XML output file. + TestEventListener* default_xml_generator_; + + // We disallow copying TestEventListeners. + GTEST_DISALLOW_COPY_AND_ASSIGN_(TestEventListeners); +}; + +// A UnitTest consists of a vector of TestCases. +// +// This is a singleton class. The only instance of UnitTest is +// created when UnitTest::GetInstance() is first called. This +// instance is never deleted. +// +// UnitTest is not copyable. +// +// This class is thread-safe as long as the methods are called +// according to their specification. +class GTEST_API_ UnitTest { + public: + // Gets the singleton UnitTest object. The first time this method + // is called, a UnitTest object is constructed and returned. + // Consecutive calls will return the same object. + static UnitTest* GetInstance(); + + // Runs all tests in this UnitTest object and prints the result. + // Returns 0 if successful, or 1 otherwise. + // + // This method can only be called from the main thread. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + int Run() GTEST_MUST_USE_RESULT_; + + // Returns the working directory when the first TEST() or TEST_F() + // was executed. The UnitTest object owns the string. + const char* original_working_dir() const; + + // Returns the TestCase object for the test that's currently running, + // or NULL if no test is running. + const TestCase* current_test_case() const; + + // Returns the TestInfo object for the test that's currently running, + // or NULL if no test is running. + const TestInfo* current_test_info() const; + + // Returns the random seed used at the start of the current test run. + int random_seed() const; + +#if GTEST_HAS_PARAM_TEST + // Returns the ParameterizedTestCaseRegistry object used to keep track of + // value-parameterized tests and instantiate and register them. + // + // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + internal::ParameterizedTestCaseRegistry& parameterized_test_registry(); +#endif // GTEST_HAS_PARAM_TEST + + // Gets the number of successful test cases. + int successful_test_case_count() const; + + // Gets the number of failed test cases. + int failed_test_case_count() const; + + // Gets the number of all test cases. + int total_test_case_count() const; + + // Gets the number of all test cases that contain at least one test + // that should run. + int test_case_to_run_count() const; + + // Gets the number of successful tests. + int successful_test_count() const; + + // Gets the number of failed tests. + int failed_test_count() const; + + // Gets the number of disabled tests. + int disabled_test_count() const; + + // Gets the number of all tests. + int total_test_count() const; + + // Gets the number of tests that should run. + int test_to_run_count() const; + + // Gets the elapsed time, in milliseconds. + TimeInMillis elapsed_time() const; + + // Returns true iff the unit test passed (i.e. all test cases passed). + bool Passed() const; + + // Returns true iff the unit test failed (i.e. some test case failed + // or something outside of all tests failed). + bool Failed() const; + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + const TestCase* GetTestCase(int i) const; + + // Returns the list of event listeners that can be used to track events + // inside Google Test. + TestEventListeners& listeners(); + + private: + // Registers and returns a global test environment. When a test + // program is run, all global test environments will be set-up in + // the order they were registered. After all tests in the program + // have finished, all global test environments will be torn-down in + // the *reverse* order they were registered. + // + // The UnitTest object takes ownership of the given environment. + // + // This method can only be called from the main thread. + Environment* AddEnvironment(Environment* env); + + // Adds a TestPartResult to the current TestResult object. All + // Google Test assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) + // eventually call this to report their results. The user code + // should use the assertion macros instead of calling this directly. + void AddTestPartResult(TestPartResult::Type result_type, + const char* file_name, + int line_number, + const internal::String& message, + const internal::String& os_stack_trace); + + // Adds a TestProperty to the current TestResult object. If the result already + // contains a property with the same key, the value will be updated. + void RecordPropertyForCurrentTest(const char* key, const char* value); + + // Gets the i-th test case among all the test cases. i can range from 0 to + // total_test_case_count() - 1. If i is not in that range, returns NULL. + TestCase* GetMutableTestCase(int i); + + // Accessors for the implementation object. + internal::UnitTestImpl* impl() { return impl_; } + const internal::UnitTestImpl* impl() const { return impl_; } + + // These classes and funcions are friends as they need to access private + // members of UnitTest. + friend class Test; + friend class internal::AssertHelper; + friend class internal::ScopedTrace; + friend Environment* AddGlobalTestEnvironment(Environment* env); + friend internal::UnitTestImpl* internal::GetUnitTestImpl(); + friend void internal::ReportFailureInUnknownLocation( + TestPartResult::Type result_type, + const internal::String& message); + + // Creates an empty UnitTest. + UnitTest(); + + // D'tor + virtual ~UnitTest(); + + // Pushes a trace defined by SCOPED_TRACE() on to the per-thread + // Google Test trace stack. + void PushGTestTrace(const internal::TraceInfo& trace); + + // Pops a trace from the per-thread Google Test trace stack. + void PopGTestTrace(); + + // Protects mutable state in *impl_. This is mutable as some const + // methods need to lock it too. + mutable internal::Mutex mutex_; + + // Opaque implementation object. This field is never changed once + // the object is constructed. We don't mark it as const here, as + // doing so will cause a warning in the constructor of UnitTest. + // Mutable state in *impl_ is protected by mutex_. + internal::UnitTestImpl* impl_; + + // We disallow copying UnitTest. + GTEST_DISALLOW_COPY_AND_ASSIGN_(UnitTest); +}; + +// A convenient wrapper for adding an environment for the test +// program. +// +// You should call this before RUN_ALL_TESTS() is called, probably in +// main(). If you use gtest_main, you need to call this before main() +// starts for it to take effect. For example, you can define a global +// variable like this: +// +// testing::Environment* const foo_env = +// testing::AddGlobalTestEnvironment(new FooEnvironment); +// +// However, we strongly recommend you to write your own main() and +// call AddGlobalTestEnvironment() there, as relying on initialization +// of global variables makes the code harder to read and may cause +// problems when you register multiple environments from different +// translation units and the environments have dependencies among them +// (remember that the compiler doesn't guarantee the order in which +// global variables from different translation units are initialized). +inline Environment* AddGlobalTestEnvironment(Environment* env) { + return UnitTest::GetInstance()->AddEnvironment(env); +} + +// Initializes Google Test. This must be called before calling +// RUN_ALL_TESTS(). In particular, it parses a command line for the +// flags that Google Test recognizes. Whenever a Google Test flag is +// seen, it is removed from argv, and *argc is decremented. +// +// No value is returned. Instead, the Google Test flag variables are +// updated. +// +// Calling the function for the second time has no user-visible effect. +GTEST_API_ void InitGoogleTest(int* argc, char** argv); + +// This overloaded version can be used in Windows programs compiled in +// UNICODE mode. +GTEST_API_ void InitGoogleTest(int* argc, wchar_t** argv); + +namespace internal { + +// Formats a comparison assertion (e.g. ASSERT_EQ, EXPECT_LT, and etc) +// operand to be used in a failure message. The type (but not value) +// of the other operand may affect the format. This allows us to +// print a char* as a raw pointer when it is compared against another +// char*, and print it as a C string when it is compared against an +// std::string object, for example. +// +// The default implementation ignores the type of the other operand. +// Some specialized versions are used to handle formatting wide or +// narrow C strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +String FormatForComparisonFailureMessage(const T1& value, + const T2& /* other_operand */) { + // C++Builder compiles this incorrectly if the namespace isn't explicitly + // given. + return ::testing::PrintToString(value); +} + +// The helper function for {ASSERT|EXPECT}_EQ. +template +AssertionResult CmpHelperEQ(const char* expected_expression, + const char* actual_expression, + const T1& expected, + const T2& actual) { +#ifdef _MSC_VER +# pragma warning(push) // Saves the current warning state. +# pragma warning(disable:4389) // Temporarily disables warning on + // signed/unsigned mismatch. +#endif + + if (expected == actual) { + return AssertionSuccess(); + } + +#ifdef _MSC_VER +# pragma warning(pop) // Restores the warning state. +#endif + + return EqFailure(expected_expression, + actual_expression, + FormatForComparisonFailureMessage(expected, actual), + FormatForComparisonFailureMessage(actual, expected), + false); +} + +// With this overloaded version, we allow anonymous enums to be used +// in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous enums +// can be implicitly cast to BiggestInt. +GTEST_API_ AssertionResult CmpHelperEQ(const char* expected_expression, + const char* actual_expression, + BiggestInt expected, + BiggestInt actual); + +// The helper class for {ASSERT|EXPECT}_EQ. The template argument +// lhs_is_null_literal is true iff the first argument to ASSERT_EQ() +// is a null pointer literal. The following default implementation is +// for lhs_is_null_literal being false. +template +class EqHelper { + public: + // This templatized version is for the general case. + template + static AssertionResult Compare(const char* expected_expression, + const char* actual_expression, + const T1& expected, + const T2& actual) { + return CmpHelperEQ(expected_expression, actual_expression, expected, + actual); + } + + // With this overloaded version, we allow anonymous enums to be used + // in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous + // enums can be implicitly cast to BiggestInt. + // + // Even though its body looks the same as the above version, we + // cannot merge the two, as it will make anonymous enums unhappy. + static AssertionResult Compare(const char* expected_expression, + const char* actual_expression, + BiggestInt expected, + BiggestInt actual) { + return CmpHelperEQ(expected_expression, actual_expression, expected, + actual); + } +}; + +// This specialization is used when the first argument to ASSERT_EQ() +// is a null pointer literal, like NULL, false, or 0. +template <> +class EqHelper { + public: + // We define two overloaded versions of Compare(). The first + // version will be picked when the second argument to ASSERT_EQ() is + // NOT a pointer, e.g. ASSERT_EQ(0, AnIntFunction()) or + // EXPECT_EQ(false, a_bool). + template + static AssertionResult Compare( + const char* expected_expression, + const char* actual_expression, + const T1& expected, + const T2& actual, + // The following line prevents this overload from being considered if T2 + // is not a pointer type. We need this because ASSERT_EQ(NULL, my_ptr) + // expands to Compare("", "", NULL, my_ptr), which requires a conversion + // to match the Secret* in the other overload, which would otherwise make + // this template match better. + typename EnableIf::value>::type* = 0) { + return CmpHelperEQ(expected_expression, actual_expression, expected, + actual); + } + + // This version will be picked when the second argument to ASSERT_EQ() is a + // pointer, e.g. ASSERT_EQ(NULL, a_pointer). + template + static AssertionResult Compare( + const char* expected_expression, + const char* actual_expression, + // We used to have a second template parameter instead of Secret*. That + // template parameter would deduce to 'long', making this a better match + // than the first overload even without the first overload's EnableIf. + // Unfortunately, gcc with -Wconversion-null warns when "passing NULL to + // non-pointer argument" (even a deduced integral argument), so the old + // implementation caused warnings in user code. + Secret* /* expected (NULL) */, + T* actual) { + // We already know that 'expected' is a null pointer. + return CmpHelperEQ(expected_expression, actual_expression, + static_cast(NULL), actual); + } +}; + +// A macro for implementing the helper functions needed to implement +// ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste +// of similar code. +// +// For each templatized helper function, we also define an overloaded +// version for BiggestInt in order to reduce code bloat and allow +// anonymous enums to be used with {ASSERT|EXPECT}_?? when compiled +// with gcc 4. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ +template \ +AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ + const T1& val1, const T2& val2) {\ + if (val1 op val2) {\ + return AssertionSuccess();\ + } else {\ + return AssertionFailure() \ + << "Expected: (" << expr1 << ") " #op " (" << expr2\ + << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ + << " vs " << FormatForComparisonFailureMessage(val2, val1);\ + }\ +}\ +GTEST_API_ AssertionResult CmpHelper##op_name(\ + const char* expr1, const char* expr2, BiggestInt val1, BiggestInt val2) + +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// Implements the helper function for {ASSERT|EXPECT}_NE +GTEST_IMPL_CMP_HELPER_(NE, !=); +// Implements the helper function for {ASSERT|EXPECT}_LE +GTEST_IMPL_CMP_HELPER_(LE, <=); +// Implements the helper function for {ASSERT|EXPECT}_LT +GTEST_IMPL_CMP_HELPER_(LT, < ); +// Implements the helper function for {ASSERT|EXPECT}_GE +GTEST_IMPL_CMP_HELPER_(GE, >=); +// Implements the helper function for {ASSERT|EXPECT}_GT +GTEST_IMPL_CMP_HELPER_(GT, > ); + +#undef GTEST_IMPL_CMP_HELPER_ + +// The helper function for {ASSERT|EXPECT}_STREQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* expected_expression, + const char* actual_expression, + const char* expected, + const char* actual); + +// The helper function for {ASSERT|EXPECT}_STRCASEEQ. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASEEQ(const char* expected_expression, + const char* actual_expression, + const char* expected, + const char* actual); + +// The helper function for {ASSERT|EXPECT}_STRNE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + +// The helper function for {ASSERT|EXPECT}_STRCASENE. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRCASENE(const char* s1_expression, + const char* s2_expression, + const char* s1, + const char* s2); + + +// Helper function for *_STREQ on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTREQ(const char* expected_expression, + const char* actual_expression, + const wchar_t* expected, + const wchar_t* actual); + +// Helper function for *_STRNE on wide strings. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult CmpHelperSTRNE(const char* s1_expression, + const char* s2_expression, + const wchar_t* s1, + const wchar_t* s2); + +} // namespace internal + +// IsSubstring() and IsNotSubstring() are intended to be used as the +// first argument to {EXPECT,ASSERT}_PRED_FORMAT2(), not by +// themselves. They check whether needle is a substring of haystack +// (NULL is considered a substring of itself only), and return an +// appropriate error message when they fail. +// +// The {needle,haystack}_expr arguments are the stringified +// expressions that generated the two real arguments. +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const char* needle, const char* haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const wchar_t* needle, const wchar_t* haystack); +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::string& needle, const ::std::string& haystack); + +#if GTEST_HAS_STD_WSTRING +GTEST_API_ AssertionResult IsSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +GTEST_API_ AssertionResult IsNotSubstring( + const char* needle_expr, const char* haystack_expr, + const ::std::wstring& needle, const ::std::wstring& haystack); +#endif // GTEST_HAS_STD_WSTRING + +namespace internal { + +// Helper template function for comparing floating-points. +// +// Template parameter: +// +// RawType: the raw floating-point type (either float or double) +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +template +AssertionResult CmpHelperFloatingPointEQ(const char* expected_expression, + const char* actual_expression, + RawType expected, + RawType actual) { + const FloatingPoint lhs(expected), rhs(actual); + + if (lhs.AlmostEquals(rhs)) { + return AssertionSuccess(); + } + + ::std::stringstream expected_ss; + expected_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << expected; + + ::std::stringstream actual_ss; + actual_ss << std::setprecision(std::numeric_limits::digits10 + 2) + << actual; + + return EqFailure(expected_expression, + actual_expression, + StringStreamToString(&expected_ss), + StringStreamToString(&actual_ss), + false); +} + +// Helper function for implementing ASSERT_NEAR. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. +GTEST_API_ AssertionResult DoubleNearPredFormat(const char* expr1, + const char* expr2, + const char* abs_error_expr, + double val1, + double val2, + double abs_error); + +// INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. +// A class that enables one to stream messages to assertion macros +class GTEST_API_ AssertHelper { + public: + // Constructor. + AssertHelper(TestPartResult::Type type, + const char* file, + int line, + const char* message); + ~AssertHelper(); + + // Message assignment is a semantic trick to enable assertion + // streaming; see the GTEST_MESSAGE_ macro below. + void operator=(const Message& message) const; + + private: + // We put our data in a struct so that the size of the AssertHelper class can + // be as small as possible. This is important because gcc is incapable of + // re-using stack space even for temporary variables, so every EXPECT_EQ + // reserves stack space for another AssertHelper. + struct AssertHelperData { + AssertHelperData(TestPartResult::Type t, + const char* srcfile, + int line_num, + const char* msg) + : type(t), file(srcfile), line(line_num), message(msg) { } + + TestPartResult::Type const type; + const char* const file; + int const line; + String const message; + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelperData); + }; + + AssertHelperData* const data_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelper); +}; + +} // namespace internal + +#if GTEST_HAS_PARAM_TEST +// The pure interface class that all value-parameterized tests inherit from. +// A value-parameterized class must inherit from both ::testing::Test and +// ::testing::WithParamInterface. In most cases that just means inheriting +// from ::testing::TestWithParam, but more complicated test hierarchies +// may need to inherit from Test and WithParamInterface at different levels. +// +// This interface has support for accessing the test parameter value via +// the GetParam() method. +// +// Use it with one of the parameter generator defining functions, like Range(), +// Values(), ValuesIn(), Bool(), and Combine(). +// +// class FooTest : public ::testing::TestWithParam { +// protected: +// FooTest() { +// // Can use GetParam() here. +// } +// virtual ~FooTest() { +// // Can use GetParam() here. +// } +// virtual void SetUp() { +// // Can use GetParam() here. +// } +// virtual void TearDown { +// // Can use GetParam() here. +// } +// }; +// TEST_P(FooTest, DoesBar) { +// // Can use GetParam() method here. +// Foo foo; +// ASSERT_TRUE(foo.DoesBar(GetParam())); +// } +// INSTANTIATE_TEST_CASE_P(OneToTenRange, FooTest, ::testing::Range(1, 10)); + +template +class WithParamInterface { + public: + typedef T ParamType; + virtual ~WithParamInterface() {} + + // The current parameter value. Is also available in the test fixture's + // constructor. This member function is non-static, even though it only + // references static data, to reduce the opportunity for incorrect uses + // like writing 'WithParamInterface::GetParam()' for a test that + // uses a fixture whose parameter type is int. + const ParamType& GetParam() const { return *parameter_; } + + private: + // Sets parameter value. The caller is responsible for making sure the value + // remains alive and unchanged throughout the current test. + static void SetParam(const ParamType* parameter) { + parameter_ = parameter; + } + + // Static value used for accessing parameter during a test lifetime. + static const ParamType* parameter_; + + // TestClass must be a subclass of WithParamInterface and Test. + template friend class internal::ParameterizedTestFactory; +}; + +template +const T* WithParamInterface::parameter_ = NULL; + +// Most value-parameterized classes can ignore the existence of +// WithParamInterface, and can just inherit from ::testing::TestWithParam. + +template +class TestWithParam : public Test, public WithParamInterface { +}; + +#endif // GTEST_HAS_PARAM_TEST + +// Macros for indicating success/failure in test code. + +// ADD_FAILURE unconditionally adds a failure to the current test. +// SUCCEED generates a success - it doesn't automatically make the +// current test successful, as a test is only successful when it has +// no failure. +// +// EXPECT_* verifies that a certain condition is satisfied. If not, +// it behaves like ADD_FAILURE. In particular: +// +// EXPECT_TRUE verifies that a Boolean condition is true. +// EXPECT_FALSE verifies that a Boolean condition is false. +// +// FAIL and ASSERT_* are similar to ADD_FAILURE and EXPECT_*, except +// that they will also abort the current function on failure. People +// usually want the fail-fast behavior of FAIL and ASSERT_*, but those +// writing data-driven tests often find themselves using ADD_FAILURE +// and EXPECT_* more. +// +// Examples: +// +// EXPECT_TRUE(server.StatusIsOK()); +// ASSERT_FALSE(server.HasPendingRequest(port)) +// << "There are still pending requests " << "on port " << port; + +// Generates a nonfatal failure with a generic message. +#define ADD_FAILURE() GTEST_NONFATAL_FAILURE_("Failed") + +// Generates a nonfatal failure at the given source file location with +// a generic message. +#define ADD_FAILURE_AT(file, line) \ + GTEST_MESSAGE_AT_(file, line, "Failed", \ + ::testing::TestPartResult::kNonFatalFailure) + +// Generates a fatal failure with a generic message. +#define GTEST_FAIL() GTEST_FATAL_FAILURE_("Failed") + +// Define this macro to 1 to omit the definition of FAIL(), which is a +// generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_FAIL +# define FAIL() GTEST_FAIL() +#endif + +// Generates a success with a generic message. +#define GTEST_SUCCEED() GTEST_SUCCESS_("Succeeded") + +// Define this macro to 1 to omit the definition of SUCCEED(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_SUCCEED +# define SUCCEED() GTEST_SUCCEED() +#endif + +// Macros for testing exceptions. +// +// * {ASSERT|EXPECT}_THROW(statement, expected_exception): +// Tests that the statement throws the expected exception. +// * {ASSERT|EXPECT}_NO_THROW(statement): +// Tests that the statement doesn't throw any exception. +// * {ASSERT|EXPECT}_ANY_THROW(statement): +// Tests that the statement throws an exception. + +#define EXPECT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_NONFATAL_FAILURE_) +#define EXPECT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define EXPECT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_NONFATAL_FAILURE_) +#define ASSERT_THROW(statement, expected_exception) \ + GTEST_TEST_THROW_(statement, expected_exception, GTEST_FATAL_FAILURE_) +#define ASSERT_NO_THROW(statement) \ + GTEST_TEST_NO_THROW_(statement, GTEST_FATAL_FAILURE_) +#define ASSERT_ANY_THROW(statement) \ + GTEST_TEST_ANY_THROW_(statement, GTEST_FATAL_FAILURE_) + +// Boolean assertions. Condition can be either a Boolean expression or an +// AssertionResult. For more information on how to use AssertionResult with +// these macros see comments on that class. +#define EXPECT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ + GTEST_NONFATAL_FAILURE_) +#define EXPECT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_NONFATAL_FAILURE_) +#define ASSERT_TRUE(condition) \ + GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ + GTEST_FATAL_FAILURE_) +#define ASSERT_FALSE(condition) \ + GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ + GTEST_FATAL_FAILURE_) + +// Includes the auto-generated header that implements a family of +// generic predicate assertion macros. +// Copyright 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is AUTOMATICALLY GENERATED on 09/24/2010 by command +// 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! +// +// Implements a family of generic predicate assertion macros. + +#ifndef GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#define GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ + +// Makes sure this header is not included before gtest.h. +#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ +# error Do not include gtest_pred_impl.h directly. Include gtest.h instead. +#endif // GTEST_INCLUDE_GTEST_GTEST_H_ + +// This header implements a family of generic predicate assertion +// macros: +// +// ASSERT_PRED_FORMAT1(pred_format, v1) +// ASSERT_PRED_FORMAT2(pred_format, v1, v2) +// ... +// +// where pred_format is a function or functor that takes n (in the +// case of ASSERT_PRED_FORMATn) values and their source expression +// text, and returns a testing::AssertionResult. See the definition +// of ASSERT_EQ in gtest.h for an example. +// +// If you don't care about formatting, you can use the more +// restrictive version: +// +// ASSERT_PRED1(pred, v1) +// ASSERT_PRED2(pred, v1, v2) +// ... +// +// where pred is an n-ary function or functor that returns bool, +// and the values v1, v2, ..., must support the << operator for +// streaming to std::ostream. +// +// We also define the EXPECT_* variations. +// +// For now we only support predicates whose arity is at most 5. +// Please email googletestframework@googlegroups.com if you need +// support for higher arities. + +// GTEST_ASSERT_ is the basic statement to which all of the assertions +// in this file reduce. Don't use this in your code. + +#define GTEST_ASSERT_(expression, on_failure) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (const ::testing::AssertionResult gtest_ar = (expression)) \ + ; \ + else \ + on_failure(gtest_ar.failure_message()) + + +// Helper function for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +template +AssertionResult AssertPred1Helper(const char* pred_text, + const char* e1, + Pred pred, + const T1& v1) { + if (pred(v1)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT1. +// Don't use this in your code. +#define GTEST_PRED_FORMAT1_(pred_format, v1, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, v1),\ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED1. Don't use +// this in your code. +#define GTEST_PRED1_(pred, v1, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred1Helper(#pred, \ + #v1, \ + pred, \ + v1), on_failure) + +// Unary predicate assertion macros. +#define EXPECT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT1(pred_format, v1) \ + GTEST_PRED_FORMAT1_(pred_format, v1, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED1(pred, v1) \ + GTEST_PRED1_(pred, v1, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +template +AssertionResult AssertPred2Helper(const char* pred_text, + const char* e1, + const char* e2, + Pred pred, + const T1& v1, + const T2& v2) { + if (pred(v1, v2)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT2. +// Don't use this in your code. +#define GTEST_PRED_FORMAT2_(pred_format, v1, v2, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, v1, v2),\ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED2. Don't use +// this in your code. +#define GTEST_PRED2_(pred, v1, v2, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred2Helper(#pred, \ + #v1, \ + #v2, \ + pred, \ + v1, \ + v2), on_failure) + +// Binary predicate assertion macros. +#define EXPECT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT2(pred_format, v1, v2) \ + GTEST_PRED_FORMAT2_(pred_format, v1, v2, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED2(pred, v1, v2) \ + GTEST_PRED2_(pred, v1, v2, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +template +AssertionResult AssertPred3Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3) { + if (pred(v1, v2, v3)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT3. +// Don't use this in your code. +#define GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, v1, v2, v3),\ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED3. Don't use +// this in your code. +#define GTEST_PRED3_(pred, v1, v2, v3, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred3Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + pred, \ + v1, \ + v2, \ + v3), on_failure) + +// Ternary predicate assertion macros. +#define EXPECT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT3(pred_format, v1, v2, v3) \ + GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED3(pred, v1, v2, v3) \ + GTEST_PRED3_(pred, v1, v2, v3, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +template +AssertionResult AssertPred4Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4) { + if (pred(v1, v2, v3, v4)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT4. +// Don't use this in your code. +#define GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, v1, v2, v3, v4),\ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED4. Don't use +// this in your code. +#define GTEST_PRED4_(pred, v1, v2, v3, v4, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred4Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4), on_failure) + +// 4-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT4(pred_format, v1, v2, v3, v4) \ + GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED4(pred, v1, v2, v3, v4) \ + GTEST_PRED4_(pred, v1, v2, v3, v4, GTEST_FATAL_FAILURE_) + + + +// Helper function for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +template +AssertionResult AssertPred5Helper(const char* pred_text, + const char* e1, + const char* e2, + const char* e3, + const char* e4, + const char* e5, + Pred pred, + const T1& v1, + const T2& v2, + const T3& v3, + const T4& v4, + const T5& v5) { + if (pred(v1, v2, v3, v4, v5)) return AssertionSuccess(); + + return AssertionFailure() << pred_text << "(" + << e1 << ", " + << e2 << ", " + << e3 << ", " + << e4 << ", " + << e5 << ") evaluates to false, where" + << "\n" << e1 << " evaluates to " << v1 + << "\n" << e2 << " evaluates to " << v2 + << "\n" << e3 << " evaluates to " << v3 + << "\n" << e4 << " evaluates to " << v4 + << "\n" << e5 << " evaluates to " << v5; +} + +// Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT5. +// Don't use this in your code. +#define GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, #v5, v1, v2, v3, v4, v5),\ + on_failure) + +// Internal macro for implementing {EXPECT|ASSERT}_PRED5. Don't use +// this in your code. +#define GTEST_PRED5_(pred, v1, v2, v3, v4, v5, on_failure)\ + GTEST_ASSERT_(::testing::AssertPred5Helper(#pred, \ + #v1, \ + #v2, \ + #v3, \ + #v4, \ + #v5, \ + pred, \ + v1, \ + v2, \ + v3, \ + v4, \ + v5), on_failure) + +// 5-ary predicate assertion macros. +#define EXPECT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define EXPECT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_NONFATAL_FAILURE_) +#define ASSERT_PRED_FORMAT5(pred_format, v1, v2, v3, v4, v5) \ + GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) +#define ASSERT_PRED5(pred, v1, v2, v3, v4, v5) \ + GTEST_PRED5_(pred, v1, v2, v3, v4, v5, GTEST_FATAL_FAILURE_) + + + +#endif // GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ + +// Macros for testing equalities and inequalities. +// +// * {ASSERT|EXPECT}_EQ(expected, actual): Tests that expected == actual +// * {ASSERT|EXPECT}_NE(v1, v2): Tests that v1 != v2 +// * {ASSERT|EXPECT}_LT(v1, v2): Tests that v1 < v2 +// * {ASSERT|EXPECT}_LE(v1, v2): Tests that v1 <= v2 +// * {ASSERT|EXPECT}_GT(v1, v2): Tests that v1 > v2 +// * {ASSERT|EXPECT}_GE(v1, v2): Tests that v1 >= v2 +// +// When they are not, Google Test prints both the tested expressions and +// their actual values. The values must be compatible built-in types, +// or you will get a compiler error. By "compatible" we mean that the +// values can be compared by the respective operator. +// +// Note: +// +// 1. It is possible to make a user-defined type work with +// {ASSERT|EXPECT}_??(), but that requires overloading the +// comparison operators and is thus discouraged by the Google C++ +// Usage Guide. Therefore, you are advised to use the +// {ASSERT|EXPECT}_TRUE() macro to assert that two objects are +// equal. +// +// 2. The {ASSERT|EXPECT}_??() macros do pointer comparisons on +// pointers (in particular, C strings). Therefore, if you use it +// with two C strings, you are testing how their locations in memory +// are related, not how their content is related. To compare two C +// strings by content, use {ASSERT|EXPECT}_STR*(). +// +// 3. {ASSERT|EXPECT}_EQ(expected, actual) is preferred to +// {ASSERT|EXPECT}_TRUE(expected == actual), as the former tells you +// what the actual value is when it fails, and similarly for the +// other comparisons. +// +// 4. Do not depend on the order in which {ASSERT|EXPECT}_??() +// evaluate their arguments, which is undefined. +// +// 5. These macros evaluate their arguments exactly once. +// +// Examples: +// +// EXPECT_NE(5, Foo()); +// EXPECT_EQ(NULL, a_pointer); +// ASSERT_LT(i, array_size); +// ASSERT_GT(records.size(), 0) << "There is no record left."; + +#define EXPECT_EQ(expected, actual) \ + EXPECT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + expected, actual) +#define EXPECT_NE(expected, actual) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperNE, expected, actual) +#define EXPECT_LE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define EXPECT_LT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define EXPECT_GE(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define EXPECT_GT(val1, val2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +#define GTEST_ASSERT_EQ(expected, actual) \ + ASSERT_PRED_FORMAT2(::testing::internal:: \ + EqHelper::Compare, \ + expected, actual) +#define GTEST_ASSERT_NE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperNE, val1, val2) +#define GTEST_ASSERT_LE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLE, val1, val2) +#define GTEST_ASSERT_LT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperLT, val1, val2) +#define GTEST_ASSERT_GE(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGE, val1, val2) +#define GTEST_ASSERT_GT(val1, val2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) + +// Define macro GTEST_DONT_DEFINE_ASSERT_XY to 1 to omit the definition of +// ASSERT_XY(), which clashes with some users' own code. + +#if !GTEST_DONT_DEFINE_ASSERT_EQ +# define ASSERT_EQ(val1, val2) GTEST_ASSERT_EQ(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_NE +# define ASSERT_NE(val1, val2) GTEST_ASSERT_NE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LE +# define ASSERT_LE(val1, val2) GTEST_ASSERT_LE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_LT +# define ASSERT_LT(val1, val2) GTEST_ASSERT_LT(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GE +# define ASSERT_GE(val1, val2) GTEST_ASSERT_GE(val1, val2) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_GT +# define ASSERT_GT(val1, val2) GTEST_ASSERT_GT(val1, val2) +#endif + +// C String Comparisons. All tests treat NULL and any non-NULL string +// as different. Two NULLs are equal. +// +// * {ASSERT|EXPECT}_STREQ(s1, s2): Tests that s1 == s2 +// * {ASSERT|EXPECT}_STRNE(s1, s2): Tests that s1 != s2 +// * {ASSERT|EXPECT}_STRCASEEQ(s1, s2): Tests that s1 == s2, ignoring case +// * {ASSERT|EXPECT}_STRCASENE(s1, s2): Tests that s1 != s2, ignoring case +// +// For wide or narrow string objects, you can use the +// {ASSERT|EXPECT}_??() macros. +// +// Don't depend on the order in which the arguments are evaluated, +// which is undefined. +// +// These macros evaluate their arguments exactly once. + +#define EXPECT_STREQ(expected, actual) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, expected, actual) +#define EXPECT_STRNE(s1, s2) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define EXPECT_STRCASEEQ(expected, actual) \ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, expected, actual) +#define EXPECT_STRCASENE(s1, s2)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +#define ASSERT_STREQ(expected, actual) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTREQ, expected, actual) +#define ASSERT_STRNE(s1, s2) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRNE, s1, s2) +#define ASSERT_STRCASEEQ(expected, actual) \ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASEEQ, expected, actual) +#define ASSERT_STRCASENE(s1, s2)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperSTRCASENE, s1, s2) + +// Macros for comparing floating-point numbers. +// +// * {ASSERT|EXPECT}_FLOAT_EQ(expected, actual): +// Tests that two float values are almost equal. +// * {ASSERT|EXPECT}_DOUBLE_EQ(expected, actual): +// Tests that two double values are almost equal. +// * {ASSERT|EXPECT}_NEAR(v1, v2, abs_error): +// Tests that v1 and v2 are within the given distance to each other. +// +// Google Test uses ULP-based comparison to automatically pick a default +// error bound that is appropriate for the operands. See the +// FloatingPoint template class in gtest-internal.h if you are +// interested in the implementation details. + +#define EXPECT_FLOAT_EQ(expected, actual)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + expected, actual) + +#define EXPECT_DOUBLE_EQ(expected, actual)\ + EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + expected, actual) + +#define ASSERT_FLOAT_EQ(expected, actual)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + expected, actual) + +#define ASSERT_DOUBLE_EQ(expected, actual)\ + ASSERT_PRED_FORMAT2(::testing::internal::CmpHelperFloatingPointEQ, \ + expected, actual) + +#define EXPECT_NEAR(val1, val2, abs_error)\ + EXPECT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +#define ASSERT_NEAR(val1, val2, abs_error)\ + ASSERT_PRED_FORMAT3(::testing::internal::DoubleNearPredFormat, \ + val1, val2, abs_error) + +// These predicate format functions work on floating-point values, and +// can be used in {ASSERT|EXPECT}_PRED_FORMAT2*(), e.g. +// +// EXPECT_PRED_FORMAT2(testing::DoubleLE, Foo(), 5.0); + +// Asserts that val1 is less than, or almost equal to, val2. Fails +// otherwise. In particular, it fails if either val1 or val2 is NaN. +GTEST_API_ AssertionResult FloatLE(const char* expr1, const char* expr2, + float val1, float val2); +GTEST_API_ AssertionResult DoubleLE(const char* expr1, const char* expr2, + double val1, double val2); + + +#if GTEST_OS_WINDOWS + +// Macros that test for HRESULT failure and success, these are only useful +// on Windows, and rely on Windows SDK macros and APIs to compile. +// +// * {ASSERT|EXPECT}_HRESULT_{SUCCEEDED|FAILED}(expr) +// +// When expr unexpectedly fails or succeeds, Google Test prints the +// expected result and the actual result with both a human-readable +// string representation of the error, if available, as well as the +// hex result code. +# define EXPECT_HRESULT_SUCCEEDED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define ASSERT_HRESULT_SUCCEEDED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTSuccess, (expr)) + +# define EXPECT_HRESULT_FAILED(expr) \ + EXPECT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +# define ASSERT_HRESULT_FAILED(expr) \ + ASSERT_PRED_FORMAT1(::testing::internal::IsHRESULTFailure, (expr)) + +#endif // GTEST_OS_WINDOWS + +// Macros that execute statement and check that it doesn't generate new fatal +// failures in the current thread. +// +// * {ASSERT|EXPECT}_NO_FATAL_FAILURE(statement); +// +// Examples: +// +// EXPECT_NO_FATAL_FAILURE(Process()); +// ASSERT_NO_FATAL_FAILURE(Process()) << "Process() failed"; +// +#define ASSERT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_FATAL_FAILURE_) +#define EXPECT_NO_FATAL_FAILURE(statement) \ + GTEST_TEST_NO_FATAL_FAILURE_(statement, GTEST_NONFATAL_FAILURE_) + +// Causes a trace (including the source file path, the current line +// number, and the given message) to be included in every test failure +// message generated by code in the current scope. The effect is +// undone when the control leaves the current scope. +// +// The message argument can be anything streamable to std::ostream. +// +// In the implementation, we include the current line number as part +// of the dummy variable name, thus allowing multiple SCOPED_TRACE()s +// to appear in the same block - as long as they are on different +// lines. +#define SCOPED_TRACE(message) \ + ::testing::internal::ScopedTrace GTEST_CONCAT_TOKEN_(gtest_trace_, __LINE__)(\ + __FILE__, __LINE__, ::testing::Message() << (message)) + +// Compile-time assertion for type equality. +// StaticAssertTypeEq() compiles iff type1 and type2 are +// the same type. The value it returns is not interesting. +// +// Instead of making StaticAssertTypeEq a class template, we make it a +// function template that invokes a helper class template. This +// prevents a user from misusing StaticAssertTypeEq by +// defining objects of that type. +// +// CAVEAT: +// +// When used inside a method of a class template, +// StaticAssertTypeEq() is effective ONLY IF the method is +// instantiated. For example, given: +// +// template class Foo { +// public: +// void Bar() { testing::StaticAssertTypeEq(); } +// }; +// +// the code: +// +// void Test1() { Foo foo; } +// +// will NOT generate a compiler error, as Foo::Bar() is never +// actually instantiated. Instead, you need: +// +// void Test2() { Foo foo; foo.Bar(); } +// +// to cause a compiler error. +template +bool StaticAssertTypeEq() { + (void)internal::StaticAssertTypeEqHelper(); + return true; +} + +// Defines a test. +// +// The first parameter is the name of the test case, and the second +// parameter is the name of the test within the test case. +// +// The convention is to end the test case name with "Test". For +// example, a test case for the Foo class can be named FooTest. +// +// The user should put his test code between braces after using this +// macro. Example: +// +// TEST(FooTest, InitializesCorrectly) { +// Foo foo; +// EXPECT_TRUE(foo.StatusIsOK()); +// } + +// Note that we call GetTestTypeId() instead of GetTypeId< +// ::testing::Test>() here to get the type ID of testing::Test. This +// is to work around a suspected linker bug when using Google Test as +// a framework on Mac OS X. The bug causes GetTypeId< +// ::testing::Test>() to return different values depending on whether +// the call is from the Google Test framework itself or from user test +// code. GetTestTypeId() is guaranteed to always return the same +// value, as it always calls GetTypeId<>() from the Google Test +// framework. +#define GTEST_TEST(test_case_name, test_name)\ + GTEST_TEST_(test_case_name, test_name, \ + ::testing::Test, ::testing::internal::GetTestTypeId()) + +// Define this macro to 1 to omit the definition of TEST(), which +// is a generic name and clashes with some other libraries. +#if !GTEST_DONT_DEFINE_TEST +# define TEST(test_case_name, test_name) GTEST_TEST(test_case_name, test_name) +#endif + +// Defines a test that uses a test fixture. +// +// The first parameter is the name of the test fixture class, which +// also doubles as the test case name. The second parameter is the +// name of the test within the test case. +// +// A test fixture class must be declared earlier. The user should put +// his test code between braces after using this macro. Example: +// +// class FooTest : public testing::Test { +// protected: +// virtual void SetUp() { b_.AddElement(3); } +// +// Foo a_; +// Foo b_; +// }; +// +// TEST_F(FooTest, InitializesCorrectly) { +// EXPECT_TRUE(a_.StatusIsOK()); +// } +// +// TEST_F(FooTest, ReturnsElementCountCorrectly) { +// EXPECT_EQ(0, a_.size()); +// EXPECT_EQ(1, b_.size()); +// } + +#define TEST_F(test_fixture, test_name)\ + GTEST_TEST_(test_fixture, test_name, test_fixture, \ + ::testing::internal::GetTypeId()) + +// Use this macro in main() to run all tests. It returns 0 if all +// tests are successful, or 1 otherwise. +// +// RUN_ALL_TESTS() should be invoked after the command line has been +// parsed by InitGoogleTest(). + +#define RUN_ALL_TESTS()\ + (::testing::UnitTest::GetInstance()->Run()) + +} // namespace testing + +#endif // GTEST_INCLUDE_GTEST_GTEST_H_ diff --git a/tests/kfdtest/include/amdp2ptest.h b/tests/kfdtest/include/amdp2ptest.h new file mode 100644 index 0000000000..b465e24790 --- /dev/null +++ b/tests/kfdtest/include/amdp2ptest.h @@ -0,0 +1,76 @@ +/* + * Copyright (C) 2015-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef AMDP2PTEST_H_ +#define AMDP2PTEST_H_ + +#include + +#define AMDP2PTEST_IOCTL_MAGIC 'A' + + +#define AMDP2PTEST_DEVICE_NAME "amdp2ptest" +#define AMDP2PTEST_DEVICE_PATH "/dev/amdp2ptest" + +struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM { + /* Input parameters */ + uint64_t addr; + uint64_t length; + + /* Output parameters */ + uint64_t page_size; +}; + +struct AMDRDMA_IOCTL_GET_PAGES_PARAM { + /* Input parameters */ + uint64_t addr; + uint64_t length; +}; + + +struct AMDRDMA_IOCTL_PUT_PAGES_PARAM { + /* Input parameters */ + uint64_t addr; + uint64_t length; +}; + +struct AMDRDMA_IOCTL_IS_GPU_ADDRESS_PARAM { + /* Input parameters */ + uint64_t addr; + uint64_t ret_value; +}; + + +#define AMD2P2PTEST_IOCTL_GET_PAGE_SIZE \ +_IOWR(AMDP2PTEST_IOCTL_MAGIC, 1, struct AMDRDMA_IOCTL_GET_PAGE_SIZE_PARAM *) + +#define AMD2P2PTEST_IOCTL_GET_PAGES \ +_IOWR(AMDP2PTEST_IOCTL_MAGIC, 2, struct AMDRDMA_IOCTL_GET_PAGES_PARAM *) + +#define AMD2P2PTEST_IOCTL_PUT_PAGES \ +_IOW(AMDP2PTEST_IOCTL_MAGIC, 3, struct AMDRDMA_IOCTL_PUT_PAGES_PARAM *) + +#define AMD2P2PTEST_IOCTL_IS_GPU_ADDRESS \ +_IOW(AMDP2PTEST_IOCTL_MAGIC, 4, struct AMDRDMA_IOCTL_IS_GPU_ADDRESS *) + + +#endif /* AMDP2PTEST_H */ diff --git a/tests/kfdtest/include/asic_reg/gfx_7_2_d.h b/tests/kfdtest/include/asic_reg/gfx_7_2_d.h new file mode 100644 index 0000000000..9f519b6835 --- /dev/null +++ b/tests/kfdtest/include/asic_reg/gfx_7_2_d.h @@ -0,0 +1,2557 @@ +/* + * Copyright (C) 2014 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef GFX_7_2_D_H +#define GFX_7_2_D_H + +#define mmCB_BLEND_RED 0xa105 +#define mmCB_BLEND_GREEN 0xa106 +#define mmCB_BLEND_BLUE 0xa107 +#define mmCB_BLEND_ALPHA 0xa108 +#define mmCB_COLOR_CONTROL 0xa202 +#define mmCB_BLEND0_CONTROL 0xa1e0 +#define mmCB_BLEND1_CONTROL 0xa1e1 +#define mmCB_BLEND2_CONTROL 0xa1e2 +#define mmCB_BLEND3_CONTROL 0xa1e3 +#define mmCB_BLEND4_CONTROL 0xa1e4 +#define mmCB_BLEND5_CONTROL 0xa1e5 +#define mmCB_BLEND6_CONTROL 0xa1e6 +#define mmCB_BLEND7_CONTROL 0xa1e7 +#define mmCB_COLOR0_BASE 0xa318 +#define mmCB_COLOR1_BASE 0xa327 +#define mmCB_COLOR2_BASE 0xa336 +#define mmCB_COLOR3_BASE 0xa345 +#define mmCB_COLOR4_BASE 0xa354 +#define mmCB_COLOR5_BASE 0xa363 +#define mmCB_COLOR6_BASE 0xa372 +#define mmCB_COLOR7_BASE 0xa381 +#define mmCB_COLOR0_PITCH 0xa319 +#define mmCB_COLOR1_PITCH 0xa328 +#define mmCB_COLOR2_PITCH 0xa337 +#define mmCB_COLOR3_PITCH 0xa346 +#define mmCB_COLOR4_PITCH 0xa355 +#define mmCB_COLOR5_PITCH 0xa364 +#define mmCB_COLOR6_PITCH 0xa373 +#define mmCB_COLOR7_PITCH 0xa382 +#define mmCB_COLOR0_SLICE 0xa31a +#define mmCB_COLOR1_SLICE 0xa329 +#define mmCB_COLOR2_SLICE 0xa338 +#define mmCB_COLOR3_SLICE 0xa347 +#define mmCB_COLOR4_SLICE 0xa356 +#define mmCB_COLOR5_SLICE 0xa365 +#define mmCB_COLOR6_SLICE 0xa374 +#define mmCB_COLOR7_SLICE 0xa383 +#define mmCB_COLOR0_VIEW 0xa31b +#define mmCB_COLOR1_VIEW 0xa32a +#define mmCB_COLOR2_VIEW 0xa339 +#define mmCB_COLOR3_VIEW 0xa348 +#define mmCB_COLOR4_VIEW 0xa357 +#define mmCB_COLOR5_VIEW 0xa366 +#define mmCB_COLOR6_VIEW 0xa375 +#define mmCB_COLOR7_VIEW 0xa384 +#define mmCB_COLOR0_INFO 0xa31c +#define mmCB_COLOR1_INFO 0xa32b +#define mmCB_COLOR2_INFO 0xa33a +#define mmCB_COLOR3_INFO 0xa349 +#define mmCB_COLOR4_INFO 0xa358 +#define mmCB_COLOR5_INFO 0xa367 +#define mmCB_COLOR6_INFO 0xa376 +#define mmCB_COLOR7_INFO 0xa385 +#define mmCB_COLOR0_ATTRIB 0xa31d +#define mmCB_COLOR1_ATTRIB 0xa32c +#define mmCB_COLOR2_ATTRIB 0xa33b +#define mmCB_COLOR3_ATTRIB 0xa34a +#define mmCB_COLOR4_ATTRIB 0xa359 +#define mmCB_COLOR5_ATTRIB 0xa368 +#define mmCB_COLOR6_ATTRIB 0xa377 +#define mmCB_COLOR7_ATTRIB 0xa386 +#define mmCB_COLOR0_CMASK 0xa31f +#define mmCB_COLOR1_CMASK 0xa32e +#define mmCB_COLOR2_CMASK 0xa33d +#define mmCB_COLOR3_CMASK 0xa34c +#define mmCB_COLOR4_CMASK 0xa35b +#define mmCB_COLOR5_CMASK 0xa36a +#define mmCB_COLOR6_CMASK 0xa379 +#define mmCB_COLOR7_CMASK 0xa388 +#define mmCB_COLOR0_CMASK_SLICE 0xa320 +#define mmCB_COLOR1_CMASK_SLICE 0xa32f +#define mmCB_COLOR2_CMASK_SLICE 0xa33e +#define mmCB_COLOR3_CMASK_SLICE 0xa34d +#define mmCB_COLOR4_CMASK_SLICE 0xa35c +#define mmCB_COLOR5_CMASK_SLICE 0xa36b +#define mmCB_COLOR6_CMASK_SLICE 0xa37a +#define mmCB_COLOR7_CMASK_SLICE 0xa389 +#define mmCB_COLOR0_FMASK 0xa321 +#define mmCB_COLOR1_FMASK 0xa330 +#define mmCB_COLOR2_FMASK 0xa33f +#define mmCB_COLOR3_FMASK 0xa34e +#define mmCB_COLOR4_FMASK 0xa35d +#define mmCB_COLOR5_FMASK 0xa36c +#define mmCB_COLOR6_FMASK 0xa37b +#define mmCB_COLOR7_FMASK 0xa38a +#define mmCB_COLOR0_FMASK_SLICE 0xa322 +#define mmCB_COLOR1_FMASK_SLICE 0xa331 +#define mmCB_COLOR2_FMASK_SLICE 0xa340 +#define mmCB_COLOR3_FMASK_SLICE 0xa34f +#define mmCB_COLOR4_FMASK_SLICE 0xa35e +#define mmCB_COLOR5_FMASK_SLICE 0xa36d +#define mmCB_COLOR6_FMASK_SLICE 0xa37c +#define mmCB_COLOR7_FMASK_SLICE 0xa38b +#define mmCB_COLOR0_CLEAR_WORD0 0xa323 +#define mmCB_COLOR1_CLEAR_WORD0 0xa332 +#define mmCB_COLOR2_CLEAR_WORD0 0xa341 +#define mmCB_COLOR3_CLEAR_WORD0 0xa350 +#define mmCB_COLOR4_CLEAR_WORD0 0xa35f +#define mmCB_COLOR5_CLEAR_WORD0 0xa36e +#define mmCB_COLOR6_CLEAR_WORD0 0xa37d +#define mmCB_COLOR7_CLEAR_WORD0 0xa38c +#define mmCB_COLOR0_CLEAR_WORD1 0xa324 +#define mmCB_COLOR1_CLEAR_WORD1 0xa333 +#define mmCB_COLOR2_CLEAR_WORD1 0xa342 +#define mmCB_COLOR3_CLEAR_WORD1 0xa351 +#define mmCB_COLOR4_CLEAR_WORD1 0xa360 +#define mmCB_COLOR5_CLEAR_WORD1 0xa36f +#define mmCB_COLOR6_CLEAR_WORD1 0xa37e +#define mmCB_COLOR7_CLEAR_WORD1 0xa38d +#define mmCB_TARGET_MASK 0xa08e +#define mmCB_SHADER_MASK 0xa08f +#define mmCB_HW_CONTROL 0x2684 +#define mmCB_HW_CONTROL_1 0x2685 +#define mmCB_HW_CONTROL_2 0x2686 +#define mmCB_HW_CONTROL_3 0x2683 +#define mmCB_PERFCOUNTER_FILTER 0xdc00 +#define mmCB_PERFCOUNTER0_SELECT 0xdc01 +#define mmCB_PERFCOUNTER0_SELECT1 0xdc02 +#define mmCB_PERFCOUNTER1_SELECT 0xdc03 +#define mmCB_PERFCOUNTER2_SELECT 0xdc04 +#define mmCB_PERFCOUNTER3_SELECT 0xdc05 +#define mmCB_PERFCOUNTER0_LO 0xd406 +#define mmCB_PERFCOUNTER1_LO 0xd408 +#define mmCB_PERFCOUNTER2_LO 0xd40a +#define mmCB_PERFCOUNTER3_LO 0xd40c +#define mmCB_PERFCOUNTER0_HI 0xd407 +#define mmCB_PERFCOUNTER1_HI 0xd409 +#define mmCB_PERFCOUNTER2_HI 0xd40b +#define mmCB_PERFCOUNTER3_HI 0xd40d +#define mmCB_CGTT_SCLK_CTRL 0xf0a8 +#define mmCB_DEBUG_BUS_1 0x2699 +#define mmCB_DEBUG_BUS_2 0x269a +#define mmCB_DEBUG_BUS_3 0x269b +#define mmCB_DEBUG_BUS_4 0x269c +#define mmCB_DEBUG_BUS_5 0x269d +#define mmCB_DEBUG_BUS_6 0x269e +#define mmCB_DEBUG_BUS_7 0x269f +#define mmCB_DEBUG_BUS_8 0x26a0 +#define mmCB_DEBUG_BUS_9 0x26a1 +#define mmCB_DEBUG_BUS_10 0x26a2 +#define mmCB_DEBUG_BUS_11 0x26a3 +#define mmCB_DEBUG_BUS_12 0x26a4 +#define mmCB_DEBUG_BUS_13 0x26a5 +#define mmCB_DEBUG_BUS_14 0x26a6 +#define mmCB_DEBUG_BUS_15 0x26a7 +#define mmCB_DEBUG_BUS_16 0x26a8 +#define mmCB_DEBUG_BUS_17 0x26a9 +#define mmCB_DEBUG_BUS_18 0x26aa +#define mmCP_DFY_CNTL 0x3020 +#define mmCP_DFY_STAT 0x3021 +#define mmCP_DFY_ADDR_HI 0x3022 +#define mmCP_DFY_ADDR_LO 0x3023 +#define mmCP_DFY_DATA_0 0x3024 +#define mmCP_DFY_DATA_1 0x3025 +#define mmCP_DFY_DATA_2 0x3026 +#define mmCP_DFY_DATA_3 0x3027 +#define mmCP_DFY_DATA_4 0x3028 +#define mmCP_DFY_DATA_5 0x3029 +#define mmCP_DFY_DATA_6 0x302a +#define mmCP_DFY_DATA_7 0x302b +#define mmCP_DFY_DATA_8 0x302c +#define mmCP_DFY_DATA_9 0x302d +#define mmCP_DFY_DATA_10 0x302e +#define mmCP_DFY_DATA_11 0x302f +#define mmCP_DFY_DATA_12 0x3030 +#define mmCP_DFY_DATA_13 0x3031 +#define mmCP_DFY_DATA_14 0x3032 +#define mmCP_DFY_DATA_15 0x3033 +#define mmCP_RB0_BASE 0x3040 +#define mmCP_RB0_BASE_HI 0x30b1 +#define mmCP_RB_BASE 0x3040 +#define mmCP_RB1_BASE 0x3060 +#define mmCP_RB1_BASE_HI 0x30b2 +#define mmCP_RB2_BASE 0x3065 +#define mmCP_RB0_CNTL 0x3041 +#define mmCP_RB_CNTL 0x3041 +#define mmCP_RB1_CNTL 0x3061 +#define mmCP_RB2_CNTL 0x3066 +#define mmCP_RB_RPTR_WR 0x3042 +#define mmCP_RB0_RPTR_ADDR 0x3043 +#define mmCP_RB_RPTR_ADDR 0x3043 +#define mmCP_RB1_RPTR_ADDR 0x3062 +#define mmCP_RB2_RPTR_ADDR 0x3067 +#define mmCP_RB0_RPTR_ADDR_HI 0x3044 +#define mmCP_RB_RPTR_ADDR_HI 0x3044 +#define mmCP_RB1_RPTR_ADDR_HI 0x3063 +#define mmCP_RB2_RPTR_ADDR_HI 0x3068 +#define mmCP_RB0_WPTR 0x3045 +#define mmCP_RB_WPTR 0x3045 +#define mmCP_RB1_WPTR 0x3064 +#define mmCP_RB2_WPTR 0x3069 +#define mmCP_RB_WPTR_POLL_ADDR_LO 0x3046 +#define mmCP_RB_WPTR_POLL_ADDR_HI 0x3047 +#define mmGC_PRIV_MODE 0x3048 +#define mmCP_INT_CNTL 0x3049 +#define mmCP_INT_CNTL_RING0 0x306a +#define mmCP_INT_CNTL_RING1 0x306b +#define mmCP_INT_CNTL_RING2 0x306c +#define mmCP_INT_STATUS 0x304a +#define mmCP_INT_STATUS_RING0 0x306d +#define mmCP_INT_STATUS_RING1 0x306e +#define mmCP_INT_STATUS_RING2 0x306f +#define mmCP_DEVICE_ID 0x304b +#define mmCP_RING_PRIORITY_CNTS 0x304c +#define mmCP_ME0_PIPE_PRIORITY_CNTS 0x304c +#define mmCP_RING0_PRIORITY 0x304d +#define mmCP_ME0_PIPE0_PRIORITY 0x304d +#define mmCP_RING1_PRIORITY 0x304e +#define mmCP_ME0_PIPE1_PRIORITY 0x304e +#define mmCP_RING2_PRIORITY 0x304f +#define mmCP_ME0_PIPE2_PRIORITY 0x304f +#define mmCP_ENDIAN_SWAP 0x3050 +#define mmCP_RB_VMID 0x3051 +#define mmCP_ME0_PIPE0_VMID 0x3052 +#define mmCP_ME0_PIPE1_VMID 0x3053 +#define mmCP_PFP_UCODE_ADDR 0x3054 +#define mmCP_PFP_UCODE_DATA 0x3055 +#define mmCP_ME_RAM_RADDR 0x3056 +#define mmCP_ME_RAM_WADDR 0x3057 +#define mmCP_ME_RAM_DATA 0x3058 +#define mmCGTT_CPC_CLK_CTRL 0xf0b2 +#define mmCGTT_CPF_CLK_CTRL 0xf0b1 +#define mmCGTT_CP_CLK_CTRL 0xf0b0 +#define mmCP_CE_UCODE_ADDR 0x305a +#define mmCP_CE_UCODE_DATA 0x305b +#define mmCP_MEC_ME1_UCODE_ADDR 0x305c +#define mmCP_MEC_ME1_UCODE_DATA 0x305d +#define mmCP_MEC_ME2_UCODE_ADDR 0x305e +#define mmCP_MEC_ME2_UCODE_DATA 0x305f +#define mmCP_PWR_CNTL 0x3078 +#define mmCP_MEM_SLP_CNTL 0x3079 +#define mmCP_ECC_FIRSTOCCURRENCE 0x307a +#define mmCP_ECC_FIRSTOCCURRENCE_RING0 0x307b +#define mmCP_ECC_FIRSTOCCURRENCE_RING1 0x307c +#define mmCP_ECC_FIRSTOCCURRENCE_RING2 0x307d +#define mmCP_CPF_DEBUG 0x3080 +#define mmCP_FETCHER_SOURCE 0x3082 +#define mmCP_PQ_WPTR_POLL_CNTL 0x3083 +#define mmCP_PQ_WPTR_POLL_CNTL1 0x3084 +#define mmCPC_INT_CNTL 0x30b4 +#define mmCP_ME1_PIPE0_INT_CNTL 0x3085 +#define mmCP_ME1_PIPE1_INT_CNTL 0x3086 +#define mmCP_ME1_PIPE2_INT_CNTL 0x3087 +#define mmCP_ME1_PIPE3_INT_CNTL 0x3088 +#define mmCP_ME2_PIPE0_INT_CNTL 0x3089 +#define mmCP_ME2_PIPE1_INT_CNTL 0x308a +#define mmCP_ME2_PIPE2_INT_CNTL 0x308b +#define mmCP_ME2_PIPE3_INT_CNTL 0x308c +#define mmCPC_INT_STATUS 0x30b5 +#define mmCP_ME1_PIPE0_INT_STATUS 0x308d +#define mmCP_ME1_PIPE1_INT_STATUS 0x308e +#define mmCP_ME1_PIPE2_INT_STATUS 0x308f +#define mmCP_ME1_PIPE3_INT_STATUS 0x3090 +#define mmCP_ME2_PIPE0_INT_STATUS 0x3091 +#define mmCP_ME2_PIPE1_INT_STATUS 0x3092 +#define mmCP_ME2_PIPE2_INT_STATUS 0x3093 +#define mmCP_ME2_PIPE3_INT_STATUS 0x3094 +#define mmCP_ME1_INT_STAT_DEBUG 0x3095 +#define mmCP_ME2_INT_STAT_DEBUG 0x3096 +#define mmCP_ME1_PIPE_PRIORITY_CNTS 0x3099 +#define mmCP_ME1_PIPE0_PRIORITY 0x309a +#define mmCP_ME1_PIPE1_PRIORITY 0x309b +#define mmCP_ME1_PIPE2_PRIORITY 0x309c +#define mmCP_ME1_PIPE3_PRIORITY 0x309d +#define mmCP_ME2_PIPE_PRIORITY_CNTS 0x309e +#define mmCP_ME2_PIPE0_PRIORITY 0x309f +#define mmCP_ME2_PIPE1_PRIORITY 0x30a0 +#define mmCP_ME2_PIPE2_PRIORITY 0x30a1 +#define mmCP_ME2_PIPE3_PRIORITY 0x30a2 +#define mmCP_CE_PRGRM_CNTR_START 0x30a3 +#define mmCP_PFP_PRGRM_CNTR_START 0x30a4 +#define mmCP_ME_PRGRM_CNTR_START 0x30a5 +#define mmCP_MEC1_PRGRM_CNTR_START 0x30a6 +#define mmCP_MEC2_PRGRM_CNTR_START 0x30a7 +#define mmCP_CE_INTR_ROUTINE_START 0x30a8 +#define mmCP_PFP_INTR_ROUTINE_START 0x30a9 +#define mmCP_ME_INTR_ROUTINE_START 0x30aa +#define mmCP_MEC1_INTR_ROUTINE_START 0x30ab +#define mmCP_MEC2_INTR_ROUTINE_START 0x30ac +#define mmCP_CONTEXT_CNTL 0x30ad +#define mmCP_MAX_CONTEXT 0x30ae +#define mmCP_IQ_WAIT_TIME1 0x30af +#define mmCP_IQ_WAIT_TIME2 0x30b0 +#define mmCP_VMID_RESET 0x30b3 +#define mmCP_VMID_PREEMPT 0x30b6 +#define mmCPC_INT_CNTX_ID 0x30b7 +#define mmCP_PQ_STATUS 0x30b8 +#define mmCP_CPC_STATUS 0x2084 +#define mmCP_CPC_BUSY_STAT 0x2085 +#define mmCP_CPC_STALLED_STAT1 0x2086 +#define mmCP_CPF_STATUS 0x2087 +#define mmCP_CPF_BUSY_STAT 0x2088 +#define mmCP_CPF_STALLED_STAT1 0x2089 +#define mmCP_CPC_MC_CNTL 0x208a +#define mmCP_CPC_GRBM_FREE_COUNT 0x208b +#define mmCP_MEC_CNTL 0x208d +#define mmCP_MEC_ME1_HEADER_DUMP 0x208e +#define mmCP_MEC_ME2_HEADER_DUMP 0x208f +#define mmCP_CPC_SCRATCH_INDEX 0x2090 +#define mmCP_CPC_SCRATCH_DATA 0x2091 +#define mmCPG_PERFCOUNTER1_SELECT 0xd800 +#define mmCPG_PERFCOUNTER1_LO 0xd000 +#define mmCPG_PERFCOUNTER1_HI 0xd001 +#define mmCPG_PERFCOUNTER0_SELECT1 0xd801 +#define mmCPG_PERFCOUNTER0_SELECT 0xd802 +#define mmCPG_PERFCOUNTER0_LO 0xd002 +#define mmCPG_PERFCOUNTER0_HI 0xd003 +#define mmCPC_PERFCOUNTER1_SELECT 0xd803 +#define mmCPC_PERFCOUNTER1_LO 0xd004 +#define mmCPC_PERFCOUNTER1_HI 0xd005 +#define mmCPC_PERFCOUNTER0_SELECT1 0xd804 +#define mmCPC_PERFCOUNTER0_SELECT 0xd809 +#define mmCPC_PERFCOUNTER0_LO 0xd006 +#define mmCPC_PERFCOUNTER0_HI 0xd007 +#define mmCPF_PERFCOUNTER1_SELECT 0xd805 +#define mmCPF_PERFCOUNTER1_LO 0xd008 +#define mmCPF_PERFCOUNTER1_HI 0xd009 +#define mmCPF_PERFCOUNTER0_SELECT1 0xd806 +#define mmCPF_PERFCOUNTER0_SELECT 0xd807 +#define mmCPF_PERFCOUNTER0_LO 0xd00a +#define mmCPF_PERFCOUNTER0_HI 0xd00b +#define mmCP_CPC_HALT_HYST_COUNT 0x20a7 +#define mmCP_DRAW_OBJECT 0xd810 +#define mmCP_DRAW_OBJECT_COUNTER 0xd811 +#define mmCP_DRAW_WINDOW_MASK_HI 0xd812 +#define mmCP_DRAW_WINDOW_HI 0xd813 +#define mmCP_DRAW_WINDOW_LO 0xd814 +#define mmCP_DRAW_WINDOW_CNTL 0xd815 +#define mmCP_PRT_LOD_STATS_CNTL0 0x20ad +#define mmCP_PRT_LOD_STATS_CNTL1 0x20ae +#define mmCP_PRT_LOD_STATS_CNTL2 0x20af +#define mmCP_CE_COMPARE_COUNT 0x20c0 +#define mmCP_CE_DE_COUNT 0x20c1 +#define mmCP_DE_CE_COUNT 0x20c2 +#define mmCP_DE_LAST_INVAL_COUNT 0x20c3 +#define mmCP_DE_DE_COUNT 0x20c4 +#define mmCP_EOP_DONE_EVENT_CNTL 0xc0d5 +#define mmCP_EOP_DONE_DATA_CNTL 0xc0d6 +#define mmCP_EOP_DONE_ADDR_LO 0xc000 +#define mmCP_EOP_DONE_ADDR_HI 0xc001 +#define mmCP_EOP_DONE_DATA_LO 0xc002 +#define mmCP_EOP_DONE_DATA_HI 0xc003 +#define mmCP_EOP_LAST_FENCE_LO 0xc004 +#define mmCP_EOP_LAST_FENCE_HI 0xc005 +#define mmCP_STREAM_OUT_ADDR_LO 0xc006 +#define mmCP_STREAM_OUT_ADDR_HI 0xc007 +#define mmCP_NUM_PRIM_WRITTEN_COUNT0_LO 0xc008 +#define mmCP_NUM_PRIM_WRITTEN_COUNT0_HI 0xc009 +#define mmCP_NUM_PRIM_NEEDED_COUNT0_LO 0xc00a +#define mmCP_NUM_PRIM_NEEDED_COUNT0_HI 0xc00b +#define mmCP_NUM_PRIM_WRITTEN_COUNT1_LO 0xc00c +#define mmCP_NUM_PRIM_WRITTEN_COUNT1_HI 0xc00d +#define mmCP_NUM_PRIM_NEEDED_COUNT1_LO 0xc00e +#define mmCP_NUM_PRIM_NEEDED_COUNT1_HI 0xc00f +#define mmCP_NUM_PRIM_WRITTEN_COUNT2_LO 0xc010 +#define mmCP_NUM_PRIM_WRITTEN_COUNT2_HI 0xc011 +#define mmCP_NUM_PRIM_NEEDED_COUNT2_LO 0xc012 +#define mmCP_NUM_PRIM_NEEDED_COUNT2_HI 0xc013 +#define mmCP_NUM_PRIM_WRITTEN_COUNT3_LO 0xc014 +#define mmCP_NUM_PRIM_WRITTEN_COUNT3_HI 0xc015 +#define mmCP_NUM_PRIM_NEEDED_COUNT3_LO 0xc016 +#define mmCP_NUM_PRIM_NEEDED_COUNT3_HI 0xc017 +#define mmCP_PIPE_STATS_ADDR_LO 0xc018 +#define mmCP_PIPE_STATS_ADDR_HI 0xc019 +#define mmCP_VGT_IAVERT_COUNT_LO 0xc01a +#define mmCP_VGT_IAVERT_COUNT_HI 0xc01b +#define mmCP_VGT_IAPRIM_COUNT_LO 0xc01c +#define mmCP_VGT_IAPRIM_COUNT_HI 0xc01d +#define mmCP_VGT_GSPRIM_COUNT_LO 0xc01e +#define mmCP_VGT_GSPRIM_COUNT_HI 0xc01f +#define mmCP_VGT_VSINVOC_COUNT_LO 0xc020 +#define mmCP_VGT_VSINVOC_COUNT_HI 0xc021 +#define mmCP_VGT_GSINVOC_COUNT_LO 0xc022 +#define mmCP_VGT_GSINVOC_COUNT_HI 0xc023 +#define mmCP_VGT_HSINVOC_COUNT_LO 0xc024 +#define mmCP_VGT_HSINVOC_COUNT_HI 0xc025 +#define mmCP_VGT_DSINVOC_COUNT_LO 0xc026 +#define mmCP_VGT_DSINVOC_COUNT_HI 0xc027 +#define mmCP_PA_CINVOC_COUNT_LO 0xc028 +#define mmCP_PA_CINVOC_COUNT_HI 0xc029 +#define mmCP_PA_CPRIM_COUNT_LO 0xc02a +#define mmCP_PA_CPRIM_COUNT_HI 0xc02b +#define mmCP_SC_PSINVOC_COUNT0_LO 0xc02c +#define mmCP_SC_PSINVOC_COUNT0_HI 0xc02d +#define mmCP_SC_PSINVOC_COUNT1_LO 0xc02e +#define mmCP_SC_PSINVOC_COUNT1_HI 0xc02f +#define mmCP_VGT_CSINVOC_COUNT_LO 0xc030 +#define mmCP_VGT_CSINVOC_COUNT_HI 0xc031 +#define mmCP_STRMOUT_CNTL 0xc03f +#define mmSCRATCH_REG0 0xc040 +#define mmSCRATCH_REG1 0xc041 +#define mmSCRATCH_REG2 0xc042 +#define mmSCRATCH_REG3 0xc043 +#define mmSCRATCH_REG4 0xc044 +#define mmSCRATCH_REG5 0xc045 +#define mmSCRATCH_REG6 0xc046 +#define mmSCRATCH_REG7 0xc047 +#define mmSCRATCH_UMSK 0xc050 +#define mmSCRATCH_ADDR 0xc051 +#define mmCP_PFP_ATOMIC_PREOP_LO 0xc052 +#define mmCP_PFP_ATOMIC_PREOP_HI 0xc053 +#define mmCP_PFP_GDS_ATOMIC0_PREOP_LO 0xc054 +#define mmCP_PFP_GDS_ATOMIC0_PREOP_HI 0xc055 +#define mmCP_PFP_GDS_ATOMIC1_PREOP_LO 0xc056 +#define mmCP_PFP_GDS_ATOMIC1_PREOP_HI 0xc057 +#define mmCP_APPEND_ADDR_LO 0xc058 +#define mmCP_APPEND_ADDR_HI 0xc059 +#define mmCP_APPEND_DATA 0xc05a +#define mmCP_APPEND_LAST_CS_FENCE 0xc05b +#define mmCP_APPEND_LAST_PS_FENCE 0xc05c +#define mmCP_ATOMIC_PREOP_LO 0xc05d +#define mmCP_ME_ATOMIC_PREOP_LO 0xc05d +#define mmCP_ATOMIC_PREOP_HI 0xc05e +#define mmCP_ME_ATOMIC_PREOP_HI 0xc05e +#define mmCP_GDS_ATOMIC0_PREOP_LO 0xc05f +#define mmCP_ME_GDS_ATOMIC0_PREOP_LO 0xc05f +#define mmCP_GDS_ATOMIC0_PREOP_HI 0xc060 +#define mmCP_ME_GDS_ATOMIC0_PREOP_HI 0xc060 +#define mmCP_GDS_ATOMIC1_PREOP_LO 0xc061 +#define mmCP_ME_GDS_ATOMIC1_PREOP_LO 0xc061 +#define mmCP_GDS_ATOMIC1_PREOP_HI 0xc062 +#define mmCP_ME_GDS_ATOMIC1_PREOP_HI 0xc062 +#define mmCP_ME_MC_WADDR_LO 0xc069 +#define mmCP_ME_MC_WADDR_HI 0xc06a +#define mmCP_ME_MC_WDATA_LO 0xc06b +#define mmCP_ME_MC_WDATA_HI 0xc06c +#define mmCP_ME_MC_RADDR_LO 0xc06d +#define mmCP_ME_MC_RADDR_HI 0xc06e +#define mmCP_SEM_WAIT_TIMER 0xc06f +#define mmCP_SIG_SEM_ADDR_LO 0xc070 +#define mmCP_SIG_SEM_ADDR_HI 0xc071 +#define mmCP_WAIT_SEM_ADDR_LO 0xc075 +#define mmCP_WAIT_SEM_ADDR_HI 0xc076 +#define mmCP_WAIT_REG_MEM_TIMEOUT 0xc074 +#define mmCP_COHER_START_DELAY 0xc07b +#define mmCP_COHER_CNTL 0xc07c +#define mmCP_COHER_SIZE 0xc07d +#define mmCP_COHER_SIZE_HI 0xc08c +#define mmCP_COHER_BASE 0xc07e +#define mmCP_COHER_BASE_HI 0xc079 +#define mmCP_COHER_STATUS 0xc07f +#define mmCOHER_DEST_BASE_0 0xa092 +#define mmCOHER_DEST_BASE_1 0xa093 +#define mmCOHER_DEST_BASE_2 0xa07e +#define mmCOHER_DEST_BASE_3 0xa07f +#define mmCOHER_DEST_BASE_HI_0 0xa07a +#define mmCOHER_DEST_BASE_HI_1 0xa07b +#define mmCOHER_DEST_BASE_HI_2 0xa07c +#define mmCOHER_DEST_BASE_HI_3 0xa07d +#define mmCP_DMA_ME_SRC_ADDR 0xc080 +#define mmCP_DMA_ME_SRC_ADDR_HI 0xc081 +#define mmCP_DMA_ME_DST_ADDR 0xc082 +#define mmCP_DMA_ME_DST_ADDR_HI 0xc083 +#define mmCP_DMA_ME_CONTROL 0xc078 +#define mmCP_DMA_ME_COMMAND 0xc084 +#define mmCP_DMA_PFP_SRC_ADDR 0xc085 +#define mmCP_DMA_PFP_SRC_ADDR_HI 0xc086 +#define mmCP_DMA_PFP_DST_ADDR 0xc087 +#define mmCP_DMA_PFP_DST_ADDR_HI 0xc088 +#define mmCP_DMA_PFP_CONTROL 0xc077 +#define mmCP_DMA_PFP_COMMAND 0xc089 +#define mmCP_DMA_CNTL 0xc08a +#define mmCP_DMA_READ_TAGS 0xc08b +#define mmCP_PFP_IB_CONTROL 0xc08d +#define mmCP_PFP_LOAD_CONTROL 0xc08e +#define mmCP_SCRATCH_INDEX 0xc08f +#define mmCP_SCRATCH_DATA 0xc090 +#define mmCP_RB_OFFSET 0xc091 +#define mmCP_IB1_OFFSET 0xc092 +#define mmCP_IB2_OFFSET 0xc093 +#define mmCP_IB1_PREAMBLE_BEGIN 0xc094 +#define mmCP_IB1_PREAMBLE_END 0xc095 +#define mmCP_IB2_PREAMBLE_BEGIN 0xc096 +#define mmCP_IB2_PREAMBLE_END 0xc097 +#define mmCP_CE_IB1_OFFSET 0xc098 +#define mmCP_CE_IB2_OFFSET 0xc099 +#define mmCP_CE_COUNTER 0xc09a +#define mmCP_STALLED_STAT1 0x219d +#define mmCP_STALLED_STAT2 0x219e +#define mmCP_STALLED_STAT3 0x219c +#define mmCP_BUSY_STAT 0x219f +#define mmCP_STAT 0x21a0 +#define mmCP_ME_HEADER_DUMP 0x21a1 +#define mmCP_PFP_HEADER_DUMP 0x21a2 +#define mmCP_GRBM_FREE_COUNT 0x21a3 +#define mmCP_CE_HEADER_DUMP 0x21a4 +#define mmCP_MC_PACK_DELAY_CNT 0x21a7 +#define mmCP_MC_TAG_CNTL 0x21a8 +#define mmCP_MC_TAG_DATA 0x21a9 +#define mmCP_CSF_STAT 0x21b4 +#define mmCP_CSF_CNTL 0x21b5 +#define mmCP_ME_CNTL 0x21b6 +#define mmCP_CNTX_STAT 0x21b8 +#define mmCP_ME_PREEMPTION 0x21b9 +#define mmCP_RB0_RPTR 0x21c0 +#define mmCP_RB_RPTR 0x21c0 +#define mmCP_RB1_RPTR 0x21bf +#define mmCP_RB2_RPTR 0x21be +#define mmCP_RB_WPTR_DELAY 0x21c1 +#define mmCP_RB_WPTR_POLL_CNTL 0x21c2 +#define mmCP_CE_INIT_BASE_LO 0xc0c3 +#define mmCP_CE_INIT_BASE_HI 0xc0c4 +#define mmCP_CE_INIT_BUFSZ 0xc0c5 +#define mmCP_CE_IB1_BASE_LO 0xc0c6 +#define mmCP_CE_IB1_BASE_HI 0xc0c7 +#define mmCP_CE_IB1_BUFSZ 0xc0c8 +#define mmCP_CE_IB2_BASE_LO 0xc0c9 +#define mmCP_CE_IB2_BASE_HI 0xc0ca +#define mmCP_CE_IB2_BUFSZ 0xc0cb +#define mmCP_IB1_BASE_LO 0xc0cc +#define mmCP_IB1_BASE_HI 0xc0cd +#define mmCP_IB1_BUFSZ 0xc0ce +#define mmCP_IB2_BASE_LO 0xc0cf +#define mmCP_IB2_BASE_HI 0xc0d0 +#define mmCP_IB2_BUFSZ 0xc0d1 +#define mmCP_ST_BASE_LO 0xc0d2 +#define mmCP_ST_BASE_HI 0xc0d3 +#define mmCP_ST_BUFSZ 0xc0d4 +#define mmCP_ROQ_THRESHOLDS 0x21bc +#define mmCP_MEQ_STQ_THRESHOLD 0x21bd +#define mmCP_ROQ1_THRESHOLDS 0x21d5 +#define mmCP_ROQ2_THRESHOLDS 0x21d6 +#define mmCP_STQ_THRESHOLDS 0x21d7 +#define mmCP_QUEUE_THRESHOLDS 0x21d8 +#define mmCP_MEQ_THRESHOLDS 0x21d9 +#define mmCP_ROQ_AVAIL 0x21da +#define mmCP_STQ_AVAIL 0x21db +#define mmCP_ROQ2_AVAIL 0x21dc +#define mmCP_MEQ_AVAIL 0x21dd +#define mmCP_CMD_INDEX 0x21de +#define mmCP_CMD_DATA 0x21df +#define mmCP_ROQ_RB_STAT 0x21e0 +#define mmCP_ROQ_IB1_STAT 0x21e1 +#define mmCP_ROQ_IB2_STAT 0x21e2 +#define mmCP_STQ_STAT 0x21e3 +#define mmCP_STQ_WR_STAT 0x21e4 +#define mmCP_MEQ_STAT 0x21e5 +#define mmCP_CEQ1_AVAIL 0x21e6 +#define mmCP_CEQ2_AVAIL 0x21e7 +#define mmCP_CE_ROQ_RB_STAT 0x21e8 +#define mmCP_CE_ROQ_IB1_STAT 0x21e9 +#define mmCP_CE_ROQ_IB2_STAT 0x21ea +#define mmCP_INT_STAT_DEBUG 0x21f7 +#define mmCP_PERFMON_CNTL 0xd808 +#define mmCP_PERFMON_CNTX_CNTL 0xa0d8 +#define mmCP_RINGID 0xa0d9 +#define mmCP_PIPEID 0xa0d9 +#define mmCP_VMID 0xa0da +#define mmCP_HPD_ROQ_OFFSETS 0x3240 +#define mmCP_HPD_EOP_BASE_ADDR 0x3241 +#define mmCP_HPD_EOP_BASE_ADDR_HI 0x3242 +#define mmCP_HPD_EOP_VMID 0x3243 +#define mmCP_HPD_EOP_CONTROL 0x3244 +#define mmCP_MQD_BASE_ADDR 0x3245 +#define mmCP_MQD_BASE_ADDR_HI 0x3246 +#define mmCP_HQD_ACTIVE 0x3247 +#define mmCP_HQD_VMID 0x3248 +#define mmCP_HQD_PERSISTENT_STATE 0x3249 +#define mmCP_HQD_PIPE_PRIORITY 0x324a +#define mmCP_HQD_QUEUE_PRIORITY 0x324b +#define mmCP_HQD_QUANTUM 0x324c +#define mmCP_HQD_PQ_BASE 0x324d +#define mmCP_HQD_PQ_BASE_HI 0x324e +#define mmCP_HQD_PQ_RPTR 0x324f +#define mmCP_HQD_PQ_RPTR_REPORT_ADDR 0x3250 +#define mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI 0x3251 +#define mmCP_HQD_PQ_WPTR_POLL_ADDR 0x3252 +#define mmCP_HQD_PQ_WPTR_POLL_ADDR_HI 0x3253 +#define mmCP_HQD_PQ_DOORBELL_CONTROL 0x3254 +#define mmCP_HQD_PQ_WPTR 0x3255 +#define mmCP_HQD_PQ_CONTROL 0x3256 +#define mmCP_HQD_IB_BASE_ADDR 0x3257 +#define mmCP_HQD_IB_BASE_ADDR_HI 0x3258 +#define mmCP_HQD_IB_RPTR 0x3259 +#define mmCP_HQD_IB_CONTROL 0x325a +#define mmCP_HQD_IQ_TIMER 0x325b +#define mmCP_HQD_IQ_RPTR 0x325c +#define mmCP_HQD_DEQUEUE_REQUEST 0x325d +#define mmCP_HQD_DMA_OFFLOAD 0x325e +#define mmCP_HQD_SEMA_CMD 0x325f +#define mmCP_HQD_MSG_TYPE 0x3260 +#define mmCP_HQD_ATOMIC0_PREOP_LO 0x3261 +#define mmCP_HQD_ATOMIC0_PREOP_HI 0x3262 +#define mmCP_HQD_ATOMIC1_PREOP_LO 0x3263 +#define mmCP_HQD_ATOMIC1_PREOP_HI 0x3264 +#define mmCP_HQD_HQ_SCHEDULER0 0x3265 +#define mmCP_HQD_HQ_SCHEDULER1 0x3266 +#define mmCP_MQD_CONTROL 0x3267 +#define mmDB_Z_READ_BASE 0xa012 +#define mmDB_STENCIL_READ_BASE 0xa013 +#define mmDB_Z_WRITE_BASE 0xa014 +#define mmDB_STENCIL_WRITE_BASE 0xa015 +#define mmDB_DEPTH_INFO 0xa00f +#define mmDB_Z_INFO 0xa010 +#define mmDB_STENCIL_INFO 0xa011 +#define mmDB_DEPTH_SIZE 0xa016 +#define mmDB_DEPTH_SLICE 0xa017 +#define mmDB_DEPTH_VIEW 0xa002 +#define mmDB_RENDER_CONTROL 0xa000 +#define mmDB_COUNT_CONTROL 0xa001 +#define mmDB_RENDER_OVERRIDE 0xa003 +#define mmDB_RENDER_OVERRIDE2 0xa004 +#define mmDB_EQAA 0xa201 +#define mmDB_SHADER_CONTROL 0xa203 +#define mmDB_DEPTH_BOUNDS_MIN 0xa008 +#define mmDB_DEPTH_BOUNDS_MAX 0xa009 +#define mmDB_STENCIL_CLEAR 0xa00a +#define mmDB_DEPTH_CLEAR 0xa00b +#define mmDB_HTILE_DATA_BASE 0xa005 +#define mmDB_HTILE_SURFACE 0xa2af +#define mmDB_PRELOAD_CONTROL 0xa2b2 +#define mmDB_STENCILREFMASK 0xa10c +#define mmDB_STENCILREFMASK_BF 0xa10d +#define mmDB_SRESULTS_COMPARE_STATE0 0xa2b0 +#define mmDB_SRESULTS_COMPARE_STATE1 0xa2b1 +#define mmDB_DEPTH_CONTROL 0xa200 +#define mmDB_STENCIL_CONTROL 0xa10b +#define mmDB_ALPHA_TO_MASK 0xa2dc +#define mmDB_PERFCOUNTER0_SELECT 0xdc40 +#define mmDB_PERFCOUNTER1_SELECT 0xdc42 +#define mmDB_PERFCOUNTER2_SELECT 0xdc44 +#define mmDB_PERFCOUNTER3_SELECT 0xdc46 +#define mmDB_PERFCOUNTER0_SELECT1 0xdc41 +#define mmDB_PERFCOUNTER1_SELECT1 0xdc43 +#define mmDB_PERFCOUNTER0_LO 0xd440 +#define mmDB_PERFCOUNTER1_LO 0xd442 +#define mmDB_PERFCOUNTER2_LO 0xd444 +#define mmDB_PERFCOUNTER3_LO 0xd446 +#define mmDB_PERFCOUNTER0_HI 0xd441 +#define mmDB_PERFCOUNTER1_HI 0xd443 +#define mmDB_PERFCOUNTER2_HI 0xd445 +#define mmDB_PERFCOUNTER3_HI 0xd447 +#define mmDB_DEBUG 0x260c +#define mmDB_DEBUG2 0x260d +#define mmDB_DEBUG3 0x260e +#define mmDB_DEBUG4 0x260f +#define mmDB_CREDIT_LIMIT 0x2614 +#define mmDB_WATERMARKS 0x2615 +#define mmDB_SUBTILE_CONTROL 0x2616 +#define mmDB_FREE_CACHELINES 0x2617 +#define mmDB_FIFO_DEPTH1 0x2618 +#define mmDB_FIFO_DEPTH2 0x2619 +#define mmDB_CGTT_CLK_CTRL_0 0xf0a4 +#define mmDB_ZPASS_COUNT_LOW 0xc3fe +#define mmDB_ZPASS_COUNT_HI 0xc3ff +#define mmDB_RING_CONTROL 0x261b +#define mmDB_READ_DEBUG_0 0x2620 +#define mmDB_READ_DEBUG_1 0x2621 +#define mmDB_READ_DEBUG_2 0x2622 +#define mmDB_READ_DEBUG_3 0x2623 +#define mmDB_READ_DEBUG_4 0x2624 +#define mmDB_READ_DEBUG_5 0x2625 +#define mmDB_READ_DEBUG_6 0x2626 +#define mmDB_READ_DEBUG_7 0x2627 +#define mmDB_READ_DEBUG_8 0x2628 +#define mmDB_READ_DEBUG_9 0x2629 +#define mmDB_READ_DEBUG_A 0x262a +#define mmDB_READ_DEBUG_B 0x262b +#define mmDB_READ_DEBUG_C 0x262c +#define mmDB_READ_DEBUG_D 0x262d +#define mmDB_READ_DEBUG_E 0x262e +#define mmDB_READ_DEBUG_F 0x262f +#define mmDB_OCCLUSION_COUNT0_LOW 0xc3c0 +#define mmDB_OCCLUSION_COUNT0_HI 0xc3c1 +#define mmDB_OCCLUSION_COUNT1_LOW 0xc3c2 +#define mmDB_OCCLUSION_COUNT1_HI 0xc3c3 +#define mmDB_OCCLUSION_COUNT2_LOW 0xc3c4 +#define mmDB_OCCLUSION_COUNT2_HI 0xc3c5 +#define mmDB_OCCLUSION_COUNT3_LOW 0xc3c6 +#define mmDB_OCCLUSION_COUNT3_HI 0xc3c7 +#define mmCC_RB_REDUNDANCY 0x263c +#define mmCC_RB_BACKEND_DISABLE 0x263d +#define mmGC_USER_RB_REDUNDANCY 0x26de +#define mmGC_USER_RB_BACKEND_DISABLE 0x26df +#define mmGB_ADDR_CONFIG 0x263e +#define mmGB_BACKEND_MAP 0x263f +#define mmGB_GPU_ID 0x2640 +#define mmCC_RB_DAISY_CHAIN 0x2641 +#define mmGB_TILE_MODE0 0x2644 +#define mmGB_TILE_MODE1 0x2645 +#define mmGB_TILE_MODE2 0x2646 +#define mmGB_TILE_MODE3 0x2647 +#define mmGB_TILE_MODE4 0x2648 +#define mmGB_TILE_MODE5 0x2649 +#define mmGB_TILE_MODE6 0x264a +#define mmGB_TILE_MODE7 0x264b +#define mmGB_TILE_MODE8 0x264c +#define mmGB_TILE_MODE9 0x264d +#define mmGB_TILE_MODE10 0x264e +#define mmGB_TILE_MODE11 0x264f +#define mmGB_TILE_MODE12 0x2650 +#define mmGB_TILE_MODE13 0x2651 +#define mmGB_TILE_MODE14 0x2652 +#define mmGB_TILE_MODE15 0x2653 +#define mmGB_TILE_MODE16 0x2654 +#define mmGB_TILE_MODE17 0x2655 +#define mmGB_TILE_MODE18 0x2656 +#define mmGB_TILE_MODE19 0x2657 +#define mmGB_TILE_MODE20 0x2658 +#define mmGB_TILE_MODE21 0x2659 +#define mmGB_TILE_MODE22 0x265a +#define mmGB_TILE_MODE23 0x265b +#define mmGB_TILE_MODE24 0x265c +#define mmGB_TILE_MODE25 0x265d +#define mmGB_TILE_MODE26 0x265e +#define mmGB_TILE_MODE27 0x265f +#define mmGB_TILE_MODE28 0x2660 +#define mmGB_TILE_MODE29 0x2661 +#define mmGB_TILE_MODE30 0x2662 +#define mmGB_TILE_MODE31 0x2663 +#define mmGB_MACROTILE_MODE0 0x2664 +#define mmGB_MACROTILE_MODE1 0x2665 +#define mmGB_MACROTILE_MODE2 0x2666 +#define mmGB_MACROTILE_MODE3 0x2667 +#define mmGB_MACROTILE_MODE4 0x2668 +#define mmGB_MACROTILE_MODE5 0x2669 +#define mmGB_MACROTILE_MODE6 0x266a +#define mmGB_MACROTILE_MODE7 0x266b +#define mmGB_MACROTILE_MODE8 0x266c +#define mmGB_MACROTILE_MODE9 0x266d +#define mmGB_MACROTILE_MODE10 0x266e +#define mmGB_MACROTILE_MODE11 0x266f +#define mmGB_MACROTILE_MODE12 0x2670 +#define mmGB_MACROTILE_MODE13 0x2671 +#define mmGB_MACROTILE_MODE14 0x2672 +#define mmGB_MACROTILE_MODE15 0x2673 +#define mmGB_EDC_MODE 0x307e +#define mmCC_GC_EDC_CONFIG 0x3098 +#define mmRAS_SIGNATURE_CONTROL 0x3380 +#define mmRAS_SIGNATURE_MASK 0x3381 +#define mmRAS_SX_SIGNATURE0 0x3382 +#define mmRAS_SX_SIGNATURE1 0x3383 +#define mmRAS_SX_SIGNATURE2 0x3384 +#define mmRAS_SX_SIGNATURE3 0x3385 +#define mmRAS_DB_SIGNATURE0 0x338b +#define mmRAS_PA_SIGNATURE0 0x338c +#define mmRAS_VGT_SIGNATURE0 0x338d +#define mmRAS_SQ_SIGNATURE0 0x338e +#define mmRAS_SC_SIGNATURE0 0x338f +#define mmRAS_SC_SIGNATURE1 0x3390 +#define mmRAS_SC_SIGNATURE2 0x3391 +#define mmRAS_SC_SIGNATURE3 0x3392 +#define mmRAS_SC_SIGNATURE4 0x3393 +#define mmRAS_SC_SIGNATURE5 0x3394 +#define mmRAS_SC_SIGNATURE6 0x3395 +#define mmRAS_SC_SIGNATURE7 0x3396 +#define mmRAS_IA_SIGNATURE0 0x3397 +#define mmRAS_IA_SIGNATURE1 0x3398 +#define mmRAS_SPI_SIGNATURE0 0x3399 +#define mmRAS_SPI_SIGNATURE1 0x339a +#define mmRAS_TA_SIGNATURE0 0x339b +#define mmRAS_TD_SIGNATURE0 0x339c +#define mmRAS_CB_SIGNATURE0 0x339d +#define mmRAS_BCI_SIGNATURE0 0x339e +#define mmRAS_BCI_SIGNATURE1 0x339f +#define mmGRBM_CAM_INDEX 0x3000 +#define mmGRBM_CAM_DATA 0x3001 +#define mmGRBM_CNTL 0x2000 +#define mmGRBM_SKEW_CNTL 0x2001 +#define mmGRBM_PWR_CNTL 0x2003 +#define mmGRBM_STATUS 0x2004 +#define mmGRBM_STATUS2 0x2002 +#define mmGRBM_STATUS_SE0 0x2005 +#define mmGRBM_STATUS_SE1 0x2006 +#define mmGRBM_STATUS_SE2 0x200e +#define mmGRBM_STATUS_SE3 0x200f +#define mmGRBM_SOFT_RESET 0x2008 +#define mmGRBM_DEBUG_CNTL 0x2009 +#define mmGRBM_DEBUG_DATA 0x200a +#define mmGRBM_GFX_INDEX 0xc200 +#define mmGRBM_GFX_CLKEN_CNTL 0x200c +#define mmGRBM_WAIT_IDLE_CLOCKS 0x200d +#define mmGRBM_DEBUG 0x2014 +#define mmGRBM_DEBUG_SNAPSHOT 0x2015 +#define mmGRBM_READ_ERROR 0x2016 +#define mmGRBM_READ_ERROR2 0x2017 +#define mmGRBM_INT_CNTL 0x2018 +#define mmGRBM_PERFCOUNTER0_SELECT 0xd840 +#define mmGRBM_PERFCOUNTER1_SELECT 0xd841 +#define mmGRBM_SE0_PERFCOUNTER_SELECT 0xd842 +#define mmGRBM_SE1_PERFCOUNTER_SELECT 0xd843 +#define mmGRBM_SE2_PERFCOUNTER_SELECT 0xd844 +#define mmGRBM_SE3_PERFCOUNTER_SELECT 0xd845 +#define mmGRBM_PERFCOUNTER0_LO 0xd040 +#define mmGRBM_PERFCOUNTER0_HI 0xd041 +#define mmGRBM_PERFCOUNTER1_LO 0xd043 +#define mmGRBM_PERFCOUNTER1_HI 0xd044 +#define mmGRBM_SE0_PERFCOUNTER_LO 0xd045 +#define mmGRBM_SE0_PERFCOUNTER_HI 0xd046 +#define mmGRBM_SE1_PERFCOUNTER_LO 0xd047 +#define mmGRBM_SE1_PERFCOUNTER_HI 0xd048 +#define mmGRBM_SE2_PERFCOUNTER_LO 0xd049 +#define mmGRBM_SE2_PERFCOUNTER_HI 0xd04a +#define mmGRBM_SE3_PERFCOUNTER_LO 0xd04b +#define mmGRBM_SE3_PERFCOUNTER_HI 0xd04c +#define mmGRBM_SCRATCH_REG0 0x2040 +#define mmGRBM_SCRATCH_REG1 0x2041 +#define mmGRBM_SCRATCH_REG2 0x2042 +#define mmGRBM_SCRATCH_REG3 0x2043 +#define mmGRBM_SCRATCH_REG4 0x2044 +#define mmGRBM_SCRATCH_REG5 0x2045 +#define mmGRBM_SCRATCH_REG6 0x2046 +#define mmGRBM_SCRATCH_REG7 0x2047 +#define mmDEBUG_INDEX 0x203c +#define mmDEBUG_DATA 0x203d +#define mmGRBM_NOWHERE 0x203f +#define mmPA_CL_VPORT_XSCALE 0xa10f +#define mmPA_CL_VPORT_XOFFSET 0xa110 +#define mmPA_CL_VPORT_YSCALE 0xa111 +#define mmPA_CL_VPORT_YOFFSET 0xa112 +#define mmPA_CL_VPORT_ZSCALE 0xa113 +#define mmPA_CL_VPORT_ZOFFSET 0xa114 +#define mmPA_CL_VPORT_XSCALE_1 0xa115 +#define mmPA_CL_VPORT_XSCALE_2 0xa11b +#define mmPA_CL_VPORT_XSCALE_3 0xa121 +#define mmPA_CL_VPORT_XSCALE_4 0xa127 +#define mmPA_CL_VPORT_XSCALE_5 0xa12d +#define mmPA_CL_VPORT_XSCALE_6 0xa133 +#define mmPA_CL_VPORT_XSCALE_7 0xa139 +#define mmPA_CL_VPORT_XSCALE_8 0xa13f +#define mmPA_CL_VPORT_XSCALE_9 0xa145 +#define mmPA_CL_VPORT_XSCALE_10 0xa14b +#define mmPA_CL_VPORT_XSCALE_11 0xa151 +#define mmPA_CL_VPORT_XSCALE_12 0xa157 +#define mmPA_CL_VPORT_XSCALE_13 0xa15d +#define mmPA_CL_VPORT_XSCALE_14 0xa163 +#define mmPA_CL_VPORT_XSCALE_15 0xa169 +#define mmPA_CL_VPORT_XOFFSET_1 0xa116 +#define mmPA_CL_VPORT_XOFFSET_2 0xa11c +#define mmPA_CL_VPORT_XOFFSET_3 0xa122 +#define mmPA_CL_VPORT_XOFFSET_4 0xa128 +#define mmPA_CL_VPORT_XOFFSET_5 0xa12e +#define mmPA_CL_VPORT_XOFFSET_6 0xa134 +#define mmPA_CL_VPORT_XOFFSET_7 0xa13a +#define mmPA_CL_VPORT_XOFFSET_8 0xa140 +#define mmPA_CL_VPORT_XOFFSET_9 0xa146 +#define mmPA_CL_VPORT_XOFFSET_10 0xa14c +#define mmPA_CL_VPORT_XOFFSET_11 0xa152 +#define mmPA_CL_VPORT_XOFFSET_12 0xa158 +#define mmPA_CL_VPORT_XOFFSET_13 0xa15e +#define mmPA_CL_VPORT_XOFFSET_14 0xa164 +#define mmPA_CL_VPORT_XOFFSET_15 0xa16a +#define mmPA_CL_VPORT_YSCALE_1 0xa117 +#define mmPA_CL_VPORT_YSCALE_2 0xa11d +#define mmPA_CL_VPORT_YSCALE_3 0xa123 +#define mmPA_CL_VPORT_YSCALE_4 0xa129 +#define mmPA_CL_VPORT_YSCALE_5 0xa12f +#define mmPA_CL_VPORT_YSCALE_6 0xa135 +#define mmPA_CL_VPORT_YSCALE_7 0xa13b +#define mmPA_CL_VPORT_YSCALE_8 0xa141 +#define mmPA_CL_VPORT_YSCALE_9 0xa147 +#define mmPA_CL_VPORT_YSCALE_10 0xa14d +#define mmPA_CL_VPORT_YSCALE_11 0xa153 +#define mmPA_CL_VPORT_YSCALE_12 0xa159 +#define mmPA_CL_VPORT_YSCALE_13 0xa15f +#define mmPA_CL_VPORT_YSCALE_14 0xa165 +#define mmPA_CL_VPORT_YSCALE_15 0xa16b +#define mmPA_CL_VPORT_YOFFSET_1 0xa118 +#define mmPA_CL_VPORT_YOFFSET_2 0xa11e +#define mmPA_CL_VPORT_YOFFSET_3 0xa124 +#define mmPA_CL_VPORT_YOFFSET_4 0xa12a +#define mmPA_CL_VPORT_YOFFSET_5 0xa130 +#define mmPA_CL_VPORT_YOFFSET_6 0xa136 +#define mmPA_CL_VPORT_YOFFSET_7 0xa13c +#define mmPA_CL_VPORT_YOFFSET_8 0xa142 +#define mmPA_CL_VPORT_YOFFSET_9 0xa148 +#define mmPA_CL_VPORT_YOFFSET_10 0xa14e +#define mmPA_CL_VPORT_YOFFSET_11 0xa154 +#define mmPA_CL_VPORT_YOFFSET_12 0xa15a +#define mmPA_CL_VPORT_YOFFSET_13 0xa160 +#define mmPA_CL_VPORT_YOFFSET_14 0xa166 +#define mmPA_CL_VPORT_YOFFSET_15 0xa16c +#define mmPA_CL_VPORT_ZSCALE_1 0xa119 +#define mmPA_CL_VPORT_ZSCALE_2 0xa11f +#define mmPA_CL_VPORT_ZSCALE_3 0xa125 +#define mmPA_CL_VPORT_ZSCALE_4 0xa12b +#define mmPA_CL_VPORT_ZSCALE_5 0xa131 +#define mmPA_CL_VPORT_ZSCALE_6 0xa137 +#define mmPA_CL_VPORT_ZSCALE_7 0xa13d +#define mmPA_CL_VPORT_ZSCALE_8 0xa143 +#define mmPA_CL_VPORT_ZSCALE_9 0xa149 +#define mmPA_CL_VPORT_ZSCALE_10 0xa14f +#define mmPA_CL_VPORT_ZSCALE_11 0xa155 +#define mmPA_CL_VPORT_ZSCALE_12 0xa15b +#define mmPA_CL_VPORT_ZSCALE_13 0xa161 +#define mmPA_CL_VPORT_ZSCALE_14 0xa167 +#define mmPA_CL_VPORT_ZSCALE_15 0xa16d +#define mmPA_CL_VPORT_ZOFFSET_1 0xa11a +#define mmPA_CL_VPORT_ZOFFSET_2 0xa120 +#define mmPA_CL_VPORT_ZOFFSET_3 0xa126 +#define mmPA_CL_VPORT_ZOFFSET_4 0xa12c +#define mmPA_CL_VPORT_ZOFFSET_5 0xa132 +#define mmPA_CL_VPORT_ZOFFSET_6 0xa138 +#define mmPA_CL_VPORT_ZOFFSET_7 0xa13e +#define mmPA_CL_VPORT_ZOFFSET_8 0xa144 +#define mmPA_CL_VPORT_ZOFFSET_9 0xa14a +#define mmPA_CL_VPORT_ZOFFSET_10 0xa150 +#define mmPA_CL_VPORT_ZOFFSET_11 0xa156 +#define mmPA_CL_VPORT_ZOFFSET_12 0xa15c +#define mmPA_CL_VPORT_ZOFFSET_13 0xa162 +#define mmPA_CL_VPORT_ZOFFSET_14 0xa168 +#define mmPA_CL_VPORT_ZOFFSET_15 0xa16e +#define mmPA_CL_VTE_CNTL 0xa206 +#define mmPA_CL_VS_OUT_CNTL 0xa207 +#define mmPA_CL_NANINF_CNTL 0xa208 +#define mmPA_CL_CLIP_CNTL 0xa204 +#define mmPA_CL_GB_VERT_CLIP_ADJ 0xa2fa +#define mmPA_CL_GB_VERT_DISC_ADJ 0xa2fb +#define mmPA_CL_GB_HORZ_CLIP_ADJ 0xa2fc +#define mmPA_CL_GB_HORZ_DISC_ADJ 0xa2fd +#define mmPA_CL_UCP_0_X 0xa16f +#define mmPA_CL_UCP_0_Y 0xa170 +#define mmPA_CL_UCP_0_Z 0xa171 +#define mmPA_CL_UCP_0_W 0xa172 +#define mmPA_CL_UCP_1_X 0xa173 +#define mmPA_CL_UCP_1_Y 0xa174 +#define mmPA_CL_UCP_1_Z 0xa175 +#define mmPA_CL_UCP_1_W 0xa176 +#define mmPA_CL_UCP_2_X 0xa177 +#define mmPA_CL_UCP_2_Y 0xa178 +#define mmPA_CL_UCP_2_Z 0xa179 +#define mmPA_CL_UCP_2_W 0xa17a +#define mmPA_CL_UCP_3_X 0xa17b +#define mmPA_CL_UCP_3_Y 0xa17c +#define mmPA_CL_UCP_3_Z 0xa17d +#define mmPA_CL_UCP_3_W 0xa17e +#define mmPA_CL_UCP_4_X 0xa17f +#define mmPA_CL_UCP_4_Y 0xa180 +#define mmPA_CL_UCP_4_Z 0xa181 +#define mmPA_CL_UCP_4_W 0xa182 +#define mmPA_CL_UCP_5_X 0xa183 +#define mmPA_CL_UCP_5_Y 0xa184 +#define mmPA_CL_UCP_5_Z 0xa185 +#define mmPA_CL_UCP_5_W 0xa186 +#define mmPA_CL_POINT_X_RAD 0xa1f5 +#define mmPA_CL_POINT_Y_RAD 0xa1f6 +#define mmPA_CL_POINT_SIZE 0xa1f7 +#define mmPA_CL_POINT_CULL_RAD 0xa1f8 +#define mmPA_CL_ENHANCE 0x2285 +#define mmPA_CL_RESET_DEBUG 0x2286 +#define mmPA_SU_VTX_CNTL 0xa2f9 +#define mmPA_SU_POINT_SIZE 0xa280 +#define mmPA_SU_POINT_MINMAX 0xa281 +#define mmPA_SU_LINE_CNTL 0xa282 +#define mmPA_SU_LINE_STIPPLE_CNTL 0xa209 +#define mmPA_SU_LINE_STIPPLE_SCALE 0xa20a +#define mmPA_SU_PRIM_FILTER_CNTL 0xa20b +#define mmPA_SU_SC_MODE_CNTL 0xa205 +#define mmPA_SU_POLY_OFFSET_DB_FMT_CNTL 0xa2de +#define mmPA_SU_POLY_OFFSET_CLAMP 0xa2df +#define mmPA_SU_POLY_OFFSET_FRONT_SCALE 0xa2e0 +#define mmPA_SU_POLY_OFFSET_FRONT_OFFSET 0xa2e1 +#define mmPA_SU_POLY_OFFSET_BACK_SCALE 0xa2e2 +#define mmPA_SU_POLY_OFFSET_BACK_OFFSET 0xa2e3 +#define mmPA_SU_HARDWARE_SCREEN_OFFSET 0xa08d +#define mmPA_SU_LINE_STIPPLE_VALUE 0xc280 +#define mmPA_SU_PERFCOUNTER0_SELECT 0xd900 +#define mmPA_SU_PERFCOUNTER0_SELECT1 0xd901 +#define mmPA_SU_PERFCOUNTER1_SELECT 0xd902 +#define mmPA_SU_PERFCOUNTER1_SELECT1 0xd903 +#define mmPA_SU_PERFCOUNTER2_SELECT 0xd904 +#define mmPA_SU_PERFCOUNTER3_SELECT 0xd905 +#define mmPA_SU_PERFCOUNTER0_LO 0xd100 +#define mmPA_SU_PERFCOUNTER0_HI 0xd101 +#define mmPA_SU_PERFCOUNTER1_LO 0xd102 +#define mmPA_SU_PERFCOUNTER1_HI 0xd103 +#define mmPA_SU_PERFCOUNTER2_LO 0xd104 +#define mmPA_SU_PERFCOUNTER2_HI 0xd105 +#define mmPA_SU_PERFCOUNTER3_LO 0xd106 +#define mmPA_SU_PERFCOUNTER3_HI 0xd107 +#define mmPA_SC_AA_CONFIG 0xa2f8 +#define mmPA_SC_AA_MASK_X0Y0_X1Y0 0xa30e +#define mmPA_SC_AA_MASK_X0Y1_X1Y1 0xa30f +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0 0xa2fe +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1 0xa2ff +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2 0xa300 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3 0xa301 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0 0xa302 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1 0xa303 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2 0xa304 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3 0xa305 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0 0xa306 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1 0xa307 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2 0xa308 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3 0xa309 +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0 0xa30a +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1 0xa30b +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2 0xa30c +#define mmPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3 0xa30d +#define mmPA_SC_CENTROID_PRIORITY_0 0xa2f5 +#define mmPA_SC_CENTROID_PRIORITY_1 0xa2f6 +#define mmPA_SC_CLIPRECT_0_TL 0xa084 +#define mmPA_SC_CLIPRECT_0_BR 0xa085 +#define mmPA_SC_CLIPRECT_1_TL 0xa086 +#define mmPA_SC_CLIPRECT_1_BR 0xa087 +#define mmPA_SC_CLIPRECT_2_TL 0xa088 +#define mmPA_SC_CLIPRECT_2_BR 0xa089 +#define mmPA_SC_CLIPRECT_3_TL 0xa08a +#define mmPA_SC_CLIPRECT_3_BR 0xa08b +#define mmPA_SC_CLIPRECT_RULE 0xa083 +#define mmPA_SC_EDGERULE 0xa08c +#define mmPA_SC_LINE_CNTL 0xa2f7 +#define mmPA_SC_LINE_STIPPLE 0xa283 +#define mmPA_SC_MODE_CNTL_0 0xa292 +#define mmPA_SC_MODE_CNTL_1 0xa293 +#define mmPA_SC_RASTER_CONFIG 0xa0d4 +#define mmPA_SC_RASTER_CONFIG_1 0xa0d5 +#define mmPA_SC_SCREEN_EXTENT_CONTROL 0xa0d6 +#define mmPA_SC_GENERIC_SCISSOR_TL 0xa090 +#define mmPA_SC_GENERIC_SCISSOR_BR 0xa091 +#define mmPA_SC_SCREEN_SCISSOR_TL 0xa00c +#define mmPA_SC_SCREEN_SCISSOR_BR 0xa00d +#define mmPA_SC_WINDOW_OFFSET 0xa080 +#define mmPA_SC_WINDOW_SCISSOR_TL 0xa081 +#define mmPA_SC_WINDOW_SCISSOR_BR 0xa082 +#define mmPA_SC_VPORT_SCISSOR_0_TL 0xa094 +#define mmPA_SC_VPORT_SCISSOR_1_TL 0xa096 +#define mmPA_SC_VPORT_SCISSOR_2_TL 0xa098 +#define mmPA_SC_VPORT_SCISSOR_3_TL 0xa09a +#define mmPA_SC_VPORT_SCISSOR_4_TL 0xa09c +#define mmPA_SC_VPORT_SCISSOR_5_TL 0xa09e +#define mmPA_SC_VPORT_SCISSOR_6_TL 0xa0a0 +#define mmPA_SC_VPORT_SCISSOR_7_TL 0xa0a2 +#define mmPA_SC_VPORT_SCISSOR_8_TL 0xa0a4 +#define mmPA_SC_VPORT_SCISSOR_9_TL 0xa0a6 +#define mmPA_SC_VPORT_SCISSOR_10_TL 0xa0a8 +#define mmPA_SC_VPORT_SCISSOR_11_TL 0xa0aa +#define mmPA_SC_VPORT_SCISSOR_12_TL 0xa0ac +#define mmPA_SC_VPORT_SCISSOR_13_TL 0xa0ae +#define mmPA_SC_VPORT_SCISSOR_14_TL 0xa0b0 +#define mmPA_SC_VPORT_SCISSOR_15_TL 0xa0b2 +#define mmPA_SC_VPORT_SCISSOR_0_BR 0xa095 +#define mmPA_SC_VPORT_SCISSOR_1_BR 0xa097 +#define mmPA_SC_VPORT_SCISSOR_2_BR 0xa099 +#define mmPA_SC_VPORT_SCISSOR_3_BR 0xa09b +#define mmPA_SC_VPORT_SCISSOR_4_BR 0xa09d +#define mmPA_SC_VPORT_SCISSOR_5_BR 0xa09f +#define mmPA_SC_VPORT_SCISSOR_6_BR 0xa0a1 +#define mmPA_SC_VPORT_SCISSOR_7_BR 0xa0a3 +#define mmPA_SC_VPORT_SCISSOR_8_BR 0xa0a5 +#define mmPA_SC_VPORT_SCISSOR_9_BR 0xa0a7 +#define mmPA_SC_VPORT_SCISSOR_10_BR 0xa0a9 +#define mmPA_SC_VPORT_SCISSOR_11_BR 0xa0ab +#define mmPA_SC_VPORT_SCISSOR_12_BR 0xa0ad +#define mmPA_SC_VPORT_SCISSOR_13_BR 0xa0af +#define mmPA_SC_VPORT_SCISSOR_14_BR 0xa0b1 +#define mmPA_SC_VPORT_SCISSOR_15_BR 0xa0b3 +#define mmPA_SC_VPORT_ZMIN_0 0xa0b4 +#define mmPA_SC_VPORT_ZMIN_1 0xa0b6 +#define mmPA_SC_VPORT_ZMIN_2 0xa0b8 +#define mmPA_SC_VPORT_ZMIN_3 0xa0ba +#define mmPA_SC_VPORT_ZMIN_4 0xa0bc +#define mmPA_SC_VPORT_ZMIN_5 0xa0be +#define mmPA_SC_VPORT_ZMIN_6 0xa0c0 +#define mmPA_SC_VPORT_ZMIN_7 0xa0c2 +#define mmPA_SC_VPORT_ZMIN_8 0xa0c4 +#define mmPA_SC_VPORT_ZMIN_9 0xa0c6 +#define mmPA_SC_VPORT_ZMIN_10 0xa0c8 +#define mmPA_SC_VPORT_ZMIN_11 0xa0ca +#define mmPA_SC_VPORT_ZMIN_12 0xa0cc +#define mmPA_SC_VPORT_ZMIN_13 0xa0ce +#define mmPA_SC_VPORT_ZMIN_14 0xa0d0 +#define mmPA_SC_VPORT_ZMIN_15 0xa0d2 +#define mmPA_SC_VPORT_ZMAX_0 0xa0b5 +#define mmPA_SC_VPORT_ZMAX_1 0xa0b7 +#define mmPA_SC_VPORT_ZMAX_2 0xa0b9 +#define mmPA_SC_VPORT_ZMAX_3 0xa0bb +#define mmPA_SC_VPORT_ZMAX_4 0xa0bd +#define mmPA_SC_VPORT_ZMAX_5 0xa0bf +#define mmPA_SC_VPORT_ZMAX_6 0xa0c1 +#define mmPA_SC_VPORT_ZMAX_7 0xa0c3 +#define mmPA_SC_VPORT_ZMAX_8 0xa0c5 +#define mmPA_SC_VPORT_ZMAX_9 0xa0c7 +#define mmPA_SC_VPORT_ZMAX_10 0xa0c9 +#define mmPA_SC_VPORT_ZMAX_11 0xa0cb +#define mmPA_SC_VPORT_ZMAX_12 0xa0cd +#define mmPA_SC_VPORT_ZMAX_13 0xa0cf +#define mmPA_SC_VPORT_ZMAX_14 0xa0d1 +#define mmPA_SC_VPORT_ZMAX_15 0xa0d3 +#define mmPA_SC_ENHANCE 0x22fc +#define mmPA_SC_FIFO_SIZE 0x22f3 +#define mmPA_SC_IF_FIFO_SIZE 0x22f5 +#define mmPA_SC_FORCE_EOV_MAX_CNTS 0x22c9 +#define mmPA_SC_LINE_STIPPLE_STATE 0xc281 +#define mmPA_SC_SCREEN_EXTENT_MIN_0 0xc284 +#define mmPA_SC_SCREEN_EXTENT_MAX_0 0xc285 +#define mmPA_SC_SCREEN_EXTENT_MIN_1 0xc286 +#define mmPA_SC_SCREEN_EXTENT_MAX_1 0xc28b +#define mmPA_SC_PERFCOUNTER0_SELECT 0xd940 +#define mmPA_SC_PERFCOUNTER0_SELECT1 0xd941 +#define mmPA_SC_PERFCOUNTER1_SELECT 0xd942 +#define mmPA_SC_PERFCOUNTER2_SELECT 0xd943 +#define mmPA_SC_PERFCOUNTER3_SELECT 0xd944 +#define mmPA_SC_PERFCOUNTER4_SELECT 0xd945 +#define mmPA_SC_PERFCOUNTER5_SELECT 0xd946 +#define mmPA_SC_PERFCOUNTER6_SELECT 0xd947 +#define mmPA_SC_PERFCOUNTER7_SELECT 0xd948 +#define mmPA_SC_PERFCOUNTER0_LO 0xd140 +#define mmPA_SC_PERFCOUNTER0_HI 0xd141 +#define mmPA_SC_PERFCOUNTER1_LO 0xd142 +#define mmPA_SC_PERFCOUNTER1_HI 0xd143 +#define mmPA_SC_PERFCOUNTER2_LO 0xd144 +#define mmPA_SC_PERFCOUNTER2_HI 0xd145 +#define mmPA_SC_PERFCOUNTER3_LO 0xd146 +#define mmPA_SC_PERFCOUNTER3_HI 0xd147 +#define mmPA_SC_PERFCOUNTER4_LO 0xd148 +#define mmPA_SC_PERFCOUNTER4_HI 0xd149 +#define mmPA_SC_PERFCOUNTER5_LO 0xd14a +#define mmPA_SC_PERFCOUNTER5_HI 0xd14b +#define mmPA_SC_PERFCOUNTER6_LO 0xd14c +#define mmPA_SC_PERFCOUNTER6_HI 0xd14d +#define mmPA_SC_PERFCOUNTER7_LO 0xd14e +#define mmPA_SC_PERFCOUNTER7_HI 0xd14f +#define mmPA_SC_P3D_TRAP_SCREEN_HV_EN 0xc2a0 +#define mmPA_SC_P3D_TRAP_SCREEN_H 0xc2a1 +#define mmPA_SC_P3D_TRAP_SCREEN_V 0xc2a2 +#define mmPA_SC_P3D_TRAP_SCREEN_OCCURRENCE 0xc2a3 +#define mmPA_SC_P3D_TRAP_SCREEN_COUNT 0xc2a4 +#define mmPA_SC_HP3D_TRAP_SCREEN_HV_EN 0xc2a8 +#define mmPA_SC_HP3D_TRAP_SCREEN_H 0xc2a9 +#define mmPA_SC_HP3D_TRAP_SCREEN_V 0xc2aa +#define mmPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE 0xc2ab +#define mmPA_SC_HP3D_TRAP_SCREEN_COUNT 0xc2ac +#define mmPA_SC_TRAP_SCREEN_HV_EN 0xc2b0 +#define mmPA_SC_TRAP_SCREEN_H 0xc2b1 +#define mmPA_SC_TRAP_SCREEN_V 0xc2b2 +#define mmPA_SC_TRAP_SCREEN_OCCURRENCE 0xc2b3 +#define mmPA_SC_TRAP_SCREEN_COUNT 0xc2b4 +#define mmPA_SC_P3D_TRAP_SCREEN_HV_LOCK 0x22c0 +#define mmPA_SC_HP3D_TRAP_SCREEN_HV_LOCK 0x22c1 +#define mmPA_SC_TRAP_SCREEN_HV_LOCK 0x22c2 +#define mmPA_CL_CNTL_STATUS 0x2284 +#define mmPA_SU_CNTL_STATUS 0x2294 +#define mmPA_SC_FIFO_DEPTH_CNTL 0x2295 +#define mmCGTT_PA_CLK_CTRL 0xf088 +#define mmCGTT_SC_CLK_CTRL 0xf089 +#define mmPA_SU_DEBUG_CNTL 0x2280 +#define mmPA_SU_DEBUG_DATA 0x2281 +#define mmPA_SC_DEBUG_CNTL 0x22f6 +#define mmPA_SC_DEBUG_DATA 0x22f7 +#define ixCLIPPER_DEBUG_REG00 0x0 +#define ixCLIPPER_DEBUG_REG01 0x1 +#define ixCLIPPER_DEBUG_REG02 0x2 +#define ixCLIPPER_DEBUG_REG03 0x3 +#define ixCLIPPER_DEBUG_REG04 0x4 +#define ixCLIPPER_DEBUG_REG05 0x5 +#define ixCLIPPER_DEBUG_REG06 0x6 +#define ixCLIPPER_DEBUG_REG07 0x7 +#define ixCLIPPER_DEBUG_REG08 0x8 +#define ixCLIPPER_DEBUG_REG09 0x9 +#define ixCLIPPER_DEBUG_REG10 0xa +#define ixCLIPPER_DEBUG_REG11 0xb +#define ixCLIPPER_DEBUG_REG12 0xc +#define ixCLIPPER_DEBUG_REG13 0xd +#define ixCLIPPER_DEBUG_REG14 0xe +#define ixCLIPPER_DEBUG_REG15 0xf +#define ixCLIPPER_DEBUG_REG16 0x10 +#define ixCLIPPER_DEBUG_REG17 0x11 +#define ixCLIPPER_DEBUG_REG18 0x12 +#define ixCLIPPER_DEBUG_REG19 0x13 +#define ixSXIFCCG_DEBUG_REG0 0x14 +#define ixSXIFCCG_DEBUG_REG1 0x15 +#define ixSXIFCCG_DEBUG_REG2 0x16 +#define ixSXIFCCG_DEBUG_REG3 0x17 +#define ixSETUP_DEBUG_REG0 0x18 +#define ixSETUP_DEBUG_REG1 0x19 +#define ixSETUP_DEBUG_REG2 0x1a +#define ixSETUP_DEBUG_REG3 0x1b +#define ixSETUP_DEBUG_REG4 0x1c +#define ixSETUP_DEBUG_REG5 0x1d +#define ixPA_SC_DEBUG_REG0 0x0 +#define ixPA_SC_DEBUG_REG1 0x1 +#define mmCOMPUTE_DISPATCH_INITIATOR 0x2e00 +#define mmCOMPUTE_DIM_X 0x2e01 +#define mmCOMPUTE_DIM_Y 0x2e02 +#define mmCOMPUTE_DIM_Z 0x2e03 +#define mmCOMPUTE_START_X 0x2e04 +#define mmCOMPUTE_START_Y 0x2e05 +#define mmCOMPUTE_START_Z 0x2e06 +#define mmCOMPUTE_NUM_THREAD_X 0x2e07 +#define mmCOMPUTE_NUM_THREAD_Y 0x2e08 +#define mmCOMPUTE_NUM_THREAD_Z 0x2e09 +#define mmCOMPUTE_PIPELINESTAT_ENABLE 0x2e0a +#define mmCOMPUTE_PERFCOUNT_ENABLE 0x2e0b +#define mmCOMPUTE_PGM_LO 0x2e0c +#define mmCOMPUTE_PGM_HI 0x2e0d +#define mmCOMPUTE_TBA_LO 0x2e0e +#define mmCOMPUTE_TBA_HI 0x2e0f +#define mmCOMPUTE_TMA_LO 0x2e10 +#define mmCOMPUTE_TMA_HI 0x2e11 +#define mmCOMPUTE_PGM_RSRC1 0x2e12 +#define mmCOMPUTE_PGM_RSRC2 0x2e13 +#define mmCOMPUTE_VMID 0x2e14 +#define mmCOMPUTE_RESOURCE_LIMITS 0x2e15 +#define mmCOMPUTE_STATIC_THREAD_MGMT_SE0 0x2e16 +#define mmCOMPUTE_STATIC_THREAD_MGMT_SE1 0x2e17 +#define mmCOMPUTE_TMPRING_SIZE 0x2e18 +#define mmCOMPUTE_STATIC_THREAD_MGMT_SE2 0x2e19 +#define mmCOMPUTE_STATIC_THREAD_MGMT_SE3 0x2e1a +#define mmCOMPUTE_RESTART_X 0x2e1b +#define mmCOMPUTE_RESTART_Y 0x2e1c +#define mmCOMPUTE_RESTART_Z 0x2e1d +#define mmCOMPUTE_THREAD_TRACE_ENABLE 0x2e1e +#define mmCOMPUTE_MISC_RESERVED 0x2e1f +#define mmCOMPUTE_USER_DATA_0 0x2e40 +#define mmCOMPUTE_USER_DATA_1 0x2e41 +#define mmCOMPUTE_USER_DATA_2 0x2e42 +#define mmCOMPUTE_USER_DATA_3 0x2e43 +#define mmCOMPUTE_USER_DATA_4 0x2e44 +#define mmCOMPUTE_USER_DATA_5 0x2e45 +#define mmCOMPUTE_USER_DATA_6 0x2e46 +#define mmCOMPUTE_USER_DATA_7 0x2e47 +#define mmCOMPUTE_USER_DATA_8 0x2e48 +#define mmCOMPUTE_USER_DATA_9 0x2e49 +#define mmCOMPUTE_USER_DATA_10 0x2e4a +#define mmCOMPUTE_USER_DATA_11 0x2e4b +#define mmCOMPUTE_USER_DATA_12 0x2e4c +#define mmCOMPUTE_USER_DATA_13 0x2e4d +#define mmCOMPUTE_USER_DATA_14 0x2e4e +#define mmCOMPUTE_USER_DATA_15 0x2e4f +#define mmCSPRIV_CONNECT 0x0 +#define mmCSPRIV_THREAD_TRACE_TG0 0x1e +#define mmCSPRIV_THREAD_TRACE_TG1 0x1e +#define mmCSPRIV_THREAD_TRACE_TG2 0x1e +#define mmCSPRIV_THREAD_TRACE_TG3 0x1e +#define mmCSPRIV_THREAD_TRACE_EVENT 0x1f +#define mmRLC_CNTL 0x30c0 +#define mmRLC_DEBUG_SELECT 0x30c1 +#define mmRLC_DEBUG 0x30c2 +#define mmRLC_MC_CNTL 0x30c3 +#define mmRLC_STAT 0x30c4 +#define mmRLC_SAFE_MODE 0x313a +#define mmRLC_SOFT_RESET_GPU 0x30c5 +#define mmRLC_MEM_SLP_CNTL 0x30c6 +#define mmRLC_PERFMON_CNTL 0xdcc0 +#define mmRLC_PERFCOUNTER0_SELECT 0xdcc1 +#define mmRLC_PERFCOUNTER1_SELECT 0xdcc2 +#define mmRLC_PERFCOUNTER0_LO 0xd480 +#define mmRLC_PERFCOUNTER1_LO 0xd482 +#define mmRLC_PERFCOUNTER0_HI 0xd481 +#define mmRLC_PERFCOUNTER1_HI 0xd483 +#define mmCGTT_RLC_CLK_CTRL 0xf0b8 +#define mmRLC_LB_CNTL 0x30d9 +#define mmRLC_LB_CNTR_MAX 0x30d2 +#define mmRLC_LB_CNTR_INIT 0x30db +#define mmRLC_LOAD_BALANCE_CNTR 0x30dc +#define mmRLC_SAVE_AND_RESTORE_BASE 0x30dd +#define mmRLC_JUMP_TABLE_RESTORE 0x30de +#define mmRLC_DRIVER_CPDMA_STATUS 0x30de +#define mmRLC_PG_DELAY_2 0x30df +#define mmRLC_GPM_DEBUG_SELECT 0x30e0 +#define mmRLC_GPM_DEBUG 0x30e1 +#define mmRLC_GPM_UCODE_ADDR 0x30e2 +#define mmRLC_GPM_UCODE_DATA 0x30e3 +#define mmRLC_GPU_CLOCK_COUNT_LSB 0x30e4 +#define mmRLC_GPU_CLOCK_COUNT_MSB 0x30e5 +#define mmRLC_CAPTURE_GPU_CLOCK_COUNT 0x30e6 +#define mmRLC_UCODE_CNTL 0x30e7 +#define mmRLC_GPM_STAT 0x3100 +#define mmRLC_GPU_CLOCK_32_RES_SEL 0x3101 +#define mmRLC_GPU_CLOCK_32 0x3102 +#define mmRLC_PG_CNTL 0x3103 +#define mmRLC_GPM_THREAD_PRIORITY 0x3104 +#define mmRLC_GPM_THREAD_ENABLE 0x3105 +#define mmRLC_GPM_VMID_THREAD0 0x3106 +#define mmRLC_GPM_VMID_THREAD1 0x3107 +#define mmRLC_CGTT_MGCG_OVERRIDE 0x3108 +#define mmRLC_CGCG_CGLS_CTRL 0x3109 +#define mmRLC_CGCG_RAMP_CTRL 0x310a +#define mmRLC_DYN_PG_STATUS 0x310b +#define mmRLC_DYN_PG_REQUEST 0x310c +#define mmRLC_PG_DELAY 0x310d +#define mmRLC_CU_STATUS 0x310e +#define mmRLC_LB_INIT_CU_MASK 0x310f +#define mmRLC_LB_ALWAYS_ACTIVE_CU_MASK 0x3110 +#define mmRLC_LB_PARAMS 0x3111 +#define mmRLC_THREAD1_DELAY 0x3112 +#define mmRLC_PG_ALWAYS_ON_CU_MASK 0x3113 +#define mmRLC_MAX_PG_CU 0x3114 +#define mmRLC_AUTO_PG_CTRL 0x3115 +#define mmRLC_SMU_GRBM_REG_SAVE_CTRL 0x3116 +#define mmRLC_SMU_PG_CTRL 0x3117 +#define mmRLC_SMU_PG_WAKE_UP_CTRL 0x3118 +#define mmRLC_SERDES_RD_MASTER_INDEX 0x3119 +#define mmRLC_SERDES_RD_DATA_0 0x311a +#define mmRLC_SERDES_RD_DATA_1 0x311b +#define mmRLC_SERDES_RD_DATA_2 0x311c +#define mmRLC_SERDES_WR_CU_MASTER_MASK 0x311d +#define mmRLC_SERDES_WR_NONCU_MASTER_MASK 0x311e +#define mmRLC_SERDES_WR_CTRL 0x311f +#define mmRLC_SERDES_WR_DATA 0x3120 +#define mmRLC_SERDES_CU_MASTER_BUSY 0x3121 +#define mmRLC_SERDES_NONCU_MASTER_BUSY 0x3122 +#define mmRLC_GPM_GENERAL_0 0x3123 +#define mmRLC_GPM_GENERAL_1 0x3124 +#define mmRLC_GPM_GENERAL_2 0x3125 +#define mmRLC_GPM_GENERAL_3 0x3126 +#define mmRLC_GPM_GENERAL_4 0x3127 +#define mmRLC_GPM_GENERAL_5 0x3128 +#define mmRLC_GPM_GENERAL_6 0x3129 +#define mmRLC_GPM_GENERAL_7 0x312a +#define mmRLC_GPM_CU_PD_TIMEOUT 0x312b +#define mmRLC_GPM_SCRATCH_ADDR 0x312c +#define mmRLC_GPM_SCRATCH_DATA 0x312d +#define mmRLC_STATIC_PG_STATUS 0x312e +#define mmRLC_GPM_PERF_COUNT_0 0x312f +#define mmRLC_GPM_PERF_COUNT_1 0x3130 +#define mmRLC_GPR_REG1 0x3139 +#define mmRLC_GPR_REG2 0x313a +#define mmRLC_SPM_VMID 0x3131 +#define mmRLC_SPM_INT_CNTL 0x3132 +#define mmRLC_SPM_INT_STATUS 0x3133 +#define mmRLC_SPM_DEBUG_SELECT 0x3134 +#define mmRLC_SPM_DEBUG 0x3135 +#define mmRLC_GPM_LOG_ADDR 0x3136 +#define mmRLC_GPM_LOG_SIZE 0x3137 +#define mmRLC_GPM_LOG_CONT 0x3138 +#define mmRLC_SPM_PERFMON_CNTL 0xdc80 +#define mmRLC_SPM_PERFMON_RING_BASE_LO 0xdc81 +#define mmRLC_SPM_PERFMON_RING_BASE_HI 0xdc82 +#define mmRLC_SPM_PERFMON_RING_SIZE 0xdc83 +#define mmRLC_SPM_PERFMON_SEGMENT_SIZE 0xdc84 +#define mmRLC_SPM_SE_MUXSEL_ADDR 0xdc85 +#define mmRLC_SPM_SE_MUXSEL_DATA 0xdc86 +#define mmRLC_SPM_CPG_PERFMON_SAMPLE_DELAY 0xdc87 +#define mmRLC_SPM_CPC_PERFMON_SAMPLE_DELAY 0xdc88 +#define mmRLC_SPM_CPF_PERFMON_SAMPLE_DELAY 0xdc89 +#define mmRLC_SPM_CB_PERFMON_SAMPLE_DELAY 0xdc8a +#define mmRLC_SPM_DB_PERFMON_SAMPLE_DELAY 0xdc8b +#define mmRLC_SPM_PA_PERFMON_SAMPLE_DELAY 0xdc8c +#define mmRLC_SPM_GDS_PERFMON_SAMPLE_DELAY 0xdc8d +#define mmRLC_SPM_IA_PERFMON_SAMPLE_DELAY 0xdc8e +#define mmRLC_SPM_SC_PERFMON_SAMPLE_DELAY 0xdc90 +#define mmRLC_SPM_TCC_PERFMON_SAMPLE_DELAY 0xdc91 +#define mmRLC_SPM_TCA_PERFMON_SAMPLE_DELAY 0xdc92 +#define mmRLC_SPM_TCP_PERFMON_SAMPLE_DELAY 0xdc93 +#define mmRLC_SPM_TA_PERFMON_SAMPLE_DELAY 0xdc94 +#define mmRLC_SPM_TD_PERFMON_SAMPLE_DELAY 0xdc95 +#define mmRLC_SPM_VGT_PERFMON_SAMPLE_DELAY 0xdc96 +#define mmRLC_SPM_SPI_PERFMON_SAMPLE_DELAY 0xdc97 +#define mmRLC_SPM_SQG_PERFMON_SAMPLE_DELAY 0xdc98 +#define mmRLC_SPM_TCS_PERFMON_SAMPLE_DELAY 0xdc99 +#define mmRLC_SPM_SX_PERFMON_SAMPLE_DELAY 0xdc9a +#define mmRLC_SPM_GLOBAL_MUXSEL_ADDR 0xdc9b +#define mmRLC_SPM_GLOBAL_MUXSEL_DATA 0xdc9c +#define mmRLC_SPM_RING_RDPTR 0xdc9d +#define mmRLC_SPM_SEGMENT_THRESHOLD 0xdc9e +#define mmRLC_SPM_DBR0_PERFMON_SAMPLE_DELAY 0xdc9f +#define mmRLC_SPM_DBR1_PERFMON_SAMPLE_DELAY 0xdca0 +#define mmRLC_SPM_CBR0_PERFMON_SAMPLE_DELAY 0xdca1 +#define mmRLC_SPM_CBR1_PERFMON_SAMPLE_DELAY 0xdca2 +#define mmSPI_PS_INPUT_CNTL_0 0xa191 +#define mmSPI_PS_INPUT_CNTL_1 0xa192 +#define mmSPI_PS_INPUT_CNTL_2 0xa193 +#define mmSPI_PS_INPUT_CNTL_3 0xa194 +#define mmSPI_PS_INPUT_CNTL_4 0xa195 +#define mmSPI_PS_INPUT_CNTL_5 0xa196 +#define mmSPI_PS_INPUT_CNTL_6 0xa197 +#define mmSPI_PS_INPUT_CNTL_7 0xa198 +#define mmSPI_PS_INPUT_CNTL_8 0xa199 +#define mmSPI_PS_INPUT_CNTL_9 0xa19a +#define mmSPI_PS_INPUT_CNTL_10 0xa19b +#define mmSPI_PS_INPUT_CNTL_11 0xa19c +#define mmSPI_PS_INPUT_CNTL_12 0xa19d +#define mmSPI_PS_INPUT_CNTL_13 0xa19e +#define mmSPI_PS_INPUT_CNTL_14 0xa19f +#define mmSPI_PS_INPUT_CNTL_15 0xa1a0 +#define mmSPI_PS_INPUT_CNTL_16 0xa1a1 +#define mmSPI_PS_INPUT_CNTL_17 0xa1a2 +#define mmSPI_PS_INPUT_CNTL_18 0xa1a3 +#define mmSPI_PS_INPUT_CNTL_19 0xa1a4 +#define mmSPI_PS_INPUT_CNTL_20 0xa1a5 +#define mmSPI_PS_INPUT_CNTL_21 0xa1a6 +#define mmSPI_PS_INPUT_CNTL_22 0xa1a7 +#define mmSPI_PS_INPUT_CNTL_23 0xa1a8 +#define mmSPI_PS_INPUT_CNTL_24 0xa1a9 +#define mmSPI_PS_INPUT_CNTL_25 0xa1aa +#define mmSPI_PS_INPUT_CNTL_26 0xa1ab +#define mmSPI_PS_INPUT_CNTL_27 0xa1ac +#define mmSPI_PS_INPUT_CNTL_28 0xa1ad +#define mmSPI_PS_INPUT_CNTL_29 0xa1ae +#define mmSPI_PS_INPUT_CNTL_30 0xa1af +#define mmSPI_PS_INPUT_CNTL_31 0xa1b0 +#define mmSPI_VS_OUT_CONFIG 0xa1b1 +#define mmSPI_PS_INPUT_ENA 0xa1b3 +#define mmSPI_PS_INPUT_ADDR 0xa1b4 +#define mmSPI_INTERP_CONTROL_0 0xa1b5 +#define mmSPI_PS_IN_CONTROL 0xa1b6 +#define mmSPI_BARYC_CNTL 0xa1b8 +#define mmSPI_TMPRING_SIZE 0xa1ba +#define mmSPI_SHADER_POS_FORMAT 0xa1c3 +#define mmSPI_SHADER_Z_FORMAT 0xa1c4 +#define mmSPI_SHADER_COL_FORMAT 0xa1c5 +#define mmSPI_ARB_PRIORITY 0x31c0 +#define mmSPI_ARB_CYCLES_0 0x31c1 +#define mmSPI_ARB_CYCLES_1 0x31c2 +#define mmSPI_CDBG_SYS_GFX 0x31c3 +#define mmSPI_CDBG_SYS_HP3D 0x31c4 +#define mmSPI_CDBG_SYS_CS0 0x31c5 +#define mmSPI_CDBG_SYS_CS1 0x31c6 +#define mmSPI_WCL_PIPE_PERCENT_GFX 0x31c7 +#define mmSPI_WCL_PIPE_PERCENT_HP3D 0x31c8 +#define mmSPI_WCL_PIPE_PERCENT_CS0 0x31c9 +#define mmSPI_WCL_PIPE_PERCENT_CS1 0x31ca +#define mmSPI_WCL_PIPE_PERCENT_CS2 0x31cb +#define mmSPI_WCL_PIPE_PERCENT_CS3 0x31cc +#define mmSPI_WCL_PIPE_PERCENT_CS4 0x31cd +#define mmSPI_WCL_PIPE_PERCENT_CS5 0x31ce +#define mmSPI_WCL_PIPE_PERCENT_CS6 0x31cf +#define mmSPI_WCL_PIPE_PERCENT_CS7 0x31d0 +#define mmSPI_GDBG_WAVE_CNTL 0x31d1 +#define mmSPI_GDBG_TRAP_CONFIG 0x31d2 +#define mmSPI_GDBG_TRAP_MASK 0x31d3 +#define mmSPI_GDBG_TBA_LO 0x31d4 +#define mmSPI_GDBG_TBA_HI 0x31d5 +#define mmSPI_GDBG_TMA_LO 0x31d6 +#define mmSPI_GDBG_TMA_HI 0x31d7 +#define mmSPI_GDBG_TRAP_DATA0 0x31d8 +#define mmSPI_GDBG_TRAP_DATA1 0x31d9 +#define mmSPI_RESET_DEBUG 0x31da +#define mmSPI_COMPUTE_QUEUE_RESET 0x31db +#define mmSPI_RESOURCE_RESERVE_CU_0 0x31dc +#define mmSPI_RESOURCE_RESERVE_CU_1 0x31dd +#define mmSPI_RESOURCE_RESERVE_CU_2 0x31de +#define mmSPI_RESOURCE_RESERVE_CU_3 0x31df +#define mmSPI_RESOURCE_RESERVE_CU_4 0x31e0 +#define mmSPI_RESOURCE_RESERVE_CU_5 0x31e1 +#define mmSPI_RESOURCE_RESERVE_CU_6 0x31e2 +#define mmSPI_RESOURCE_RESERVE_CU_7 0x31e3 +#define mmSPI_RESOURCE_RESERVE_CU_8 0x31e4 +#define mmSPI_RESOURCE_RESERVE_CU_9 0x31e5 +#define mmSPI_RESOURCE_RESERVE_CU_10 0x31f0 +#define mmSPI_RESOURCE_RESERVE_CU_11 0x31f1 +#define mmSPI_RESOURCE_RESERVE_EN_CU_0 0x31e6 +#define mmSPI_RESOURCE_RESERVE_EN_CU_1 0x31e7 +#define mmSPI_RESOURCE_RESERVE_EN_CU_2 0x31e8 +#define mmSPI_RESOURCE_RESERVE_EN_CU_3 0x31e9 +#define mmSPI_RESOURCE_RESERVE_EN_CU_4 0x31ea +#define mmSPI_RESOURCE_RESERVE_EN_CU_5 0x31eb +#define mmSPI_RESOURCE_RESERVE_EN_CU_6 0x31ec +#define mmSPI_RESOURCE_RESERVE_EN_CU_7 0x31ed +#define mmSPI_RESOURCE_RESERVE_EN_CU_8 0x31ee +#define mmSPI_RESOURCE_RESERVE_EN_CU_9 0x31ef +#define mmSPI_RESOURCE_RESERVE_EN_CU_10 0x31f2 +#define mmSPI_RESOURCE_RESERVE_EN_CU_11 0x31f3 +#define mmSPI_PS_MAX_WAVE_ID 0x243a +#define mmSPI_CONFIG_CNTL 0x2440 +#define mmSPI_DEBUG_CNTL 0x2441 +#define mmSPI_DEBUG_READ 0x2442 +#define mmSPI_PERFCOUNTER0_SELECT 0xd980 +#define mmSPI_PERFCOUNTER1_SELECT 0xd981 +#define mmSPI_PERFCOUNTER2_SELECT 0xd982 +#define mmSPI_PERFCOUNTER3_SELECT 0xd983 +#define mmSPI_PERFCOUNTER0_SELECT1 0xd984 +#define mmSPI_PERFCOUNTER1_SELECT1 0xd985 +#define mmSPI_PERFCOUNTER2_SELECT1 0xd986 +#define mmSPI_PERFCOUNTER3_SELECT1 0xd987 +#define mmSPI_PERFCOUNTER4_SELECT 0xd988 +#define mmSPI_PERFCOUNTER5_SELECT 0xd989 +#define mmSPI_PERFCOUNTER_BINS 0xd98a +#define mmSPI_PERFCOUNTER0_HI 0xd180 +#define mmSPI_PERFCOUNTER0_LO 0xd181 +#define mmSPI_PERFCOUNTER1_HI 0xd182 +#define mmSPI_PERFCOUNTER1_LO 0xd183 +#define mmSPI_PERFCOUNTER2_HI 0xd184 +#define mmSPI_PERFCOUNTER2_LO 0xd185 +#define mmSPI_PERFCOUNTER3_HI 0xd186 +#define mmSPI_PERFCOUNTER3_LO 0xd187 +#define mmSPI_PERFCOUNTER4_HI 0xd188 +#define mmSPI_PERFCOUNTER4_LO 0xd189 +#define mmSPI_PERFCOUNTER5_HI 0xd18a +#define mmSPI_PERFCOUNTER5_LO 0xd18b +#define mmSPI_CONFIG_CNTL_1 0x244f +#define mmSPI_DEBUG_BUSY 0x2450 +#define mmCGTS_SM_CTRL_REG 0xf000 +#define mmCGTS_RD_CTRL_REG 0xf001 +#define mmCGTS_RD_REG 0xf002 +#define mmCGTS_TCC_DISABLE 0xf003 +#define mmCGTS_USER_TCC_DISABLE 0xf004 +#define mmCGTS_CU0_SP0_CTRL_REG 0xf008 +#define mmCGTS_CU0_LDS_SQ_CTRL_REG 0xf009 +#define mmCGTS_CU0_TA_SQC_CTRL_REG 0xf00a +#define mmCGTS_CU0_SP1_CTRL_REG 0xf00b +#define mmCGTS_CU0_TD_TCP_CTRL_REG 0xf00c +#define mmCGTS_CU1_SP0_CTRL_REG 0xf00d +#define mmCGTS_CU1_LDS_SQ_CTRL_REG 0xf00e +#define mmCGTS_CU1_TA_CTRL_REG 0xf00f +#define mmCGTS_CU1_SP1_CTRL_REG 0xf010 +#define mmCGTS_CU1_TD_TCP_CTRL_REG 0xf011 +#define mmCGTS_CU2_SP0_CTRL_REG 0xf012 +#define mmCGTS_CU2_LDS_SQ_CTRL_REG 0xf013 +#define mmCGTS_CU2_TA_CTRL_REG 0xf014 +#define mmCGTS_CU2_SP1_CTRL_REG 0xf015 +#define mmCGTS_CU2_TD_TCP_CTRL_REG 0xf016 +#define mmCGTS_CU3_SP0_CTRL_REG 0xf017 +#define mmCGTS_CU3_LDS_SQ_CTRL_REG 0xf018 +#define mmCGTS_CU3_TA_CTRL_REG 0xf019 +#define mmCGTS_CU3_SP1_CTRL_REG 0xf01a +#define mmCGTS_CU3_TD_TCP_CTRL_REG 0xf01b +#define mmCGTS_CU4_SP0_CTRL_REG 0xf01c +#define mmCGTS_CU4_LDS_SQ_CTRL_REG 0xf01d +#define mmCGTS_CU4_TA_SQC_CTRL_REG 0xf01e +#define mmCGTS_CU4_SP1_CTRL_REG 0xf01f +#define mmCGTS_CU4_TD_TCP_CTRL_REG 0xf020 +#define mmCGTS_CU5_SP0_CTRL_REG 0xf021 +#define mmCGTS_CU5_LDS_SQ_CTRL_REG 0xf022 +#define mmCGTS_CU5_TA_CTRL_REG 0xf023 +#define mmCGTS_CU5_SP1_CTRL_REG 0xf024 +#define mmCGTS_CU5_TD_TCP_CTRL_REG 0xf025 +#define mmCGTS_CU6_SP0_CTRL_REG 0xf026 +#define mmCGTS_CU6_LDS_SQ_CTRL_REG 0xf027 +#define mmCGTS_CU6_TA_CTRL_REG 0xf028 +#define mmCGTS_CU6_SP1_CTRL_REG 0xf029 +#define mmCGTS_CU6_TD_TCP_CTRL_REG 0xf02a +#define mmCGTS_CU7_SP0_CTRL_REG 0xf02b +#define mmCGTS_CU7_LDS_SQ_CTRL_REG 0xf02c +#define mmCGTS_CU7_TA_CTRL_REG 0xf02d +#define mmCGTS_CU7_SP1_CTRL_REG 0xf02e +#define mmCGTS_CU7_TD_TCP_CTRL_REG 0xf02f +#define mmCGTS_CU8_SP0_CTRL_REG 0xf030 +#define mmCGTS_CU8_LDS_SQ_CTRL_REG 0xf031 +#define mmCGTS_CU8_TA_SQC_CTRL_REG 0xf032 +#define mmCGTS_CU8_SP1_CTRL_REG 0xf033 +#define mmCGTS_CU8_TD_TCP_CTRL_REG 0xf034 +#define mmCGTS_CU9_SP0_CTRL_REG 0xf035 +#define mmCGTS_CU9_LDS_SQ_CTRL_REG 0xf036 +#define mmCGTS_CU9_TA_CTRL_REG 0xf037 +#define mmCGTS_CU9_SP1_CTRL_REG 0xf038 +#define mmCGTS_CU9_TD_TCP_CTRL_REG 0xf039 +#define mmCGTS_CU10_SP0_CTRL_REG 0xf03a +#define mmCGTS_CU10_LDS_SQ_CTRL_REG 0xf03b +#define mmCGTS_CU10_TA_CTRL_REG 0xf03c +#define mmCGTS_CU10_SP1_CTRL_REG 0xf03d +#define mmCGTS_CU10_TD_TCP_CTRL_REG 0xf03e +#define mmCGTS_CU11_SP0_CTRL_REG 0xf03f +#define mmCGTS_CU11_LDS_SQ_CTRL_REG 0xf040 +#define mmCGTS_CU11_TA_CTRL_REG 0xf041 +#define mmCGTS_CU11_SP1_CTRL_REG 0xf042 +#define mmCGTS_CU11_TD_TCP_CTRL_REG 0xf043 +#define mmCGTS_CU12_SP0_CTRL_REG 0xf044 +#define mmCGTS_CU12_LDS_SQ_CTRL_REG 0xf045 +#define mmCGTS_CU12_TA_SQC_CTRL_REG 0xf046 +#define mmCGTS_CU12_SP1_CTRL_REG 0xf047 +#define mmCGTS_CU12_TD_TCP_CTRL_REG 0xf048 +#define mmCGTS_CU13_SP0_CTRL_REG 0xf049 +#define mmCGTS_CU13_LDS_SQ_CTRL_REG 0xf04a +#define mmCGTS_CU13_TA_CTRL_REG 0xf04b +#define mmCGTS_CU13_SP1_CTRL_REG 0xf04c +#define mmCGTS_CU13_TD_TCP_CTRL_REG 0xf04d +#define mmCGTS_CU14_SP0_CTRL_REG 0xf04e +#define mmCGTS_CU14_LDS_SQ_CTRL_REG 0xf04f +#define mmCGTS_CU14_TA_CTRL_REG 0xf050 +#define mmCGTS_CU14_SP1_CTRL_REG 0xf051 +#define mmCGTS_CU14_TD_TCP_CTRL_REG 0xf052 +#define mmCGTS_CU15_SP0_CTRL_REG 0xf053 +#define mmCGTS_CU15_LDS_SQ_CTRL_REG 0xf054 +#define mmCGTS_CU15_TA_CTRL_REG 0xf055 +#define mmCGTS_CU15_SP1_CTRL_REG 0xf056 +#define mmCGTS_CU15_TD_TCP_CTRL_REG 0xf057 +#define mmCGTT_SPI_CLK_CTRL 0xf080 +#define mmCGTT_PC_CLK_CTRL 0xf081 +#define mmCGTT_BCI_CLK_CTRL 0xf082 +#define mmSPI_WF_LIFETIME_CNTL 0x24aa +#define mmSPI_WF_LIFETIME_LIMIT_0 0x24ab +#define mmSPI_WF_LIFETIME_LIMIT_1 0x24ac +#define mmSPI_WF_LIFETIME_LIMIT_2 0x24ad +#define mmSPI_WF_LIFETIME_LIMIT_3 0x24ae +#define mmSPI_WF_LIFETIME_LIMIT_4 0x24af +#define mmSPI_WF_LIFETIME_LIMIT_5 0x24b0 +#define mmSPI_WF_LIFETIME_LIMIT_6 0x24b1 +#define mmSPI_WF_LIFETIME_LIMIT_7 0x24b2 +#define mmSPI_WF_LIFETIME_LIMIT_8 0x24b3 +#define mmSPI_WF_LIFETIME_LIMIT_9 0x24b4 +#define mmSPI_WF_LIFETIME_STATUS_0 0x24b5 +#define mmSPI_WF_LIFETIME_STATUS_1 0x24b6 +#define mmSPI_WF_LIFETIME_STATUS_2 0x24b7 +#define mmSPI_WF_LIFETIME_STATUS_3 0x24b8 +#define mmSPI_WF_LIFETIME_STATUS_4 0x24b9 +#define mmSPI_WF_LIFETIME_STATUS_5 0x24ba +#define mmSPI_WF_LIFETIME_STATUS_6 0x24bb +#define mmSPI_WF_LIFETIME_STATUS_7 0x24bc +#define mmSPI_WF_LIFETIME_STATUS_8 0x24bd +#define mmSPI_WF_LIFETIME_STATUS_9 0x24be +#define mmSPI_WF_LIFETIME_STATUS_10 0x24bf +#define mmSPI_WF_LIFETIME_STATUS_11 0x24c0 +#define mmSPI_WF_LIFETIME_STATUS_12 0x24c1 +#define mmSPI_WF_LIFETIME_STATUS_13 0x24c2 +#define mmSPI_WF_LIFETIME_STATUS_14 0x24c3 +#define mmSPI_WF_LIFETIME_STATUS_15 0x24c4 +#define mmSPI_WF_LIFETIME_STATUS_16 0x24c5 +#define mmSPI_WF_LIFETIME_STATUS_17 0x24c6 +#define mmSPI_WF_LIFETIME_STATUS_18 0x24c7 +#define mmSPI_WF_LIFETIME_STATUS_19 0x24c8 +#define mmSPI_WF_LIFETIME_STATUS_20 0x24c9 +#define mmSPI_WF_LIFETIME_DEBUG 0x24ca +#define mmSPI_SLAVE_DEBUG_BUSY 0x24d3 +#define mmSPI_LB_CTR_CTRL 0x24d4 +#define mmSPI_LB_CU_MASK 0x24d5 +#define mmSPI_LB_DATA_REG 0x24d6 +#define mmSPI_PG_ENABLE_STATIC_CU_MASK 0x24d7 +#define mmSPI_GDS_CREDITS 0x24d8 +#define mmSPI_SX_EXPORT_BUFFER_SIZES 0x24d9 +#define mmSPI_SX_SCOREBOARD_BUFFER_SIZES 0x24da +#define mmSPI_CSQ_WF_ACTIVE_STATUS 0x24db +#define mmSPI_CSQ_WF_ACTIVE_COUNT_0 0x24dc +#define mmSPI_CSQ_WF_ACTIVE_COUNT_1 0x24dd +#define mmSPI_CSQ_WF_ACTIVE_COUNT_2 0x24de +#define mmSPI_CSQ_WF_ACTIVE_COUNT_3 0x24df +#define mmSPI_CSQ_WF_ACTIVE_COUNT_4 0x24e0 +#define mmSPI_CSQ_WF_ACTIVE_COUNT_5 0x24e1 +#define mmSPI_CSQ_WF_ACTIVE_COUNT_6 0x24e2 +#define mmSPI_CSQ_WF_ACTIVE_COUNT_7 0x24e3 +#define mmBCI_DEBUG_READ 0x24eb +#define mmSPI_P0_TRAP_SCREEN_PSBA_LO 0x24ec +#define mmSPI_P0_TRAP_SCREEN_PSBA_HI 0x24ed +#define mmSPI_P0_TRAP_SCREEN_PSMA_LO 0x24ee +#define mmSPI_P0_TRAP_SCREEN_PSMA_HI 0x24ef +#define mmSPI_P0_TRAP_SCREEN_GPR_MIN 0x24f0 +#define mmSPI_P1_TRAP_SCREEN_PSBA_LO 0x24f1 +#define mmSPI_P1_TRAP_SCREEN_PSBA_HI 0x24f2 +#define mmSPI_P1_TRAP_SCREEN_PSMA_LO 0x24f3 +#define mmSPI_P1_TRAP_SCREEN_PSMA_HI 0x24f4 +#define mmSPI_P1_TRAP_SCREEN_GPR_MIN 0x24f5 +#define mmSPI_SHADER_TBA_LO_PS 0x2c00 +#define mmSPI_SHADER_TBA_HI_PS 0x2c01 +#define mmSPI_SHADER_TMA_LO_PS 0x2c02 +#define mmSPI_SHADER_TMA_HI_PS 0x2c03 +#define mmSPI_SHADER_PGM_LO_PS 0x2c08 +#define mmSPI_SHADER_PGM_HI_PS 0x2c09 +#define mmSPI_SHADER_PGM_RSRC1_PS 0x2c0a +#define mmSPI_SHADER_PGM_RSRC2_PS 0x2c0b +#define mmSPI_SHADER_PGM_RSRC3_PS 0x2c07 +#define mmSPI_SHADER_USER_DATA_PS_0 0x2c0c +#define mmSPI_SHADER_USER_DATA_PS_1 0x2c0d +#define mmSPI_SHADER_USER_DATA_PS_2 0x2c0e +#define mmSPI_SHADER_USER_DATA_PS_3 0x2c0f +#define mmSPI_SHADER_USER_DATA_PS_4 0x2c10 +#define mmSPI_SHADER_USER_DATA_PS_5 0x2c11 +#define mmSPI_SHADER_USER_DATA_PS_6 0x2c12 +#define mmSPI_SHADER_USER_DATA_PS_7 0x2c13 +#define mmSPI_SHADER_USER_DATA_PS_8 0x2c14 +#define mmSPI_SHADER_USER_DATA_PS_9 0x2c15 +#define mmSPI_SHADER_USER_DATA_PS_10 0x2c16 +#define mmSPI_SHADER_USER_DATA_PS_11 0x2c17 +#define mmSPI_SHADER_USER_DATA_PS_12 0x2c18 +#define mmSPI_SHADER_USER_DATA_PS_13 0x2c19 +#define mmSPI_SHADER_USER_DATA_PS_14 0x2c1a +#define mmSPI_SHADER_USER_DATA_PS_15 0x2c1b +#define mmSPI_SHADER_TBA_LO_VS 0x2c40 +#define mmSPI_SHADER_TBA_HI_VS 0x2c41 +#define mmSPI_SHADER_TMA_LO_VS 0x2c42 +#define mmSPI_SHADER_TMA_HI_VS 0x2c43 +#define mmSPI_SHADER_PGM_LO_VS 0x2c48 +#define mmSPI_SHADER_PGM_HI_VS 0x2c49 +#define mmSPI_SHADER_PGM_RSRC1_VS 0x2c4a +#define mmSPI_SHADER_PGM_RSRC2_VS 0x2c4b +#define mmSPI_SHADER_PGM_RSRC3_VS 0x2c46 +#define mmSPI_SHADER_LATE_ALLOC_VS 0x2c47 +#define mmSPI_SHADER_USER_DATA_VS_0 0x2c4c +#define mmSPI_SHADER_USER_DATA_VS_1 0x2c4d +#define mmSPI_SHADER_USER_DATA_VS_2 0x2c4e +#define mmSPI_SHADER_USER_DATA_VS_3 0x2c4f +#define mmSPI_SHADER_USER_DATA_VS_4 0x2c50 +#define mmSPI_SHADER_USER_DATA_VS_5 0x2c51 +#define mmSPI_SHADER_USER_DATA_VS_6 0x2c52 +#define mmSPI_SHADER_USER_DATA_VS_7 0x2c53 +#define mmSPI_SHADER_USER_DATA_VS_8 0x2c54 +#define mmSPI_SHADER_USER_DATA_VS_9 0x2c55 +#define mmSPI_SHADER_USER_DATA_VS_10 0x2c56 +#define mmSPI_SHADER_USER_DATA_VS_11 0x2c57 +#define mmSPI_SHADER_USER_DATA_VS_12 0x2c58 +#define mmSPI_SHADER_USER_DATA_VS_13 0x2c59 +#define mmSPI_SHADER_USER_DATA_VS_14 0x2c5a +#define mmSPI_SHADER_USER_DATA_VS_15 0x2c5b +#define mmSPI_SHADER_PGM_RSRC2_ES_VS 0x2c7c +#define mmSPI_SHADER_PGM_RSRC2_LS_VS 0x2c7d +#define mmSPI_SHADER_TBA_LO_GS 0x2c80 +#define mmSPI_SHADER_TBA_HI_GS 0x2c81 +#define mmSPI_SHADER_TMA_LO_GS 0x2c82 +#define mmSPI_SHADER_TMA_HI_GS 0x2c83 +#define mmSPI_SHADER_PGM_LO_GS 0x2c88 +#define mmSPI_SHADER_PGM_HI_GS 0x2c89 +#define mmSPI_SHADER_PGM_RSRC1_GS 0x2c8a +#define mmSPI_SHADER_PGM_RSRC2_GS 0x2c8b +#define mmSPI_SHADER_PGM_RSRC3_GS 0x2c87 +#define mmSPI_SHADER_USER_DATA_GS_0 0x2c8c +#define mmSPI_SHADER_USER_DATA_GS_1 0x2c8d +#define mmSPI_SHADER_USER_DATA_GS_2 0x2c8e +#define mmSPI_SHADER_USER_DATA_GS_3 0x2c8f +#define mmSPI_SHADER_USER_DATA_GS_4 0x2c90 +#define mmSPI_SHADER_USER_DATA_GS_5 0x2c91 +#define mmSPI_SHADER_USER_DATA_GS_6 0x2c92 +#define mmSPI_SHADER_USER_DATA_GS_7 0x2c93 +#define mmSPI_SHADER_USER_DATA_GS_8 0x2c94 +#define mmSPI_SHADER_USER_DATA_GS_9 0x2c95 +#define mmSPI_SHADER_USER_DATA_GS_10 0x2c96 +#define mmSPI_SHADER_USER_DATA_GS_11 0x2c97 +#define mmSPI_SHADER_USER_DATA_GS_12 0x2c98 +#define mmSPI_SHADER_USER_DATA_GS_13 0x2c99 +#define mmSPI_SHADER_USER_DATA_GS_14 0x2c9a +#define mmSPI_SHADER_USER_DATA_GS_15 0x2c9b +#define mmSPI_SHADER_PGM_RSRC2_ES_GS 0x2cbc +#define mmSPI_SHADER_TBA_LO_ES 0x2cc0 +#define mmSPI_SHADER_TBA_HI_ES 0x2cc1 +#define mmSPI_SHADER_TMA_LO_ES 0x2cc2 +#define mmSPI_SHADER_TMA_HI_ES 0x2cc3 +#define mmSPI_SHADER_PGM_LO_ES 0x2cc8 +#define mmSPI_SHADER_PGM_HI_ES 0x2cc9 +#define mmSPI_SHADER_PGM_RSRC1_ES 0x2cca +#define mmSPI_SHADER_PGM_RSRC2_ES 0x2ccb +#define mmSPI_SHADER_PGM_RSRC3_ES 0x2cc7 +#define mmSPI_SHADER_USER_DATA_ES_0 0x2ccc +#define mmSPI_SHADER_USER_DATA_ES_1 0x2ccd +#define mmSPI_SHADER_USER_DATA_ES_2 0x2cce +#define mmSPI_SHADER_USER_DATA_ES_3 0x2ccf +#define mmSPI_SHADER_USER_DATA_ES_4 0x2cd0 +#define mmSPI_SHADER_USER_DATA_ES_5 0x2cd1 +#define mmSPI_SHADER_USER_DATA_ES_6 0x2cd2 +#define mmSPI_SHADER_USER_DATA_ES_7 0x2cd3 +#define mmSPI_SHADER_USER_DATA_ES_8 0x2cd4 +#define mmSPI_SHADER_USER_DATA_ES_9 0x2cd5 +#define mmSPI_SHADER_USER_DATA_ES_10 0x2cd6 +#define mmSPI_SHADER_USER_DATA_ES_11 0x2cd7 +#define mmSPI_SHADER_USER_DATA_ES_12 0x2cd8 +#define mmSPI_SHADER_USER_DATA_ES_13 0x2cd9 +#define mmSPI_SHADER_USER_DATA_ES_14 0x2cda +#define mmSPI_SHADER_USER_DATA_ES_15 0x2cdb +#define mmSPI_SHADER_PGM_RSRC2_LS_ES 0x2cfd +#define mmSPI_SHADER_TBA_LO_HS 0x2d00 +#define mmSPI_SHADER_TBA_HI_HS 0x2d01 +#define mmSPI_SHADER_TMA_LO_HS 0x2d02 +#define mmSPI_SHADER_TMA_HI_HS 0x2d03 +#define mmSPI_SHADER_PGM_LO_HS 0x2d08 +#define mmSPI_SHADER_PGM_HI_HS 0x2d09 +#define mmSPI_SHADER_PGM_RSRC1_HS 0x2d0a +#define mmSPI_SHADER_PGM_RSRC2_HS 0x2d0b +#define mmSPI_SHADER_PGM_RSRC3_HS 0x2d07 +#define mmSPI_SHADER_USER_DATA_HS_0 0x2d0c +#define mmSPI_SHADER_USER_DATA_HS_1 0x2d0d +#define mmSPI_SHADER_USER_DATA_HS_2 0x2d0e +#define mmSPI_SHADER_USER_DATA_HS_3 0x2d0f +#define mmSPI_SHADER_USER_DATA_HS_4 0x2d10 +#define mmSPI_SHADER_USER_DATA_HS_5 0x2d11 +#define mmSPI_SHADER_USER_DATA_HS_6 0x2d12 +#define mmSPI_SHADER_USER_DATA_HS_7 0x2d13 +#define mmSPI_SHADER_USER_DATA_HS_8 0x2d14 +#define mmSPI_SHADER_USER_DATA_HS_9 0x2d15 +#define mmSPI_SHADER_USER_DATA_HS_10 0x2d16 +#define mmSPI_SHADER_USER_DATA_HS_11 0x2d17 +#define mmSPI_SHADER_USER_DATA_HS_12 0x2d18 +#define mmSPI_SHADER_USER_DATA_HS_13 0x2d19 +#define mmSPI_SHADER_USER_DATA_HS_14 0x2d1a +#define mmSPI_SHADER_USER_DATA_HS_15 0x2d1b +#define mmSPI_SHADER_PGM_RSRC2_LS_HS 0x2d3d +#define mmSPI_SHADER_TBA_LO_LS 0x2d40 +#define mmSPI_SHADER_TBA_HI_LS 0x2d41 +#define mmSPI_SHADER_TMA_LO_LS 0x2d42 +#define mmSPI_SHADER_TMA_HI_LS 0x2d43 +#define mmSPI_SHADER_PGM_LO_LS 0x2d48 +#define mmSPI_SHADER_PGM_HI_LS 0x2d49 +#define mmSPI_SHADER_PGM_RSRC1_LS 0x2d4a +#define mmSPI_SHADER_PGM_RSRC2_LS 0x2d4b +#define mmSPI_SHADER_PGM_RSRC3_LS 0x2d47 +#define mmSPI_SHADER_USER_DATA_LS_0 0x2d4c +#define mmSPI_SHADER_USER_DATA_LS_1 0x2d4d +#define mmSPI_SHADER_USER_DATA_LS_2 0x2d4e +#define mmSPI_SHADER_USER_DATA_LS_3 0x2d4f +#define mmSPI_SHADER_USER_DATA_LS_4 0x2d50 +#define mmSPI_SHADER_USER_DATA_LS_5 0x2d51 +#define mmSPI_SHADER_USER_DATA_LS_6 0x2d52 +#define mmSPI_SHADER_USER_DATA_LS_7 0x2d53 +#define mmSPI_SHADER_USER_DATA_LS_8 0x2d54 +#define mmSPI_SHADER_USER_DATA_LS_9 0x2d55 +#define mmSPI_SHADER_USER_DATA_LS_10 0x2d56 +#define mmSPI_SHADER_USER_DATA_LS_11 0x2d57 +#define mmSPI_SHADER_USER_DATA_LS_12 0x2d58 +#define mmSPI_SHADER_USER_DATA_LS_13 0x2d59 +#define mmSPI_SHADER_USER_DATA_LS_14 0x2d5a +#define mmSPI_SHADER_USER_DATA_LS_15 0x2d5b +#define mmSQ_CONFIG 0x2300 +#define mmSQC_CONFIG 0x2301 +#define mmSQC_CACHES 0xc348 +#define mmSQ_RANDOM_WAVE_PRI 0x2303 +#define mmSQ_REG_CREDITS 0x2304 +#define mmSQ_FIFO_SIZES 0x2305 +#define mmSQ_INTERRUPT_AUTO_MASK 0x2314 +#define mmSQ_INTERRUPT_MSG_CTRL 0x2315 +#define mmSQ_PERFCOUNTER_CTRL 0xd9e0 +#define mmSQ_PERFCOUNTER_MASK 0xd9e1 +#define mmSQ_PERFCOUNTER_CTRL2 0xd9e2 +#define mmCC_SQC_BANK_DISABLE 0x2307 +#define mmUSER_SQC_BANK_DISABLE 0x2308 +#define mmSQ_PERFCOUNTER0_LO 0xd1c0 +#define mmSQ_PERFCOUNTER1_LO 0xd1c2 +#define mmSQ_PERFCOUNTER2_LO 0xd1c4 +#define mmSQ_PERFCOUNTER3_LO 0xd1c6 +#define mmSQ_PERFCOUNTER4_LO 0xd1c8 +#define mmSQ_PERFCOUNTER5_LO 0xd1ca +#define mmSQ_PERFCOUNTER6_LO 0xd1cc +#define mmSQ_PERFCOUNTER7_LO 0xd1ce +#define mmSQ_PERFCOUNTER8_LO 0xd1d0 +#define mmSQ_PERFCOUNTER9_LO 0xd1d2 +#define mmSQ_PERFCOUNTER10_LO 0xd1d4 +#define mmSQ_PERFCOUNTER11_LO 0xd1d6 +#define mmSQ_PERFCOUNTER12_LO 0xd1d8 +#define mmSQ_PERFCOUNTER13_LO 0xd1da +#define mmSQ_PERFCOUNTER14_LO 0xd1dc +#define mmSQ_PERFCOUNTER15_LO 0xd1de +#define mmSQ_PERFCOUNTER0_HI 0xd1c1 +#define mmSQ_PERFCOUNTER1_HI 0xd1c3 +#define mmSQ_PERFCOUNTER2_HI 0xd1c5 +#define mmSQ_PERFCOUNTER3_HI 0xd1c7 +#define mmSQ_PERFCOUNTER4_HI 0xd1c9 +#define mmSQ_PERFCOUNTER5_HI 0xd1cb +#define mmSQ_PERFCOUNTER6_HI 0xd1cd +#define mmSQ_PERFCOUNTER7_HI 0xd1cf +#define mmSQ_PERFCOUNTER8_HI 0xd1d1 +#define mmSQ_PERFCOUNTER9_HI 0xd1d3 +#define mmSQ_PERFCOUNTER10_HI 0xd1d5 +#define mmSQ_PERFCOUNTER11_HI 0xd1d7 +#define mmSQ_PERFCOUNTER12_HI 0xd1d9 +#define mmSQ_PERFCOUNTER13_HI 0xd1db +#define mmSQ_PERFCOUNTER14_HI 0xd1dd +#define mmSQ_PERFCOUNTER15_HI 0xd1df +#define mmSQ_PERFCOUNTER0_SELECT 0xd9c0 +#define mmSQ_PERFCOUNTER1_SELECT 0xd9c1 +#define mmSQ_PERFCOUNTER2_SELECT 0xd9c2 +#define mmSQ_PERFCOUNTER3_SELECT 0xd9c3 +#define mmSQ_PERFCOUNTER4_SELECT 0xd9c4 +#define mmSQ_PERFCOUNTER5_SELECT 0xd9c5 +#define mmSQ_PERFCOUNTER6_SELECT 0xd9c6 +#define mmSQ_PERFCOUNTER7_SELECT 0xd9c7 +#define mmSQ_PERFCOUNTER8_SELECT 0xd9c8 +#define mmSQ_PERFCOUNTER9_SELECT 0xd9c9 +#define mmSQ_PERFCOUNTER10_SELECT 0xd9ca +#define mmSQ_PERFCOUNTER11_SELECT 0xd9cb +#define mmSQ_PERFCOUNTER12_SELECT 0xd9cc +#define mmSQ_PERFCOUNTER13_SELECT 0xd9cd +#define mmSQ_PERFCOUNTER14_SELECT 0xd9ce +#define mmSQ_PERFCOUNTER15_SELECT 0xd9cf +#define mmCGTT_SQ_CLK_CTRL 0xf08c +#define mmCGTT_SQG_CLK_CTRL 0xf08d +#define mmSQ_ALU_CLK_CTRL 0xf08e +#define mmSQ_TEX_CLK_CTRL 0xf08f +#define mmSQ_LDS_CLK_CTRL 0xf090 +#define mmSQ_POWER_THROTTLE 0xf091 +#define mmSQ_POWER_THROTTLE2 0xf092 +#define mmSQ_TIME_HI 0x237c +#define mmSQ_TIME_LO 0x237d +#define mmSQ_THREAD_TRACE_BASE 0x2380 +#define mmSQ_THREAD_TRACE_BASE2 0x2385 +#define mmSQ_THREAD_TRACE_SIZE 0x2381 +#define mmSQ_THREAD_TRACE_MASK 0x2382 +#define mmSQ_THREAD_TRACE_USERDATA_0 0xc340 +#define mmSQ_THREAD_TRACE_USERDATA_1 0xc341 +#define mmSQ_THREAD_TRACE_USERDATA_2 0xc342 +#define mmSQ_THREAD_TRACE_USERDATA_3 0xc343 +#define mmSQ_THREAD_TRACE_MODE 0x238e +#define mmSQ_THREAD_TRACE_CTRL 0x238f +#define mmSQ_THREAD_TRACE_TOKEN_MASK 0x2383 +#define mmSQ_THREAD_TRACE_TOKEN_MASK2 0x2386 +#define mmSQ_THREAD_TRACE_PERF_MASK 0x2384 +#define mmSQ_THREAD_TRACE_WPTR 0x238c +#define mmSQ_THREAD_TRACE_STATUS 0x238d +#define mmSQ_THREAD_TRACE_CNTR 0x2390 +#define mmSQ_THREAD_TRACE_HIWATER 0x2392 +#define mmSQ_LB_CTR_CTRL 0x2398 +#define mmSQ_LB_DATA_ALU_CYCLES 0x2399 +#define mmSQ_LB_DATA_TEX_CYCLES 0x239a +#define mmSQ_LB_DATA_ALU_STALLS 0x239b +#define mmSQ_LB_DATA_TEX_STALLS 0x239c +#define mmSQC_SECDED_CNT 0x23a0 +#define mmSQ_SEC_CNT 0x23a1 +#define mmSQ_DED_CNT 0x23a2 +#define mmSQ_DED_INFO 0x23a3 +#define mmSQ_BUF_RSRC_WORD0 0x23c0 +#define mmSQ_BUF_RSRC_WORD1 0x23c1 +#define mmSQ_BUF_RSRC_WORD2 0x23c2 +#define mmSQ_BUF_RSRC_WORD3 0x23c3 +#define mmSQ_IMG_RSRC_WORD0 0x23c4 +#define mmSQ_IMG_RSRC_WORD1 0x23c5 +#define mmSQ_IMG_RSRC_WORD2 0x23c6 +#define mmSQ_IMG_RSRC_WORD3 0x23c7 +#define mmSQ_IMG_RSRC_WORD4 0x23c8 +#define mmSQ_IMG_RSRC_WORD5 0x23c9 +#define mmSQ_IMG_RSRC_WORD6 0x23ca +#define mmSQ_IMG_RSRC_WORD7 0x23cb +#define mmSQ_IMG_SAMP_WORD0 0x23cc +#define mmSQ_IMG_SAMP_WORD1 0x23cd +#define mmSQ_IMG_SAMP_WORD2 0x23ce +#define mmSQ_IMG_SAMP_WORD3 0x23cf +#define mmSQ_FLAT_SCRATCH_WORD0 0x23d0 +#define mmSQ_FLAT_SCRATCH_WORD1 0x23d1 +#define mmSQ_IND_INDEX 0x2378 +#define mmSQ_IND_CMD 0x237a +#define mmSQ_CMD 0x237b +#define mmSQ_IND_DATA 0x2379 +#define mmSQ_REG_TIMESTAMP 0x2374 +#define mmSQ_CMD_TIMESTAMP 0x2375 +#define mmSQ_HV_VMID_CTRL 0xf840 +#define ixSQ_WAVE_INST_DW0 0x1a +#define ixSQ_WAVE_INST_DW1 0x1b +#define ixSQ_WAVE_PC_LO 0x18 +#define ixSQ_WAVE_PC_HI 0x19 +#define ixSQ_WAVE_IB_DBG0 0x1c +#define ixSQ_WAVE_EXEC_LO 0x27e +#define ixSQ_WAVE_EXEC_HI 0x27f +#define ixSQ_WAVE_STATUS 0x12 +#define ixSQ_WAVE_MODE 0x11 +#define ixSQ_WAVE_TRAPSTS 0x13 +#define ixSQ_WAVE_HW_ID 0x14 +#define ixSQ_WAVE_GPR_ALLOC 0x15 +#define ixSQ_WAVE_LDS_ALLOC 0x16 +#define ixSQ_WAVE_IB_STS 0x17 +#define ixSQ_WAVE_M0 0x27c +#define ixSQ_WAVE_TBA_LO 0x26c +#define ixSQ_WAVE_TBA_HI 0x26d +#define ixSQ_WAVE_TMA_LO 0x26e +#define ixSQ_WAVE_TMA_HI 0x26f +#define ixSQ_WAVE_TTMP0 0x270 +#define ixSQ_WAVE_TTMP1 0x271 +#define ixSQ_WAVE_TTMP2 0x272 +#define ixSQ_WAVE_TTMP3 0x273 +#define ixSQ_WAVE_TTMP4 0x274 +#define ixSQ_WAVE_TTMP5 0x275 +#define ixSQ_WAVE_TTMP6 0x276 +#define ixSQ_WAVE_TTMP7 0x277 +#define ixSQ_WAVE_TTMP8 0x278 +#define ixSQ_WAVE_TTMP9 0x279 +#define ixSQ_WAVE_TTMP10 0x27a +#define ixSQ_WAVE_TTMP11 0x27b +#define mmSQ_DEBUG_STS_GLOBAL 0x2309 +#define mmSQ_DEBUG_STS_GLOBAL2 0x2310 +#define mmSQ_DEBUG_STS_GLOBAL3 0x2311 +#define ixSQ_DEBUG_STS_LOCAL 0x8 +#define ixSQ_DEBUG_CTRL_LOCAL 0x9 +#define mmSH_MEM_BASES 0x230a +#define mmSH_MEM_APE1_BASE 0x230b +#define mmSH_MEM_APE1_LIMIT 0x230c +#define mmSH_MEM_CONFIG 0x230d +#define mmSQC_POLICY 0x230e +#define mmSQC_VOLATILE 0x230f +#define mmSQ_THREAD_TRACE_WORD_CMN 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_INST 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_INST_PC_1_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_INST_PC_2_OF_2 0x23b1 +#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2 0x23b1 +#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2 0x23b1 +#define mmSQ_THREAD_TRACE_WORD_WAVE 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_MISC 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_WAVE_START 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_REG_1_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_REG_2_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_REG_CS_1_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_REG_CS_2_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_EVENT 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_ISSUE 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_PERF_1_OF_2 0x23b0 +#define mmSQ_THREAD_TRACE_WORD_PERF_2_OF_2 0x23b1 +#define ixSQ_INTERRUPT_WORD_CMN 0x20c0 +#define ixSQ_INTERRUPT_WORD_AUTO 0x20c0 +#define ixSQ_INTERRUPT_WORD_WAVE 0x20c0 +#define mmSQ_SOP2 0x237f +#define mmSQ_VOP1 0x237f +#define mmSQ_MTBUF_1 0x237f +#define mmSQ_EXP_1 0x237f +#define mmSQ_MUBUF_1 0x237f +#define mmSQ_INST 0x237f +#define mmSQ_EXP_0 0x237f +#define mmSQ_MUBUF_0 0x237f +#define mmSQ_VOP3_0 0x237f +#define mmSQ_VOP2 0x237f +#define mmSQ_MTBUF_0 0x237f +#define mmSQ_SOPP 0x237f +#define mmSQ_FLAT_0 0x237f +#define mmSQ_VOP3_0_SDST_ENC 0x237f +#define mmSQ_MIMG_1 0x237f +#define mmSQ_SMRD 0x237f +#define mmSQ_SOP1 0x237f +#define mmSQ_SOPC 0x237f +#define mmSQ_FLAT_1 0x237f +#define mmSQ_DS_1 0x237f +#define mmSQ_VOP3_1 0x237f +#define mmSQ_MIMG_0 0x237f +#define mmSQ_SOPK 0x237f +#define mmSQ_DS_0 0x237f +#define mmSQ_VOPC 0x237f +#define mmSQ_VINTRP 0x237f +#define mmCGTT_SX_CLK_CTRL0 0xf094 +#define mmCGTT_SX_CLK_CTRL1 0xf095 +#define mmCGTT_SX_CLK_CTRL2 0xf096 +#define mmCGTT_SX_CLK_CTRL3 0xf097 +#define mmCGTT_SX_CLK_CTRL4 0xf098 +#define mmSX_DEBUG_BUSY 0x2414 +#define mmSX_DEBUG_BUSY_2 0x2415 +#define mmSX_DEBUG_BUSY_3 0x2416 +#define mmSX_DEBUG_BUSY_4 0x2417 +#define mmSX_DEBUG_1 0x2418 +#define mmSX_PERFCOUNTER0_SELECT 0xda40 +#define mmSX_PERFCOUNTER1_SELECT 0xda41 +#define mmSX_PERFCOUNTER2_SELECT 0xda42 +#define mmSX_PERFCOUNTER3_SELECT 0xda43 +#define mmSX_PERFCOUNTER0_SELECT1 0xda44 +#define mmSX_PERFCOUNTER1_SELECT1 0xda45 +#define mmSX_PERFCOUNTER0_LO 0xd240 +#define mmSX_PERFCOUNTER0_HI 0xd241 +#define mmSX_PERFCOUNTER1_LO 0xd242 +#define mmSX_PERFCOUNTER1_HI 0xd243 +#define mmSX_PERFCOUNTER2_LO 0xd244 +#define mmSX_PERFCOUNTER2_HI 0xd245 +#define mmSX_PERFCOUNTER3_LO 0xd246 +#define mmSX_PERFCOUNTER3_HI 0xd247 +#define mmTCC_CTRL 0x2b80 +#define mmTCC_EDC_COUNTER 0x2b82 +#define mmTCC_REDUNDANCY 0x2b83 +#define mmTCC_CGTT_SCLK_CTRL 0xf0ac +#define mmTCA_CGTT_SCLK_CTRL 0xf0ad +#define mmTCS_CGTT_SCLK_CTRL 0xf0ae +#define mmTCC_PERFCOUNTER0_SELECT 0xdb80 +#define mmTCC_PERFCOUNTER1_SELECT 0xdb82 +#define mmTCC_PERFCOUNTER0_SELECT1 0xdb81 +#define mmTCC_PERFCOUNTER1_SELECT1 0xdb83 +#define mmTCC_PERFCOUNTER2_SELECT 0xdb84 +#define mmTCC_PERFCOUNTER3_SELECT 0xdb85 +#define mmTCC_PERFCOUNTER0_LO 0xd380 +#define mmTCC_PERFCOUNTER1_LO 0xd382 +#define mmTCC_PERFCOUNTER2_LO 0xd384 +#define mmTCC_PERFCOUNTER3_LO 0xd386 +#define mmTCC_PERFCOUNTER0_HI 0xd381 +#define mmTCC_PERFCOUNTER1_HI 0xd383 +#define mmTCC_PERFCOUNTER2_HI 0xd385 +#define mmTCC_PERFCOUNTER3_HI 0xd387 +#define mmTCA_CTRL 0x2bc0 +#define mmTCA_PERFCOUNTER0_SELECT 0xdb90 +#define mmTCA_PERFCOUNTER1_SELECT 0xdb92 +#define mmTCA_PERFCOUNTER0_SELECT1 0xdb91 +#define mmTCA_PERFCOUNTER1_SELECT1 0xdb93 +#define mmTCA_PERFCOUNTER2_SELECT 0xdb94 +#define mmTCA_PERFCOUNTER3_SELECT 0xdb95 +#define mmTCA_PERFCOUNTER0_LO 0xd390 +#define mmTCA_PERFCOUNTER1_LO 0xd392 +#define mmTCA_PERFCOUNTER2_LO 0xd394 +#define mmTCA_PERFCOUNTER3_LO 0xd396 +#define mmTCA_PERFCOUNTER0_HI 0xd391 +#define mmTCA_PERFCOUNTER1_HI 0xd393 +#define mmTCA_PERFCOUNTER2_HI 0xd395 +#define mmTCA_PERFCOUNTER3_HI 0xd397 +#define mmTCS_CTRL 0x2be0 +#define mmTCS_PERFCOUNTER0_SELECT 0xdba0 +#define mmTCS_PERFCOUNTER0_SELECT1 0xdba1 +#define mmTCS_PERFCOUNTER1_SELECT 0xdba2 +#define mmTCS_PERFCOUNTER2_SELECT 0xdba3 +#define mmTCS_PERFCOUNTER3_SELECT 0xdba4 +#define mmTCS_PERFCOUNTER0_LO 0xd3a0 +#define mmTCS_PERFCOUNTER1_LO 0xd3a2 +#define mmTCS_PERFCOUNTER2_LO 0xd3a4 +#define mmTCS_PERFCOUNTER3_LO 0xd3a6 +#define mmTCS_PERFCOUNTER0_HI 0xd3a1 +#define mmTCS_PERFCOUNTER1_HI 0xd3a3 +#define mmTCS_PERFCOUNTER2_HI 0xd3a5 +#define mmTCS_PERFCOUNTER3_HI 0xd3a7 +#define mmTA_BC_BASE_ADDR 0xa020 +#define mmTA_BC_BASE_ADDR_HI 0xa021 +#define mmTD_CNTL 0x2525 +#define mmTD_STATUS 0x2526 +#define mmTD_DEBUG_INDEX 0x2528 +#define mmTD_DEBUG_DATA 0x2529 +#define mmTD_PERFCOUNTER0_SELECT 0xdb00 +#define mmTD_PERFCOUNTER1_SELECT 0xdb02 +#define mmTD_PERFCOUNTER0_SELECT1 0xdb01 +#define mmTD_PERFCOUNTER0_LO 0xd300 +#define mmTD_PERFCOUNTER1_LO 0xd302 +#define mmTD_PERFCOUNTER0_HI 0xd301 +#define mmTD_PERFCOUNTER1_HI 0xd303 +#define mmTD_SCRATCH 0x2533 +#define mmTA_CNTL 0x2541 +#define mmTA_CNTL_AUX 0x2542 +#define mmTA_RESERVED_010C 0x2543 +#define mmTA_CS_BC_BASE_ADDR 0xc380 +#define mmTA_CS_BC_BASE_ADDR_HI 0xc381 +#define mmTA_STATUS 0x2548 +#define mmTA_DEBUG_INDEX 0x254c +#define mmTA_DEBUG_DATA 0x254d +#define mmTA_PERFCOUNTER0_SELECT 0xdac0 +#define mmTA_PERFCOUNTER1_SELECT 0xdac2 +#define mmTA_PERFCOUNTER0_SELECT1 0xdac1 +#define mmTA_PERFCOUNTER0_LO 0xd2c0 +#define mmTA_PERFCOUNTER1_LO 0xd2c2 +#define mmTA_PERFCOUNTER0_HI 0xd2c1 +#define mmTA_PERFCOUNTER1_HI 0xd2c3 +#define mmTA_SCRATCH 0x2564 +#define mmSH_HIDDEN_PRIVATE_BASE_VMID 0x2580 +#define mmSH_STATIC_MEM_CONFIG 0x2581 +#define mmTCP_INVALIDATE 0x2b00 +#define mmTCP_STATUS 0x2b01 +#define mmTCP_CNTL 0x2b02 +#define mmTCP_CHAN_STEER_LO 0x2b03 +#define mmTCP_CHAN_STEER_HI 0x2b04 +#define mmTCP_ADDR_CONFIG 0x2b05 +#define mmTCP_CREDIT 0x2b06 +#define mmTCP_PERFCOUNTER0_SELECT 0xdb40 +#define mmTCP_PERFCOUNTER1_SELECT 0xdb42 +#define mmTCP_PERFCOUNTER0_SELECT1 0xdb41 +#define mmTCP_PERFCOUNTER1_SELECT1 0xdb43 +#define mmTCP_PERFCOUNTER2_SELECT 0xdb44 +#define mmTCP_PERFCOUNTER3_SELECT 0xdb45 +#define mmTCP_PERFCOUNTER0_LO 0xd340 +#define mmTCP_PERFCOUNTER1_LO 0xd342 +#define mmTCP_PERFCOUNTER2_LO 0xd344 +#define mmTCP_PERFCOUNTER3_LO 0xd346 +#define mmTCP_PERFCOUNTER0_HI 0xd341 +#define mmTCP_PERFCOUNTER1_HI 0xd343 +#define mmTCP_PERFCOUNTER2_HI 0xd345 +#define mmTCP_PERFCOUNTER3_HI 0xd347 +#define mmTCP_BUFFER_ADDR_HASH_CNTL 0x2b16 +#define mmTCP_EDC_COUNTER 0x2b17 +#define mmTC_CFG_L1_LOAD_POLICY0 0x2b1a +#define mmTC_CFG_L1_LOAD_POLICY1 0x2b1b +#define mmTC_CFG_L1_STORE_POLICY 0x2b1c +#define mmTC_CFG_L2_LOAD_POLICY0 0x2b1d +#define mmTC_CFG_L2_LOAD_POLICY1 0x2b1e +#define mmTC_CFG_L2_STORE_POLICY0 0x2b1f +#define mmTC_CFG_L2_STORE_POLICY1 0x2b20 +#define mmTC_CFG_L2_ATOMIC_POLICY 0x2b21 +#define mmTC_CFG_L1_VOLATILE 0x2b22 +#define mmTC_CFG_L2_VOLATILE 0x2b23 +#define mmTCP_WATCH0_ADDR_H 0x32a0 +#define mmTCP_WATCH1_ADDR_H 0x32a3 +#define mmTCP_WATCH2_ADDR_H 0x32a6 +#define mmTCP_WATCH3_ADDR_H 0x32a9 +#define mmTCP_WATCH0_ADDR_L 0x32a1 +#define mmTCP_WATCH1_ADDR_L 0x32a4 +#define mmTCP_WATCH2_ADDR_L 0x32a7 +#define mmTCP_WATCH3_ADDR_L 0x32aa +#define mmTCP_WATCH0_CNTL 0x32a2 +#define mmTCP_WATCH1_CNTL 0x32a5 +#define mmTCP_WATCH2_CNTL 0x32a8 +#define mmTCP_WATCH3_CNTL 0x32ab +#define mmTD_CGTT_CTRL 0xf09c +#define mmTA_CGTT_CTRL 0xf09d +#define mmCGTT_TCP_CLK_CTRL 0xf09e +#define mmCGTT_TCI_CLK_CTRL 0xf09f +#define mmTCI_STATUS 0x2b61 +#define mmTCI_CNTL_1 0x2b62 +#define mmTCI_CNTL_2 0x2b63 +#define mmGDS_CONFIG 0x25c0 +#define mmGDS_CNTL_STATUS 0x25c1 +#define mmGDS_ENHANCE2 0x25c2 +#define mmGDS_PROTECTION_FAULT 0x25c3 +#define mmGDS_VM_PROTECTION_FAULT 0x25c4 +#define mmGDS_SECDED_CNT 0x25c5 +#define mmGDS_GRBM_SECDED_CNT 0x25c6 +#define mmGDS_OA_DED 0x25c7 +#define mmGDS_DEBUG_CNTL 0x25c8 +#define mmGDS_DEBUG_DATA 0x25c9 +#define mmCGTT_GDS_CLK_CTRL 0xf0a0 +#define mmGDS_RD_ADDR 0xc400 +#define mmGDS_RD_DATA 0xc401 +#define mmGDS_RD_BURST_ADDR 0xc402 +#define mmGDS_RD_BURST_COUNT 0xc403 +#define mmGDS_RD_BURST_DATA 0xc404 +#define mmGDS_WR_ADDR 0xc405 +#define mmGDS_WR_DATA 0xc406 +#define mmGDS_WR_BURST_ADDR 0xc407 +#define mmGDS_WR_BURST_DATA 0xc408 +#define mmGDS_WRITE_COMPLETE 0xc409 +#define mmGDS_ATOM_CNTL 0xc40a +#define mmGDS_ATOM_COMPLETE 0xc40b +#define mmGDS_ATOM_BASE 0xc40c +#define mmGDS_ATOM_SIZE 0xc40d +#define mmGDS_ATOM_OFFSET0 0xc40e +#define mmGDS_ATOM_OFFSET1 0xc40f +#define mmGDS_ATOM_DST 0xc410 +#define mmGDS_ATOM_OP 0xc411 +#define mmGDS_ATOM_SRC0 0xc412 +#define mmGDS_ATOM_SRC0_U 0xc413 +#define mmGDS_ATOM_SRC1 0xc414 +#define mmGDS_ATOM_SRC1_U 0xc415 +#define mmGDS_ATOM_READ0 0xc416 +#define mmGDS_ATOM_READ0_U 0xc417 +#define mmGDS_ATOM_READ1 0xc418 +#define mmGDS_ATOM_READ1_U 0xc419 +#define mmGDS_GWS_RESOURCE_CNTL 0xc41a +#define mmGDS_GWS_RESOURCE 0xc41b +#define mmGDS_GWS_RESOURCE_CNT 0xc41c +#define mmGDS_OA_CNTL 0xc41d +#define mmGDS_OA_COUNTER 0xc41e +#define mmGDS_OA_ADDRESS 0xc41f +#define mmGDS_OA_INCDEC 0xc420 +#define mmGDS_OA_RING_SIZE 0xc421 +#define ixGDS_DEBUG_REG0 0x0 +#define ixGDS_DEBUG_REG1 0x1 +#define ixGDS_DEBUG_REG2 0x2 +#define ixGDS_DEBUG_REG3 0x3 +#define ixGDS_DEBUG_REG4 0x4 +#define ixGDS_DEBUG_REG5 0x5 +#define ixGDS_DEBUG_REG6 0x6 +#define mmGDS_PERFCOUNTER0_SELECT 0xda80 +#define mmGDS_PERFCOUNTER1_SELECT 0xda81 +#define mmGDS_PERFCOUNTER2_SELECT 0xda82 +#define mmGDS_PERFCOUNTER3_SELECT 0xda83 +#define mmGDS_PERFCOUNTER0_LO 0xd280 +#define mmGDS_PERFCOUNTER1_LO 0xd282 +#define mmGDS_PERFCOUNTER2_LO 0xd284 +#define mmGDS_PERFCOUNTER3_LO 0xd286 +#define mmGDS_PERFCOUNTER0_HI 0xd281 +#define mmGDS_PERFCOUNTER1_HI 0xd283 +#define mmGDS_PERFCOUNTER2_HI 0xd285 +#define mmGDS_PERFCOUNTER3_HI 0xd287 +#define mmGDS_PERFCOUNTER0_SELECT1 0xda84 +#define mmGDS_VMID0_BASE 0x3300 +#define mmGDS_VMID1_BASE 0x3302 +#define mmGDS_VMID2_BASE 0x3304 +#define mmGDS_VMID3_BASE 0x3306 +#define mmGDS_VMID4_BASE 0x3308 +#define mmGDS_VMID5_BASE 0x330a +#define mmGDS_VMID6_BASE 0x330c +#define mmGDS_VMID7_BASE 0x330e +#define mmGDS_VMID8_BASE 0x3310 +#define mmGDS_VMID9_BASE 0x3312 +#define mmGDS_VMID10_BASE 0x3314 +#define mmGDS_VMID11_BASE 0x3316 +#define mmGDS_VMID12_BASE 0x3318 +#define mmGDS_VMID13_BASE 0x331a +#define mmGDS_VMID14_BASE 0x331c +#define mmGDS_VMID15_BASE 0x331e +#define mmGDS_VMID0_SIZE 0x3301 +#define mmGDS_VMID1_SIZE 0x3303 +#define mmGDS_VMID2_SIZE 0x3305 +#define mmGDS_VMID3_SIZE 0x3307 +#define mmGDS_VMID4_SIZE 0x3309 +#define mmGDS_VMID5_SIZE 0x330b +#define mmGDS_VMID6_SIZE 0x330d +#define mmGDS_VMID7_SIZE 0x330f +#define mmGDS_VMID8_SIZE 0x3311 +#define mmGDS_VMID9_SIZE 0x3313 +#define mmGDS_VMID10_SIZE 0x3315 +#define mmGDS_VMID11_SIZE 0x3317 +#define mmGDS_VMID12_SIZE 0x3319 +#define mmGDS_VMID13_SIZE 0x331b +#define mmGDS_VMID14_SIZE 0x331d +#define mmGDS_VMID15_SIZE 0x331f +#define mmGDS_GWS_VMID0 0x3320 +#define mmGDS_GWS_VMID1 0x3321 +#define mmGDS_GWS_VMID2 0x3322 +#define mmGDS_GWS_VMID3 0x3323 +#define mmGDS_GWS_VMID4 0x3324 +#define mmGDS_GWS_VMID5 0x3325 +#define mmGDS_GWS_VMID6 0x3326 +#define mmGDS_GWS_VMID7 0x3327 +#define mmGDS_GWS_VMID8 0x3328 +#define mmGDS_GWS_VMID9 0x3329 +#define mmGDS_GWS_VMID10 0x332a +#define mmGDS_GWS_VMID11 0x332b +#define mmGDS_GWS_VMID12 0x332c +#define mmGDS_GWS_VMID13 0x332d +#define mmGDS_GWS_VMID14 0x332e +#define mmGDS_GWS_VMID15 0x332f +#define mmGDS_OA_VMID0 0x3330 +#define mmGDS_OA_VMID1 0x3331 +#define mmGDS_OA_VMID2 0x3332 +#define mmGDS_OA_VMID3 0x3333 +#define mmGDS_OA_VMID4 0x3334 +#define mmGDS_OA_VMID5 0x3335 +#define mmGDS_OA_VMID6 0x3336 +#define mmGDS_OA_VMID7 0x3337 +#define mmGDS_OA_VMID8 0x3338 +#define mmGDS_OA_VMID9 0x3339 +#define mmGDS_OA_VMID10 0x333a +#define mmGDS_OA_VMID11 0x333b +#define mmGDS_OA_VMID12 0x333c +#define mmGDS_OA_VMID13 0x333d +#define mmGDS_OA_VMID14 0x333e +#define mmGDS_OA_VMID15 0x333f +#define mmGDS_GWS_RESET0 0x3344 +#define mmGDS_GWS_RESET1 0x3345 +#define mmGDS_GWS_RESOURCE_RESET 0x3346 +#define mmGDS_COMPUTE_MAX_WAVE_ID 0x3348 +#define mmGDS_OA_RESET_MASK 0x3349 +#define mmGDS_OA_RESET 0x334a +#define mmGDS_ENHANCE 0x334b +#define mmGDS_OA_CGPG_RESTORE 0x334c +#define mmCS_COPY_STATE 0xa1f3 +#define mmGFX_COPY_STATE 0xa1f4 +#define mmVGT_DRAW_INITIATOR 0xa1fc +#define mmVGT_EVENT_INITIATOR 0xa2a4 +#define mmVGT_EVENT_ADDRESS_REG 0xa1fe +#define mmVGT_DMA_BASE_HI 0xa1f9 +#define mmVGT_DMA_BASE 0xa1fa +#define mmVGT_DMA_INDEX_TYPE 0xa29f +#define mmVGT_DMA_NUM_INSTANCES 0xa2a2 +#define mmIA_ENHANCE 0xa29c +#define mmVGT_DMA_SIZE 0xa29d +#define mmVGT_DMA_MAX_SIZE 0xa29e +#define mmVGT_DMA_PRIMITIVE_TYPE 0x2271 +#define mmVGT_DMA_CONTROL 0x2272 +#define mmVGT_IMMED_DATA 0xa1fd +#define mmVGT_INDEX_TYPE 0xc243 +#define mmVGT_NUM_INDICES 0xc24c +#define mmVGT_NUM_INSTANCES 0xc24d +#define mmVGT_PRIMITIVE_TYPE 0xc242 +#define mmVGT_PRIMITIVEID_EN 0xa2a1 +#define mmVGT_PRIMITIVEID_RESET 0xa2a3 +#define mmVGT_VTX_CNT_EN 0xa2ae +#define mmVGT_REUSE_OFF 0xa2ad +#define mmVGT_INSTANCE_STEP_RATE_0 0xa2a8 +#define mmVGT_INSTANCE_STEP_RATE_1 0xa2a9 +#define mmVGT_MAX_VTX_INDX 0xa100 +#define mmVGT_MIN_VTX_INDX 0xa101 +#define mmVGT_INDX_OFFSET 0xa102 +#define mmVGT_VERTEX_REUSE_BLOCK_CNTL 0xa316 +#define mmVGT_OUT_DEALLOC_CNTL 0xa317 +#define mmVGT_MULTI_PRIM_IB_RESET_INDX 0xa103 +#define mmVGT_MULTI_PRIM_IB_RESET_EN 0xa2a5 +#define mmVGT_ENHANCE 0xa294 +#define mmVGT_OUTPUT_PATH_CNTL 0xa284 +#define mmVGT_HOS_CNTL 0xa285 +#define mmVGT_HOS_MAX_TESS_LEVEL 0xa286 +#define mmVGT_HOS_MIN_TESS_LEVEL 0xa287 +#define mmVGT_HOS_REUSE_DEPTH 0xa288 +#define mmVGT_GROUP_PRIM_TYPE 0xa289 +#define mmVGT_GROUP_FIRST_DECR 0xa28a +#define mmVGT_GROUP_DECR 0xa28b +#define mmVGT_GROUP_VECT_0_CNTL 0xa28c +#define mmVGT_GROUP_VECT_1_CNTL 0xa28d +#define mmVGT_GROUP_VECT_0_FMT_CNTL 0xa28e +#define mmVGT_GROUP_VECT_1_FMT_CNTL 0xa28f +#define mmVGT_VTX_VECT_EJECT_REG 0x222c +#define mmVGT_DMA_DATA_FIFO_DEPTH 0x222d +#define mmVGT_DMA_REQ_FIFO_DEPTH 0x222e +#define mmVGT_DRAW_INIT_FIFO_DEPTH 0x222f +#define mmVGT_LAST_COPY_STATE 0x2230 +#define mmCC_GC_SHADER_ARRAY_CONFIG 0x226f +#define mmGC_USER_SHADER_ARRAY_CONFIG 0x2270 +#define mmVGT_GS_MODE 0xa290 +#define mmVGT_GS_ONCHIP_CNTL 0xa291 +#define mmVGT_GS_OUT_PRIM_TYPE 0xa29b +#define mmVGT_CACHE_INVALIDATION 0x2231 +#define mmVGT_RESET_DEBUG 0x2232 +#define mmVGT_STRMOUT_DELAY 0x2233 +#define mmVGT_FIFO_DEPTHS 0x2234 +#define mmVGT_GS_PER_ES 0xa295 +#define mmVGT_ES_PER_GS 0xa296 +#define mmVGT_GS_PER_VS 0xa297 +#define mmVGT_GS_VERTEX_REUSE 0x2235 +#define mmVGT_MC_LAT_CNTL 0x2236 +#define mmIA_CNTL_STATUS 0x2237 +#define mmVGT_STRMOUT_CONFIG 0xa2e5 +#define mmVGT_STRMOUT_BUFFER_SIZE_0 0xa2b4 +#define mmVGT_STRMOUT_BUFFER_SIZE_1 0xa2b8 +#define mmVGT_STRMOUT_BUFFER_SIZE_2 0xa2bc +#define mmVGT_STRMOUT_BUFFER_SIZE_3 0xa2c0 +#define mmVGT_STRMOUT_BUFFER_OFFSET_0 0xa2b7 +#define mmVGT_STRMOUT_BUFFER_OFFSET_1 0xa2bb +#define mmVGT_STRMOUT_BUFFER_OFFSET_2 0xa2bf +#define mmVGT_STRMOUT_BUFFER_OFFSET_3 0xa2c3 +#define mmVGT_STRMOUT_VTX_STRIDE_0 0xa2b5 +#define mmVGT_STRMOUT_VTX_STRIDE_1 0xa2b9 +#define mmVGT_STRMOUT_VTX_STRIDE_2 0xa2bd +#define mmVGT_STRMOUT_VTX_STRIDE_3 0xa2c1 +#define mmVGT_STRMOUT_BUFFER_CONFIG 0xa2e6 +#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_0 0xc244 +#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_1 0xc245 +#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_2 0xc246 +#define mmVGT_STRMOUT_BUFFER_FILLED_SIZE_3 0xc247 +#define mmVGT_STRMOUT_DRAW_OPAQUE_OFFSET 0xa2ca +#define mmVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE 0xa2cb +#define mmVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE 0xa2cc +#define mmVGT_GS_MAX_VERT_OUT 0xa2ce +#define mmIA_VMID_OVERRIDE 0x2260 +#define mmVGT_SHADER_STAGES_EN 0xa2d5 +#define mmVGT_DISPATCH_DRAW_INDEX 0xa2dd +#define mmVGT_LS_HS_CONFIG 0xa2d6 +#define mmVGT_DMA_LS_HS_CONFIG 0x2273 +#define mmVGT_TF_PARAM 0xa2db +#define mmVGT_TF_RING_SIZE 0xc24e +#define mmVGT_SYS_CONFIG 0x2263 +#define mmVGT_HS_OFFCHIP_PARAM 0xc24f +#define mmVGT_TF_MEMORY_BASE 0xc250 +#define mmVGT_GS_INSTANCE_CNT 0xa2e4 +#define mmIA_MULTI_VGT_PARAM 0xa2aa +#define mmVGT_VS_MAX_WAVE_ID 0x2268 +#define mmVGT_ESGS_RING_SIZE 0xc240 +#define mmVGT_GSVS_RING_SIZE 0xc241 +#define mmVGT_GSVS_RING_OFFSET_1 0xa298 +#define mmVGT_GSVS_RING_OFFSET_2 0xa299 +#define mmVGT_GSVS_RING_OFFSET_3 0xa29a +#define mmVGT_ESGS_RING_ITEMSIZE 0xa2ab +#define mmVGT_GSVS_RING_ITEMSIZE 0xa2ac +#define mmVGT_GS_VERT_ITEMSIZE 0xa2d7 +#define mmVGT_GS_VERT_ITEMSIZE_1 0xa2d8 +#define mmVGT_GS_VERT_ITEMSIZE_2 0xa2d9 +#define mmVGT_GS_VERT_ITEMSIZE_3 0xa2da +#define mmWD_CNTL_STATUS 0x223f +#define mmWD_ENHANCE 0xa2a0 +#define mmGFX_PIPE_CONTROL 0x226d +#define mmGFX_PIPE_PRIORITY 0xf87f +#define mmCGTT_VGT_CLK_CTRL 0xf084 +#define mmCGTT_IA_CLK_CTRL 0xf085 +#define mmCGTT_WD_CLK_CTRL 0xf086 +#define mmVGT_DEBUG_CNTL 0x2238 +#define mmVGT_DEBUG_DATA 0x2239 +#define mmIA_DEBUG_CNTL 0x223a +#define mmIA_DEBUG_DATA 0x223b +#define mmVGT_CNTL_STATUS 0x223c +#define mmWD_DEBUG_CNTL 0x223d +#define mmWD_DEBUG_DATA 0x223e +#define mmCC_GC_PRIM_CONFIG 0x2240 +#define mmGC_USER_PRIM_CONFIG 0x2241 +#define ixWD_DEBUG_REG0 0x0 +#define ixWD_DEBUG_REG1 0x1 +#define ixWD_DEBUG_REG2 0x2 +#define ixWD_DEBUG_REG3 0x3 +#define ixWD_DEBUG_REG4 0x4 +#define ixWD_DEBUG_REG5 0x5 +#define ixIA_DEBUG_REG0 0x0 +#define ixIA_DEBUG_REG1 0x1 +#define ixIA_DEBUG_REG2 0x2 +#define ixIA_DEBUG_REG3 0x3 +#define ixIA_DEBUG_REG4 0x4 +#define ixIA_DEBUG_REG5 0x5 +#define ixIA_DEBUG_REG6 0x6 +#define ixIA_DEBUG_REG7 0x7 +#define ixIA_DEBUG_REG8 0x8 +#define ixIA_DEBUG_REG9 0x9 +#define ixVGT_DEBUG_REG0 0x0 +#define ixVGT_DEBUG_REG1 0x1 +#define ixVGT_DEBUG_REG2 0x1e +#define ixVGT_DEBUG_REG3 0x1f +#define ixVGT_DEBUG_REG4 0x20 +#define ixVGT_DEBUG_REG5 0x21 +#define ixVGT_DEBUG_REG6 0x22 +#define ixVGT_DEBUG_REG7 0x23 +#define ixVGT_DEBUG_REG8 0x8 +#define ixVGT_DEBUG_REG9 0x9 +#define ixVGT_DEBUG_REG10 0xa +#define ixVGT_DEBUG_REG11 0xb +#define ixVGT_DEBUG_REG12 0xc +#define ixVGT_DEBUG_REG13 0xd +#define ixVGT_DEBUG_REG14 0xe +#define ixVGT_DEBUG_REG15 0xf +#define ixVGT_DEBUG_REG16 0x10 +#define ixVGT_DEBUG_REG17 0x11 +#define ixVGT_DEBUG_REG18 0x7 +#define ixVGT_DEBUG_REG19 0x13 +#define ixVGT_DEBUG_REG20 0x14 +#define ixVGT_DEBUG_REG21 0x15 +#define ixVGT_DEBUG_REG22 0x16 +#define ixVGT_DEBUG_REG23 0x17 +#define ixVGT_DEBUG_REG24 0x18 +#define ixVGT_DEBUG_REG25 0x19 +#define ixVGT_DEBUG_REG26 0x24 +#define ixVGT_DEBUG_REG27 0x1b +#define ixVGT_DEBUG_REG28 0x1c +#define ixVGT_DEBUG_REG29 0x1d +#define ixVGT_DEBUG_REG30 0x25 +#define ixVGT_DEBUG_REG31 0x26 +#define ixVGT_DEBUG_REG32 0x27 +#define ixVGT_DEBUG_REG33 0x28 +#define ixVGT_DEBUG_REG34 0x29 +#define ixVGT_DEBUG_REG35 0x2a +#define mmVGT_PERFCOUNTER_SEID_MASK 0xd894 +#define mmVGT_PERFCOUNTER0_SELECT 0xd88c +#define mmVGT_PERFCOUNTER1_SELECT 0xd88d +#define mmVGT_PERFCOUNTER2_SELECT 0xd88e +#define mmVGT_PERFCOUNTER3_SELECT 0xd88f +#define mmVGT_PERFCOUNTER0_SELECT1 0xd890 +#define mmVGT_PERFCOUNTER1_SELECT1 0xd891 +#define mmVGT_PERFCOUNTER0_LO 0xd090 +#define mmVGT_PERFCOUNTER1_LO 0xd092 +#define mmVGT_PERFCOUNTER2_LO 0xd094 +#define mmVGT_PERFCOUNTER3_LO 0xd096 +#define mmVGT_PERFCOUNTER0_HI 0xd091 +#define mmVGT_PERFCOUNTER1_HI 0xd093 +#define mmVGT_PERFCOUNTER2_HI 0xd095 +#define mmVGT_PERFCOUNTER3_HI 0xd097 +#define mmIA_PERFCOUNTER0_SELECT 0xd884 +#define mmIA_PERFCOUNTER1_SELECT 0xd885 +#define mmIA_PERFCOUNTER2_SELECT 0xd886 +#define mmIA_PERFCOUNTER3_SELECT 0xd887 +#define mmIA_PERFCOUNTER0_SELECT1 0xd888 +#define mmIA_PERFCOUNTER0_LO 0xd088 +#define mmIA_PERFCOUNTER1_LO 0xd08a +#define mmIA_PERFCOUNTER2_LO 0xd08c +#define mmIA_PERFCOUNTER3_LO 0xd08e +#define mmIA_PERFCOUNTER0_HI 0xd089 +#define mmIA_PERFCOUNTER1_HI 0xd08b +#define mmIA_PERFCOUNTER2_HI 0xd08d +#define mmIA_PERFCOUNTER3_HI 0xd08f +#define mmWD_PERFCOUNTER0_SELECT 0xd880 +#define mmWD_PERFCOUNTER1_SELECT 0xd881 +#define mmWD_PERFCOUNTER2_SELECT 0xd882 +#define mmWD_PERFCOUNTER3_SELECT 0xd883 +#define mmWD_PERFCOUNTER0_LO 0xd080 +#define mmWD_PERFCOUNTER1_LO 0xd082 +#define mmWD_PERFCOUNTER2_LO 0xd084 +#define mmWD_PERFCOUNTER3_LO 0xd086 +#define mmWD_PERFCOUNTER0_HI 0xd081 +#define mmWD_PERFCOUNTER1_HI 0xd083 +#define mmWD_PERFCOUNTER2_HI 0xd085 +#define mmWD_PERFCOUNTER3_HI 0xd087 +#define mmDIDT_IND_INDEX 0x3280 +#define mmDIDT_IND_DATA 0x3281 +#define ixDIDT_SQ_CTRL0 0x0 +#define ixDIDT_SQ_CTRL1 0x1 +#define ixDIDT_SQ_CTRL2 0x2 +#define ixDIDT_SQ_WEIGHT0_3 0x10 +#define ixDIDT_SQ_WEIGHT4_7 0x11 +#define ixDIDT_SQ_WEIGHT8_11 0x12 +#define ixDIDT_DB_CTRL0 0x20 +#define ixDIDT_DB_CTRL1 0x21 +#define ixDIDT_DB_CTRL2 0x22 +#define ixDIDT_DB_WEIGHT0_3 0x30 +#define ixDIDT_DB_WEIGHT4_7 0x31 +#define ixDIDT_DB_WEIGHT8_11 0x32 +#define ixDIDT_TD_CTRL0 0x40 +#define ixDIDT_TD_CTRL1 0x41 +#define ixDIDT_TD_CTRL2 0x42 +#define ixDIDT_TD_WEIGHT0_3 0x50 +#define ixDIDT_TD_WEIGHT4_7 0x51 +#define ixDIDT_TD_WEIGHT8_11 0x52 +#define ixDIDT_TCP_CTRL0 0x60 +#define ixDIDT_TCP_CTRL1 0x61 +#define ixDIDT_TCP_CTRL2 0x62 +#define ixDIDT_TCP_WEIGHT0_3 0x70 +#define ixDIDT_TCP_WEIGHT4_7 0x71 +#define ixDIDT_TCP_WEIGHT8_11 0x72 + +#endif /* GFX_7_2_D_H */ diff --git a/tests/kfdtest/include/asic_reg/gfx_7_2_enum.h b/tests/kfdtest/include/asic_reg/gfx_7_2_enum.h new file mode 100644 index 0000000000..a0a698390c --- /dev/null +++ b/tests/kfdtest/include/asic_reg/gfx_7_2_enum.h @@ -0,0 +1,6280 @@ +/* + * Copyright (C) 2014 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef GFX_7_2_ENUM_H +#define GFX_7_2_ENUM_H + +typedef enum SurfaceNumber { + NUMBER_UNORM = 0x0, + NUMBER_SNORM = 0x1, + NUMBER_USCALED = 0x2, + NUMBER_SSCALED = 0x3, + NUMBER_UINT = 0x4, + NUMBER_SINT = 0x5, + NUMBER_SRGB = 0x6, + NUMBER_FLOAT = 0x7, +} SurfaceNumber; +typedef enum SurfaceSwap { + SWAP_STD = 0x0, + SWAP_ALT = 0x1, + SWAP_STD_REV = 0x2, + SWAP_ALT_REV = 0x3, +} SurfaceSwap; +typedef enum CBMode { + CB_DISABLE = 0x0, + CB_NORMAL = 0x1, + CB_ELIMINATE_FAST_CLEAR = 0x2, + CB_RESOLVE = 0x3, + CB_DECOMPRESS = 0x4, + CB_FMASK_DECOMPRESS = 0x5, +} CBMode; +typedef enum RoundMode { + ROUND_BY_HALF = 0x0, + ROUND_TRUNCATE = 0x1, +} RoundMode; +typedef enum SourceFormat { + EXPORT_4C_32BPC = 0x0, + EXPORT_4C_16BPC = 0x1, + EXPORT_2C_32BPC_GR = 0x2, + EXPORT_2C_32BPC_AR = 0x3, +} SourceFormat; +typedef enum BlendOp { + BLEND_ZERO = 0x0, + BLEND_ONE = 0x1, + BLEND_SRC_COLOR = 0x2, + BLEND_ONE_MINUS_SRC_COLOR = 0x3, + BLEND_SRC_ALPHA = 0x4, + BLEND_ONE_MINUS_SRC_ALPHA = 0x5, + BLEND_DST_ALPHA = 0x6, + BLEND_ONE_MINUS_DST_ALPHA = 0x7, + BLEND_DST_COLOR = 0x8, + BLEND_ONE_MINUS_DST_COLOR = 0x9, + BLEND_SRC_ALPHA_SATURATE = 0xa, + BLEND_BOTH_SRC_ALPHA = 0xb, + BLEND_BOTH_INV_SRC_ALPHA = 0xc, + BLEND_CONSTANT_COLOR = 0xd, + BLEND_ONE_MINUS_CONSTANT_COLOR = 0xe, + BLEND_SRC1_COLOR = 0xf, + BLEND_INV_SRC1_COLOR = 0x10, + BLEND_SRC1_ALPHA = 0x11, + BLEND_INV_SRC1_ALPHA = 0x12, + BLEND_CONSTANT_ALPHA = 0x13, + BLEND_ONE_MINUS_CONSTANT_ALPHA = 0x14, +} BlendOp; +typedef enum CombFunc { + COMB_DST_PLUS_SRC = 0x0, + COMB_SRC_MINUS_DST = 0x1, + COMB_MIN_DST_SRC = 0x2, + COMB_MAX_DST_SRC = 0x3, + COMB_DST_MINUS_SRC = 0x4, +} CombFunc; +typedef enum BlendOpt { + FORCE_OPT_AUTO = 0x0, + FORCE_OPT_DISABLE = 0x1, + FORCE_OPT_ENABLE_IF_SRC_A_0 = 0x2, + FORCE_OPT_ENABLE_IF_SRC_RGB_0 = 0x3, + FORCE_OPT_ENABLE_IF_SRC_ARGB_0 = 0x4, + FORCE_OPT_ENABLE_IF_SRC_A_1 = 0x5, + FORCE_OPT_ENABLE_IF_SRC_RGB_1 = 0x6, + FORCE_OPT_ENABLE_IF_SRC_ARGB_1 = 0x7, +} BlendOpt; +typedef enum CmaskCode { + CMASK_CLR00_F0 = 0x0, + CMASK_CLR00_F1 = 0x1, + CMASK_CLR00_F2 = 0x2, + CMASK_CLR00_FX = 0x3, + CMASK_CLR01_F0 = 0x4, + CMASK_CLR01_F1 = 0x5, + CMASK_CLR01_F2 = 0x6, + CMASK_CLR01_FX = 0x7, + CMASK_CLR10_F0 = 0x8, + CMASK_CLR10_F1 = 0x9, + CMASK_CLR10_F2 = 0xa, + CMASK_CLR10_FX = 0xb, + CMASK_CLR11_F0 = 0xc, + CMASK_CLR11_F1 = 0xd, + CMASK_CLR11_F2 = 0xe, + CMASK_CLR11_FX = 0xf, +} CmaskCode; +typedef enum CBPerfSel { + CB_PERF_SEL_NONE = 0x0, + CB_PERF_SEL_BUSY = 0x1, + CB_PERF_SEL_CORE_SCLK_VLD = 0x2, + CB_PERF_SEL_REG_SCLK0_VLD = 0x3, + CB_PERF_SEL_REG_SCLK1_VLD = 0x4, + CB_PERF_SEL_DRAWN_QUAD = 0x5, + CB_PERF_SEL_DRAWN_PIXEL = 0x6, + CB_PERF_SEL_DRAWN_QUAD_FRAGMENT = 0x7, + CB_PERF_SEL_DRAWN_TILE = 0x8, + CB_PERF_SEL_DB_CB_TILE_VALID_READY = 0x9, + CB_PERF_SEL_DB_CB_TILE_VALID_READYB = 0xa, + CB_PERF_SEL_DB_CB_TILE_VALIDB_READY = 0xb, + CB_PERF_SEL_DB_CB_TILE_VALIDB_READYB = 0xc, + CB_PERF_SEL_CM_FC_TILE_VALID_READY = 0xd, + CB_PERF_SEL_CM_FC_TILE_VALID_READYB = 0xe, + CB_PERF_SEL_CM_FC_TILE_VALIDB_READY = 0xf, + CB_PERF_SEL_CM_FC_TILE_VALIDB_READYB = 0x10, + CB_PERF_SEL_MERGE_TILE_ONLY_VALID_READY = 0x11, + CB_PERF_SEL_MERGE_TILE_ONLY_VALID_READYB = 0x12, + CB_PERF_SEL_DB_CB_LQUAD_VALID_READY = 0x13, + CB_PERF_SEL_DB_CB_LQUAD_VALID_READYB = 0x14, + CB_PERF_SEL_DB_CB_LQUAD_VALIDB_READY = 0x15, + CB_PERF_SEL_DB_CB_LQUAD_VALIDB_READYB = 0x16, + CB_PERF_SEL_LQUAD_NO_TILE = 0x17, + CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_R = 0x18, + CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_AR = 0x19, + CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_GR = 0x1a, + CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_32_ABGR = 0x1b, + CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_FP16_ABGR = 0x1c, + CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_SIGNED16_ABGR = 0x1d, + CB_PERF_SEL_LQUAD_FORMAT_IS_EXPORT_UNSIGNED16_ABGR= 0x1e, + CB_PERF_SEL_QUAD_KILLED_BY_EXTRA_PIXEL_EXPORT = 0x1f, + CB_PERF_SEL_QUAD_KILLED_BY_COLOR_INVALID = 0x20, + CB_PERF_SEL_QUAD_KILLED_BY_NULL_TARGET_SHADER_MASK= 0x21, + CB_PERF_SEL_QUAD_KILLED_BY_NULL_SAMPLE_MASK = 0x22, + CB_PERF_SEL_QUAD_KILLED_BY_DISCARD_PIXEL = 0x23, + CB_PERF_SEL_FC_CLEAR_QUAD_VALID_READY = 0x24, + CB_PERF_SEL_FC_CLEAR_QUAD_VALID_READYB = 0x25, + CB_PERF_SEL_FC_CLEAR_QUAD_VALIDB_READY = 0x26, + CB_PERF_SEL_FC_CLEAR_QUAD_VALIDB_READYB = 0x27, + CB_PERF_SEL_FOP_IN_VALID_READY = 0x28, + CB_PERF_SEL_FOP_IN_VALID_READYB = 0x29, + CB_PERF_SEL_FOP_IN_VALIDB_READY = 0x2a, + CB_PERF_SEL_FOP_IN_VALIDB_READYB = 0x2b, + CB_PERF_SEL_FC_CC_QUADFRAG_VALID_READY = 0x2c, + CB_PERF_SEL_FC_CC_QUADFRAG_VALID_READYB = 0x2d, + CB_PERF_SEL_FC_CC_QUADFRAG_VALIDB_READY = 0x2e, + CB_PERF_SEL_FC_CC_QUADFRAG_VALIDB_READYB = 0x2f, + CB_PERF_SEL_CC_IB_SR_FRAG_VALID_READY = 0x30, + CB_PERF_SEL_CC_IB_SR_FRAG_VALID_READYB = 0x31, + CB_PERF_SEL_CC_IB_SR_FRAG_VALIDB_READY = 0x32, + CB_PERF_SEL_CC_IB_SR_FRAG_VALIDB_READYB = 0x33, + CB_PERF_SEL_CC_IB_TB_FRAG_VALID_READY = 0x34, + CB_PERF_SEL_CC_IB_TB_FRAG_VALID_READYB = 0x35, + CB_PERF_SEL_CC_IB_TB_FRAG_VALIDB_READY = 0x36, + CB_PERF_SEL_CC_IB_TB_FRAG_VALIDB_READYB = 0x37, + CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALID_READY = 0x38, + CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALID_READYB = 0x39, + CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALIDB_READY = 0x3a, + CB_PERF_SEL_CC_RB_BC_EVENFRAG_VALIDB_READYB = 0x3b, + CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALID_READY = 0x3c, + CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALID_READYB = 0x3d, + CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALIDB_READY = 0x3e, + CB_PERF_SEL_CC_RB_BC_ODDFRAG_VALIDB_READYB = 0x3f, + CB_PERF_SEL_CC_BC_CS_FRAG_VALID = 0x40, + CB_PERF_SEL_CM_CACHE_HIT = 0x41, + CB_PERF_SEL_CM_CACHE_TAG_MISS = 0x42, + CB_PERF_SEL_CM_CACHE_SECTOR_MISS = 0x43, + CB_PERF_SEL_CM_CACHE_REEVICTION_STALL = 0x44, + CB_PERF_SEL_CM_CACHE_EVICT_NONZERO_INFLIGHT_STALL= 0x45, + CB_PERF_SEL_CM_CACHE_REPLACE_PENDING_EVICT_STALL = 0x46, + CB_PERF_SEL_CM_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL= 0x47, + CB_PERF_SEL_CM_CACHE_READ_OUTPUT_STALL = 0x48, + CB_PERF_SEL_CM_CACHE_WRITE_OUTPUT_STALL = 0x49, + CB_PERF_SEL_CM_CACHE_ACK_OUTPUT_STALL = 0x4a, + CB_PERF_SEL_CM_CACHE_STALL = 0x4b, + CB_PERF_SEL_CM_CACHE_FLUSH = 0x4c, + CB_PERF_SEL_CM_CACHE_TAGS_FLUSHED = 0x4d, + CB_PERF_SEL_CM_CACHE_SECTORS_FLUSHED = 0x4e, + CB_PERF_SEL_CM_CACHE_DIRTY_SECTORS_FLUSHED = 0x4f, + CB_PERF_SEL_FC_CACHE_HIT = 0x50, + CB_PERF_SEL_FC_CACHE_TAG_MISS = 0x51, + CB_PERF_SEL_FC_CACHE_SECTOR_MISS = 0x52, + CB_PERF_SEL_FC_CACHE_REEVICTION_STALL = 0x53, + CB_PERF_SEL_FC_CACHE_EVICT_NONZERO_INFLIGHT_STALL= 0x54, + CB_PERF_SEL_FC_CACHE_REPLACE_PENDING_EVICT_STALL = 0x55, + CB_PERF_SEL_FC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL= 0x56, + CB_PERF_SEL_FC_CACHE_READ_OUTPUT_STALL = 0x57, + CB_PERF_SEL_FC_CACHE_WRITE_OUTPUT_STALL = 0x58, + CB_PERF_SEL_FC_CACHE_ACK_OUTPUT_STALL = 0x59, + CB_PERF_SEL_FC_CACHE_STALL = 0x5a, + CB_PERF_SEL_FC_CACHE_FLUSH = 0x5b, + CB_PERF_SEL_FC_CACHE_TAGS_FLUSHED = 0x5c, + CB_PERF_SEL_FC_CACHE_SECTORS_FLUSHED = 0x5d, + CB_PERF_SEL_FC_CACHE_DIRTY_SECTORS_FLUSHED = 0x5e, + CB_PERF_SEL_CC_CACHE_HIT = 0x5f, + CB_PERF_SEL_CC_CACHE_TAG_MISS = 0x60, + CB_PERF_SEL_CC_CACHE_SECTOR_MISS = 0x61, + CB_PERF_SEL_CC_CACHE_REEVICTION_STALL = 0x62, + CB_PERF_SEL_CC_CACHE_EVICT_NONZERO_INFLIGHT_STALL= 0x63, + CB_PERF_SEL_CC_CACHE_REPLACE_PENDING_EVICT_STALL = 0x64, + CB_PERF_SEL_CC_CACHE_INFLIGHT_COUNTER_MAXIMUM_STALL= 0x65, + CB_PERF_SEL_CC_CACHE_READ_OUTPUT_STALL = 0x66, + CB_PERF_SEL_CC_CACHE_WRITE_OUTPUT_STALL = 0x67, + CB_PERF_SEL_CC_CACHE_ACK_OUTPUT_STALL = 0x68, + CB_PERF_SEL_CC_CACHE_STALL = 0x69, + CB_PERF_SEL_CC_CACHE_FLUSH = 0x6a, + CB_PERF_SEL_CC_CACHE_TAGS_FLUSHED = 0x6b, + CB_PERF_SEL_CC_CACHE_SECTORS_FLUSHED = 0x6c, + CB_PERF_SEL_CC_CACHE_DIRTY_SECTORS_FLUSHED = 0x6d, + CB_PERF_SEL_CC_CACHE_WA_TO_RMW_CONVERSION = 0x6e, + CB_PERF_SEL_CB_TAP_WRREQ_VALID_READY = 0x6f, + CB_PERF_SEL_CB_TAP_WRREQ_VALID_READYB = 0x70, + CB_PERF_SEL_CB_TAP_WRREQ_VALIDB_READY = 0x71, + CB_PERF_SEL_CB_TAP_WRREQ_VALIDB_READYB = 0x72, + CB_PERF_SEL_CM_MC_WRITE_REQUEST = 0x73, + CB_PERF_SEL_FC_MC_WRITE_REQUEST = 0x74, + CB_PERF_SEL_CC_MC_WRITE_REQUEST = 0x75, + CB_PERF_SEL_CM_MC_WRITE_REQUESTS_IN_FLIGHT = 0x76, + CB_PERF_SEL_FC_MC_WRITE_REQUESTS_IN_FLIGHT = 0x77, + CB_PERF_SEL_CC_MC_WRITE_REQUESTS_IN_FLIGHT = 0x78, + CB_PERF_SEL_CB_TAP_RDREQ_VALID_READY = 0x79, + CB_PERF_SEL_CB_TAP_RDREQ_VALID_READYB = 0x7a, + CB_PERF_SEL_CB_TAP_RDREQ_VALIDB_READY = 0x7b, + CB_PERF_SEL_CB_TAP_RDREQ_VALIDB_READYB = 0x7c, + CB_PERF_SEL_CM_MC_READ_REQUEST = 0x7d, + CB_PERF_SEL_FC_MC_READ_REQUEST = 0x7e, + CB_PERF_SEL_CC_MC_READ_REQUEST = 0x7f, + CB_PERF_SEL_CM_MC_READ_REQUESTS_IN_FLIGHT = 0x80, + CB_PERF_SEL_FC_MC_READ_REQUESTS_IN_FLIGHT = 0x81, + CB_PERF_SEL_CC_MC_READ_REQUESTS_IN_FLIGHT = 0x82, + CB_PERF_SEL_CM_TQ_FULL = 0x83, + CB_PERF_SEL_CM_TQ_FIFO_TILE_RESIDENCY_STALL = 0x84, + CB_PERF_SEL_FC_QUAD_RDLAT_FIFO_FULL = 0x85, + CB_PERF_SEL_FC_TILE_RDLAT_FIFO_FULL = 0x86, + CB_PERF_SEL_FC_RDLAT_FIFO_QUAD_RESIDENCY_STALL = 0x87, + CB_PERF_SEL_FOP_FMASK_RAW_STALL = 0x88, + CB_PERF_SEL_FOP_FMASK_BYPASS_STALL = 0x89, + CB_PERF_SEL_CC_SF_FULL = 0x8a, + CB_PERF_SEL_CC_RB_FULL = 0x8b, + CB_PERF_SEL_CC_EVENFIFO_QUAD_RESIDENCY_STALL = 0x8c, + CB_PERF_SEL_CC_ODDFIFO_QUAD_RESIDENCY_STALL = 0x8d, + CB_PERF_SEL_BLENDER_RAW_HAZARD_STALL = 0x8e, + CB_PERF_SEL_EVENT = 0x8f, + CB_PERF_SEL_EVENT_CACHE_FLUSH_TS = 0x90, + CB_PERF_SEL_EVENT_CONTEXT_DONE = 0x91, + CB_PERF_SEL_EVENT_CACHE_FLUSH = 0x92, + CB_PERF_SEL_EVENT_CACHE_FLUSH_AND_INV_TS_EVENT = 0x93, + CB_PERF_SEL_EVENT_CACHE_FLUSH_AND_INV_EVENT = 0x94, + CB_PERF_SEL_EVENT_FLUSH_AND_INV_CB_DATA_TS = 0x95, + CB_PERF_SEL_EVENT_FLUSH_AND_INV_CB_META = 0x96, + CB_PERF_SEL_CC_SURFACE_SYNC = 0x97, + CB_PERF_SEL_CMASK_READ_DATA_0xC = 0x98, + CB_PERF_SEL_CMASK_READ_DATA_0xD = 0x99, + CB_PERF_SEL_CMASK_READ_DATA_0xE = 0x9a, + CB_PERF_SEL_CMASK_READ_DATA_0xF = 0x9b, + CB_PERF_SEL_CMASK_WRITE_DATA_0xC = 0x9c, + CB_PERF_SEL_CMASK_WRITE_DATA_0xD = 0x9d, + CB_PERF_SEL_CMASK_WRITE_DATA_0xE = 0x9e, + CB_PERF_SEL_CMASK_WRITE_DATA_0xF = 0x9f, + CB_PERF_SEL_TWO_PROBE_QUAD_FRAGMENT = 0xa0, + CB_PERF_SEL_EXPORT_32_ABGR_QUAD_FRAGMENT = 0xa1, + CB_PERF_SEL_DUAL_SOURCE_COLOR_QUAD_FRAGMENT = 0xa2, + CB_PERF_SEL_QUAD_HAS_1_FRAGMENT_BEFORE_UPDATE = 0xa3, + CB_PERF_SEL_QUAD_HAS_2_FRAGMENTS_BEFORE_UPDATE = 0xa4, + CB_PERF_SEL_QUAD_HAS_3_FRAGMENTS_BEFORE_UPDATE = 0xa5, + CB_PERF_SEL_QUAD_HAS_4_FRAGMENTS_BEFORE_UPDATE = 0xa6, + CB_PERF_SEL_QUAD_HAS_5_FRAGMENTS_BEFORE_UPDATE = 0xa7, + CB_PERF_SEL_QUAD_HAS_6_FRAGMENTS_BEFORE_UPDATE = 0xa8, + CB_PERF_SEL_QUAD_HAS_7_FRAGMENTS_BEFORE_UPDATE = 0xa9, + CB_PERF_SEL_QUAD_HAS_8_FRAGMENTS_BEFORE_UPDATE = 0xaa, + CB_PERF_SEL_QUAD_HAS_1_FRAGMENT_AFTER_UPDATE = 0xab, + CB_PERF_SEL_QUAD_HAS_2_FRAGMENTS_AFTER_UPDATE = 0xac, + CB_PERF_SEL_QUAD_HAS_3_FRAGMENTS_AFTER_UPDATE = 0xad, + CB_PERF_SEL_QUAD_HAS_4_FRAGMENTS_AFTER_UPDATE = 0xae, + CB_PERF_SEL_QUAD_HAS_5_FRAGMENTS_AFTER_UPDATE = 0xaf, + CB_PERF_SEL_QUAD_HAS_6_FRAGMENTS_AFTER_UPDATE = 0xb0, + CB_PERF_SEL_QUAD_HAS_7_FRAGMENTS_AFTER_UPDATE = 0xb1, + CB_PERF_SEL_QUAD_HAS_8_FRAGMENTS_AFTER_UPDATE = 0xb2, + CB_PERF_SEL_QUAD_ADDED_1_FRAGMENT = 0xb3, + CB_PERF_SEL_QUAD_ADDED_2_FRAGMENTS = 0xb4, + CB_PERF_SEL_QUAD_ADDED_3_FRAGMENTS = 0xb5, + CB_PERF_SEL_QUAD_ADDED_4_FRAGMENTS = 0xb6, + CB_PERF_SEL_QUAD_ADDED_5_FRAGMENTS = 0xb7, + CB_PERF_SEL_QUAD_ADDED_6_FRAGMENTS = 0xb8, + CB_PERF_SEL_QUAD_ADDED_7_FRAGMENTS = 0xb9, + CB_PERF_SEL_QUAD_REMOVED_1_FRAGMENT = 0xba, + CB_PERF_SEL_QUAD_REMOVED_2_FRAGMENTS = 0xbb, + CB_PERF_SEL_QUAD_REMOVED_3_FRAGMENTS = 0xbc, + CB_PERF_SEL_QUAD_REMOVED_4_FRAGMENTS = 0xbd, + CB_PERF_SEL_QUAD_REMOVED_5_FRAGMENTS = 0xbe, + CB_PERF_SEL_QUAD_REMOVED_6_FRAGMENTS = 0xbf, + CB_PERF_SEL_QUAD_REMOVED_7_FRAGMENTS = 0xc0, + CB_PERF_SEL_QUAD_READS_FRAGMENT_0 = 0xc1, + CB_PERF_SEL_QUAD_READS_FRAGMENT_1 = 0xc2, + CB_PERF_SEL_QUAD_READS_FRAGMENT_2 = 0xc3, + CB_PERF_SEL_QUAD_READS_FRAGMENT_3 = 0xc4, + CB_PERF_SEL_QUAD_READS_FRAGMENT_4 = 0xc5, + CB_PERF_SEL_QUAD_READS_FRAGMENT_5 = 0xc6, + CB_PERF_SEL_QUAD_READS_FRAGMENT_6 = 0xc7, + CB_PERF_SEL_QUAD_READS_FRAGMENT_7 = 0xc8, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_0 = 0xc9, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_1 = 0xca, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_2 = 0xcb, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_3 = 0xcc, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_4 = 0xcd, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_5 = 0xce, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_6 = 0xcf, + CB_PERF_SEL_QUAD_WRITES_FRAGMENT_7 = 0xd0, + CB_PERF_SEL_QUAD_BLEND_OPT_DONT_READ_DST = 0xd1, + CB_PERF_SEL_QUAD_BLEND_OPT_BLEND_BYPASS = 0xd2, + CB_PERF_SEL_QUAD_BLEND_OPT_DISCARD_PIXELS = 0xd3, + CB_PERF_SEL_QUAD_DST_READ_COULD_HAVE_BEEN_OPTIMIZED= 0xd4, + CB_PERF_SEL_QUAD_BLENDING_COULD_HAVE_BEEN_BYPASSED= 0xd5, + CB_PERF_SEL_QUAD_COULD_HAVE_BEEN_DISCARDED = 0xd6, + CB_PERF_SEL_BLEND_OPT_PIXELS_RESULT_EQ_DEST = 0xd7, + CB_PERF_SEL_DRAWN_BUSY = 0xd8, + CB_PERF_SEL_TILE_TO_CMR_REGION_BUSY = 0xd9, + CB_PERF_SEL_CMR_TO_FCR_REGION_BUSY = 0xda, + CB_PERF_SEL_FCR_TO_CCR_REGION_BUSY = 0xdb, + CB_PERF_SEL_CCR_TO_CCW_REGION_BUSY = 0xdc, + CB_PERF_SEL_FC_PF_SLOW_MODE_QUAD_EMPTY_HALF_DROPPED= 0xdd, + CB_PERF_SEL_FC_SEQUENCER_CLEAR = 0xde, + CB_PERF_SEL_FC_SEQUENCER_ELIMINATE_FAST_CLEAR = 0xdf, + CB_PERF_SEL_FC_SEQUENCER_FMASK_DECOMPRESS = 0xe0, + CB_PERF_SEL_FC_SEQUENCER_FMASK_COMPRESSION_DISABLE= 0xe1, +} CBPerfSel; +typedef enum CBPerfOpFilterSel { + CB_PERF_OP_FILTER_SEL_WRITE_ONLY = 0x0, + CB_PERF_OP_FILTER_SEL_NEEDS_DESTINATION = 0x1, + CB_PERF_OP_FILTER_SEL_RESOLVE = 0x2, + CB_PERF_OP_FILTER_SEL_DECOMPRESS = 0x3, + CB_PERF_OP_FILTER_SEL_FMASK_DECOMPRESS = 0x4, + CB_PERF_OP_FILTER_SEL_ELIMINATE_FAST_CLEAR = 0x5, +} CBPerfOpFilterSel; +typedef enum CBPerfClearFilterSel { + CB_PERF_CLEAR_FILTER_SEL_NONCLEAR = 0x0, + CB_PERF_CLEAR_FILTER_SEL_CLEAR = 0x1, +} CBPerfClearFilterSel; +typedef enum CP_RING_ID { + RINGID0 = 0x0, + RINGID1 = 0x1, + RINGID2 = 0x2, + RINGID3 = 0x3, +} CP_RING_ID; +typedef enum CP_PIPE_ID { + PIPE_ID0 = 0x0, + PIPE_ID1 = 0x1, + PIPE_ID2 = 0x2, + PIPE_ID3 = 0x3, +} CP_PIPE_ID; +typedef enum CP_ME_ID { + ME_ID0 = 0x0, + ME_ID1 = 0x1, + ME_ID2 = 0x2, + ME_ID3 = 0x3, +} CP_ME_ID; +typedef enum SPM_PERFMON_STATE { + STRM_PERFMON_STATE_DISABLE_AND_RESET = 0x0, + STRM_PERFMON_STATE_START_COUNTING = 0x1, + STRM_PERFMON_STATE_STOP_COUNTING = 0x2, + STRM_PERFMON_STATE_RESERVED_3 = 0x3, + STRM_PERFMON_STATE_DISABLE_AND_RESET_PHANTOM = 0x4, + STRM_PERFMON_STATE_COUNT_AND_DUMP_PHANTOM = 0x5, +} SPM_PERFMON_STATE; +typedef enum CP_PERFMON_STATE { + CP_PERFMON_STATE_DISABLE_AND_RESET = 0x0, + CP_PERFMON_STATE_START_COUNTING = 0x1, + CP_PERFMON_STATE_STOP_COUNTING = 0x2, + CP_PERFMON_STATE_RESERVED_3 = 0x3, + CP_PERFMON_STATE_DISABLE_AND_RESET_PHANTOM = 0x4, + CP_PERFMON_STATE_COUNT_AND_DUMP_PHANTOM = 0x5, +} CP_PERFMON_STATE; +typedef enum CP_PERFMON_ENABLE_MODE { + CP_PERFMON_ENABLE_MODE_ALWAYS_COUNT = 0x0, + CP_PERFMON_ENABLE_MODE_RESERVED_1 = 0x1, + CP_PERFMON_ENABLE_MODE_COUNT_CONTEXT_TRUE = 0x2, + CP_PERFMON_ENABLE_MODE_COUNT_CONTEXT_FALSE = 0x3, +} CP_PERFMON_ENABLE_MODE; +typedef enum CPG_PERFCOUNT_SEL { + CPG_PERF_SEL_ALWAYS_COUNT = 0x0, + CPG_PERF_SEL_RBIU_FIFO_FULL = 0x1, + CPG_PERF_SEL_CSF_RTS_BUT_MIU_NOT_RTR = 0x2, + CPG_PERF_SEL_CSF_ST_BASE_SIZE_FIFO_FULL = 0x3, + CPG_PERF_SEL_CP_GRBM_DWORDS_SENT = 0x4, + CPG_PERF_SEL_ME_PARSER_BUSY = 0x5, + CPG_PERF_SEL_COUNT_TYPE0_PACKETS = 0x6, + CPG_PERF_SEL_COUNT_TYPE3_PACKETS = 0x7, + CPG_PERF_SEL_CSF_FETCHING_CMD_BUFFERS = 0x8, + CPG_PERF_SEL_CP_GRBM_OUT_OF_CREDITS = 0x9, + CPG_PERF_SEL_CP_PFP_GRBM_OUT_OF_CREDITS = 0xa, + CPG_PERF_SEL_CP_GDS_GRBM_OUT_OF_CREDITS = 0xb, + CPG_PERF_SEL_RCIU_STALLED_ON_ME_READ = 0xc, + CPG_PERF_SEL_RCIU_STALLED_ON_DMA_READ = 0xd, + CPG_PERF_SEL_SSU_STALLED_ON_ACTIVE_CNTX = 0xe, + CPG_PERF_SEL_SSU_STALLED_ON_CLEAN_SIGNALS = 0xf, + CPG_PERF_SEL_QU_STALLED_ON_EOP_DONE_PULSE = 0x10, + CPG_PERF_SEL_QU_STALLED_ON_EOP_DONE_WR_CONFIRM = 0x11, + CPG_PERF_SEL_PFP_STALLED_ON_CSF_READY = 0x12, + CPG_PERF_SEL_PFP_STALLED_ON_MEQ_READY = 0x13, + CPG_PERF_SEL_PFP_STALLED_ON_RCIU_READY = 0x14, + CPG_PERF_SEL_PFP_STALLED_FOR_DATA_FROM_ROQ = 0x15, + CPG_PERF_SEL_ME_STALLED_FOR_DATA_FROM_PFP = 0x16, + CPG_PERF_SEL_ME_STALLED_FOR_DATA_FROM_STQ = 0x17, + CPG_PERF_SEL_ME_STALLED_ON_NO_AVAIL_GFX_CNTX = 0x18, + CPG_PERF_SEL_ME_STALLED_WRITING_TO_RCIU = 0x19, + CPG_PERF_SEL_ME_STALLED_WRITING_CONSTANTS = 0x1a, + CPG_PERF_SEL_ME_STALLED_ON_PARTIAL_FLUSH = 0x1b, + CPG_PERF_SEL_ME_WAIT_ON_CE_COUNTER = 0x1c, + CPG_PERF_SEL_ME_WAIT_ON_AVAIL_BUFFER = 0x1d, + CPG_PERF_SEL_SEMAPHORE_BUSY_POLLING_FOR_PASS = 0x1e, + CPG_PERF_SEL_LOAD_STALLED_ON_SET_COHERENCY = 0x1f, + CPG_PERF_SEL_DYNAMIC_CLK_VALID = 0x20, + CPG_PERF_SEL_REGISTER_CLK_VALID = 0x21, + CPG_PERF_SEL_MIU_WRITE_REQUEST_SENT = 0x22, + CPG_PERF_SEL_MIU_READ_REQUEST_SENT = 0x23, + CPG_PERF_SEL_CE_STALL_RAM_DUMP = 0x24, + CPG_PERF_SEL_CE_STALL_RAM_WRITE = 0x25, + CPG_PERF_SEL_CE_STALL_ON_INC_FIFO = 0x26, + CPG_PERF_SEL_CE_STALL_ON_WR_RAM_FIFO = 0x27, + CPG_PERF_SEL_CE_STALL_ON_DATA_FROM_MIU = 0x28, + CPG_PERF_SEL_CE_STALL_ON_DATA_FROM_ROQ = 0x29, + CPG_PERF_SEL_CE_STALL_ON_CE_BUFFER_FLAG = 0x2a, + CPG_PERF_SEL_CE_STALL_ON_DE_COUNTER = 0x2b, + CPG_PERF_SEL_TCIU_STALL_WAIT_ON_FREE = 0x2c, + CPG_PERF_SEL_TCIU_STALL_WAIT_ON_TAGS = 0x2d, +} CPG_PERFCOUNT_SEL; +typedef enum CPF_PERFCOUNT_SEL { + CPF_PERF_SEL_ALWAYS_COUNT = 0x0, + CPF_PERF_SEL_MIU_STALLED_WAITING_RDREQ_FREE = 0x1, + CPF_PERF_SEL_TCIU_STALLED_WAITING_ON_FREE = 0x2, + CPF_PERF_SEL_TCIU_STALLED_WAITING_ON_TAGS = 0x3, + CPF_PERF_SEL_CSF_BUSY_FOR_FETCHING_RING = 0x4, + CPF_PERF_SEL_CSF_BUSY_FOR_FETCHING_IB1 = 0x5, + CPF_PERF_SEL_CSF_BUSY_FOR_FETCHING_IB2 = 0x6, + CPF_PERF_SEL_CSF_BUSY_FOR_FECTHINC_STATE = 0x7, + CPF_PERF_SEL_MIU_BUSY_FOR_OUTSTANDING_TAGS = 0x8, + CPF_PERF_SEL_CSF_RTS_MIU_NOT_RTR = 0x9, + CPF_PERF_SEL_CSF_STATE_FIFO_NOT_RTR = 0xa, + CPF_PERF_SEL_CSF_FETCHING_CMD_BUFFERS = 0xb, + CPF_PERF_SEL_GRBM_DWORDS_SENT = 0xc, + CPF_PERF_SEL_DYNAMIC_CLOCK_VALID = 0xd, + CPF_PERF_SEL_REGISTER_CLOCK_VALID = 0xe, + CPF_PERF_SEL_MIU_WRITE_REQUEST_SEND = 0xf, + CPF_PERF_SEL_MIU_READ_REQUEST_SEND = 0x10, +} CPF_PERFCOUNT_SEL; +typedef enum CPC_PERFCOUNT_SEL { + CPC_PERF_SEL_ALWAYS_COUNT = 0x0, + CPC_PERF_SEL_RCIU_STALL_WAIT_ON_FREE = 0x1, + CPC_PERF_SEL_RCIU_STALL_PRIV_VIOLATION = 0x2, + CPC_PERF_SEL_MIU_STALL_ON_RDREQ_FREE = 0x3, + CPC_PERF_SEL_MIU_STALL_ON_WRREQ_FREE = 0x4, + CPC_PERF_SEL_TCIU_STALL_WAIT_ON_FREE = 0x5, + CPC_PERF_SEL_ME1_STALL_WAIT_ON_RCIU_READY = 0x6, + CPC_PERF_SEL_ME1_STALL_WAIT_ON_RCIU_READY_PERF = 0x7, + CPC_PERF_SEL_ME1_STALL_WAIT_ON_RCIU_READ = 0x8, + CPC_PERF_SEL_ME1_STALL_WAIT_ON_MIU_READ = 0x9, + CPC_PERF_SEL_ME1_STALL_WAIT_ON_MIU_WRITE = 0xa, + CPC_PERF_SEL_ME1_STALL_ON_DATA_FROM_ROQ = 0xb, + CPC_PERF_SEL_ME1_STALL_ON_DATA_FROM_ROQ_PERF = 0xc, + CPC_PERF_SEL_ME1_BUSY_FOR_PACKET_DECODE = 0xd, + CPC_PERF_SEL_ME2_STALL_WAIT_ON_RCIU_READY = 0xe, + CPC_PERF_SEL_ME2_STALL_WAIT_ON_RCIU_READY_PERF = 0xf, + CPC_PERF_SEL_ME2_STALL_WAIT_ON_RCIU_READ = 0x10, + CPC_PERF_SEL_ME2_STALL_WAIT_ON_MIU_READ = 0x11, + CPC_PERF_SEL_ME2_STALL_WAIT_ON_MIU_WRITE = 0x12, + CPC_PERF_SEL_ME2_STALL_ON_DATA_FROM_ROQ = 0x13, + CPC_PERF_SEL_ME2_STALL_ON_DATA_FROM_ROQ_PERF = 0x14, + CPC_PERF_SEL_ME2_BUSY_FOR_PACKET_DECODE = 0x15, +} CPC_PERFCOUNT_SEL; +typedef enum CP_ALPHA_TAG_RAM_SEL { + CPG_TAG_RAM = 0x0, + CPC_TAG_RAM = 0x1, + CPF_TAG_RAM = 0x2, + RSV_TAG_RAM = 0x3, +} CP_ALPHA_TAG_RAM_SEL; +#define SEM_ECC_ERROR 0x0 +#define SEM_RESERVED 0x1 +#define SEM_FAILED 0x2 +#define SEM_PASSED 0x3 +#define IQ_QUEUE_SLEEP 0x0 +#define IQ_OFFLOAD_RETRY 0x1 +#define IQ_SCH_WAVE_MSG 0x2 +#define IQ_SEM_REARM 0x3 +#define IQ_DEQUEUE_RETRY 0x4 +#define IQ_INTR_TYPE_PQ 0x0 +#define IQ_INTR_TYPE_IB 0x1 +#define IQ_INTR_TYPE_MQD 0x2 +#define VMID_SZ 0x4 +#define CONFIG_SPACE_START 0x2000 +#define CONFIG_SPACE_END 0x9fff +#define CONFIG_SPACE1_START 0x2000 +#define CONFIG_SPACE1_END 0x2bff +#define CONFIG_SPACE2_START 0x3000 +#define CONFIG_SPACE2_END 0x9fff +#define UCONFIG_SPACE_START 0xc000 +#define UCONFIG_SPACE_END 0xffff +#define PERSISTENT_SPACE_START 0x2c00 +#define PERSISTENT_SPACE_END 0x2fff +#define CONTEXT_SPACE_START 0xa000 +#define CONTEXT_SPACE_END 0xbfff +typedef enum ForceControl { + FORCE_OFF = 0x0, + FORCE_ENABLE = 0x1, + FORCE_DISABLE = 0x2, + FORCE_RESERVED = 0x3, +} ForceControl; +typedef enum ZSamplePosition { + Z_SAMPLE_CENTER = 0x0, + Z_SAMPLE_CENTROID = 0x1, +} ZSamplePosition; +typedef enum ZOrder { + LATE_Z = 0x0, + EARLY_Z_THEN_LATE_Z = 0x1, + RE_Z = 0x2, + EARLY_Z_THEN_RE_Z = 0x3, +} ZOrder; +typedef enum ZpassControl { + ZPASS_DISABLE = 0x0, + ZPASS_SAMPLES = 0x1, + ZPASS_PIXELS = 0x2, +} ZpassControl; +typedef enum ZModeForce { + NO_FORCE = 0x0, + FORCE_EARLY_Z = 0x1, + FORCE_LATE_Z = 0x2, + FORCE_RE_Z = 0x3, +} ZModeForce; +typedef enum ZLimitSumm { + FORCE_SUMM_OFF = 0x0, + FORCE_SUMM_MINZ = 0x1, + FORCE_SUMM_MAXZ = 0x2, + FORCE_SUMM_BOTH = 0x3, +} ZLimitSumm; +typedef enum CompareFrag { + FRAG_NEVER = 0x0, + FRAG_LESS = 0x1, + FRAG_EQUAL = 0x2, + FRAG_LEQUAL = 0x3, + FRAG_GREATER = 0x4, + FRAG_NOTEQUAL = 0x5, + FRAG_GEQUAL = 0x6, + FRAG_ALWAYS = 0x7, +} CompareFrag; +typedef enum StencilOp { + STENCIL_KEEP = 0x0, + STENCIL_ZERO = 0x1, + STENCIL_ONES = 0x2, + STENCIL_REPLACE_TEST = 0x3, + STENCIL_REPLACE_OP = 0x4, + STENCIL_ADD_CLAMP = 0x5, + STENCIL_SUB_CLAMP = 0x6, + STENCIL_INVERT = 0x7, + STENCIL_ADD_WRAP = 0x8, + STENCIL_SUB_WRAP = 0x9, + STENCIL_AND = 0xa, + STENCIL_OR = 0xb, + STENCIL_XOR = 0xc, + STENCIL_NAND = 0xd, + STENCIL_NOR = 0xe, + STENCIL_XNOR = 0xf, +} StencilOp; +typedef enum ConservativeZExport { + EXPORT_ANY_Z = 0x0, + EXPORT_LESS_THAN_Z = 0x1, + EXPORT_GREATER_THAN_Z = 0x2, + EXPORT_RESERVED = 0x3, +} ConservativeZExport; +typedef enum DbPSLControl { + PSLC_AUTO = 0x0, + PSLC_ON_HANG_ONLY = 0x1, + PSLC_ASAP = 0x2, + PSLC_COUNTDOWN = 0x3, +} DbPSLControl; +typedef enum PerfCounter_Vals { + DB_PERF_SEL_SC_DB_tile_sends = 0x0, + DB_PERF_SEL_SC_DB_tile_busy = 0x1, + DB_PERF_SEL_SC_DB_tile_stalls = 0x2, + DB_PERF_SEL_SC_DB_tile_events = 0x3, + DB_PERF_SEL_SC_DB_tile_tiles = 0x4, + DB_PERF_SEL_SC_DB_tile_covered = 0x5, + DB_PERF_SEL_hiz_tc_read_starved = 0x6, + DB_PERF_SEL_hiz_tc_write_stall = 0x7, + DB_PERF_SEL_hiz_qtiles_culled = 0x8, + DB_PERF_SEL_his_qtiles_culled = 0x9, + DB_PERF_SEL_DB_SC_tile_sends = 0xa, + DB_PERF_SEL_DB_SC_tile_busy = 0xb, + DB_PERF_SEL_DB_SC_tile_stalls = 0xc, + DB_PERF_SEL_DB_SC_tile_df_stalls = 0xd, + DB_PERF_SEL_DB_SC_tile_tiles = 0xe, + DB_PERF_SEL_DB_SC_tile_culled = 0xf, + DB_PERF_SEL_DB_SC_tile_hier_kill = 0x10, + DB_PERF_SEL_DB_SC_tile_fast_ops = 0x11, + DB_PERF_SEL_DB_SC_tile_no_ops = 0x12, + DB_PERF_SEL_DB_SC_tile_tile_rate = 0x13, + DB_PERF_SEL_DB_SC_tile_ssaa_kill = 0x14, + DB_PERF_SEL_DB_SC_tile_fast_z_ops = 0x15, + DB_PERF_SEL_DB_SC_tile_fast_stencil_ops = 0x16, + DB_PERF_SEL_SC_DB_quad_sends = 0x17, + DB_PERF_SEL_SC_DB_quad_busy = 0x18, + DB_PERF_SEL_SC_DB_quad_squads = 0x19, + DB_PERF_SEL_SC_DB_quad_tiles = 0x1a, + DB_PERF_SEL_SC_DB_quad_pixels = 0x1b, + DB_PERF_SEL_SC_DB_quad_killed_tiles = 0x1c, + DB_PERF_SEL_DB_SC_quad_sends = 0x1d, + DB_PERF_SEL_DB_SC_quad_busy = 0x1e, + DB_PERF_SEL_DB_SC_quad_stalls = 0x1f, + DB_PERF_SEL_DB_SC_quad_tiles = 0x20, + DB_PERF_SEL_DB_SC_quad_lit_quad = 0x21, + DB_PERF_SEL_DB_CB_tile_sends = 0x22, + DB_PERF_SEL_DB_CB_tile_busy = 0x23, + DB_PERF_SEL_DB_CB_tile_stalls = 0x24, + DB_PERF_SEL_SX_DB_quad_sends = 0x25, + DB_PERF_SEL_SX_DB_quad_busy = 0x26, + DB_PERF_SEL_SX_DB_quad_stalls = 0x27, + DB_PERF_SEL_SX_DB_quad_quads = 0x28, + DB_PERF_SEL_SX_DB_quad_pixels = 0x29, + DB_PERF_SEL_SX_DB_quad_exports = 0x2a, + DB_PERF_SEL_SH_quads_outstanding_sum = 0x2b, + DB_PERF_SEL_DB_CB_lquad_sends = 0x2c, + DB_PERF_SEL_DB_CB_lquad_busy = 0x2d, + DB_PERF_SEL_DB_CB_lquad_stalls = 0x2e, + DB_PERF_SEL_DB_CB_lquad_quads = 0x2f, + DB_PERF_SEL_tile_rd_sends = 0x30, + DB_PERF_SEL_mi_tile_rd_outstanding_sum = 0x31, + DB_PERF_SEL_quad_rd_sends = 0x32, + DB_PERF_SEL_quad_rd_busy = 0x33, + DB_PERF_SEL_quad_rd_mi_stall = 0x34, + DB_PERF_SEL_quad_rd_rw_collision = 0x35, + DB_PERF_SEL_quad_rd_tag_stall = 0x36, + DB_PERF_SEL_quad_rd_32byte_reqs = 0x37, + DB_PERF_SEL_quad_rd_panic = 0x38, + DB_PERF_SEL_mi_quad_rd_outstanding_sum = 0x39, + DB_PERF_SEL_quad_rdret_sends = 0x3a, + DB_PERF_SEL_quad_rdret_busy = 0x3b, + DB_PERF_SEL_tile_wr_sends = 0x3c, + DB_PERF_SEL_tile_wr_acks = 0x3d, + DB_PERF_SEL_mi_tile_wr_outstanding_sum = 0x3e, + DB_PERF_SEL_quad_wr_sends = 0x3f, + DB_PERF_SEL_quad_wr_busy = 0x40, + DB_PERF_SEL_quad_wr_mi_stall = 0x41, + DB_PERF_SEL_quad_wr_coherency_stall = 0x42, + DB_PERF_SEL_quad_wr_acks = 0x43, + DB_PERF_SEL_mi_quad_wr_outstanding_sum = 0x44, + DB_PERF_SEL_Tile_Cache_misses = 0x45, + DB_PERF_SEL_Tile_Cache_hits = 0x46, + DB_PERF_SEL_Tile_Cache_flushes = 0x47, + DB_PERF_SEL_Tile_Cache_surface_stall = 0x48, + DB_PERF_SEL_Tile_Cache_starves = 0x49, + DB_PERF_SEL_Tile_Cache_mem_return_starve = 0x4a, + DB_PERF_SEL_tcp_dispatcher_reads = 0x4b, + DB_PERF_SEL_tcp_prefetcher_reads = 0x4c, + DB_PERF_SEL_tcp_preloader_reads = 0x4d, + DB_PERF_SEL_tcp_dispatcher_flushes = 0x4e, + DB_PERF_SEL_tcp_prefetcher_flushes = 0x4f, + DB_PERF_SEL_tcp_preloader_flushes = 0x50, + DB_PERF_SEL_Depth_Tile_Cache_sends = 0x51, + DB_PERF_SEL_Depth_Tile_Cache_busy = 0x52, + DB_PERF_SEL_Depth_Tile_Cache_starves = 0x53, + DB_PERF_SEL_Depth_Tile_Cache_dtile_locked = 0x54, + DB_PERF_SEL_Depth_Tile_Cache_alloc_stall = 0x55, + DB_PERF_SEL_Depth_Tile_Cache_misses = 0x56, + DB_PERF_SEL_Depth_Tile_Cache_hits = 0x57, + DB_PERF_SEL_Depth_Tile_Cache_flushes = 0x58, + DB_PERF_SEL_Depth_Tile_Cache_noop_tile = 0x59, + DB_PERF_SEL_Depth_Tile_Cache_detailed_noop = 0x5a, + DB_PERF_SEL_Depth_Tile_Cache_event = 0x5b, + DB_PERF_SEL_Depth_Tile_Cache_tile_frees = 0x5c, + DB_PERF_SEL_Depth_Tile_Cache_data_frees = 0x5d, + DB_PERF_SEL_Depth_Tile_Cache_mem_return_starve = 0x5e, + DB_PERF_SEL_Stencil_Cache_misses = 0x5f, + DB_PERF_SEL_Stencil_Cache_hits = 0x60, + DB_PERF_SEL_Stencil_Cache_flushes = 0x61, + DB_PERF_SEL_Stencil_Cache_starves = 0x62, + DB_PERF_SEL_Stencil_Cache_frees = 0x63, + DB_PERF_SEL_Z_Cache_separate_Z_misses = 0x64, + DB_PERF_SEL_Z_Cache_separate_Z_hits = 0x65, + DB_PERF_SEL_Z_Cache_separate_Z_flushes = 0x66, + DB_PERF_SEL_Z_Cache_separate_Z_starves = 0x67, + DB_PERF_SEL_Z_Cache_pmask_misses = 0x68, + DB_PERF_SEL_Z_Cache_pmask_hits = 0x69, + DB_PERF_SEL_Z_Cache_pmask_flushes = 0x6a, + DB_PERF_SEL_Z_Cache_pmask_starves = 0x6b, + DB_PERF_SEL_Z_Cache_frees = 0x6c, + DB_PERF_SEL_Plane_Cache_misses = 0x6d, + DB_PERF_SEL_Plane_Cache_hits = 0x6e, + DB_PERF_SEL_Plane_Cache_flushes = 0x6f, + DB_PERF_SEL_Plane_Cache_starves = 0x70, + DB_PERF_SEL_Plane_Cache_frees = 0x71, + DB_PERF_SEL_flush_expanded_stencil = 0x72, + DB_PERF_SEL_flush_compressed_stencil = 0x73, + DB_PERF_SEL_flush_single_stencil = 0x74, + DB_PERF_SEL_planes_flushed = 0x75, + DB_PERF_SEL_flush_1plane = 0x76, + DB_PERF_SEL_flush_2plane = 0x77, + DB_PERF_SEL_flush_3plane = 0x78, + DB_PERF_SEL_flush_4plane = 0x79, + DB_PERF_SEL_flush_5plane = 0x7a, + DB_PERF_SEL_flush_6plane = 0x7b, + DB_PERF_SEL_flush_7plane = 0x7c, + DB_PERF_SEL_flush_8plane = 0x7d, + DB_PERF_SEL_flush_9plane = 0x7e, + DB_PERF_SEL_flush_10plane = 0x7f, + DB_PERF_SEL_flush_11plane = 0x80, + DB_PERF_SEL_flush_12plane = 0x81, + DB_PERF_SEL_flush_13plane = 0x82, + DB_PERF_SEL_flush_14plane = 0x83, + DB_PERF_SEL_flush_15plane = 0x84, + DB_PERF_SEL_flush_16plane = 0x85, + DB_PERF_SEL_flush_expanded_z = 0x86, + DB_PERF_SEL_earlyZ_waiting_for_postZ_done = 0x87, + DB_PERF_SEL_reZ_waiting_for_postZ_done = 0x88, + DB_PERF_SEL_dk_tile_sends = 0x89, + DB_PERF_SEL_dk_tile_busy = 0x8a, + DB_PERF_SEL_dk_tile_quad_starves = 0x8b, + DB_PERF_SEL_dk_tile_stalls = 0x8c, + DB_PERF_SEL_dk_squad_sends = 0x8d, + DB_PERF_SEL_dk_squad_busy = 0x8e, + DB_PERF_SEL_dk_squad_stalls = 0x8f, + DB_PERF_SEL_Op_Pipe_Busy = 0x90, + DB_PERF_SEL_Op_Pipe_MC_Read_stall = 0x91, + DB_PERF_SEL_qc_busy = 0x92, + DB_PERF_SEL_qc_xfc = 0x93, + DB_PERF_SEL_qc_conflicts = 0x94, + DB_PERF_SEL_qc_full_stall = 0x95, + DB_PERF_SEL_qc_in_preZ_tile_stalls_postZ = 0x96, + DB_PERF_SEL_qc_in_postZ_tile_stalls_preZ = 0x97, + DB_PERF_SEL_tsc_insert_summarize_stall = 0x98, + DB_PERF_SEL_tl_busy = 0x99, + DB_PERF_SEL_tl_dtc_read_starved = 0x9a, + DB_PERF_SEL_tl_z_fetch_stall = 0x9b, + DB_PERF_SEL_tl_stencil_stall = 0x9c, + DB_PERF_SEL_tl_z_decompress_stall = 0x9d, + DB_PERF_SEL_tl_stencil_locked_stall = 0x9e, + DB_PERF_SEL_tl_events = 0x9f, + DB_PERF_SEL_tl_summarize_squads = 0xa0, + DB_PERF_SEL_tl_flush_expand_squads = 0xa1, + DB_PERF_SEL_tl_expand_squads = 0xa2, + DB_PERF_SEL_tl_preZ_squads = 0xa3, + DB_PERF_SEL_tl_postZ_squads = 0xa4, + DB_PERF_SEL_tl_preZ_noop_squads = 0xa5, + DB_PERF_SEL_tl_postZ_noop_squads = 0xa6, + DB_PERF_SEL_tl_tile_ops = 0xa7, + DB_PERF_SEL_tl_in_xfc = 0xa8, + DB_PERF_SEL_tl_in_single_stencil_expand_stall = 0xa9, + DB_PERF_SEL_tl_in_fast_z_stall = 0xaa, + DB_PERF_SEL_tl_out_xfc = 0xab, + DB_PERF_SEL_tl_out_squads = 0xac, + DB_PERF_SEL_zf_plane_multicycle = 0xad, + DB_PERF_SEL_PostZ_Samples_passing_Z = 0xae, + DB_PERF_SEL_PostZ_Samples_failing_Z = 0xaf, + DB_PERF_SEL_PostZ_Samples_failing_S = 0xb0, + DB_PERF_SEL_PreZ_Samples_passing_Z = 0xb1, + DB_PERF_SEL_PreZ_Samples_failing_Z = 0xb2, + DB_PERF_SEL_PreZ_Samples_failing_S = 0xb3, + DB_PERF_SEL_ts_tc_update_stall = 0xb4, + DB_PERF_SEL_sc_kick_start = 0xb5, + DB_PERF_SEL_sc_kick_end = 0xb6, + DB_PERF_SEL_clock_reg_active = 0xb7, + DB_PERF_SEL_clock_main_active = 0xb8, + DB_PERF_SEL_clock_mem_export_active = 0xb9, + DB_PERF_SEL_esr_ps_out_busy = 0xba, + DB_PERF_SEL_esr_ps_lqf_busy = 0xbb, + DB_PERF_SEL_esr_ps_lqf_stall = 0xbc, + DB_PERF_SEL_etr_out_send = 0xbd, + DB_PERF_SEL_etr_out_busy = 0xbe, + DB_PERF_SEL_etr_out_ltile_probe_fifo_full_stall = 0xbf, + DB_PERF_SEL_etr_out_cb_tile_stall = 0xc0, + DB_PERF_SEL_etr_out_esr_stall = 0xc1, + DB_PERF_SEL_esr_ps_sqq_busy = 0xc2, + DB_PERF_SEL_esr_ps_sqq_stall = 0xc3, + DB_PERF_SEL_esr_eot_fwd_busy = 0xc4, + DB_PERF_SEL_esr_eot_fwd_holding_squad = 0xc5, + DB_PERF_SEL_esr_eot_fwd_forward = 0xc6, + DB_PERF_SEL_esr_sqq_zi_busy = 0xc7, + DB_PERF_SEL_esr_sqq_zi_stall = 0xc8, + DB_PERF_SEL_postzl_sq_pt_busy = 0xc9, + DB_PERF_SEL_postzl_sq_pt_stall = 0xca, + DB_PERF_SEL_postzl_se_busy = 0xcb, + DB_PERF_SEL_postzl_se_stall = 0xcc, + DB_PERF_SEL_postzl_partial_launch = 0xcd, + DB_PERF_SEL_postzl_full_launch = 0xce, + DB_PERF_SEL_postzl_partial_waiting = 0xcf, + DB_PERF_SEL_postzl_tile_mem_stall = 0xd0, + DB_PERF_SEL_postzl_tile_init_stall = 0xd1, + DB_PEFF_SEL_prezl_tile_mem_stall = 0xd2, + DB_PERF_SEL_prezl_tile_init_stall = 0xd3, + DB_PERF_SEL_dtt_sm_clash_stall = 0xd4, + DB_PERF_SEL_dtt_sm_slot_stall = 0xd5, + DB_PERF_SEL_dtt_sm_miss_stall = 0xd6, + DB_PERF_SEL_mi_rdreq_busy = 0xd7, + DB_PERF_SEL_mi_rdreq_stall = 0xd8, + DB_PERF_SEL_mi_wrreq_busy = 0xd9, + DB_PERF_SEL_mi_wrreq_stall = 0xda, + DB_PERF_SEL_recomp_tile_to_1zplane_no_fastop = 0xdb, + DB_PERF_SEL_dkg_tile_rate_tile = 0xdc, + DB_PERF_SEL_prezl_src_in_sends = 0xdd, + DB_PERF_SEL_prezl_src_in_stall = 0xde, + DB_PERF_SEL_prezl_src_in_squads = 0xdf, + DB_PERF_SEL_prezl_src_in_squads_unrolled = 0xe0, + DB_PERF_SEL_prezl_src_in_tile_rate = 0xe1, + DB_PERF_SEL_prezl_src_in_tile_rate_unrolled = 0xe2, + DB_PERF_SEL_prezl_src_out_stall = 0xe3, + DB_PERF_SEL_postzl_src_in_sends = 0xe4, + DB_PERF_SEL_postzl_src_in_stall = 0xe5, + DB_PERF_SEL_postzl_src_in_squads = 0xe6, + DB_PERF_SEL_postzl_src_in_squads_unrolled = 0xe7, + DB_PERF_SEL_postzl_src_in_tile_rate = 0xe8, + DB_PERF_SEL_postzl_src_in_tile_rate_unrolled = 0xe9, + DB_PERF_SEL_postzl_src_out_stall = 0xea, + DB_PERF_SEL_esr_ps_src_in_sends = 0xeb, + DB_PERF_SEL_esr_ps_src_in_stall = 0xec, + DB_PERF_SEL_esr_ps_src_in_squads = 0xed, + DB_PERF_SEL_esr_ps_src_in_squads_unrolled = 0xee, + DB_PERF_SEL_esr_ps_src_in_tile_rate = 0xef, + DB_PERF_SEL_esr_ps_src_in_tile_rate_unrolled = 0xf0, + DB_PERF_SEL_esr_ps_src_in_tile_rate_unrolled_to_pixel_rate= 0xf1, + DB_PERF_SEL_esr_ps_src_out_stall = 0xf2, + DB_PERF_SEL_depth_bounds_qtiles_culled = 0xf3, + DB_PERF_SEL_PreZ_Samples_failing_DB = 0xf4, + DB_PERF_SEL_PostZ_Samples_failing_DB = 0xf5, + DB_PERF_SEL_flush_compressed = 0xf6, + DB_PERF_SEL_flush_plane_le4 = 0xf7, + DB_PERF_SEL_tiles_z_fully_summarized = 0xf8, + DB_PERF_SEL_tiles_stencil_fully_summarized = 0xf9, + DB_PERF_SEL_tiles_z_clear_on_expclear = 0xfa, + DB_PERF_SEL_tiles_s_clear_on_expclear = 0xfb, + DB_PERF_SEL_tiles_decomp_on_expclear = 0xfc, + DB_PERF_SEL_tiles_compressed_to_decompressed = 0xfd, + DB_PERF_SEL_Op_Pipe_Prez_Busy = 0xfe, + DB_PERF_SEL_Op_Pipe_Postz_Busy = 0xff, + DB_PERF_SEL_di_dt_stall = 0x100, +} PerfCounter_Vals; +typedef enum RingCounterControl { + COUNTER_RING_SPLIT = 0x0, + COUNTER_RING_0 = 0x1, + COUNTER_RING_1 = 0x2, +} RingCounterControl; +typedef enum PixelPipeCounterId { + PIXEL_PIPE_OCCLUSION_COUNT_0 = 0x0, + PIXEL_PIPE_OCCLUSION_COUNT_1 = 0x1, + PIXEL_PIPE_OCCLUSION_COUNT_2 = 0x2, + PIXEL_PIPE_OCCLUSION_COUNT_3 = 0x3, + PIXEL_PIPE_SCREEN_MIN_EXTENTS_0 = 0x4, + PIXEL_PIPE_SCREEN_MAX_EXTENTS_0 = 0x5, + PIXEL_PIPE_SCREEN_MIN_EXTENTS_1 = 0x6, + PIXEL_PIPE_SCREEN_MAX_EXTENTS_1 = 0x7, +} PixelPipeCounterId; +typedef enum PixelPipeStride { + PIXEL_PIPE_STRIDE_32_BITS = 0x0, + PIXEL_PIPE_STRIDE_64_BITS = 0x1, + PIXEL_PIPE_STRIDE_128_BITS = 0x2, + PIXEL_PIPE_STRIDE_256_BITS = 0x3, +} PixelPipeStride; +typedef enum GB_EDC_DED_MODE { + GB_EDC_DED_MODE_LOG = 0x0, + GB_EDC_DED_MODE_HALT = 0x1, + GB_EDC_DED_MODE_INT_HALT = 0x2, +} GB_EDC_DED_MODE; +#define GB_TILING_CONFIG_TABLE_SIZE 0x20 +#define GB_TILING_CONFIG_MACROTABLE_SIZE 0x10 +typedef enum GRBM_PERF_SEL { + GRBM_PERF_SEL_COUNT = 0x0, + GRBM_PERF_SEL_USER_DEFINED = 0x1, + GRBM_PERF_SEL_GUI_ACTIVE = 0x2, + GRBM_PERF_SEL_CP_BUSY = 0x3, + GRBM_PERF_SEL_CP_COHER_BUSY = 0x4, + GRBM_PERF_SEL_CP_DMA_BUSY = 0x5, + GRBM_PERF_SEL_CB_BUSY = 0x6, + GRBM_PERF_SEL_DB_BUSY = 0x7, + GRBM_PERF_SEL_PA_BUSY = 0x8, + GRBM_PERF_SEL_SC_BUSY = 0x9, + GRBM_PERF_SEL_RESERVED_6 = 0xa, + GRBM_PERF_SEL_SPI_BUSY = 0xb, + GRBM_PERF_SEL_SX_BUSY = 0xc, + GRBM_PERF_SEL_TA_BUSY = 0xd, + GRBM_PERF_SEL_CB_CLEAN = 0xe, + GRBM_PERF_SEL_DB_CLEAN = 0xf, + GRBM_PERF_SEL_RESERVED_5 = 0x10, + GRBM_PERF_SEL_VGT_BUSY = 0x11, + GRBM_PERF_SEL_RESERVED_4 = 0x12, + GRBM_PERF_SEL_RESERVED_3 = 0x13, + GRBM_PERF_SEL_RESERVED_2 = 0x14, + GRBM_PERF_SEL_RESERVED_1 = 0x15, + GRBM_PERF_SEL_RESERVED_0 = 0x16, + GRBM_PERF_SEL_IA_BUSY = 0x17, + GRBM_PERF_SEL_IA_NO_DMA_BUSY = 0x18, + GRBM_PERF_SEL_GDS_BUSY = 0x19, + GRBM_PERF_SEL_BCI_BUSY = 0x1a, + GRBM_PERF_SEL_RLC_BUSY = 0x1b, + GRBM_PERF_SEL_TC_BUSY = 0x1c, + GRBM_PERF_SEL_CPG_BUSY = 0x1d, + GRBM_PERF_SEL_CPC_BUSY = 0x1e, + GRBM_PERF_SEL_CPF_BUSY = 0x1f, + GRBM_PERF_SEL_WD_BUSY = 0x20, + GRBM_PERF_SEL_WD_NO_DMA_BUSY = 0x21, +} GRBM_PERF_SEL; +typedef enum GRBM_SE0_PERF_SEL { + GRBM_SE0_PERF_SEL_COUNT = 0x0, + GRBM_SE0_PERF_SEL_USER_DEFINED = 0x1, + GRBM_SE0_PERF_SEL_CB_BUSY = 0x2, + GRBM_SE0_PERF_SEL_DB_BUSY = 0x3, + GRBM_SE0_PERF_SEL_SC_BUSY = 0x4, + GRBM_SE0_PERF_SEL_RESERVED_1 = 0x5, + GRBM_SE0_PERF_SEL_SPI_BUSY = 0x6, + GRBM_SE0_PERF_SEL_SX_BUSY = 0x7, + GRBM_SE0_PERF_SEL_TA_BUSY = 0x8, + GRBM_SE0_PERF_SEL_CB_CLEAN = 0x9, + GRBM_SE0_PERF_SEL_DB_CLEAN = 0xa, + GRBM_SE0_PERF_SEL_RESERVED_0 = 0xb, + GRBM_SE0_PERF_SEL_PA_BUSY = 0xc, + GRBM_SE0_PERF_SEL_VGT_BUSY = 0xd, + GRBM_SE0_PERF_SEL_BCI_BUSY = 0xe, +} GRBM_SE0_PERF_SEL; +typedef enum GRBM_SE1_PERF_SEL { + GRBM_SE1_PERF_SEL_COUNT = 0x0, + GRBM_SE1_PERF_SEL_USER_DEFINED = 0x1, + GRBM_SE1_PERF_SEL_CB_BUSY = 0x2, + GRBM_SE1_PERF_SEL_DB_BUSY = 0x3, + GRBM_SE1_PERF_SEL_SC_BUSY = 0x4, + GRBM_SE1_PERF_SEL_RESERVED_1 = 0x5, + GRBM_SE1_PERF_SEL_SPI_BUSY = 0x6, + GRBM_SE1_PERF_SEL_SX_BUSY = 0x7, + GRBM_SE1_PERF_SEL_TA_BUSY = 0x8, + GRBM_SE1_PERF_SEL_CB_CLEAN = 0x9, + GRBM_SE1_PERF_SEL_DB_CLEAN = 0xa, + GRBM_SE1_PERF_SEL_RESERVED_0 = 0xb, + GRBM_SE1_PERF_SEL_PA_BUSY = 0xc, + GRBM_SE1_PERF_SEL_VGT_BUSY = 0xd, + GRBM_SE1_PERF_SEL_BCI_BUSY = 0xe, +} GRBM_SE1_PERF_SEL; +typedef enum GRBM_SE2_PERF_SEL { + GRBM_SE2_PERF_SEL_COUNT = 0x0, + GRBM_SE2_PERF_SEL_USER_DEFINED = 0x1, + GRBM_SE2_PERF_SEL_CB_BUSY = 0x2, + GRBM_SE2_PERF_SEL_DB_BUSY = 0x3, + GRBM_SE2_PERF_SEL_SC_BUSY = 0x4, + GRBM_SE2_PERF_SEL_RESERVED_1 = 0x5, + GRBM_SE2_PERF_SEL_SPI_BUSY = 0x6, + GRBM_SE2_PERF_SEL_SX_BUSY = 0x7, + GRBM_SE2_PERF_SEL_TA_BUSY = 0x8, + GRBM_SE2_PERF_SEL_CB_CLEAN = 0x9, + GRBM_SE2_PERF_SEL_DB_CLEAN = 0xa, + GRBM_SE2_PERF_SEL_RESERVED_0 = 0xb, + GRBM_SE2_PERF_SEL_PA_BUSY = 0xc, + GRBM_SE2_PERF_SEL_VGT_BUSY = 0xd, + GRBM_SE2_PERF_SEL_BCI_BUSY = 0xe, +} GRBM_SE2_PERF_SEL; +typedef enum GRBM_SE3_PERF_SEL { + GRBM_SE3_PERF_SEL_COUNT = 0x0, + GRBM_SE3_PERF_SEL_USER_DEFINED = 0x1, + GRBM_SE3_PERF_SEL_CB_BUSY = 0x2, + GRBM_SE3_PERF_SEL_DB_BUSY = 0x3, + GRBM_SE3_PERF_SEL_SC_BUSY = 0x4, + GRBM_SE3_PERF_SEL_RESERVED_1 = 0x5, + GRBM_SE3_PERF_SEL_SPI_BUSY = 0x6, + GRBM_SE3_PERF_SEL_SX_BUSY = 0x7, + GRBM_SE3_PERF_SEL_TA_BUSY = 0x8, + GRBM_SE3_PERF_SEL_CB_CLEAN = 0x9, + GRBM_SE3_PERF_SEL_DB_CLEAN = 0xa, + GRBM_SE3_PERF_SEL_RESERVED_0 = 0xb, + GRBM_SE3_PERF_SEL_PA_BUSY = 0xc, + GRBM_SE3_PERF_SEL_VGT_BUSY = 0xd, + GRBM_SE3_PERF_SEL_BCI_BUSY = 0xe, +} GRBM_SE3_PERF_SEL; +typedef enum SU_PERFCNT_SEL { + PERF_PAPC_PASX_REQ = 0x0, + PERF_PAPC_PASX_DISABLE_PIPE = 0x1, + PERF_PAPC_PASX_FIRST_VECTOR = 0x2, + PERF_PAPC_PASX_SECOND_VECTOR = 0x3, + PERF_PAPC_PASX_FIRST_DEAD = 0x4, + PERF_PAPC_PASX_SECOND_DEAD = 0x5, + PERF_PAPC_PASX_VTX_KILL_DISCARD = 0x6, + PERF_PAPC_PASX_VTX_NAN_DISCARD = 0x7, + PERF_PAPC_PA_INPUT_PRIM = 0x8, + PERF_PAPC_PA_INPUT_NULL_PRIM = 0x9, + PERF_PAPC_PA_INPUT_EVENT_FLAG = 0xa, + PERF_PAPC_PA_INPUT_FIRST_PRIM_SLOT = 0xb, + PERF_PAPC_PA_INPUT_END_OF_PACKET = 0xc, + PERF_PAPC_PA_INPUT_EXTENDED_EVENT = 0xd, + PERF_PAPC_CLPR_CULL_PRIM = 0xe, + PERF_PAPC_CLPR_VVUCP_CULL_PRIM = 0xf, + PERF_PAPC_CLPR_VV_CULL_PRIM = 0x10, + PERF_PAPC_CLPR_UCP_CULL_PRIM = 0x11, + PERF_PAPC_CLPR_VTX_KILL_CULL_PRIM = 0x12, + PERF_PAPC_CLPR_VTX_NAN_CULL_PRIM = 0x13, + PERF_PAPC_CLPR_CULL_TO_NULL_PRIM = 0x14, + PERF_PAPC_CLPR_VVUCP_CLIP_PRIM = 0x15, + PERF_PAPC_CLPR_VV_CLIP_PRIM = 0x16, + PERF_PAPC_CLPR_UCP_CLIP_PRIM = 0x17, + PERF_PAPC_CLPR_POINT_CLIP_CANDIDATE = 0x18, + PERF_PAPC_CLPR_CLIP_PLANE_CNT_1 = 0x19, + PERF_PAPC_CLPR_CLIP_PLANE_CNT_2 = 0x1a, + PERF_PAPC_CLPR_CLIP_PLANE_CNT_3 = 0x1b, + PERF_PAPC_CLPR_CLIP_PLANE_CNT_4 = 0x1c, + PERF_PAPC_CLPR_CLIP_PLANE_CNT_5_8 = 0x1d, + PERF_PAPC_CLPR_CLIP_PLANE_CNT_9_12 = 0x1e, + PERF_PAPC_CLPR_CLIP_PLANE_NEAR = 0x1f, + PERF_PAPC_CLPR_CLIP_PLANE_FAR = 0x20, + PERF_PAPC_CLPR_CLIP_PLANE_LEFT = 0x21, + PERF_PAPC_CLPR_CLIP_PLANE_RIGHT = 0x22, + PERF_PAPC_CLPR_CLIP_PLANE_TOP = 0x23, + PERF_PAPC_CLPR_CLIP_PLANE_BOTTOM = 0x24, + PERF_PAPC_CLPR_GSC_KILL_CULL_PRIM = 0x25, + PERF_PAPC_CLPR_RASTER_KILL_CULL_PRIM = 0x26, + PERF_PAPC_CLSM_NULL_PRIM = 0x27, + PERF_PAPC_CLSM_TOTALLY_VISIBLE_PRIM = 0x28, + PERF_PAPC_CLSM_CULL_TO_NULL_PRIM = 0x29, + PERF_PAPC_CLSM_OUT_PRIM_CNT_1 = 0x2a, + PERF_PAPC_CLSM_OUT_PRIM_CNT_2 = 0x2b, + PERF_PAPC_CLSM_OUT_PRIM_CNT_3 = 0x2c, + PERF_PAPC_CLSM_OUT_PRIM_CNT_4 = 0x2d, + PERF_PAPC_CLSM_OUT_PRIM_CNT_5_8 = 0x2e, + PERF_PAPC_CLSM_OUT_PRIM_CNT_9_13 = 0x2f, + PERF_PAPC_CLIPGA_VTE_KILL_PRIM = 0x30, + PERF_PAPC_SU_INPUT_PRIM = 0x31, + PERF_PAPC_SU_INPUT_CLIP_PRIM = 0x32, + PERF_PAPC_SU_INPUT_NULL_PRIM = 0x33, + PERF_PAPC_SU_INPUT_PRIM_DUAL = 0x34, + PERF_PAPC_SU_INPUT_CLIP_PRIM_DUAL = 0x35, + PERF_PAPC_SU_ZERO_AREA_CULL_PRIM = 0x36, + PERF_PAPC_SU_BACK_FACE_CULL_PRIM = 0x37, + PERF_PAPC_SU_FRONT_FACE_CULL_PRIM = 0x38, + PERF_PAPC_SU_POLYMODE_FACE_CULL = 0x39, + PERF_PAPC_SU_POLYMODE_BACK_CULL = 0x3a, + PERF_PAPC_SU_POLYMODE_FRONT_CULL = 0x3b, + PERF_PAPC_SU_POLYMODE_INVALID_FILL = 0x3c, + PERF_PAPC_SU_OUTPUT_PRIM = 0x3d, + PERF_PAPC_SU_OUTPUT_CLIP_PRIM = 0x3e, + PERF_PAPC_SU_OUTPUT_NULL_PRIM = 0x3f, + PERF_PAPC_SU_OUTPUT_EVENT_FLAG = 0x40, + PERF_PAPC_SU_OUTPUT_FIRST_PRIM_SLOT = 0x41, + PERF_PAPC_SU_OUTPUT_END_OF_PACKET = 0x42, + PERF_PAPC_SU_OUTPUT_POLYMODE_FACE = 0x43, + PERF_PAPC_SU_OUTPUT_POLYMODE_BACK = 0x44, + PERF_PAPC_SU_OUTPUT_POLYMODE_FRONT = 0x45, + PERF_PAPC_SU_OUT_CLIP_POLYMODE_FACE = 0x46, + PERF_PAPC_SU_OUT_CLIP_POLYMODE_BACK = 0x47, + PERF_PAPC_SU_OUT_CLIP_POLYMODE_FRONT = 0x48, + PERF_PAPC_SU_OUTPUT_PRIM_DUAL = 0x49, + PERF_PAPC_SU_OUTPUT_CLIP_PRIM_DUAL = 0x4a, + PERF_PAPC_SU_OUTPUT_POLYMODE_DUAL = 0x4b, + PERF_PAPC_SU_OUTPUT_CLIP_POLYMODE_DUAL = 0x4c, + PERF_PAPC_PASX_REQ_IDLE = 0x4d, + PERF_PAPC_PASX_REQ_BUSY = 0x4e, + PERF_PAPC_PASX_REQ_STALLED = 0x4f, + PERF_PAPC_PASX_REC_IDLE = 0x50, + PERF_PAPC_PASX_REC_BUSY = 0x51, + PERF_PAPC_PASX_REC_STARVED_SX = 0x52, + PERF_PAPC_PASX_REC_STALLED = 0x53, + PERF_PAPC_PASX_REC_STALLED_POS_MEM = 0x54, + PERF_PAPC_PASX_REC_STALLED_CCGSM_IN = 0x55, + PERF_PAPC_CCGSM_IDLE = 0x56, + PERF_PAPC_CCGSM_BUSY = 0x57, + PERF_PAPC_CCGSM_STALLED = 0x58, + PERF_PAPC_CLPRIM_IDLE = 0x59, + PERF_PAPC_CLPRIM_BUSY = 0x5a, + PERF_PAPC_CLPRIM_STALLED = 0x5b, + PERF_PAPC_CLPRIM_STARVED_CCGSM = 0x5c, + PERF_PAPC_CLIPSM_IDLE = 0x5d, + PERF_PAPC_CLIPSM_BUSY = 0x5e, + PERF_PAPC_CLIPSM_WAIT_CLIP_VERT_ENGH = 0x5f, + PERF_PAPC_CLIPSM_WAIT_HIGH_PRI_SEQ = 0x60, + PERF_PAPC_CLIPSM_WAIT_CLIPGA = 0x61, + PERF_PAPC_CLIPSM_WAIT_AVAIL_VTE_CLIP = 0x62, + PERF_PAPC_CLIPSM_WAIT_CLIP_OUTSM = 0x63, + PERF_PAPC_CLIPGA_IDLE = 0x64, + PERF_PAPC_CLIPGA_BUSY = 0x65, + PERF_PAPC_CLIPGA_STARVED_VTE_CLIP = 0x66, + PERF_PAPC_CLIPGA_STALLED = 0x67, + PERF_PAPC_CLIP_IDLE = 0x68, + PERF_PAPC_CLIP_BUSY = 0x69, + PERF_PAPC_SU_IDLE = 0x6a, + PERF_PAPC_SU_BUSY = 0x6b, + PERF_PAPC_SU_STARVED_CLIP = 0x6c, + PERF_PAPC_SU_STALLED_SC = 0x6d, + PERF_PAPC_CL_DYN_SCLK_VLD = 0x6e, + PERF_PAPC_SU_DYN_SCLK_VLD = 0x6f, + PERF_PAPC_PA_REG_SCLK_VLD = 0x70, + PERF_PAPC_SU_MULTI_GPU_PRIM_FILTER_CULL = 0x71, + PERF_PAPC_PASX_SE0_REQ = 0x72, + PERF_PAPC_PASX_SE1_REQ = 0x73, + PERF_PAPC_PASX_SE0_FIRST_VECTOR = 0x74, + PERF_PAPC_PASX_SE0_SECOND_VECTOR = 0x75, + PERF_PAPC_PASX_SE1_FIRST_VECTOR = 0x76, + PERF_PAPC_PASX_SE1_SECOND_VECTOR = 0x77, + PERF_PAPC_SU_SE0_PRIM_FILTER_CULL = 0x78, + PERF_PAPC_SU_SE1_PRIM_FILTER_CULL = 0x79, + PERF_PAPC_SU_SE01_PRIM_FILTER_CULL = 0x7a, + PERF_PAPC_SU_SE0_OUTPUT_PRIM = 0x7b, + PERF_PAPC_SU_SE1_OUTPUT_PRIM = 0x7c, + PERF_PAPC_SU_SE01_OUTPUT_PRIM = 0x7d, + PERF_PAPC_SU_SE0_OUTPUT_NULL_PRIM = 0x7e, + PERF_PAPC_SU_SE1_OUTPUT_NULL_PRIM = 0x7f, + PERF_PAPC_SU_SE01_OUTPUT_NULL_PRIM = 0x80, + PERF_PAPC_SU_SE0_OUTPUT_FIRST_PRIM_SLOT = 0x81, + PERF_PAPC_SU_SE1_OUTPUT_FIRST_PRIM_SLOT = 0x82, + PERF_PAPC_SU_SE0_STALLED_SC = 0x83, + PERF_PAPC_SU_SE1_STALLED_SC = 0x84, + PERF_PAPC_SU_SE01_STALLED_SC = 0x85, + PERF_PAPC_CLSM_CLIPPING_PRIM = 0x86, + PERF_PAPC_SU_CULLED_PRIM = 0x87, + PERF_PAPC_SU_OUTPUT_EOPG = 0x88, + PERF_PAPC_SU_SE2_PRIM_FILTER_CULL = 0x89, + PERF_PAPC_SU_SE3_PRIM_FILTER_CULL = 0x8a, + PERF_PAPC_SU_SE2_OUTPUT_PRIM = 0x8b, + PERF_PAPC_SU_SE3_OUTPUT_PRIM = 0x8c, + PERF_PAPC_SU_SE2_OUTPUT_NULL_PRIM = 0x8d, + PERF_PAPC_SU_SE3_OUTPUT_NULL_PRIM = 0x8e, + PERF_PAPC_SU_SE0_OUTPUT_END_OF_PACKET = 0x8f, + PERF_PAPC_SU_SE1_OUTPUT_END_OF_PACKET = 0x90, + PERF_PAPC_SU_SE2_OUTPUT_END_OF_PACKET = 0x91, + PERF_PAPC_SU_SE3_OUTPUT_END_OF_PACKET = 0x92, + PERF_PAPC_SU_SE0_OUTPUT_EOPG = 0x93, + PERF_PAPC_SU_SE1_OUTPUT_EOPG = 0x94, + PERF_PAPC_SU_SE2_OUTPUT_EOPG = 0x95, + PERF_PAPC_SU_SE3_OUTPUT_EOPG = 0x96, + PERF_PAPC_SU_SE2_STALLED_SC = 0x97, + PERF_PAPC_SU_SE3_STALLED_SC = 0x98, +} SU_PERFCNT_SEL; +typedef enum SC_PERFCNT_SEL { + SC_SRPS_WINDOW_VALID = 0x0, + SC_PSSW_WINDOW_VALID = 0x1, + SC_TPQZ_WINDOW_VALID = 0x2, + SC_QZQP_WINDOW_VALID = 0x3, + SC_TRPK_WINDOW_VALID = 0x4, + SC_SRPS_WINDOW_VALID_BUSY = 0x5, + SC_PSSW_WINDOW_VALID_BUSY = 0x6, + SC_TPQZ_WINDOW_VALID_BUSY = 0x7, + SC_QZQP_WINDOW_VALID_BUSY = 0x8, + SC_TRPK_WINDOW_VALID_BUSY = 0x9, + SC_STARVED_BY_PA = 0xa, + SC_STALLED_BY_PRIMFIFO = 0xb, + SC_STALLED_BY_DB_TILE = 0xc, + SC_STARVED_BY_DB_TILE = 0xd, + SC_STALLED_BY_TILEORDERFIFO = 0xe, + SC_STALLED_BY_TILEFIFO = 0xf, + SC_STALLED_BY_DB_QUAD = 0x10, + SC_STARVED_BY_DB_QUAD = 0x11, + SC_STALLED_BY_QUADFIFO = 0x12, + SC_STALLED_BY_BCI = 0x13, + SC_STALLED_BY_SPI = 0x14, + SC_SCISSOR_DISCARD = 0x15, + SC_BB_DISCARD = 0x16, + SC_SUPERTILE_COUNT = 0x17, + SC_SUPERTILE_PER_PRIM_H0 = 0x18, + SC_SUPERTILE_PER_PRIM_H1 = 0x19, + SC_SUPERTILE_PER_PRIM_H2 = 0x1a, + SC_SUPERTILE_PER_PRIM_H3 = 0x1b, + SC_SUPERTILE_PER_PRIM_H4 = 0x1c, + SC_SUPERTILE_PER_PRIM_H5 = 0x1d, + SC_SUPERTILE_PER_PRIM_H6 = 0x1e, + SC_SUPERTILE_PER_PRIM_H7 = 0x1f, + SC_SUPERTILE_PER_PRIM_H8 = 0x20, + SC_SUPERTILE_PER_PRIM_H9 = 0x21, + SC_SUPERTILE_PER_PRIM_H10 = 0x22, + SC_SUPERTILE_PER_PRIM_H11 = 0x23, + SC_SUPERTILE_PER_PRIM_H12 = 0x24, + SC_SUPERTILE_PER_PRIM_H13 = 0x25, + SC_SUPERTILE_PER_PRIM_H14 = 0x26, + SC_SUPERTILE_PER_PRIM_H15 = 0x27, + SC_SUPERTILE_PER_PRIM_H16 = 0x28, + SC_TILE_PER_PRIM_H0 = 0x29, + SC_TILE_PER_PRIM_H1 = 0x2a, + SC_TILE_PER_PRIM_H2 = 0x2b, + SC_TILE_PER_PRIM_H3 = 0x2c, + SC_TILE_PER_PRIM_H4 = 0x2d, + SC_TILE_PER_PRIM_H5 = 0x2e, + SC_TILE_PER_PRIM_H6 = 0x2f, + SC_TILE_PER_PRIM_H7 = 0x30, + SC_TILE_PER_PRIM_H8 = 0x31, + SC_TILE_PER_PRIM_H9 = 0x32, + SC_TILE_PER_PRIM_H10 = 0x33, + SC_TILE_PER_PRIM_H11 = 0x34, + SC_TILE_PER_PRIM_H12 = 0x35, + SC_TILE_PER_PRIM_H13 = 0x36, + SC_TILE_PER_PRIM_H14 = 0x37, + SC_TILE_PER_PRIM_H15 = 0x38, + SC_TILE_PER_PRIM_H16 = 0x39, + SC_TILE_PER_SUPERTILE_H0 = 0x3a, + SC_TILE_PER_SUPERTILE_H1 = 0x3b, + SC_TILE_PER_SUPERTILE_H2 = 0x3c, + SC_TILE_PER_SUPERTILE_H3 = 0x3d, + SC_TILE_PER_SUPERTILE_H4 = 0x3e, + SC_TILE_PER_SUPERTILE_H5 = 0x3f, + SC_TILE_PER_SUPERTILE_H6 = 0x40, + SC_TILE_PER_SUPERTILE_H7 = 0x41, + SC_TILE_PER_SUPERTILE_H8 = 0x42, + SC_TILE_PER_SUPERTILE_H9 = 0x43, + SC_TILE_PER_SUPERTILE_H10 = 0x44, + SC_TILE_PER_SUPERTILE_H11 = 0x45, + SC_TILE_PER_SUPERTILE_H12 = 0x46, + SC_TILE_PER_SUPERTILE_H13 = 0x47, + SC_TILE_PER_SUPERTILE_H14 = 0x48, + SC_TILE_PER_SUPERTILE_H15 = 0x49, + SC_TILE_PER_SUPERTILE_H16 = 0x4a, + SC_TILE_PICKED_H1 = 0x4b, + SC_TILE_PICKED_H2 = 0x4c, + SC_TILE_PICKED_H3 = 0x4d, + SC_TILE_PICKED_H4 = 0x4e, + SC_QZ0_MULTI_GPU_TILE_DISCARD = 0x4f, + SC_QZ1_MULTI_GPU_TILE_DISCARD = 0x50, + SC_QZ2_MULTI_GPU_TILE_DISCARD = 0x51, + SC_QZ3_MULTI_GPU_TILE_DISCARD = 0x52, + SC_QZ0_TILE_COUNT = 0x53, + SC_QZ1_TILE_COUNT = 0x54, + SC_QZ2_TILE_COUNT = 0x55, + SC_QZ3_TILE_COUNT = 0x56, + SC_QZ0_TILE_COVERED_COUNT = 0x57, + SC_QZ1_TILE_COVERED_COUNT = 0x58, + SC_QZ2_TILE_COVERED_COUNT = 0x59, + SC_QZ3_TILE_COVERED_COUNT = 0x5a, + SC_QZ0_TILE_NOT_COVERED_COUNT = 0x5b, + SC_QZ1_TILE_NOT_COVERED_COUNT = 0x5c, + SC_QZ2_TILE_NOT_COVERED_COUNT = 0x5d, + SC_QZ3_TILE_NOT_COVERED_COUNT = 0x5e, + SC_QZ0_QUAD_PER_TILE_H0 = 0x5f, + SC_QZ0_QUAD_PER_TILE_H1 = 0x60, + SC_QZ0_QUAD_PER_TILE_H2 = 0x61, + SC_QZ0_QUAD_PER_TILE_H3 = 0x62, + SC_QZ0_QUAD_PER_TILE_H4 = 0x63, + SC_QZ0_QUAD_PER_TILE_H5 = 0x64, + SC_QZ0_QUAD_PER_TILE_H6 = 0x65, + SC_QZ0_QUAD_PER_TILE_H7 = 0x66, + SC_QZ0_QUAD_PER_TILE_H8 = 0x67, + SC_QZ0_QUAD_PER_TILE_H9 = 0x68, + SC_QZ0_QUAD_PER_TILE_H10 = 0x69, + SC_QZ0_QUAD_PER_TILE_H11 = 0x6a, + SC_QZ0_QUAD_PER_TILE_H12 = 0x6b, + SC_QZ0_QUAD_PER_TILE_H13 = 0x6c, + SC_QZ0_QUAD_PER_TILE_H14 = 0x6d, + SC_QZ0_QUAD_PER_TILE_H15 = 0x6e, + SC_QZ0_QUAD_PER_TILE_H16 = 0x6f, + SC_QZ1_QUAD_PER_TILE_H0 = 0x70, + SC_QZ1_QUAD_PER_TILE_H1 = 0x71, + SC_QZ1_QUAD_PER_TILE_H2 = 0x72, + SC_QZ1_QUAD_PER_TILE_H3 = 0x73, + SC_QZ1_QUAD_PER_TILE_H4 = 0x74, + SC_QZ1_QUAD_PER_TILE_H5 = 0x75, + SC_QZ1_QUAD_PER_TILE_H6 = 0x76, + SC_QZ1_QUAD_PER_TILE_H7 = 0x77, + SC_QZ1_QUAD_PER_TILE_H8 = 0x78, + SC_QZ1_QUAD_PER_TILE_H9 = 0x79, + SC_QZ1_QUAD_PER_TILE_H10 = 0x7a, + SC_QZ1_QUAD_PER_TILE_H11 = 0x7b, + SC_QZ1_QUAD_PER_TILE_H12 = 0x7c, + SC_QZ1_QUAD_PER_TILE_H13 = 0x7d, + SC_QZ1_QUAD_PER_TILE_H14 = 0x7e, + SC_QZ1_QUAD_PER_TILE_H15 = 0x7f, + SC_QZ1_QUAD_PER_TILE_H16 = 0x80, + SC_QZ2_QUAD_PER_TILE_H0 = 0x81, + SC_QZ2_QUAD_PER_TILE_H1 = 0x82, + SC_QZ2_QUAD_PER_TILE_H2 = 0x83, + SC_QZ2_QUAD_PER_TILE_H3 = 0x84, + SC_QZ2_QUAD_PER_TILE_H4 = 0x85, + SC_QZ2_QUAD_PER_TILE_H5 = 0x86, + SC_QZ2_QUAD_PER_TILE_H6 = 0x87, + SC_QZ2_QUAD_PER_TILE_H7 = 0x88, + SC_QZ2_QUAD_PER_TILE_H8 = 0x89, + SC_QZ2_QUAD_PER_TILE_H9 = 0x8a, + SC_QZ2_QUAD_PER_TILE_H10 = 0x8b, + SC_QZ2_QUAD_PER_TILE_H11 = 0x8c, + SC_QZ2_QUAD_PER_TILE_H12 = 0x8d, + SC_QZ2_QUAD_PER_TILE_H13 = 0x8e, + SC_QZ2_QUAD_PER_TILE_H14 = 0x8f, + SC_QZ2_QUAD_PER_TILE_H15 = 0x90, + SC_QZ2_QUAD_PER_TILE_H16 = 0x91, + SC_QZ3_QUAD_PER_TILE_H0 = 0x92, + SC_QZ3_QUAD_PER_TILE_H1 = 0x93, + SC_QZ3_QUAD_PER_TILE_H2 = 0x94, + SC_QZ3_QUAD_PER_TILE_H3 = 0x95, + SC_QZ3_QUAD_PER_TILE_H4 = 0x96, + SC_QZ3_QUAD_PER_TILE_H5 = 0x97, + SC_QZ3_QUAD_PER_TILE_H6 = 0x98, + SC_QZ3_QUAD_PER_TILE_H7 = 0x99, + SC_QZ3_QUAD_PER_TILE_H8 = 0x9a, + SC_QZ3_QUAD_PER_TILE_H9 = 0x9b, + SC_QZ3_QUAD_PER_TILE_H10 = 0x9c, + SC_QZ3_QUAD_PER_TILE_H11 = 0x9d, + SC_QZ3_QUAD_PER_TILE_H12 = 0x9e, + SC_QZ3_QUAD_PER_TILE_H13 = 0x9f, + SC_QZ3_QUAD_PER_TILE_H14 = 0xa0, + SC_QZ3_QUAD_PER_TILE_H15 = 0xa1, + SC_QZ3_QUAD_PER_TILE_H16 = 0xa2, + SC_QZ0_QUAD_COUNT = 0xa3, + SC_QZ1_QUAD_COUNT = 0xa4, + SC_QZ2_QUAD_COUNT = 0xa5, + SC_QZ3_QUAD_COUNT = 0xa6, + SC_P0_HIZ_TILE_COUNT = 0xa7, + SC_P1_HIZ_TILE_COUNT = 0xa8, + SC_P2_HIZ_TILE_COUNT = 0xa9, + SC_P3_HIZ_TILE_COUNT = 0xaa, + SC_P0_HIZ_QUAD_PER_TILE_H0 = 0xab, + SC_P0_HIZ_QUAD_PER_TILE_H1 = 0xac, + SC_P0_HIZ_QUAD_PER_TILE_H2 = 0xad, + SC_P0_HIZ_QUAD_PER_TILE_H3 = 0xae, + SC_P0_HIZ_QUAD_PER_TILE_H4 = 0xaf, + SC_P0_HIZ_QUAD_PER_TILE_H5 = 0xb0, + SC_P0_HIZ_QUAD_PER_TILE_H6 = 0xb1, + SC_P0_HIZ_QUAD_PER_TILE_H7 = 0xb2, + SC_P0_HIZ_QUAD_PER_TILE_H8 = 0xb3, + SC_P0_HIZ_QUAD_PER_TILE_H9 = 0xb4, + SC_P0_HIZ_QUAD_PER_TILE_H10 = 0xb5, + SC_P0_HIZ_QUAD_PER_TILE_H11 = 0xb6, + SC_P0_HIZ_QUAD_PER_TILE_H12 = 0xb7, + SC_P0_HIZ_QUAD_PER_TILE_H13 = 0xb8, + SC_P0_HIZ_QUAD_PER_TILE_H14 = 0xb9, + SC_P0_HIZ_QUAD_PER_TILE_H15 = 0xba, + SC_P0_HIZ_QUAD_PER_TILE_H16 = 0xbb, + SC_P1_HIZ_QUAD_PER_TILE_H0 = 0xbc, + SC_P1_HIZ_QUAD_PER_TILE_H1 = 0xbd, + SC_P1_HIZ_QUAD_PER_TILE_H2 = 0xbe, + SC_P1_HIZ_QUAD_PER_TILE_H3 = 0xbf, + SC_P1_HIZ_QUAD_PER_TILE_H4 = 0xc0, + SC_P1_HIZ_QUAD_PER_TILE_H5 = 0xc1, + SC_P1_HIZ_QUAD_PER_TILE_H6 = 0xc2, + SC_P1_HIZ_QUAD_PER_TILE_H7 = 0xc3, + SC_P1_HIZ_QUAD_PER_TILE_H8 = 0xc4, + SC_P1_HIZ_QUAD_PER_TILE_H9 = 0xc5, + SC_P1_HIZ_QUAD_PER_TILE_H10 = 0xc6, + SC_P1_HIZ_QUAD_PER_TILE_H11 = 0xc7, + SC_P1_HIZ_QUAD_PER_TILE_H12 = 0xc8, + SC_P1_HIZ_QUAD_PER_TILE_H13 = 0xc9, + SC_P1_HIZ_QUAD_PER_TILE_H14 = 0xca, + SC_P1_HIZ_QUAD_PER_TILE_H15 = 0xcb, + SC_P1_HIZ_QUAD_PER_TILE_H16 = 0xcc, + SC_P2_HIZ_QUAD_PER_TILE_H0 = 0xcd, + SC_P2_HIZ_QUAD_PER_TILE_H1 = 0xce, + SC_P2_HIZ_QUAD_PER_TILE_H2 = 0xcf, + SC_P2_HIZ_QUAD_PER_TILE_H3 = 0xd0, + SC_P2_HIZ_QUAD_PER_TILE_H4 = 0xd1, + SC_P2_HIZ_QUAD_PER_TILE_H5 = 0xd2, + SC_P2_HIZ_QUAD_PER_TILE_H6 = 0xd3, + SC_P2_HIZ_QUAD_PER_TILE_H7 = 0xd4, + SC_P2_HIZ_QUAD_PER_TILE_H8 = 0xd5, + SC_P2_HIZ_QUAD_PER_TILE_H9 = 0xd6, + SC_P2_HIZ_QUAD_PER_TILE_H10 = 0xd7, + SC_P2_HIZ_QUAD_PER_TILE_H11 = 0xd8, + SC_P2_HIZ_QUAD_PER_TILE_H12 = 0xd9, + SC_P2_HIZ_QUAD_PER_TILE_H13 = 0xda, + SC_P2_HIZ_QUAD_PER_TILE_H14 = 0xdb, + SC_P2_HIZ_QUAD_PER_TILE_H15 = 0xdc, + SC_P2_HIZ_QUAD_PER_TILE_H16 = 0xdd, + SC_P3_HIZ_QUAD_PER_TILE_H0 = 0xde, + SC_P3_HIZ_QUAD_PER_TILE_H1 = 0xdf, + SC_P3_HIZ_QUAD_PER_TILE_H2 = 0xe0, + SC_P3_HIZ_QUAD_PER_TILE_H3 = 0xe1, + SC_P3_HIZ_QUAD_PER_TILE_H4 = 0xe2, + SC_P3_HIZ_QUAD_PER_TILE_H5 = 0xe3, + SC_P3_HIZ_QUAD_PER_TILE_H6 = 0xe4, + SC_P3_HIZ_QUAD_PER_TILE_H7 = 0xe5, + SC_P3_HIZ_QUAD_PER_TILE_H8 = 0xe6, + SC_P3_HIZ_QUAD_PER_TILE_H9 = 0xe7, + SC_P3_HIZ_QUAD_PER_TILE_H10 = 0xe8, + SC_P3_HIZ_QUAD_PER_TILE_H11 = 0xe9, + SC_P3_HIZ_QUAD_PER_TILE_H12 = 0xea, + SC_P3_HIZ_QUAD_PER_TILE_H13 = 0xeb, + SC_P3_HIZ_QUAD_PER_TILE_H14 = 0xec, + SC_P3_HIZ_QUAD_PER_TILE_H15 = 0xed, + SC_P3_HIZ_QUAD_PER_TILE_H16 = 0xee, + SC_P0_HIZ_QUAD_COUNT = 0xef, + SC_P1_HIZ_QUAD_COUNT = 0xf0, + SC_P2_HIZ_QUAD_COUNT = 0xf1, + SC_P3_HIZ_QUAD_COUNT = 0xf2, + SC_P0_DETAIL_QUAD_COUNT = 0xf3, + SC_P1_DETAIL_QUAD_COUNT = 0xf4, + SC_P2_DETAIL_QUAD_COUNT = 0xf5, + SC_P3_DETAIL_QUAD_COUNT = 0xf6, + SC_P0_DETAIL_QUAD_WITH_1_PIX = 0xf7, + SC_P0_DETAIL_QUAD_WITH_2_PIX = 0xf8, + SC_P0_DETAIL_QUAD_WITH_3_PIX = 0xf9, + SC_P0_DETAIL_QUAD_WITH_4_PIX = 0xfa, + SC_P1_DETAIL_QUAD_WITH_1_PIX = 0xfb, + SC_P1_DETAIL_QUAD_WITH_2_PIX = 0xfc, + SC_P1_DETAIL_QUAD_WITH_3_PIX = 0xfd, + SC_P1_DETAIL_QUAD_WITH_4_PIX = 0xfe, + SC_P2_DETAIL_QUAD_WITH_1_PIX = 0xff, + SC_P2_DETAIL_QUAD_WITH_2_PIX = 0x100, + SC_P2_DETAIL_QUAD_WITH_3_PIX = 0x101, + SC_P2_DETAIL_QUAD_WITH_4_PIX = 0x102, + SC_P3_DETAIL_QUAD_WITH_1_PIX = 0x103, + SC_P3_DETAIL_QUAD_WITH_2_PIX = 0x104, + SC_P3_DETAIL_QUAD_WITH_3_PIX = 0x105, + SC_P3_DETAIL_QUAD_WITH_4_PIX = 0x106, + SC_EARLYZ_QUAD_COUNT = 0x107, + SC_EARLYZ_QUAD_WITH_1_PIX = 0x108, + SC_EARLYZ_QUAD_WITH_2_PIX = 0x109, + SC_EARLYZ_QUAD_WITH_3_PIX = 0x10a, + SC_EARLYZ_QUAD_WITH_4_PIX = 0x10b, + SC_PKR_QUAD_PER_ROW_H1 = 0x10c, + SC_PKR_QUAD_PER_ROW_H2 = 0x10d, + SC_PKR_QUAD_PER_ROW_H3 = 0x10e, + SC_PKR_QUAD_PER_ROW_H4 = 0x10f, + SC_PKR_END_OF_VECTOR = 0x110, + SC_PKR_CONTROL_XFER = 0x111, + SC_PKR_DBHANG_FORCE_EOV = 0x112, + SC_REG_SCLK_BUSY = 0x113, + SC_GRP0_DYN_SCLK_BUSY = 0x114, + SC_GRP1_DYN_SCLK_BUSY = 0x115, + SC_GRP2_DYN_SCLK_BUSY = 0x116, + SC_GRP3_DYN_SCLK_BUSY = 0x117, + SC_GRP4_DYN_SCLK_BUSY = 0x118, + SC_PA0_SC_DATA_FIFO_RD = 0x119, + SC_PA0_SC_DATA_FIFO_WE = 0x11a, + SC_PA1_SC_DATA_FIFO_RD = 0x11b, + SC_PA1_SC_DATA_FIFO_WE = 0x11c, + SC_PS_ARB_XFC_ALL_EVENT_OR_PRIM_CYCLES = 0x11d, + SC_PS_ARB_XFC_ONLY_PRIM_CYCLES = 0x11e, + SC_PS_ARB_XFC_ONLY_ONE_INC_PER_PRIM = 0x11f, + SC_PS_ARB_STALLED_FROM_BELOW = 0x120, + SC_PS_ARB_STARVED_FROM_ABOVE = 0x121, + SC_PS_ARB_SC_BUSY = 0x122, + SC_PS_ARB_PA_SC_BUSY = 0x123, + SC_PA2_SC_DATA_FIFO_RD = 0x124, + SC_PA2_SC_DATA_FIFO_WE = 0x125, + SC_PA3_SC_DATA_FIFO_RD = 0x126, + SC_PA3_SC_DATA_FIFO_WE = 0x127, + SC_PA_SC_DEALLOC_0_0_WE = 0x128, + SC_PA_SC_DEALLOC_0_1_WE = 0x129, + SC_PA_SC_DEALLOC_1_0_WE = 0x12a, + SC_PA_SC_DEALLOC_1_1_WE = 0x12b, + SC_PA_SC_DEALLOC_2_0_WE = 0x12c, + SC_PA_SC_DEALLOC_2_1_WE = 0x12d, + SC_PA_SC_DEALLOC_3_0_WE = 0x12e, + SC_PA_SC_DEALLOC_3_1_WE = 0x12f, + SC_PA0_SC_EOP_WE = 0x130, + SC_PA0_SC_EOPG_WE = 0x131, + SC_PA0_SC_EVENT_WE = 0x132, + SC_PA1_SC_EOP_WE = 0x133, + SC_PA1_SC_EOPG_WE = 0x134, + SC_PA1_SC_EVENT_WE = 0x135, + SC_PA2_SC_EOP_WE = 0x136, + SC_PA2_SC_EOPG_WE = 0x137, + SC_PA2_SC_EVENT_WE = 0x138, + SC_PA3_SC_EOP_WE = 0x139, + SC_PA3_SC_EOPG_WE = 0x13a, + SC_PA3_SC_EVENT_WE = 0x13b, + SC_PS_ARB_OOO_THRESHOLD_SWITCH_TO_DESIRED_FIFO = 0x13c, + SC_PS_ARB_OOO_FIFO_EMPTY_SWITCH = 0x13d, + SC_PS_ARB_NULL_PRIM_BUBBLE_POP = 0x13e, + SC_PS_ARB_EOP_POP_SYNC_POP = 0x13f, + SC_PS_ARB_EVENT_SYNC_POP = 0x140, + SC_SC_PS_ENG_MULTICYCLE_BUBBLE = 0x141, + SC_PA0_SC_FPOV_WE = 0x142, + SC_PA1_SC_FPOV_WE = 0x143, + SC_PA2_SC_FPOV_WE = 0x144, + SC_PA3_SC_FPOV_WE = 0x145, + SC_PA0_SC_LPOV_WE = 0x146, + SC_PA1_SC_LPOV_WE = 0x147, + SC_PA2_SC_LPOV_WE = 0x148, + SC_PA3_SC_LPOV_WE = 0x149, + SC_SC_SPI_DEALLOC_0_0 = 0x14a, + SC_SC_SPI_DEALLOC_0_1 = 0x14b, + SC_SC_SPI_DEALLOC_0_2 = 0x14c, + SC_SC_SPI_DEALLOC_1_0 = 0x14d, + SC_SC_SPI_DEALLOC_1_1 = 0x14e, + SC_SC_SPI_DEALLOC_1_2 = 0x14f, + SC_SC_SPI_DEALLOC_2_0 = 0x150, + SC_SC_SPI_DEALLOC_2_1 = 0x151, + SC_SC_SPI_DEALLOC_2_2 = 0x152, + SC_SC_SPI_DEALLOC_3_0 = 0x153, + SC_SC_SPI_DEALLOC_3_1 = 0x154, + SC_SC_SPI_DEALLOC_3_2 = 0x155, + SC_SC_SPI_FPOV_0 = 0x156, + SC_SC_SPI_FPOV_1 = 0x157, + SC_SC_SPI_FPOV_2 = 0x158, + SC_SC_SPI_FPOV_3 = 0x159, + SC_SC_SPI_EVENT = 0x15a, + SC_PS_TS_EVENT_FIFO_PUSH = 0x15b, + SC_PS_TS_EVENT_FIFO_POP = 0x15c, + SC_PS_CTX_DONE_FIFO_PUSH = 0x15d, + SC_PS_CTX_DONE_FIFO_POP = 0x15e, + SC_MULTICYCLE_BUBBLE_FREEZE = 0x15f, + SC_EOP_SYNC_WINDOW = 0x160, + SC_PA0_SC_NULL_WE = 0x161, + SC_PA0_SC_NULL_DEALLOC_WE = 0x162, + SC_PA0_SC_DATA_FIFO_EOPG_RD = 0x163, + SC_PA0_SC_DATA_FIFO_EOP_RD = 0x164, + SC_PA0_SC_DEALLOC_0_RD = 0x165, + SC_PA0_SC_DEALLOC_1_RD = 0x166, + SC_PA1_SC_DATA_FIFO_EOPG_RD = 0x167, + SC_PA1_SC_DATA_FIFO_EOP_RD = 0x168, + SC_PA1_SC_DEALLOC_0_RD = 0x169, + SC_PA1_SC_DEALLOC_1_RD = 0x16a, + SC_PA1_SC_NULL_WE = 0x16b, + SC_PA1_SC_NULL_DEALLOC_WE = 0x16c, + SC_PA2_SC_DATA_FIFO_EOPG_RD = 0x16d, + SC_PA2_SC_DATA_FIFO_EOP_RD = 0x16e, + SC_PA2_SC_DEALLOC_0_RD = 0x16f, + SC_PA2_SC_DEALLOC_1_RD = 0x170, + SC_PA2_SC_NULL_WE = 0x171, + SC_PA2_SC_NULL_DEALLOC_WE = 0x172, + SC_PA3_SC_DATA_FIFO_EOPG_RD = 0x173, + SC_PA3_SC_DATA_FIFO_EOP_RD = 0x174, + SC_PA3_SC_DEALLOC_0_RD = 0x175, + SC_PA3_SC_DEALLOC_1_RD = 0x176, + SC_PA3_SC_NULL_WE = 0x177, + SC_PA3_SC_NULL_DEALLOC_WE = 0x178, + SC_PS_PA0_SC_FIFO_EMPTY = 0x179, + SC_PS_PA0_SC_FIFO_FULL = 0x17a, + SC_PA0_PS_DATA_SEND = 0x17b, + SC_PS_PA1_SC_FIFO_EMPTY = 0x17c, + SC_PS_PA1_SC_FIFO_FULL = 0x17d, + SC_PA1_PS_DATA_SEND = 0x17e, + SC_PS_PA2_SC_FIFO_EMPTY = 0x17f, + SC_PS_PA2_SC_FIFO_FULL = 0x180, + SC_PA2_PS_DATA_SEND = 0x181, + SC_PS_PA3_SC_FIFO_EMPTY = 0x182, + SC_PS_PA3_SC_FIFO_FULL = 0x183, + SC_PA3_PS_DATA_SEND = 0x184, + SC_BUSY_PROCESSING_MULTICYCLE_PRIM = 0x185, + SC_BUSY_CNT_NOT_ZERO = 0x186, + SC_BM_BUSY = 0x187, + SC_BACKEND_BUSY = 0x188, + SC_SCF_SCB_INTERFACE_BUSY = 0x189, + SC_SCB_BUSY = 0x18a, +} SC_PERFCNT_SEL; +typedef enum SePairXsel { + RASTER_CONFIG_SE_PAIR_XSEL_8_WIDE_TILE = 0x0, + RASTER_CONFIG_SE_PAIR_XSEL_16_WIDE_TILE = 0x1, + RASTER_CONFIG_SE_PAIR_XSEL_32_WIDE_TILE = 0x2, + RASTER_CONFIG_SE_PAIR_XSEL_64_WIDE_TILE = 0x3, +} SePairXsel; +typedef enum SePairYsel { + RASTER_CONFIG_SE_PAIR_YSEL_8_WIDE_TILE = 0x0, + RASTER_CONFIG_SE_PAIR_YSEL_16_WIDE_TILE = 0x1, + RASTER_CONFIG_SE_PAIR_YSEL_32_WIDE_TILE = 0x2, + RASTER_CONFIG_SE_PAIR_YSEL_64_WIDE_TILE = 0x3, +} SePairYsel; +typedef enum SePairMap { + RASTER_CONFIG_SE_PAIR_MAP_0 = 0x0, + RASTER_CONFIG_SE_PAIR_MAP_1 = 0x1, + RASTER_CONFIG_SE_PAIR_MAP_2 = 0x2, + RASTER_CONFIG_SE_PAIR_MAP_3 = 0x3, +} SePairMap; +typedef enum SeXsel { + RASTER_CONFIG_SE_XSEL_8_WIDE_TILE = 0x0, + RASTER_CONFIG_SE_XSEL_16_WIDE_TILE = 0x1, + RASTER_CONFIG_SE_XSEL_32_WIDE_TILE = 0x2, + RASTER_CONFIG_SE_XSEL_64_WIDE_TILE = 0x3, +} SeXsel; +typedef enum SeYsel { + RASTER_CONFIG_SE_YSEL_8_WIDE_TILE = 0x0, + RASTER_CONFIG_SE_YSEL_16_WIDE_TILE = 0x1, + RASTER_CONFIG_SE_YSEL_32_WIDE_TILE = 0x2, + RASTER_CONFIG_SE_YSEL_64_WIDE_TILE = 0x3, +} SeYsel; +typedef enum SeMap { + RASTER_CONFIG_SE_MAP_0 = 0x0, + RASTER_CONFIG_SE_MAP_1 = 0x1, + RASTER_CONFIG_SE_MAP_2 = 0x2, + RASTER_CONFIG_SE_MAP_3 = 0x3, +} SeMap; +typedef enum ScXsel { + RASTER_CONFIG_SC_XSEL_8_WIDE_TILE = 0x0, + RASTER_CONFIG_SC_XSEL_16_WIDE_TILE = 0x1, + RASTER_CONFIG_SC_XSEL_32_WIDE_TILE = 0x2, + RASTER_CONFIG_SC_XSEL_64_WIDE_TILE = 0x3, +} ScXsel; +typedef enum ScYsel { + RASTER_CONFIG_SC_YSEL_8_WIDE_TILE = 0x0, + RASTER_CONFIG_SC_YSEL_16_WIDE_TILE = 0x1, + RASTER_CONFIG_SC_YSEL_32_WIDE_TILE = 0x2, + RASTER_CONFIG_SC_YSEL_64_WIDE_TILE = 0x3, +} ScYsel; +typedef enum ScMap { + RASTER_CONFIG_SC_MAP_0 = 0x0, + RASTER_CONFIG_SC_MAP_1 = 0x1, + RASTER_CONFIG_SC_MAP_2 = 0x2, + RASTER_CONFIG_SC_MAP_3 = 0x3, +} ScMap; +typedef enum PkrXsel2 { + RASTER_CONFIG_PKR_XSEL2_0 = 0x0, + RASTER_CONFIG_PKR_XSEL2_1 = 0x1, + RASTER_CONFIG_PKR_XSEL2_2 = 0x2, + RASTER_CONFIG_PKR_XSEL2_3 = 0x3, +} PkrXsel2; +typedef enum PkrXsel { + RASTER_CONFIG_PKR_XSEL_0 = 0x0, + RASTER_CONFIG_PKR_XSEL_1 = 0x1, + RASTER_CONFIG_PKR_XSEL_2 = 0x2, + RASTER_CONFIG_PKR_XSEL_3 = 0x3, +} PkrXsel; +typedef enum PkrYsel { + RASTER_CONFIG_PKR_YSEL_0 = 0x0, + RASTER_CONFIG_PKR_YSEL_1 = 0x1, + RASTER_CONFIG_PKR_YSEL_2 = 0x2, + RASTER_CONFIG_PKR_YSEL_3 = 0x3, +} PkrYsel; +typedef enum PkrMap { + RASTER_CONFIG_PKR_MAP_0 = 0x0, + RASTER_CONFIG_PKR_MAP_1 = 0x1, + RASTER_CONFIG_PKR_MAP_2 = 0x2, + RASTER_CONFIG_PKR_MAP_3 = 0x3, +} PkrMap; +typedef enum RbXsel { + RASTER_CONFIG_RB_XSEL_0 = 0x0, + RASTER_CONFIG_RB_XSEL_1 = 0x1, +} RbXsel; +typedef enum RbYsel { + RASTER_CONFIG_RB_YSEL_0 = 0x0, + RASTER_CONFIG_RB_YSEL_1 = 0x1, +} RbYsel; +typedef enum RbXsel2 { + RASTER_CONFIG_RB_XSEL2_0 = 0x0, + RASTER_CONFIG_RB_XSEL2_1 = 0x1, + RASTER_CONFIG_RB_XSEL2_2 = 0x2, + RASTER_CONFIG_RB_XSEL2_3 = 0x3, +} RbXsel2; +typedef enum RbMap { + RASTER_CONFIG_RB_MAP_0 = 0x0, + RASTER_CONFIG_RB_MAP_1 = 0x1, + RASTER_CONFIG_RB_MAP_2 = 0x2, + RASTER_CONFIG_RB_MAP_3 = 0x3, +} RbMap; +typedef enum CSDATA_TYPE { + CSDATA_TYPE_TG = 0x0, + CSDATA_TYPE_STATE = 0x1, + CSDATA_TYPE_EVENT = 0x2, + CSDATA_TYPE_PRIVATE = 0x3, +} CSDATA_TYPE; +#define CSDATA_TYPE_WIDTH 0x2 +#define CSDATA_ADDR_WIDTH 0x7 +#define CSDATA_DATA_WIDTH 0x20 +typedef enum SPI_SAMPLE_CNTL { + CENTROIDS_ONLY = 0x0, + CENTERS_ONLY = 0x1, + CENTROIDS_AND_CENTERS = 0x2, + UNDEF = 0x3, +} SPI_SAMPLE_CNTL; +typedef enum SPI_FOG_MODE { + SPI_FOG_NONE = 0x0, + SPI_FOG_EXP = 0x1, + SPI_FOG_EXP2 = 0x2, + SPI_FOG_LINEAR = 0x3, +} SPI_FOG_MODE; +typedef enum SPI_PNT_SPRITE_OVERRIDE { + SPI_PNT_SPRITE_SEL_0 = 0x0, + SPI_PNT_SPRITE_SEL_1 = 0x1, + SPI_PNT_SPRITE_SEL_S = 0x2, + SPI_PNT_SPRITE_SEL_T = 0x3, + SPI_PNT_SPRITE_SEL_NONE = 0x4, +} SPI_PNT_SPRITE_OVERRIDE; +typedef enum SPI_PERFCNT_SEL { + SPI_PERF_VS_WINDOW_VALID = 0x0, + SPI_PERF_VS_BUSY = 0x1, + SPI_PERF_VS_FIRST_WAVE = 0x2, + SPI_PERF_VS_LAST_WAVE = 0x3, + SPI_PERF_VS_LSHS_DEALLOC = 0x4, + SPI_PERF_VS_PC_STALL = 0x5, + SPI_PERF_VS_POS0_STALL = 0x6, + SPI_PERF_VS_POS1_STALL = 0x7, + SPI_PERF_VS_CRAWLER_STALL = 0x8, + SPI_PERF_VS_EVENT_WAVE = 0x9, + SPI_PERF_VS_WAVE = 0xa, + SPI_PERF_VS_PERS_UPD_FULL0 = 0xb, + SPI_PERF_VS_PERS_UPD_FULL1 = 0xc, + SPI_PERF_VS_LATE_ALLOC_FULL = 0xd, + SPI_PERF_VS_FIRST_SUBGRP = 0xe, + SPI_PERF_VS_LAST_SUBGRP = 0xf, + SPI_PERF_GS_WINDOW_VALID = 0x10, + SPI_PERF_GS_BUSY = 0x11, + SPI_PERF_GS_CRAWLER_STALL = 0x12, + SPI_PERF_GS_EVENT_WAVE = 0x13, + SPI_PERF_GS_WAVE = 0x14, + SPI_PERF_GS_PERS_UPD_FULL0 = 0x15, + SPI_PERF_GS_PERS_UPD_FULL1 = 0x16, + SPI_PERF_GS_FIRST_SUBGRP = 0x17, + SPI_PERF_GS_LAST_SUBGRP = 0x18, + SPI_PERF_ES_WINDOW_VALID = 0x19, + SPI_PERF_ES_BUSY = 0x1a, + SPI_PERF_ES_CRAWLER_STALL = 0x1b, + SPI_PERF_ES_FIRST_WAVE = 0x1c, + SPI_PERF_ES_LAST_WAVE = 0x1d, + SPI_PERF_ES_LSHS_DEALLOC = 0x1e, + SPI_PERF_ES_EVENT_WAVE = 0x1f, + SPI_PERF_ES_WAVE = 0x20, + SPI_PERF_ES_PERS_UPD_FULL0 = 0x21, + SPI_PERF_ES_PERS_UPD_FULL1 = 0x22, + SPI_PERF_ES_FIRST_SUBGRP = 0x23, + SPI_PERF_ES_LAST_SUBGRP = 0x24, + SPI_PERF_HS_WINDOW_VALID = 0x25, + SPI_PERF_HS_BUSY = 0x26, + SPI_PERF_HS_CRAWLER_STALL = 0x27, + SPI_PERF_HS_FIRST_WAVE = 0x28, + SPI_PERF_HS_LAST_WAVE = 0x29, + SPI_PERF_HS_LSHS_DEALLOC = 0x2a, + SPI_PERF_HS_EVENT_WAVE = 0x2b, + SPI_PERF_HS_WAVE = 0x2c, + SPI_PERF_HS_PERS_UPD_FULL0 = 0x2d, + SPI_PERF_HS_PERS_UPD_FULL1 = 0x2e, + SPI_PERF_LS_WINDOW_VALID = 0x2f, + SPI_PERF_LS_BUSY = 0x30, + SPI_PERF_LS_CRAWLER_STALL = 0x31, + SPI_PERF_LS_FIRST_WAVE = 0x32, + SPI_PERF_LS_LAST_WAVE = 0x33, + SPI_PERF_OFFCHIP_LDS_STALL_LS = 0x34, + SPI_PERF_LS_EVENT_WAVE = 0x35, + SPI_PERF_LS_WAVE = 0x36, + SPI_PERF_LS_PERS_UPD_FULL0 = 0x37, + SPI_PERF_LS_PERS_UPD_FULL1 = 0x38, + SPI_PERF_CSG_WINDOW_VALID = 0x39, + SPI_PERF_CSG_BUSY = 0x3a, + SPI_PERF_CSG_NUM_THREADGROUPS = 0x3b, + SPI_PERF_CSG_CRAWLER_STALL = 0x3c, + SPI_PERF_CSG_EVENT_WAVE = 0x3d, + SPI_PERF_CSG_WAVE = 0x3e, + SPI_PERF_CSN_WINDOW_VALID = 0x3f, + SPI_PERF_CSN_BUSY = 0x40, + SPI_PERF_CSN_NUM_THREADGROUPS = 0x41, + SPI_PERF_CSN_CRAWLER_STALL = 0x42, + SPI_PERF_CSN_EVENT_WAVE = 0x43, + SPI_PERF_CSN_WAVE = 0x44, + SPI_PERF_PS_CTL_WINDOW_VALID = 0x45, + SPI_PERF_PS_CTL_BUSY = 0x46, + SPI_PERF_PS_CTL_ACTIVE = 0x47, + SPI_PERF_PS_CTL_DEALLOC_BIN0 = 0x48, + SPI_PERF_PS_CTL_FPOS_BIN1_STALL = 0x49, + SPI_PERF_PS_CTL_EVENT_WAVE = 0x4a, + SPI_PERF_PS_CTL_WAVE = 0x4b, + SPI_PERF_PS_CTL_OPT_WAVE = 0x4c, + SPI_PERF_PS_CTL_PASS_BIN0 = 0x4d, + SPI_PERF_PS_CTL_PASS_BIN1 = 0x4e, + SPI_PERF_PS_CTL_FPOS_BIN2 = 0x4f, + SPI_PERF_PS_CTL_PRIM_BIN0 = 0x50, + SPI_PERF_PS_CTL_PRIM_BIN1 = 0x51, + SPI_PERF_PS_CTL_CNF_BIN2 = 0x52, + SPI_PERF_PS_CTL_CNF_BIN3 = 0x53, + SPI_PERF_PS_CTL_CRAWLER_STALL = 0x54, + SPI_PERF_PS_CTL_LDS_RES_FULL = 0x55, + SPI_PERF_PS_PERS_UPD_FULL0 = 0x56, + SPI_PERF_PS_PERS_UPD_FULL1 = 0x57, + SPI_PERF_PIX_ALLOC_PEND_CNT = 0x58, + SPI_PERF_PIX_ALLOC_SCB_STALL = 0x59, + SPI_PERF_PIX_ALLOC_DB0_STALL = 0x5a, + SPI_PERF_PIX_ALLOC_DB1_STALL = 0x5b, + SPI_PERF_PIX_ALLOC_DB2_STALL = 0x5c, + SPI_PERF_PIX_ALLOC_DB3_STALL = 0x5d, + SPI_PERF_LDS0_PC_VALID = 0x5e, + SPI_PERF_LDS1_PC_VALID = 0x5f, + SPI_PERF_RA_PIPE_REQ_BIN2 = 0x60, + SPI_PERF_RA_TASK_REQ_BIN3 = 0x61, + SPI_PERF_RA_WR_CTL_FULL = 0x62, + SPI_PERF_RA_REQ_NO_ALLOC = 0x63, + SPI_PERF_RA_REQ_NO_ALLOC_PS = 0x64, + SPI_PERF_RA_REQ_NO_ALLOC_VS = 0x65, + SPI_PERF_RA_REQ_NO_ALLOC_GS = 0x66, + SPI_PERF_RA_REQ_NO_ALLOC_ES = 0x67, + SPI_PERF_RA_REQ_NO_ALLOC_HS = 0x68, + SPI_PERF_RA_REQ_NO_ALLOC_LS = 0x69, + SPI_PERF_RA_REQ_NO_ALLOC_CSG = 0x6a, + SPI_PERF_RA_REQ_NO_ALLOC_CSN = 0x6b, + SPI_PERF_RA_RES_STALL_PS = 0x6c, + SPI_PERF_RA_RES_STALL_VS = 0x6d, + SPI_PERF_RA_RES_STALL_GS = 0x6e, + SPI_PERF_RA_RES_STALL_ES = 0x6f, + SPI_PERF_RA_RES_STALL_HS = 0x70, + SPI_PERF_RA_RES_STALL_LS = 0x71, + SPI_PERF_RA_RES_STALL_CSG = 0x72, + SPI_PERF_RA_RES_STALL_CSN = 0x73, + SPI_PERF_RA_TMP_STALL_PS = 0x74, + SPI_PERF_RA_TMP_STALL_VS = 0x75, + SPI_PERF_RA_TMP_STALL_GS = 0x76, + SPI_PERF_RA_TMP_STALL_ES = 0x77, + SPI_PERF_RA_TMP_STALL_HS = 0x78, + SPI_PERF_RA_TMP_STALL_LS = 0x79, + SPI_PERF_RA_TMP_STALL_CSG = 0x7a, + SPI_PERF_RA_TMP_STALL_CSN = 0x7b, + SPI_PERF_RA_WAVE_SIMD_FULL_PS = 0x7c, + SPI_PERF_RA_WAVE_SIMD_FULL_VS = 0x7d, + SPI_PERF_RA_WAVE_SIMD_FULL_GS = 0x7e, + SPI_PERF_RA_WAVE_SIMD_FULL_ES = 0x7f, + SPI_PERF_RA_WAVE_SIMD_FULL_HS = 0x80, + SPI_PERF_RA_WAVE_SIMD_FULL_LS = 0x81, + SPI_PERF_RA_WAVE_SIMD_FULL_CSG = 0x82, + SPI_PERF_RA_WAVE_SIMD_FULL_CSN = 0x83, + SPI_PERF_RA_VGPR_SIMD_FULL_PS = 0x84, + SPI_PERF_RA_VGPR_SIMD_FULL_VS = 0x85, + SPI_PERF_RA_VGPR_SIMD_FULL_GS = 0x86, + SPI_PERF_RA_VGPR_SIMD_FULL_ES = 0x87, + SPI_PERF_RA_VGPR_SIMD_FULL_HS = 0x88, + SPI_PERF_RA_VGPR_SIMD_FULL_LS = 0x89, + SPI_PERF_RA_VGPR_SIMD_FULL_CSG = 0x8a, + SPI_PERF_RA_VGPR_SIMD_FULL_CSN = 0x8b, + SPI_PERF_RA_SGPR_SIMD_FULL_PS = 0x8c, + SPI_PERF_RA_SGPR_SIMD_FULL_VS = 0x8d, + SPI_PERF_RA_SGPR_SIMD_FULL_GS = 0x8e, + SPI_PERF_RA_SGPR_SIMD_FULL_ES = 0x8f, + SPI_PERF_RA_SGPR_SIMD_FULL_HS = 0x90, + SPI_PERF_RA_SGPR_SIMD_FULL_LS = 0x91, + SPI_PERF_RA_SGPR_SIMD_FULL_CSG = 0x92, + SPI_PERF_RA_SGPR_SIMD_FULL_CSN = 0x93, + SPI_PERF_RA_LDS_CU_FULL_PS = 0x94, + SPI_PERF_RA_LDS_CU_FULL_LS = 0x95, + SPI_PERF_RA_LDS_CU_FULL_ES = 0x96, + SPI_PERF_RA_LDS_CU_FULL_CSG = 0x97, + SPI_PERF_RA_LDS_CU_FULL_CSN = 0x98, + SPI_PERF_RA_BAR_CU_FULL_HS = 0x99, + SPI_PERF_RA_BAR_CU_FULL_CSG = 0x9a, + SPI_PERF_RA_BAR_CU_FULL_CSN = 0x9b, + SPI_PERF_RA_BULKY_CU_FULL_CSG = 0x9c, + SPI_PERF_RA_BULKY_CU_FULL_CSN = 0x9d, + SPI_PERF_RA_TGLIM_CU_FULL_CSG = 0x9e, + SPI_PERF_RA_TGLIM_CU_FULL_CSN = 0x9f, + SPI_PERF_RA_WVLIM_STALL_PS = 0xa0, + SPI_PERF_RA_WVLIM_STALL_VS = 0xa1, + SPI_PERF_RA_WVLIM_STALL_GS = 0xa2, + SPI_PERF_RA_WVLIM_STALL_ES = 0xa3, + SPI_PERF_RA_WVLIM_STALL_HS = 0xa4, + SPI_PERF_RA_WVLIM_STALL_LS = 0xa5, + SPI_PERF_RA_WVLIM_STALL_CSG = 0xa6, + SPI_PERF_RA_WVLIM_STALL_CSN = 0xa7, + SPI_PERF_RA_PS_LOCK = 0xa8, + SPI_PERF_RA_VS_LOCK = 0xa9, + SPI_PERF_RA_GS_LOCK = 0xaa, + SPI_PERF_RA_ES_LOCK = 0xab, + SPI_PERF_RA_HS_LOCK = 0xac, + SPI_PERF_RA_LS_LOCK = 0xad, + SPI_PERF_RA_CSG_LOCK = 0xae, + SPI_PERF_RA_CSN_LOCK = 0xaf, + SPI_PERF_RA_RSV_UPD = 0xb0, + SPI_PERF_EXP_ARB_COL_CNT = 0xb1, + SPI_PERF_EXP_ARB_PAR_CNT = 0xb2, + SPI_PERF_EXP_ARB_POS_CNT = 0xb3, + SPI_PERF_EXP_ARB_GDS_CNT = 0xb4, + SPI_PERF_CLKGATE_BUSY_STALL = 0xb5, + SPI_PERF_CLKGATE_ACTIVE_STALL = 0xb6, + SPI_PERF_CLKGATE_ALL_CLOCKS_ON = 0xb7, + SPI_PERF_CLKGATE_CGTT_DYN_ON = 0xb8, + SPI_PERF_CLKGATE_CGTT_REG_ON = 0xb9, +} SPI_PERFCNT_SEL; +typedef enum SPI_SHADER_FORMAT { + SPI_SHADER_NONE = 0x0, + SPI_SHADER_1COMP = 0x1, + SPI_SHADER_2COMP = 0x2, + SPI_SHADER_4COMPRESS = 0x3, + SPI_SHADER_4COMP = 0x4, +} SPI_SHADER_FORMAT; +typedef enum SPI_SHADER_EX_FORMAT { + SPI_SHADER_ZERO = 0x0, + SPI_SHADER_32_R = 0x1, + SPI_SHADER_32_GR = 0x2, + SPI_SHADER_32_AR = 0x3, + SPI_SHADER_FP16_ABGR = 0x4, + SPI_SHADER_UNORM16_ABGR = 0x5, + SPI_SHADER_SNORM16_ABGR = 0x6, + SPI_SHADER_UINT16_ABGR = 0x7, + SPI_SHADER_SINT16_ABGR = 0x8, + SPI_SHADER_32_ABGR = 0x9, +} SPI_SHADER_EX_FORMAT; +typedef enum CLKGATE_SM_MODE { + ON_SEQ = 0x0, + OFF_SEQ = 0x1, + PROG_SEQ = 0x2, + READ_SEQ = 0x3, + SM_MODE_RESERVED = 0x4, +} CLKGATE_SM_MODE; +typedef enum CLKGATE_BASE_MODE { + MULT_8 = 0x0, + MULT_16 = 0x1, +} CLKGATE_BASE_MODE; +typedef enum SQ_TEX_CLAMP { + SQ_TEX_WRAP = 0x0, + SQ_TEX_MIRROR = 0x1, + SQ_TEX_CLAMP_LAST_TEXEL = 0x2, + SQ_TEX_MIRROR_ONCE_LAST_TEXEL = 0x3, + SQ_TEX_CLAMP_HALF_BORDER = 0x4, + SQ_TEX_MIRROR_ONCE_HALF_BORDER = 0x5, + SQ_TEX_CLAMP_BORDER = 0x6, + SQ_TEX_MIRROR_ONCE_BORDER = 0x7, +} SQ_TEX_CLAMP; +typedef enum SQ_TEX_XY_FILTER { + SQ_TEX_XY_FILTER_POINT = 0x0, + SQ_TEX_XY_FILTER_BILINEAR = 0x1, + SQ_TEX_XY_FILTER_ANISO_POINT = 0x2, + SQ_TEX_XY_FILTER_ANISO_BILINEAR = 0x3, +} SQ_TEX_XY_FILTER; +typedef enum SQ_TEX_Z_FILTER { + SQ_TEX_Z_FILTER_NONE = 0x0, + SQ_TEX_Z_FILTER_POINT = 0x1, + SQ_TEX_Z_FILTER_LINEAR = 0x2, +} SQ_TEX_Z_FILTER; +typedef enum SQ_TEX_MIP_FILTER { + SQ_TEX_MIP_FILTER_NONE = 0x0, + SQ_TEX_MIP_FILTER_POINT = 0x1, + SQ_TEX_MIP_FILTER_LINEAR = 0x2, +} SQ_TEX_MIP_FILTER; +typedef enum SQ_TEX_ANISO_RATIO { + SQ_TEX_ANISO_RATIO_1 = 0x0, + SQ_TEX_ANISO_RATIO_2 = 0x1, + SQ_TEX_ANISO_RATIO_4 = 0x2, + SQ_TEX_ANISO_RATIO_8 = 0x3, + SQ_TEX_ANISO_RATIO_16 = 0x4, +} SQ_TEX_ANISO_RATIO; +typedef enum SQ_TEX_DEPTH_COMPARE { + SQ_TEX_DEPTH_COMPARE_NEVER = 0x0, + SQ_TEX_DEPTH_COMPARE_LESS = 0x1, + SQ_TEX_DEPTH_COMPARE_EQUAL = 0x2, + SQ_TEX_DEPTH_COMPARE_LESSEQUAL = 0x3, + SQ_TEX_DEPTH_COMPARE_GREATER = 0x4, + SQ_TEX_DEPTH_COMPARE_NOTEQUAL = 0x5, + SQ_TEX_DEPTH_COMPARE_GREATEREQUAL = 0x6, + SQ_TEX_DEPTH_COMPARE_ALWAYS = 0x7, +} SQ_TEX_DEPTH_COMPARE; +typedef enum SQ_TEX_BORDER_COLOR { + SQ_TEX_BORDER_COLOR_TRANS_BLACK = 0x0, + SQ_TEX_BORDER_COLOR_OPAQUE_BLACK = 0x1, + SQ_TEX_BORDER_COLOR_OPAQUE_WHITE = 0x2, + SQ_TEX_BORDER_COLOR_REGISTER = 0x3, +} SQ_TEX_BORDER_COLOR; +typedef enum SQ_RSRC_BUF_TYPE { + SQ_RSRC_BUF = 0x0, + SQ_RSRC_BUF_RSVD_1 = 0x1, + SQ_RSRC_BUF_RSVD_2 = 0x2, + SQ_RSRC_BUF_RSVD_3 = 0x3, +} SQ_RSRC_BUF_TYPE; +typedef enum SQ_RSRC_IMG_TYPE { + SQ_RSRC_IMG_RSVD_0 = 0x0, + SQ_RSRC_IMG_RSVD_1 = 0x1, + SQ_RSRC_IMG_RSVD_2 = 0x2, + SQ_RSRC_IMG_RSVD_3 = 0x3, + SQ_RSRC_IMG_RSVD_4 = 0x4, + SQ_RSRC_IMG_RSVD_5 = 0x5, + SQ_RSRC_IMG_RSVD_6 = 0x6, + SQ_RSRC_IMG_RSVD_7 = 0x7, + SQ_RSRC_IMG_1D = 0x8, + SQ_RSRC_IMG_2D = 0x9, + SQ_RSRC_IMG_3D = 0xa, + SQ_RSRC_IMG_CUBE = 0xb, + SQ_RSRC_IMG_1D_ARRAY = 0xc, + SQ_RSRC_IMG_2D_ARRAY = 0xd, + SQ_RSRC_IMG_2D_MSAA = 0xe, + SQ_RSRC_IMG_2D_MSAA_ARRAY = 0xf, +} SQ_RSRC_IMG_TYPE; +typedef enum SQ_RSRC_FLAT_TYPE { + SQ_RSRC_FLAT_RSVD_0 = 0x0, + SQ_RSRC_FLAT = 0x1, + SQ_RSRC_FLAT_RSVD_2 = 0x2, + SQ_RSRC_FLAT_RSVD_3 = 0x3, +} SQ_RSRC_FLAT_TYPE; +typedef enum SQ_IMG_FILTER_TYPE { + SQ_IMG_FILTER_MODE_BLEND = 0x0, + SQ_IMG_FILTER_MODE_MIN = 0x1, + SQ_IMG_FILTER_MODE_MAX = 0x2, +} SQ_IMG_FILTER_TYPE; +typedef enum SQ_SEL_XYZW01 { + SQ_SEL_0 = 0x0, + SQ_SEL_1 = 0x1, + SQ_SEL_RESERVED_0 = 0x2, + SQ_SEL_RESERVED_1 = 0x3, + SQ_SEL_X = 0x4, + SQ_SEL_Y = 0x5, + SQ_SEL_Z = 0x6, + SQ_SEL_W = 0x7, +} SQ_SEL_XYZW01; +typedef enum SQ_WAVE_TYPE { + SQ_WAVE_TYPE_PS = 0x0, + SQ_WAVE_TYPE_VS = 0x1, + SQ_WAVE_TYPE_GS = 0x2, + SQ_WAVE_TYPE_ES = 0x3, + SQ_WAVE_TYPE_HS = 0x4, + SQ_WAVE_TYPE_LS = 0x5, + SQ_WAVE_TYPE_CS = 0x6, + SQ_WAVE_TYPE_PS1 = 0x7, +} SQ_WAVE_TYPE; +typedef enum SQ_THREAD_TRACE_TOKEN_TYPE { + SQ_THREAD_TRACE_TOKEN_MISC = 0x0, + SQ_THREAD_TRACE_TOKEN_TIMESTAMP = 0x1, + SQ_THREAD_TRACE_TOKEN_REG = 0x2, + SQ_THREAD_TRACE_TOKEN_WAVE_START = 0x3, + SQ_THREAD_TRACE_TOKEN_WAVE_ALLOC = 0x4, + SQ_THREAD_TRACE_TOKEN_REG_CSPRIV = 0x5, + SQ_THREAD_TRACE_TOKEN_WAVE_END = 0x6, + SQ_THREAD_TRACE_TOKEN_EVENT = 0x7, + SQ_THREAD_TRACE_TOKEN_EVENT_CS = 0x8, + SQ_THREAD_TRACE_TOKEN_EVENT_GFX1 = 0x9, + SQ_THREAD_TRACE_TOKEN_INST = 0xa, + SQ_THREAD_TRACE_TOKEN_INST_PC = 0xb, + SQ_THREAD_TRACE_TOKEN_INST_USERDATA = 0xc, + SQ_THREAD_TRACE_TOKEN_ISSUE = 0xd, + SQ_THREAD_TRACE_TOKEN_PERF = 0xe, + SQ_THREAD_TRACE_TOKEN_REG_CS = 0xf, +} SQ_THREAD_TRACE_TOKEN_TYPE; +typedef enum SQ_THREAD_TRACE_MISC_TOKEN_TYPE { + SQ_THREAD_TRACE_MISC_TOKEN_TIME = 0x0, + SQ_THREAD_TRACE_MISC_TOKEN_TIME_RESET = 0x1, + SQ_THREAD_TRACE_MISC_TOKEN_PACKET_LOST = 0x2, + SQ_THREAD_TRACE_MISC_TOKEN_SURF_SYNC = 0x3, + SQ_THREAD_TRACE_MISC_TOKEN_TTRACE_STALL_BEGIN = 0x4, + SQ_THREAD_TRACE_MISC_TOKEN_TTRACE_STALL_END = 0x5, +} SQ_THREAD_TRACE_MISC_TOKEN_TYPE; +typedef enum SQ_THREAD_TRACE_INST_TYPE { + SQ_THREAD_TRACE_INST_TYPE_SMEM = 0x0, + SQ_THREAD_TRACE_INST_TYPE_SALU = 0x1, + SQ_THREAD_TRACE_INST_TYPE_VMEM_RD = 0x2, + SQ_THREAD_TRACE_INST_TYPE_VMEM_WR = 0x3, + SQ_THREAD_TRACE_INST_TYPE_FLAT_WR = 0x4, + SQ_THREAD_TRACE_INST_TYPE_VALU = 0x5, + SQ_THREAD_TRACE_INST_TYPE_LDS = 0x6, + SQ_THREAD_TRACE_INST_TYPE_PC = 0x7, + SQ_THREAD_TRACE_INST_TYPE_EXPREQ_GDS = 0x8, + SQ_THREAD_TRACE_INST_TYPE_EXPREQ_GFX = 0x9, + SQ_THREAD_TRACE_INST_TYPE_EXPGNT_PAR_COL = 0xa, + SQ_THREAD_TRACE_INST_TYPE_EXPGNT_POS_GDS = 0xb, + SQ_THREAD_TRACE_INST_TYPE_JUMP = 0xc, + SQ_THREAD_TRACE_INST_TYPE_NEXT = 0xd, + SQ_THREAD_TRACE_INST_TYPE_FLAT_RD = 0xe, + SQ_THREAD_TRACE_INST_TYPE_OTHER_MSG = 0xf, +} SQ_THREAD_TRACE_INST_TYPE; +typedef enum SQ_THREAD_TRACE_REG_TYPE { + SQ_THREAD_TRACE_REG_TYPE_EVENT = 0x0, + SQ_THREAD_TRACE_REG_TYPE_DRAW = 0x1, + SQ_THREAD_TRACE_REG_TYPE_DISPATCH = 0x2, + SQ_THREAD_TRACE_REG_TYPE_USERDATA = 0x3, + SQ_THREAD_TRACE_REG_TYPE_MARKER = 0x4, + SQ_THREAD_TRACE_REG_TYPE_GFXDEC = 0x5, + SQ_THREAD_TRACE_REG_TYPE_SHDEC = 0x6, + SQ_THREAD_TRACE_REG_TYPE_OTHER = 0x7, +} SQ_THREAD_TRACE_REG_TYPE; +typedef enum SQ_THREAD_TRACE_REG_OP { + SQ_THREAD_TRACE_REG_OP_READ = 0x0, + SQ_THREAD_TRACE_REG_OP_WRITE = 0x1, +} SQ_THREAD_TRACE_REG_OP; +typedef enum SQ_THREAD_TRACE_MODE_SEL { + SQ_THREAD_TRACE_MODE_OFF = 0x0, + SQ_THREAD_TRACE_MODE_ON = 0x1, + SQ_THREAD_TRACE_MODE_RANDOM = 0x2, +} SQ_THREAD_TRACE_MODE_SEL; +typedef enum SQ_THREAD_TRACE_CAPTURE_MODE { + SQ_THREAD_TRACE_CAPTURE_MODE_ALL = 0x0, + SQ_THREAD_TRACE_CAPTURE_MODE_SELECT = 0x1, + SQ_THREAD_TRACE_CAPTURE_MODE_SELECT_DETAIL = 0x2, +} SQ_THREAD_TRACE_CAPTURE_MODE; +typedef enum SQ_THREAD_TRACE_VM_ID_MASK { + SQ_THREAD_TRACE_VM_ID_MASK_SINGLE = 0x0, + SQ_THREAD_TRACE_VM_ID_MASK_ALL = 0x1, + SQ_THREAD_TRACE_VM_ID_MASK_SINGLE_DETAIL = 0x2, +} SQ_THREAD_TRACE_VM_ID_MASK; +typedef enum SQ_THREAD_TRACE_WAVE_MASK { + SQ_THREAD_TRACE_WAVE_MASK_NONE = 0x0, + SQ_THREAD_TRACE_WAVE_MASK_ALL = 0x1, + SQ_THREAD_TRACE_WAVE_MASK_1_2 = 0x2, + SQ_THREAD_TRACE_WAVE_MASK_1_4 = 0x3, + SQ_THREAD_TRACE_WAVE_MASK_1_8 = 0x4, + SQ_THREAD_TRACE_WAVE_MASK_1_16 = 0x5, + SQ_THREAD_TRACE_WAVE_MASK_1_32 = 0x6, + SQ_THREAD_TRACE_WAVE_MASK_1_64 = 0x7, +} SQ_THREAD_TRACE_WAVE_MASK; +typedef enum SQ_THREAD_TRACE_ISSUE { + SQ_THREAD_TRACE_ISSUE_NULL = 0x0, + SQ_THREAD_TRACE_ISSUE_STALL = 0x1, + SQ_THREAD_TRACE_ISSUE_INST = 0x2, + SQ_THREAD_TRACE_ISSUE_IMMED = 0x3, +} SQ_THREAD_TRACE_ISSUE; +typedef enum SQ_THREAD_TRACE_ISSUE_MASK { + SQ_THREAD_TRACE_ISSUE_MASK_ALL = 0x0, + SQ_THREAD_TRACE_ISSUE_MASK_STALLED = 0x1, + SQ_THREAD_TRACE_ISSUE_MASK_STALLED_AND_IMMED = 0x2, + SQ_THREAD_TRACE_ISSUE_MASK_IMMED = 0x3, +} SQ_THREAD_TRACE_ISSUE_MASK; +typedef enum SQ_PERF_SEL { + SQ_PERF_SEL_NONE = 0x0, + SQ_PERF_SEL_ACCUM_PREV = 0x1, + SQ_PERF_SEL_CYCLES = 0x2, + SQ_PERF_SEL_BUSY_CYCLES = 0x3, + SQ_PERF_SEL_WAVES = 0x4, + SQ_PERF_SEL_LEVEL_WAVES = 0x5, + SQ_PERF_SEL_WAVES_EQ_64 = 0x6, + SQ_PERF_SEL_WAVES_LT_64 = 0x7, + SQ_PERF_SEL_WAVES_LT_48 = 0x8, + SQ_PERF_SEL_WAVES_LT_32 = 0x9, + SQ_PERF_SEL_WAVES_LT_16 = 0xa, + SQ_PERF_SEL_WAVES_CU = 0xb, + SQ_PERF_SEL_LEVEL_WAVES_CU = 0xc, + SQ_PERF_SEL_BUSY_CU_CYCLES = 0xd, + SQ_PERF_SEL_ITEMS = 0xe, + SQ_PERF_SEL_QUADS = 0xf, + SQ_PERF_SEL_EVENTS = 0x10, + SQ_PERF_SEL_SURF_SYNCS = 0x11, + SQ_PERF_SEL_TTRACE_REQS = 0x12, + SQ_PERF_SEL_TTRACE_INFLIGHT_REQS = 0x13, + SQ_PERF_SEL_TTRACE_STALL = 0x14, + SQ_PERF_SEL_MSG_CNTR = 0x15, + SQ_PERF_SEL_MSG_PERF = 0x16, + SQ_PERF_SEL_MSG_GSCNT = 0x17, + SQ_PERF_SEL_MSG_INTERRUPT = 0x18, + SQ_PERF_SEL_INSTS = 0x19, + SQ_PERF_SEL_INSTS_VALU = 0x1a, + SQ_PERF_SEL_INSTS_VMEM_WR = 0x1b, + SQ_PERF_SEL_INSTS_VMEM_RD = 0x1c, + SQ_PERF_SEL_INSTS_VMEM = 0x1d, + SQ_PERF_SEL_INSTS_SALU = 0x1e, + SQ_PERF_SEL_INSTS_SMEM = 0x1f, + SQ_PERF_SEL_INSTS_FLAT = 0x20, + SQ_PERF_SEL_INSTS_FLAT_LDS_ONLY = 0x21, + SQ_PERF_SEL_INSTS_LDS = 0x22, + SQ_PERF_SEL_INSTS_GDS = 0x23, + SQ_PERF_SEL_INSTS_EXP = 0x24, + SQ_PERF_SEL_INSTS_EXP_GDS = 0x25, + SQ_PERF_SEL_INSTS_BRANCH = 0x26, + SQ_PERF_SEL_INSTS_SENDMSG = 0x27, + SQ_PERF_SEL_INSTS_VSKIPPED = 0x28, + SQ_PERF_SEL_INST_LEVEL_VMEM = 0x29, + SQ_PERF_SEL_INST_LEVEL_SMEM = 0x2a, + SQ_PERF_SEL_INST_LEVEL_LDS = 0x2b, + SQ_PERF_SEL_INST_LEVEL_GDS = 0x2c, + SQ_PERF_SEL_INST_LEVEL_EXP = 0x2d, + SQ_PERF_SEL_WAVE_CYCLES = 0x2e, + SQ_PERF_SEL_WAVE_READY = 0x2f, + SQ_PERF_SEL_WAIT_CNT_VM = 0x30, + SQ_PERF_SEL_WAIT_CNT_LGKM = 0x31, + SQ_PERF_SEL_WAIT_CNT_EXP = 0x32, + SQ_PERF_SEL_WAIT_CNT_ANY = 0x33, + SQ_PERF_SEL_WAIT_BARRIER = 0x34, + SQ_PERF_SEL_WAIT_EXP_ALLOC = 0x35, + SQ_PERF_SEL_WAIT_SLEEP = 0x36, + SQ_PERF_SEL_WAIT_OTHER = 0x37, + SQ_PERF_SEL_WAIT_ANY = 0x38, + SQ_PERF_SEL_WAIT_TTRACE = 0x39, + SQ_PERF_SEL_WAIT_IFETCH = 0x3a, + SQ_PERF_SEL_WAIT_INST_VMEM = 0x3b, + SQ_PERF_SEL_WAIT_INST_SCA = 0x3c, + SQ_PERF_SEL_WAIT_INST_LDS = 0x3d, + SQ_PERF_SEL_WAIT_INST_VALU = 0x3e, + SQ_PERF_SEL_WAIT_INST_EXP_GDS = 0x3f, + SQ_PERF_SEL_WAIT_INST_MISC = 0x40, + SQ_PERF_SEL_WAIT_INST_FLAT = 0x41, + SQ_PERF_SEL_ACTIVE_INST_ANY = 0x42, + SQ_PERF_SEL_ACTIVE_INST_VMEM = 0x43, + SQ_PERF_SEL_ACTIVE_INST_LDS = 0x44, + SQ_PERF_SEL_ACTIVE_INST_VALU = 0x45, + SQ_PERF_SEL_ACTIVE_INST_SCA = 0x46, + SQ_PERF_SEL_ACTIVE_INST_EXP_GDS = 0x47, + SQ_PERF_SEL_ACTIVE_INST_MISC = 0x48, + SQ_PERF_SEL_ACTIVE_INST_FLAT = 0x49, + SQ_PERF_SEL_INST_CYCLES_VMEM_WR = 0x4a, + SQ_PERF_SEL_INST_CYCLES_VMEM_RD = 0x4b, + SQ_PERF_SEL_INST_CYCLES_VMEM_ADDR = 0x4c, + SQ_PERF_SEL_INST_CYCLES_VMEM_DATA = 0x4d, + SQ_PERF_SEL_INST_CYCLES_VMEM_CMD = 0x4e, + SQ_PERF_SEL_INST_CYCLES_VMEM = 0x4f, + SQ_PERF_SEL_INST_CYCLES_LDS = 0x50, + SQ_PERF_SEL_INST_CYCLES_VALU = 0x51, + SQ_PERF_SEL_INST_CYCLES_EXP = 0x52, + SQ_PERF_SEL_INST_CYCLES_GDS = 0x53, + SQ_PERF_SEL_INST_CYCLES_SCA = 0x54, + SQ_PERF_SEL_INST_CYCLES_SMEM = 0x55, + SQ_PERF_SEL_INST_CYCLES_SALU = 0x56, + SQ_PERF_SEL_INST_CYCLES_EXP_GDS = 0x57, + SQ_PERF_SEL_INST_CYCLES_MISC = 0x58, + SQ_PERF_SEL_THREAD_CYCLES_VALU = 0x59, + SQ_PERF_SEL_THREAD_CYCLES_VALU_MAX = 0x5a, + SQ_PERF_SEL_IFETCH = 0x5b, + SQ_PERF_SEL_IFETCH_LEVEL = 0x5c, + SQ_PERF_SEL_CBRANCH_FORK = 0x5d, + SQ_PERF_SEL_CBRANCH_FORK_SPLIT = 0x5e, + SQ_PERF_SEL_VALU_LDS_DIRECT_RD = 0x5f, + SQ_PERF_SEL_VALU_LDS_INTERP_OP = 0x60, + SQ_PERF_SEL_LDS_BANK_CONFLICT = 0x61, + SQ_PERF_SEL_LDS_ADDR_CONFLICT = 0x62, + SQ_PERF_SEL_LDS_UNALIGNED_STALL = 0x63, + SQ_PERF_SEL_LDS_MEM_VIOLATIONS = 0x64, + SQ_PERF_SEL_LDS_ATOMIC_RETURN = 0x65, + SQ_PERF_SEL_LDS_IDX_ACTIVE = 0x66, + SQ_PERF_SEL_VALU_DEP_STALL = 0x67, + SQ_PERF_SEL_VALU_STARVE = 0x68, + SQ_PERF_SEL_EXP_REQ_FIFO_FULL = 0x69, + SQ_PERF_SEL_LDS_BACK2BACK_STALL = 0x6a, + SQ_PERF_SEL_LDS_DATA_FIFO_FULL = 0x6b, + SQ_PERF_SEL_LDS_CMD_FIFO_FULL = 0x6c, + SQ_PERF_SEL_VMEM_BACK2BACK_STALL = 0x6d, + SQ_PERF_SEL_VMEM_TA_ADDR_FIFO_FULL = 0x6e, + SQ_PERF_SEL_VMEM_TA_CMD_FIFO_FULL = 0x6f, + SQ_PERF_SEL_VMEM_EX_DATA_REG_BUSY = 0x70, + SQ_PERF_SEL_VMEM_WR_BACK2BACK_STALL = 0x71, + SQ_PERF_SEL_VMEM_WR_TA_DATA_FIFO_FULL = 0x72, + SQ_PERF_SEL_VALU_SRC_C_CONFLICT = 0x73, + SQ_PERF_SEL_VMEM_RD_SRC_CD_CONFLICT = 0x74, + SQ_PERF_SEL_VMEM_WR_SRC_CD_CONFLICT = 0x75, + SQ_PERF_SEL_FLAT_SRC_CD_CONFLICT = 0x76, + SQ_PERF_SEL_LDS_SRC_CD_CONFLICT = 0x77, + SQ_PERF_SEL_SRC_CD_BUSY = 0x78, + SQ_PERF_SEL_PT_POWER_STALL = 0x79, + SQ_PERF_SEL_USER0 = 0x7a, + SQ_PERF_SEL_USER1 = 0x7b, + SQ_PERF_SEL_USER2 = 0x7c, + SQ_PERF_SEL_USER3 = 0x7d, + SQ_PERF_SEL_USER4 = 0x7e, + SQ_PERF_SEL_USER5 = 0x7f, + SQ_PERF_SEL_USER6 = 0x80, + SQ_PERF_SEL_USER7 = 0x81, + SQ_PERF_SEL_USER8 = 0x82, + SQ_PERF_SEL_USER9 = 0x83, + SQ_PERF_SEL_USER10 = 0x84, + SQ_PERF_SEL_USER11 = 0x85, + SQ_PERF_SEL_USER12 = 0x86, + SQ_PERF_SEL_USER13 = 0x87, + SQ_PERF_SEL_USER14 = 0x88, + SQ_PERF_SEL_USER15 = 0x89, + SQ_PERF_SEL_USER_LEVEL0 = 0x8a, + SQ_PERF_SEL_USER_LEVEL1 = 0x8b, + SQ_PERF_SEL_USER_LEVEL2 = 0x8c, + SQ_PERF_SEL_USER_LEVEL3 = 0x8d, + SQ_PERF_SEL_USER_LEVEL4 = 0x8e, + SQ_PERF_SEL_USER_LEVEL5 = 0x8f, + SQ_PERF_SEL_USER_LEVEL6 = 0x90, + SQ_PERF_SEL_USER_LEVEL7 = 0x91, + SQ_PERF_SEL_USER_LEVEL8 = 0x92, + SQ_PERF_SEL_USER_LEVEL9 = 0x93, + SQ_PERF_SEL_USER_LEVEL10 = 0x94, + SQ_PERF_SEL_USER_LEVEL11 = 0x95, + SQ_PERF_SEL_USER_LEVEL12 = 0x96, + SQ_PERF_SEL_USER_LEVEL13 = 0x97, + SQ_PERF_SEL_USER_LEVEL14 = 0x98, + SQ_PERF_SEL_USER_LEVEL15 = 0x99, + SQ_PERF_SEL_POWER_VALU = 0x9a, + SQ_PERF_SEL_POWER_VALU0 = 0x9b, + SQ_PERF_SEL_POWER_VALU1 = 0x9c, + SQ_PERF_SEL_POWER_VALU2 = 0x9d, + SQ_PERF_SEL_POWER_GPR_RD = 0x9e, + SQ_PERF_SEL_POWER_GPR_WR = 0x9f, + SQ_PERF_SEL_POWER_LDS_BUSY = 0xa0, + SQ_PERF_SEL_POWER_ALU_BUSY = 0xa1, + SQ_PERF_SEL_POWER_TEX_BUSY = 0xa2, + SQ_PERF_SEL_ACCUM_PREV_HIRES = 0xa3, + SQ_PERF_SEL_DUMMY_LAST = 0xa7, + SQC_PERF_SEL_ICACHE_INPUT_VALID_READY = 0xa8, + SQC_PERF_SEL_ICACHE_INPUT_VALID_READYB = 0xa9, + SQC_PERF_SEL_ICACHE_INPUT_VALIDB = 0xaa, + SQC_PERF_SEL_DCACHE_INPUT_VALID_READY = 0xab, + SQC_PERF_SEL_DCACHE_INPUT_VALID_READYB = 0xac, + SQC_PERF_SEL_DCACHE_INPUT_VALIDB = 0xad, + SQC_PERF_SEL_TC_REQ = 0xae, + SQC_PERF_SEL_TC_INST_REQ = 0xaf, + SQC_PERF_SEL_TC_DATA_REQ = 0xb0, + SQC_PERF_SEL_TC_STALL = 0xb1, + SQC_PERF_SEL_TC_STARVE = 0xb2, + SQC_PERF_SEL_ICACHE_BUSY_CYCLES = 0xb3, + SQC_PERF_SEL_ICACHE_REQ = 0xb4, + SQC_PERF_SEL_ICACHE_HITS = 0xb5, + SQC_PERF_SEL_ICACHE_MISSES = 0xb6, + SQC_PERF_SEL_ICACHE_MISSES_DUPLICATE = 0xb7, + SQC_PERF_SEL_ICACHE_UNCACHED = 0xb8, + SQC_PERF_SEL_ICACHE_VOLATILE = 0xb9, + SQC_PERF_SEL_ICACHE_INVAL_INST = 0xba, + SQC_PERF_SEL_ICACHE_INVAL_ASYNC = 0xbb, + SQC_PERF_SEL_ICACHE_INVAL_VOLATILE_INST = 0xbc, + SQC_PERF_SEL_ICACHE_INVAL_VOLATILE_ASYNC = 0xbd, + SQC_PERF_SEL_ICACHE_INPUT_STALL_ARB_NO_GRANT = 0xbe, + SQC_PERF_SEL_ICACHE_INPUT_STALL_BANK_READYB = 0xbf, + SQC_PERF_SEL_ICACHE_CACHE_STALLED = 0xc0, + SQC_PERF_SEL_ICACHE_CACHE_STALL_INFLIGHT_NONZERO = 0xc1, + SQC_PERF_SEL_ICACHE_CACHE_STALL_INFLIGHT_MAX = 0xc2, + SQC_PERF_SEL_ICACHE_CACHE_STALL_VOLATILE_MISMATCH= 0xc3, + SQC_PERF_SEL_ICACHE_CACHE_STALL_UNCACHED_HIT = 0xc4, + SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT = 0xc5, + SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT_MISS_FIFO = 0xc6, + SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT_HIT_FIFO = 0xc7, + SQC_PERF_SEL_ICACHE_CACHE_STALL_OUTPUT_TC_IF = 0xc8, + SQC_PERF_SEL_ICACHE_STALL_OUTXBAR_ARB_NO_GRANT = 0xc9, + SQC_PERF_SEL_DCACHE_BUSY_CYCLES = 0xca, + SQC_PERF_SEL_DCACHE_REQ = 0xcb, + SQC_PERF_SEL_DCACHE_HITS = 0xcc, + SQC_PERF_SEL_DCACHE_MISSES = 0xcd, + SQC_PERF_SEL_DCACHE_MISSES_DUPLICATE = 0xce, + SQC_PERF_SEL_DCACHE_UNCACHED = 0xcf, + SQC_PERF_SEL_DCACHE_VOLATILE = 0xd0, + SQC_PERF_SEL_DCACHE_INVAL_INST = 0xd1, + SQC_PERF_SEL_DCACHE_INVAL_ASYNC = 0xd2, + SQC_PERF_SEL_DCACHE_INVAL_VOLATILE_INST = 0xd3, + SQC_PERF_SEL_DCACHE_INVAL_VOLATILE_ASYNC = 0xd4, + SQC_PERF_SEL_DCACHE_INPUT_STALL_ARB_NO_GRANT = 0xd5, + SQC_PERF_SEL_DCACHE_INPUT_STALL_BANK_READYB = 0xd6, + SQC_PERF_SEL_DCACHE_CACHE_STALLED = 0xd7, + SQC_PERF_SEL_DCACHE_CACHE_STALL_INFLIGHT_NONZERO = 0xd8, + SQC_PERF_SEL_DCACHE_CACHE_STALL_INFLIGHT_MAX = 0xd9, + SQC_PERF_SEL_DCACHE_CACHE_STALL_VOLATILE_MISMATCH= 0xda, + SQC_PERF_SEL_DCACHE_CACHE_STALL_UNCACHED_HIT = 0xdb, + SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT = 0xdc, + SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT_MISS_FIFO = 0xdd, + SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT_HIT_FIFO = 0xde, + SQC_PERF_SEL_DCACHE_CACHE_STALL_OUTPUT_TC_IF = 0xdf, + SQC_PERF_SEL_DCACHE_STALL_OUTXBAR_ARB_NO_GRANT = 0xe0, + SQC_PERF_SEL_DCACHE_REQ_1 = 0xe1, + SQC_PERF_SEL_DCACHE_REQ_2 = 0xe2, + SQC_PERF_SEL_DCACHE_REQ_4 = 0xe3, + SQC_PERF_SEL_DCACHE_REQ_8 = 0xe4, + SQC_PERF_SEL_DCACHE_REQ_16 = 0xe5, + SQC_PERF_SEL_DCACHE_REQ_TIME = 0xe6, + SQC_PERF_SEL_SQ_DCACHE_REQS = 0xe7, + SQC_PERF_SEL_DCACHE_FLAT_REQ = 0xe8, + SQC_PERF_SEL_DCACHE_NONFLAT_REQ = 0xe9, + SQC_PERF_SEL_ICACHE_INFLIGHT_LEVEL = 0xea, + SQC_PERF_SEL_ICACHE_PRE_CC_LEVEL = 0xeb, + SQC_PERF_SEL_ICACHE_POST_CC_LEVEL = 0xec, + SQC_PERF_SEL_ICACHE_POST_CC_HIT_LEVEL = 0xed, + SQC_PERF_SEL_ICACHE_POST_CC_MISS_LEVEL = 0xee, + SQC_PERF_SEL_DCACHE_INFLIGHT_LEVEL = 0xef, + SQC_PERF_SEL_DCACHE_PRE_CC_LEVEL = 0xf0, + SQC_PERF_SEL_DCACHE_POST_CC_LEVEL = 0xf1, + SQC_PERF_SEL_DCACHE_POST_CC_HIT_LEVEL = 0xf2, + SQC_PERF_SEL_DCACHE_POST_CC_MISS_LEVEL = 0xf3, + SQC_PERF_SEL_TC_INFLIGHT_LEVEL = 0xf4, + SQC_PERF_SEL_ICACHE_TC_INFLIGHT_LEVEL = 0xf5, + SQC_PERF_SEL_DCACHE_TC_INFLIGHT_LEVEL = 0xf6, + SQC_PERF_SEL_ERR_DCACHE_REQ_2_GPR_ADDR_UNALIGNED = 0xf7, + SQC_PERF_SEL_ERR_DCACHE_REQ_4_GPR_ADDR_UNALIGNED = 0xf8, + SQC_PERF_SEL_ERR_DCACHE_REQ_8_GPR_ADDR_UNALIGNED = 0xf9, + SQC_PERF_SEL_ERR_DCACHE_REQ_16_GPR_ADDR_UNALIGNED= 0xfa, + SQC_PERF_SEL_DUMMY_LAST = 0xfb, +} SQ_PERF_SEL; +typedef enum SQC_DATA_CACHE_POLICIES { + SQC_DATA_CACHE_POLICY_HIT_LRU = 0x0, + SQC_DATA_CACHE_POLICY_MISS_EVICT = 0x1, +} SQC_DATA_CACHE_POLICIES; +typedef enum SQ_CAC_POWER_SEL { + SQ_CAC_POWER_VALU = 0x0, + SQ_CAC_POWER_VALU0 = 0x1, + SQ_CAC_POWER_VALU1 = 0x2, + SQ_CAC_POWER_VALU2 = 0x3, + SQ_CAC_POWER_GPR_RD = 0x4, + SQ_CAC_POWER_GPR_WR = 0x5, + SQ_CAC_POWER_LDS_BUSY = 0x6, + SQ_CAC_POWER_ALU_BUSY = 0x7, + SQ_CAC_POWER_TEX_BUSY = 0x8, +} SQ_CAC_POWER_SEL; +typedef enum SQ_IND_CMD_CMD { + SQ_IND_CMD_CMD_NULL = 0x0, + SQ_IND_CMD_CMD_HALT = 0x1, + SQ_IND_CMD_CMD_RESUME = 0x2, + SQ_IND_CMD_CMD_KILL = 0x3, + SQ_IND_CMD_CMD_DEBUG = 0x4, + SQ_IND_CMD_CMD_TRAP = 0x5, +} SQ_IND_CMD_CMD; +typedef enum SQ_IND_CMD_MODE { + SQ_IND_CMD_MODE_SINGLE = 0x0, + SQ_IND_CMD_MODE_BROADCAST = 0x1, + SQ_IND_CMD_MODE_BROADCAST_QUEUE = 0x2, + SQ_IND_CMD_MODE_BROADCAST_PIPE = 0x3, + SQ_IND_CMD_MODE_BROADCAST_ME = 0x4, +} SQ_IND_CMD_MODE; +typedef enum SQ_DED_INFO_SOURCE { + SQ_DED_INFO_SOURCE_INVALID = 0x0, + SQ_DED_INFO_SOURCE_INST = 0x1, + SQ_DED_INFO_SOURCE_SGPR = 0x2, + SQ_DED_INFO_SOURCE_VGPR = 0x3, + SQ_DED_INFO_SOURCE_LDS = 0x4, + SQ_DED_INFO_SOURCE_GDS = 0x5, + SQ_DED_INFO_SOURCE_TA = 0x6, +} SQ_DED_INFO_SOURCE; +typedef enum SQ_ROUND_MODE { + SQ_ROUND_NEAREST_EVEN = 0x0, + SQ_ROUND_PLUS_INFINITY = 0x1, + SQ_ROUND_MINUS_INFINITY = 0x2, + SQ_ROUND_TO_ZERO = 0x3, +} SQ_ROUND_MODE; +typedef enum SQ_INTERRUPT_WORD_ENCODING { + SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0, + SQ_INTERRUPT_WORD_ENCODING_INST = 0x1, + SQ_INTERRUPT_WORD_ENCODING_ERROR = 0x2, +} SQ_INTERRUPT_WORD_ENCODING; +typedef enum ENUM_SQ_EXPORT_RAT_INST { + SQ_EXPORT_RAT_INST_NOP = 0x0, + SQ_EXPORT_RAT_INST_STORE_TYPED = 0x1, + SQ_EXPORT_RAT_INST_STORE_RAW = 0x2, + SQ_EXPORT_RAT_INST_STORE_RAW_FDENORM = 0x3, + SQ_EXPORT_RAT_INST_CMPXCHG_INT = 0x4, + SQ_EXPORT_RAT_INST_CMPXCHG_FLT = 0x5, + SQ_EXPORT_RAT_INST_CMPXCHG_FDENORM = 0x6, + SQ_EXPORT_RAT_INST_ADD = 0x7, + SQ_EXPORT_RAT_INST_SUB = 0x8, + SQ_EXPORT_RAT_INST_RSUB = 0x9, + SQ_EXPORT_RAT_INST_MIN_INT = 0xa, + SQ_EXPORT_RAT_INST_MIN_UINT = 0xb, + SQ_EXPORT_RAT_INST_MAX_INT = 0xc, + SQ_EXPORT_RAT_INST_MAX_UINT = 0xd, + SQ_EXPORT_RAT_INST_AND = 0xe, + SQ_EXPORT_RAT_INST_OR = 0xf, + SQ_EXPORT_RAT_INST_XOR = 0x10, + SQ_EXPORT_RAT_INST_MSKOR = 0x11, + SQ_EXPORT_RAT_INST_INC_UINT = 0x12, + SQ_EXPORT_RAT_INST_DEC_UINT = 0x13, + SQ_EXPORT_RAT_INST_STORE_DWORD = 0x14, + SQ_EXPORT_RAT_INST_STORE_SHORT = 0x15, + SQ_EXPORT_RAT_INST_STORE_BYTE = 0x16, + SQ_EXPORT_RAT_INST_NOP_RTN = 0x20, + SQ_EXPORT_RAT_INST_XCHG_RTN = 0x22, + SQ_EXPORT_RAT_INST_XCHG_FDENORM_RTN = 0x23, + SQ_EXPORT_RAT_INST_CMPXCHG_INT_RTN = 0x24, + SQ_EXPORT_RAT_INST_CMPXCHG_FLT_RTN = 0x25, + SQ_EXPORT_RAT_INST_CMPXCHG_FDENORM_RTN = 0x26, + SQ_EXPORT_RAT_INST_ADD_RTN = 0x27, + SQ_EXPORT_RAT_INST_SUB_RTN = 0x28, + SQ_EXPORT_RAT_INST_RSUB_RTN = 0x29, + SQ_EXPORT_RAT_INST_MIN_INT_RTN = 0x2a, + SQ_EXPORT_RAT_INST_MIN_UINT_RTN = 0x2b, + SQ_EXPORT_RAT_INST_MAX_INT_RTN = 0x2c, + SQ_EXPORT_RAT_INST_MAX_UINT_RTN = 0x2d, + SQ_EXPORT_RAT_INST_AND_RTN = 0x2e, + SQ_EXPORT_RAT_INST_OR_RTN = 0x2f, + SQ_EXPORT_RAT_INST_XOR_RTN = 0x30, + SQ_EXPORT_RAT_INST_MSKOR_RTN = 0x31, + SQ_EXPORT_RAT_INST_INC_UINT_RTN = 0x32, + SQ_EXPORT_RAT_INST_DEC_UINT_RTN = 0x33, +} ENUM_SQ_EXPORT_RAT_INST; +typedef enum SQ_IBUF_ST { + SQ_IBUF_IB_IDLE = 0x0, + SQ_IBUF_IB_INI_WAIT_GNT = 0x1, + SQ_IBUF_IB_INI_WAIT_DRET = 0x2, + SQ_IBUF_IB_LE_4DW = 0x3, + SQ_IBUF_IB_WAIT_DRET = 0x4, + SQ_IBUF_IB_EMPTY_WAIT_DRET = 0x5, + SQ_IBUF_IB_DRET = 0x6, + SQ_IBUF_IB_EMPTY_WAIT_GNT = 0x7, +} SQ_IBUF_ST; +typedef enum SQ_INST_STR_ST { + SQ_INST_STR_IB_WAVE_NORML = 0x0, + SQ_INST_STR_IB_WAVE2ID_NORMAL_INST_AV = 0x1, + SQ_INST_STR_IB_WAVE_INTERNAL_INST_AV = 0x2, + SQ_INST_STR_IB_WAVE_INST_SKIP_AV = 0x3, + SQ_INST_STR_IB_WAVE_SETVSKIP_ST0 = 0x4, + SQ_INST_STR_IB_WAVE_SETVSKIP_ST1 = 0x5, + SQ_INST_STR_IB_WAVE_NOP_SLEEP_WAIT = 0x6, + SQ_INST_STR_IB_WAVE_PC_FROM_SGPR_MSG_WAIT = 0x7, +} SQ_INST_STR_ST; +typedef enum SQ_WAVE_IB_ECC_ST { + SQ_WAVE_IB_ECC_CLEAN = 0x0, + SQ_WAVE_IB_ECC_ERR_CONTINUE = 0x1, + SQ_WAVE_IB_ECC_ERR_HALT = 0x2, + SQ_WAVE_IB_ECC_WITH_ERR_MSG = 0x3, +} SQ_WAVE_IB_ECC_ST; +typedef enum SH_MEM_ALIGNMENT_MODE { + SH_MEM_ALIGNMENT_MODE_DWORD = 0x0, + SH_MEM_ALIGNMENT_MODE_DWORD_STRICT = 0x1, + SH_MEM_ALIGNMENT_MODE_STRICT = 0x2, + SH_MEM_ALIGNMENT_MODE_UNALIGNED = 0x3, +} SH_MEM_ALIGNMENT_MODE; +#define SQ_WAVE_TYPE_PS0 0x0 +#define SQ_THREAD_TRACE_LFSR_PS 0x8016 +#define SQ_THREAD_TRACE_LFSR_VS 0x801c +#define SQ_THREAD_TRACE_LFSR_GS 0x801f +#define SQ_THREAD_TRACE_LFSR_ES 0x8029 +#define SQ_THREAD_TRACE_LFSR_HS 0x805e +#define SQ_THREAD_TRACE_LFSR_LS 0x806b +#define SQ_THREAD_TRACE_LFSR_CS 0x8097 +#define SQIND_GLOBAL_REGS_OFFSET 0x0 +#define SQIND_GLOBAL_REGS_SIZE 0x8 +#define SQIND_LOCAL_REGS_OFFSET 0x8 +#define SQIND_LOCAL_REGS_SIZE 0x8 +#define SQIND_WAVE_HWREGS_OFFSET 0x10 +#define SQIND_WAVE_HWREGS_SIZE 0x1f0 +#define SQIND_WAVE_SGPRS_OFFSET 0x200 +#define SQIND_WAVE_SGPRS_SIZE 0x200 +#define SQ_GFXDEC_BEGIN 0xa000 +#define SQ_GFXDEC_END 0xc000 +#define SQ_GFXDEC_STATE_ID_SHIFT 0xa +#define SQDEC_BEGIN 0x2300 +#define SQDEC_END 0x23ff +#define SQPERFSDEC_BEGIN 0xd9c0 +#define SQPERFSDEC_END 0xda40 +#define SQPERFDDEC_BEGIN 0xd1c0 +#define SQPERFDDEC_END 0xd240 +#define SQGFXUDEC_BEGIN 0xc340 +#define SQGFXUDEC_END 0xc380 +#define SQPWRDEC_BEGIN 0xf08c +#define SQPWRDEC_END 0xf094 +#define SQ_DISPATCHER_GFX_MIN 0x10 +#define SQ_DISPATCHER_GFX_CNT_PER_RING 0x8 +#define SQ_MAX_PGM_SGPRS 0x68 +#define SQ_MAX_PGM_VGPRS 0x100 +#define SQ_THREAD_TRACE_TIME_UNIT 0x4 +#define SQ_INTERRUPT_ID 0xef +#define SQ_EX_MODE_EXCP_VALU_BASE 0x0 +#define SQ_EX_MODE_EXCP_VALU_SIZE 0x7 +#define SQ_EX_MODE_EXCP_INVALID 0x0 +#define SQ_EX_MODE_EXCP_INPUT_DENORM 0x1 +#define SQ_EX_MODE_EXCP_DIV0 0x2 +#define SQ_EX_MODE_EXCP_OVERFLOW 0x3 +#define SQ_EX_MODE_EXCP_UNDERFLOW 0x4 +#define SQ_EX_MODE_EXCP_INEXACT 0x5 +#define SQ_EX_MODE_EXCP_INT_DIV0 0x6 +#define SQ_EX_MODE_EXCP_ADDR_WATCH 0x7 +#define SQ_EX_MODE_EXCP_MEM_VIOL 0x8 +#define INST_ID_ECC_INTERRUPT_MSG 0xfffffff0 +#define INST_ID_TTRACE_NEW_PC_MSG 0xfffffff1 +#define INST_ID_HW_TRAP 0xfffffff2 +#define INST_ID_KILL_SEQ 0xfffffff3 +#define INST_ID_HOST_REG_TRAP_MSG 0xfffffffe +#define SQ_ENC_SOP1_BITS 0xbe800000 +#define SQ_ENC_SOP1_MASK 0xff800000 +#define SQ_ENC_SOP1_FIELD 0x17d +#define SQ_ENC_SOPC_BITS 0xbf000000 +#define SQ_ENC_SOPC_MASK 0xff800000 +#define SQ_ENC_SOPC_FIELD 0x17e +#define SQ_ENC_SOPP_BITS 0xbf800000 +#define SQ_ENC_SOPP_MASK 0xff800000 +#define SQ_ENC_SOPP_FIELD 0x17f +#define SQ_ENC_SOPK_BITS 0xb0000000 +#define SQ_ENC_SOPK_MASK 0xf0000000 +#define SQ_ENC_SOPK_FIELD 0xb +#define SQ_ENC_SOP2_BITS 0x80000000 +#define SQ_ENC_SOP2_MASK 0xc0000000 +#define SQ_ENC_SOP2_FIELD 0x2 +#define SQ_ENC_SMRD_BITS 0xc0000000 +#define SQ_ENC_SMRD_MASK 0xf8000000 +#define SQ_ENC_SMRD_FIELD 0x18 +#define SQ_ENC_VOP1_BITS 0x7e000000 +#define SQ_ENC_VOP1_MASK 0xfe000000 +#define SQ_ENC_VOP1_FIELD 0x3f +#define SQ_ENC_VOPC_BITS 0x7c000000 +#define SQ_ENC_VOPC_MASK 0xfe000000 +#define SQ_ENC_VOPC_FIELD 0x3e +#define SQ_ENC_VOP2_BITS 0x0 +#define SQ_ENC_VOP2_MASK 0x80000000 +#define SQ_ENC_VOP2_FIELD 0x0 +#define SQ_ENC_VINTRP_BITS 0xc8000000 +#define SQ_ENC_VINTRP_MASK 0xfc000000 +#define SQ_ENC_VINTRP_FIELD 0x32 +#define SQ_ENC_VOP3_BITS 0xd0000000 +#define SQ_ENC_VOP3_MASK 0xfc000000 +#define SQ_ENC_VOP3_FIELD 0x34 +#define SQ_ENC_DS_BITS 0xd8000000 +#define SQ_ENC_DS_MASK 0xfc000000 +#define SQ_ENC_DS_FIELD 0x36 +#define SQ_ENC_MUBUF_BITS 0xe0000000 +#define SQ_ENC_MUBUF_MASK 0xfc000000 +#define SQ_ENC_MUBUF_FIELD 0x38 +#define SQ_ENC_MTBUF_BITS 0xe8000000 +#define SQ_ENC_MTBUF_MASK 0xfc000000 +#define SQ_ENC_MTBUF_FIELD 0x3a +#define SQ_ENC_MIMG_BITS 0xf0000000 +#define SQ_ENC_MIMG_MASK 0xfc000000 +#define SQ_ENC_MIMG_FIELD 0x3c +#define SQ_ENC_EXP_BITS 0xf8000000 +#define SQ_ENC_EXP_MASK 0xfc000000 +#define SQ_ENC_EXP_FIELD 0x3e +#define SQ_ENC_FLAT_BITS 0xdc000000 +#define SQ_ENC_FLAT_MASK 0xfc000000 +#define SQ_ENC_FLAT_FIELD 0x37 +#define SQ_WAITCNT_VM_SHIFT 0x0 +#define SQ_SENDMSG_STREAMID_SIZE 0x2 +#define SQ_V_OPC_COUNT 0x100 +#define SQ_HWREG_OFFSET_SIZE 0x5 +#define SQ_HWREG_OFFSET_SHIFT 0x6 +#define SQ_NUM_ATTR 0x21 +#define SQ_NUM_VGPR 0x100 +#define SQ_SENDMSG_MSG_SIZE 0x4 +#define SQ_NUM_TTMP 0xc +#define SQ_HWREG_ID_SIZE 0x6 +#define SQ_SENDMSG_GSOP_SIZE 0x2 +#define SQ_NUM_SGPR 0x68 +#define SQ_EXP_NUM_MRT 0x8 +#define SQ_SENDMSG_SYSTEM_SIZE 0x3 +#define SQ_WAITCNT_LGKM_SHIFT 0x8 +#define SQ_WAITCNT_EXP_SIZE 0x3 +#define SQ_SENDMSG_SYSTEM_SHIFT 0x4 +#define SQ_HWREG_SIZE_SHIFT 0xb +#define SQ_EXP_NUM_GDS 0x5 +#define SQ_SENDMSG_MSG_SHIFT 0x0 +#define SQ_WAITCNT_EXP_SHIFT 0x4 +#define SQ_WAITCNT_VM_SIZE 0x4 +#define SQ_SENDMSG_GSOP_SHIFT 0x4 +#define SQ_SRC_VGPR_BIT 0x100 +#define SQ_V_OP2_COUNT 0x40 +#define SQ_EXP_NUM_PARAM 0x20 +#define SQ_SENDMSG_STREAMID_SHIFT 0x8 +#define SQ_V_OP1_COUNT 0x80 +#define SQ_WAITCNT_LGKM_SIZE 0x5 +#define SQ_EXP_NUM_POS 0x4 +#define SQ_HWREG_SIZE_SIZE 0x5 +#define SQ_HWREG_ID_SHIFT 0x0 +#define SQ_S_MOV_B32 0x3 +#define SQ_S_MOV_B64 0x4 +#define SQ_S_CMOV_B32 0x5 +#define SQ_S_CMOV_B64 0x6 +#define SQ_S_NOT_B32 0x7 +#define SQ_S_NOT_B64 0x8 +#define SQ_S_WQM_B32 0x9 +#define SQ_S_WQM_B64 0xa +#define SQ_S_BREV_B32 0xb +#define SQ_S_BREV_B64 0xc +#define SQ_S_BCNT0_I32_B32 0xd +#define SQ_S_BCNT0_I32_B64 0xe +#define SQ_S_BCNT1_I32_B32 0xf +#define SQ_S_BCNT1_I32_B64 0x10 +#define SQ_S_FF0_I32_B32 0x11 +#define SQ_S_FF0_I32_B64 0x12 +#define SQ_S_FF1_I32_B32 0x13 +#define SQ_S_FF1_I32_B64 0x14 +#define SQ_S_FLBIT_I32_B32 0x15 +#define SQ_S_FLBIT_I32_B64 0x16 +#define SQ_S_FLBIT_I32 0x17 +#define SQ_S_FLBIT_I32_I64 0x18 +#define SQ_S_SEXT_I32_I8 0x19 +#define SQ_S_SEXT_I32_I16 0x1a +#define SQ_S_BITSET0_B32 0x1b +#define SQ_S_BITSET0_B64 0x1c +#define SQ_S_BITSET1_B32 0x1d +#define SQ_S_BITSET1_B64 0x1e +#define SQ_S_GETPC_B64 0x1f +#define SQ_S_SETPC_B64 0x20 +#define SQ_S_SWAPPC_B64 0x21 +#define SQ_S_RFE_B64 0x22 +#define SQ_S_AND_SAVEEXEC_B64 0x24 +#define SQ_S_OR_SAVEEXEC_B64 0x25 +#define SQ_S_XOR_SAVEEXEC_B64 0x26 +#define SQ_S_ANDN2_SAVEEXEC_B64 0x27 +#define SQ_S_ORN2_SAVEEXEC_B64 0x28 +#define SQ_S_NAND_SAVEEXEC_B64 0x29 +#define SQ_S_NOR_SAVEEXEC_B64 0x2a +#define SQ_S_XNOR_SAVEEXEC_B64 0x2b +#define SQ_S_QUADMASK_B32 0x2c +#define SQ_S_QUADMASK_B64 0x2d +#define SQ_S_MOVRELS_B32 0x2e +#define SQ_S_MOVRELS_B64 0x2f +#define SQ_S_MOVRELD_B32 0x30 +#define SQ_S_MOVRELD_B64 0x31 +#define SQ_S_CBRANCH_JOIN 0x32 +#define SQ_S_MOV_REGRD_B32 0x33 +#define SQ_S_ABS_I32 0x34 +#define SQ_S_MOV_FED_B32 0x35 +#define SQ_ATTR0 0x0 +#define SQ_S_MOVK_I32 0x0 +#define SQ_S_CMOVK_I32 0x2 +#define SQ_S_CMPK_EQ_I32 0x3 +#define SQ_S_CMPK_LG_I32 0x4 +#define SQ_S_CMPK_GT_I32 0x5 +#define SQ_S_CMPK_GE_I32 0x6 +#define SQ_S_CMPK_LT_I32 0x7 +#define SQ_S_CMPK_LE_I32 0x8 +#define SQ_S_CMPK_EQ_U32 0x9 +#define SQ_S_CMPK_LG_U32 0xa +#define SQ_S_CMPK_GT_U32 0xb +#define SQ_S_CMPK_GE_U32 0xc +#define SQ_S_CMPK_LT_U32 0xd +#define SQ_S_CMPK_LE_U32 0xe +#define SQ_S_ADDK_I32 0xf +#define SQ_S_MULK_I32 0x10 +#define SQ_S_CBRANCH_I_FORK 0x11 +#define SQ_S_GETREG_B32 0x12 +#define SQ_S_SETREG_B32 0x13 +#define SQ_S_GETREG_REGRD_B32 0x14 +#define SQ_S_SETREG_IMM32_B32 0x15 +#define SQ_TBA_LO 0x6c +#define SQ_TBA_HI 0x6d +#define SQ_TMA_LO 0x6e +#define SQ_TMA_HI 0x6f +#define SQ_TTMP0 0x70 +#define SQ_TTMP1 0x71 +#define SQ_TTMP2 0x72 +#define SQ_TTMP3 0x73 +#define SQ_TTMP4 0x74 +#define SQ_TTMP5 0x75 +#define SQ_TTMP6 0x76 +#define SQ_TTMP7 0x77 +#define SQ_TTMP8 0x78 +#define SQ_TTMP9 0x79 +#define SQ_TTMP10 0x7a +#define SQ_TTMP11 0x7b +#define SQ_VGPR0 0x0 +#define SQ_EXP 0x0 +#define SQ_EXP_MRT0 0x0 +#define SQ_EXP_MRTZ 0x8 +#define SQ_EXP_NULL 0x9 +#define SQ_EXP_POS0 0xc +#define SQ_EXP_PARAM0 0x20 +#define SQ_CNT1 0x0 +#define SQ_CNT2 0x1 +#define SQ_CNT3 0x2 +#define SQ_CNT4 0x3 +#define SQ_F 0x0 +#define SQ_LT 0x1 +#define SQ_EQ 0x2 +#define SQ_LE 0x3 +#define SQ_GT 0x4 +#define SQ_LG 0x5 +#define SQ_GE 0x6 +#define SQ_O 0x7 +#define SQ_U 0x8 +#define SQ_NGE 0x9 +#define SQ_NLG 0xa +#define SQ_NGT 0xb +#define SQ_NLE 0xc +#define SQ_NEQ 0xd +#define SQ_NLT 0xe +#define SQ_TRU 0xf +#define SQ_V_CMP_F_F32 0x0 +#define SQ_V_CMP_LT_F32 0x1 +#define SQ_V_CMP_EQ_F32 0x2 +#define SQ_V_CMP_LE_F32 0x3 +#define SQ_V_CMP_GT_F32 0x4 +#define SQ_V_CMP_LG_F32 0x5 +#define SQ_V_CMP_GE_F32 0x6 +#define SQ_V_CMP_O_F32 0x7 +#define SQ_V_CMP_U_F32 0x8 +#define SQ_V_CMP_NGE_F32 0x9 +#define SQ_V_CMP_NLG_F32 0xa +#define SQ_V_CMP_NGT_F32 0xb +#define SQ_V_CMP_NLE_F32 0xc +#define SQ_V_CMP_NEQ_F32 0xd +#define SQ_V_CMP_NLT_F32 0xe +#define SQ_V_CMP_TRU_F32 0xf +#define SQ_V_CMPX_F_F32 0x10 +#define SQ_V_CMPX_LT_F32 0x11 +#define SQ_V_CMPX_EQ_F32 0x12 +#define SQ_V_CMPX_LE_F32 0x13 +#define SQ_V_CMPX_GT_F32 0x14 +#define SQ_V_CMPX_LG_F32 0x15 +#define SQ_V_CMPX_GE_F32 0x16 +#define SQ_V_CMPX_O_F32 0x17 +#define SQ_V_CMPX_U_F32 0x18 +#define SQ_V_CMPX_NGE_F32 0x19 +#define SQ_V_CMPX_NLG_F32 0x1a +#define SQ_V_CMPX_NGT_F32 0x1b +#define SQ_V_CMPX_NLE_F32 0x1c +#define SQ_V_CMPX_NEQ_F32 0x1d +#define SQ_V_CMPX_NLT_F32 0x1e +#define SQ_V_CMPX_TRU_F32 0x1f +#define SQ_V_CMP_F_F64 0x20 +#define SQ_V_CMP_LT_F64 0x21 +#define SQ_V_CMP_EQ_F64 0x22 +#define SQ_V_CMP_LE_F64 0x23 +#define SQ_V_CMP_GT_F64 0x24 +#define SQ_V_CMP_LG_F64 0x25 +#define SQ_V_CMP_GE_F64 0x26 +#define SQ_V_CMP_O_F64 0x27 +#define SQ_V_CMP_U_F64 0x28 +#define SQ_V_CMP_NGE_F64 0x29 +#define SQ_V_CMP_NLG_F64 0x2a +#define SQ_V_CMP_NGT_F64 0x2b +#define SQ_V_CMP_NLE_F64 0x2c +#define SQ_V_CMP_NEQ_F64 0x2d +#define SQ_V_CMP_NLT_F64 0x2e +#define SQ_V_CMP_TRU_F64 0x2f +#define SQ_V_CMPX_F_F64 0x30 +#define SQ_V_CMPX_LT_F64 0x31 +#define SQ_V_CMPX_EQ_F64 0x32 +#define SQ_V_CMPX_LE_F64 0x33 +#define SQ_V_CMPX_GT_F64 0x34 +#define SQ_V_CMPX_LG_F64 0x35 +#define SQ_V_CMPX_GE_F64 0x36 +#define SQ_V_CMPX_O_F64 0x37 +#define SQ_V_CMPX_U_F64 0x38 +#define SQ_V_CMPX_NGE_F64 0x39 +#define SQ_V_CMPX_NLG_F64 0x3a +#define SQ_V_CMPX_NGT_F64 0x3b +#define SQ_V_CMPX_NLE_F64 0x3c +#define SQ_V_CMPX_NEQ_F64 0x3d +#define SQ_V_CMPX_NLT_F64 0x3e +#define SQ_V_CMPX_TRU_F64 0x3f +#define SQ_V_CMPS_F_F32 0x40 +#define SQ_V_CMPS_LT_F32 0x41 +#define SQ_V_CMPS_EQ_F32 0x42 +#define SQ_V_CMPS_LE_F32 0x43 +#define SQ_V_CMPS_GT_F32 0x44 +#define SQ_V_CMPS_LG_F32 0x45 +#define SQ_V_CMPS_GE_F32 0x46 +#define SQ_V_CMPS_O_F32 0x47 +#define SQ_V_CMPS_U_F32 0x48 +#define SQ_V_CMPS_NGE_F32 0x49 +#define SQ_V_CMPS_NLG_F32 0x4a +#define SQ_V_CMPS_NGT_F32 0x4b +#define SQ_V_CMPS_NLE_F32 0x4c +#define SQ_V_CMPS_NEQ_F32 0x4d +#define SQ_V_CMPS_NLT_F32 0x4e +#define SQ_V_CMPS_TRU_F32 0x4f +#define SQ_V_CMPSX_F_F32 0x50 +#define SQ_V_CMPSX_LT_F32 0x51 +#define SQ_V_CMPSX_EQ_F32 0x52 +#define SQ_V_CMPSX_LE_F32 0x53 +#define SQ_V_CMPSX_GT_F32 0x54 +#define SQ_V_CMPSX_LG_F32 0x55 +#define SQ_V_CMPSX_GE_F32 0x56 +#define SQ_V_CMPSX_O_F32 0x57 +#define SQ_V_CMPSX_U_F32 0x58 +#define SQ_V_CMPSX_NGE_F32 0x59 +#define SQ_V_CMPSX_NLG_F32 0x5a +#define SQ_V_CMPSX_NGT_F32 0x5b +#define SQ_V_CMPSX_NLE_F32 0x5c +#define SQ_V_CMPSX_NEQ_F32 0x5d +#define SQ_V_CMPSX_NLT_F32 0x5e +#define SQ_V_CMPSX_TRU_F32 0x5f +#define SQ_V_CMPS_F_F64 0x60 +#define SQ_V_CMPS_LT_F64 0x61 +#define SQ_V_CMPS_EQ_F64 0x62 +#define SQ_V_CMPS_LE_F64 0x63 +#define SQ_V_CMPS_GT_F64 0x64 +#define SQ_V_CMPS_LG_F64 0x65 +#define SQ_V_CMPS_GE_F64 0x66 +#define SQ_V_CMPS_O_F64 0x67 +#define SQ_V_CMPS_U_F64 0x68 +#define SQ_V_CMPS_NGE_F64 0x69 +#define SQ_V_CMPS_NLG_F64 0x6a +#define SQ_V_CMPS_NGT_F64 0x6b +#define SQ_V_CMPS_NLE_F64 0x6c +#define SQ_V_CMPS_NEQ_F64 0x6d +#define SQ_V_CMPS_NLT_F64 0x6e +#define SQ_V_CMPS_TRU_F64 0x6f +#define SQ_V_CMPSX_F_F64 0x70 +#define SQ_V_CMPSX_LT_F64 0x71 +#define SQ_V_CMPSX_EQ_F64 0x72 +#define SQ_V_CMPSX_LE_F64 0x73 +#define SQ_V_CMPSX_GT_F64 0x74 +#define SQ_V_CMPSX_LG_F64 0x75 +#define SQ_V_CMPSX_GE_F64 0x76 +#define SQ_V_CMPSX_O_F64 0x77 +#define SQ_V_CMPSX_U_F64 0x78 +#define SQ_V_CMPSX_NGE_F64 0x79 +#define SQ_V_CMPSX_NLG_F64 0x7a +#define SQ_V_CMPSX_NGT_F64 0x7b +#define SQ_V_CMPSX_NLE_F64 0x7c +#define SQ_V_CMPSX_NEQ_F64 0x7d +#define SQ_V_CMPSX_NLT_F64 0x7e +#define SQ_V_CMPSX_TRU_F64 0x7f +#define SQ_V_CMP_F_I32 0x80 +#define SQ_V_CMP_LT_I32 0x81 +#define SQ_V_CMP_EQ_I32 0x82 +#define SQ_V_CMP_LE_I32 0x83 +#define SQ_V_CMP_GT_I32 0x84 +#define SQ_V_CMP_NE_I32 0x85 +#define SQ_V_CMP_GE_I32 0x86 +#define SQ_V_CMP_T_I32 0x87 +#define SQ_V_CMPX_F_I32 0x90 +#define SQ_V_CMPX_LT_I32 0x91 +#define SQ_V_CMPX_EQ_I32 0x92 +#define SQ_V_CMPX_LE_I32 0x93 +#define SQ_V_CMPX_GT_I32 0x94 +#define SQ_V_CMPX_NE_I32 0x95 +#define SQ_V_CMPX_GE_I32 0x96 +#define SQ_V_CMPX_T_I32 0x97 +#define SQ_V_CMP_F_I64 0xa0 +#define SQ_V_CMP_LT_I64 0xa1 +#define SQ_V_CMP_EQ_I64 0xa2 +#define SQ_V_CMP_LE_I64 0xa3 +#define SQ_V_CMP_GT_I64 0xa4 +#define SQ_V_CMP_NE_I64 0xa5 +#define SQ_V_CMP_GE_I64 0xa6 +#define SQ_V_CMP_T_I64 0xa7 +#define SQ_V_CMPX_F_I64 0xb0 +#define SQ_V_CMPX_LT_I64 0xb1 +#define SQ_V_CMPX_EQ_I64 0xb2 +#define SQ_V_CMPX_LE_I64 0xb3 +#define SQ_V_CMPX_GT_I64 0xb4 +#define SQ_V_CMPX_NE_I64 0xb5 +#define SQ_V_CMPX_GE_I64 0xb6 +#define SQ_V_CMPX_T_I64 0xb7 +#define SQ_V_CMP_F_U32 0xc0 +#define SQ_V_CMP_LT_U32 0xc1 +#define SQ_V_CMP_EQ_U32 0xc2 +#define SQ_V_CMP_LE_U32 0xc3 +#define SQ_V_CMP_GT_U32 0xc4 +#define SQ_V_CMP_NE_U32 0xc5 +#define SQ_V_CMP_GE_U32 0xc6 +#define SQ_V_CMP_T_U32 0xc7 +#define SQ_V_CMPX_F_U32 0xd0 +#define SQ_V_CMPX_LT_U32 0xd1 +#define SQ_V_CMPX_EQ_U32 0xd2 +#define SQ_V_CMPX_LE_U32 0xd3 +#define SQ_V_CMPX_GT_U32 0xd4 +#define SQ_V_CMPX_NE_U32 0xd5 +#define SQ_V_CMPX_GE_U32 0xd6 +#define SQ_V_CMPX_T_U32 0xd7 +#define SQ_V_CMP_F_U64 0xe0 +#define SQ_V_CMP_LT_U64 0xe1 +#define SQ_V_CMP_EQ_U64 0xe2 +#define SQ_V_CMP_LE_U64 0xe3 +#define SQ_V_CMP_GT_U64 0xe4 +#define SQ_V_CMP_NE_U64 0xe5 +#define SQ_V_CMP_GE_U64 0xe6 +#define SQ_V_CMP_T_U64 0xe7 +#define SQ_V_CMPX_F_U64 0xf0 +#define SQ_V_CMPX_LT_U64 0xf1 +#define SQ_V_CMPX_EQ_U64 0xf2 +#define SQ_V_CMPX_LE_U64 0xf3 +#define SQ_V_CMPX_GT_U64 0xf4 +#define SQ_V_CMPX_NE_U64 0xf5 +#define SQ_V_CMPX_GE_U64 0xf6 +#define SQ_V_CMPX_T_U64 0xf7 +#define SQ_V_CMP_CLASS_F32 0x88 +#define SQ_V_CMPX_CLASS_F32 0x98 +#define SQ_V_CMP_CLASS_F64 0xa8 +#define SQ_V_CMPX_CLASS_F64 0xb8 +#define SQ_SGPR0 0x0 +#define SQ_F 0x0 +#define SQ_LT 0x1 +#define SQ_EQ 0x2 +#define SQ_LE 0x3 +#define SQ_GT 0x4 +#define SQ_NE 0x5 +#define SQ_GE 0x6 +#define SQ_T 0x7 +#define SQ_SRC_64_INT 0xc0 +#define SQ_SRC_M_1_INT 0xc1 +#define SQ_SRC_M_2_INT 0xc2 +#define SQ_SRC_M_3_INT 0xc3 +#define SQ_SRC_M_4_INT 0xc4 +#define SQ_SRC_M_5_INT 0xc5 +#define SQ_SRC_M_6_INT 0xc6 +#define SQ_SRC_M_7_INT 0xc7 +#define SQ_SRC_M_8_INT 0xc8 +#define SQ_SRC_M_9_INT 0xc9 +#define SQ_SRC_M_10_INT 0xca +#define SQ_SRC_M_11_INT 0xcb +#define SQ_SRC_M_12_INT 0xcc +#define SQ_SRC_M_13_INT 0xcd +#define SQ_SRC_M_14_INT 0xce +#define SQ_SRC_M_15_INT 0xcf +#define SQ_SRC_M_16_INT 0xd0 +#define SQ_SRC_0_5 0xf0 +#define SQ_SRC_M_0_5 0xf1 +#define SQ_SRC_1 0xf2 +#define SQ_SRC_M_1 0xf3 +#define SQ_SRC_2 0xf4 +#define SQ_SRC_M_2 0xf5 +#define SQ_SRC_4 0xf6 +#define SQ_SRC_M_4 0xf7 +#define SQ_SRC_0 0x80 +#define SQ_SRC_1_INT 0x81 +#define SQ_SRC_2_INT 0x82 +#define SQ_SRC_3_INT 0x83 +#define SQ_SRC_4_INT 0x84 +#define SQ_SRC_5_INT 0x85 +#define SQ_SRC_6_INT 0x86 +#define SQ_SRC_7_INT 0x87 +#define SQ_SRC_8_INT 0x88 +#define SQ_SRC_9_INT 0x89 +#define SQ_SRC_10_INT 0x8a +#define SQ_SRC_11_INT 0x8b +#define SQ_SRC_12_INT 0x8c +#define SQ_SRC_13_INT 0x8d +#define SQ_SRC_14_INT 0x8e +#define SQ_SRC_15_INT 0x8f +#define SQ_SRC_16_INT 0x90 +#define SQ_SRC_17_INT 0x91 +#define SQ_SRC_18_INT 0x92 +#define SQ_SRC_19_INT 0x93 +#define SQ_SRC_20_INT 0x94 +#define SQ_SRC_21_INT 0x95 +#define SQ_SRC_22_INT 0x96 +#define SQ_SRC_23_INT 0x97 +#define SQ_SRC_24_INT 0x98 +#define SQ_SRC_25_INT 0x99 +#define SQ_SRC_26_INT 0x9a +#define SQ_SRC_27_INT 0x9b +#define SQ_SRC_28_INT 0x9c +#define SQ_SRC_29_INT 0x9d +#define SQ_SRC_30_INT 0x9e +#define SQ_SRC_31_INT 0x9f +#define SQ_SRC_32_INT 0xa0 +#define SQ_SRC_33_INT 0xa1 +#define SQ_SRC_34_INT 0xa2 +#define SQ_SRC_35_INT 0xa3 +#define SQ_SRC_36_INT 0xa4 +#define SQ_SRC_37_INT 0xa5 +#define SQ_SRC_38_INT 0xa6 +#define SQ_SRC_39_INT 0xa7 +#define SQ_SRC_40_INT 0xa8 +#define SQ_SRC_41_INT 0xa9 +#define SQ_SRC_42_INT 0xaa +#define SQ_SRC_43_INT 0xab +#define SQ_SRC_44_INT 0xac +#define SQ_SRC_45_INT 0xad +#define SQ_SRC_46_INT 0xae +#define SQ_SRC_47_INT 0xaf +#define SQ_SRC_48_INT 0xb0 +#define SQ_SRC_49_INT 0xb1 +#define SQ_SRC_50_INT 0xb2 +#define SQ_SRC_51_INT 0xb3 +#define SQ_SRC_52_INT 0xb4 +#define SQ_SRC_53_INT 0xb5 +#define SQ_SRC_54_INT 0xb6 +#define SQ_SRC_55_INT 0xb7 +#define SQ_SRC_56_INT 0xb8 +#define SQ_SRC_57_INT 0xb9 +#define SQ_SRC_58_INT 0xba +#define SQ_SRC_59_INT 0xbb +#define SQ_SRC_60_INT 0xbc +#define SQ_SRC_61_INT 0xbd +#define SQ_SRC_62_INT 0xbe +#define SQ_SRC_63_INT 0xbf +#define SQ_BUFFER_LOAD_FORMAT_X 0x0 +#define SQ_BUFFER_LOAD_FORMAT_XY 0x1 +#define SQ_BUFFER_LOAD_FORMAT_XYZ 0x2 +#define SQ_BUFFER_LOAD_FORMAT_XYZW 0x3 +#define SQ_BUFFER_STORE_FORMAT_X 0x4 +#define SQ_BUFFER_STORE_FORMAT_XY 0x5 +#define SQ_BUFFER_STORE_FORMAT_XYZ 0x6 +#define SQ_BUFFER_STORE_FORMAT_XYZW 0x7 +#define SQ_BUFFER_LOAD_UBYTE 0x8 +#define SQ_BUFFER_LOAD_SBYTE 0x9 +#define SQ_BUFFER_LOAD_USHORT 0xa +#define SQ_BUFFER_LOAD_SSHORT 0xb +#define SQ_BUFFER_LOAD_DWORD 0xc +#define SQ_BUFFER_LOAD_DWORDX2 0xd +#define SQ_BUFFER_LOAD_DWORDX4 0xe +#define SQ_BUFFER_LOAD_DWORDX3 0xf +#define SQ_BUFFER_STORE_BYTE 0x18 +#define SQ_BUFFER_STORE_SHORT 0x1a +#define SQ_BUFFER_STORE_DWORD 0x1c +#define SQ_BUFFER_STORE_DWORDX2 0x1d +#define SQ_BUFFER_STORE_DWORDX4 0x1e +#define SQ_BUFFER_STORE_DWORDX3 0x1f +#define SQ_BUFFER_ATOMIC_SWAP 0x30 +#define SQ_BUFFER_ATOMIC_CMPSWAP 0x31 +#define SQ_BUFFER_ATOMIC_ADD 0x32 +#define SQ_BUFFER_ATOMIC_SUB 0x33 +#define SQ_BUFFER_ATOMIC_SMIN 0x35 +#define SQ_BUFFER_ATOMIC_UMIN 0x36 +#define SQ_BUFFER_ATOMIC_SMAX 0x37 +#define SQ_BUFFER_ATOMIC_UMAX 0x38 +#define SQ_BUFFER_ATOMIC_AND 0x39 +#define SQ_BUFFER_ATOMIC_OR 0x3a +#define SQ_BUFFER_ATOMIC_XOR 0x3b +#define SQ_BUFFER_ATOMIC_INC 0x3c +#define SQ_BUFFER_ATOMIC_DEC 0x3d +#define SQ_BUFFER_ATOMIC_FCMPSWAP 0x3e +#define SQ_BUFFER_ATOMIC_FMIN 0x3f +#define SQ_BUFFER_ATOMIC_FMAX 0x40 +#define SQ_BUFFER_ATOMIC_SWAP_X2 0x50 +#define SQ_BUFFER_ATOMIC_CMPSWAP_X2 0x51 +#define SQ_BUFFER_ATOMIC_ADD_X2 0x52 +#define SQ_BUFFER_ATOMIC_SUB_X2 0x53 +#define SQ_BUFFER_ATOMIC_SMIN_X2 0x55 +#define SQ_BUFFER_ATOMIC_UMIN_X2 0x56 +#define SQ_BUFFER_ATOMIC_SMAX_X2 0x57 +#define SQ_BUFFER_ATOMIC_UMAX_X2 0x58 +#define SQ_BUFFER_ATOMIC_AND_X2 0x59 +#define SQ_BUFFER_ATOMIC_OR_X2 0x5a +#define SQ_BUFFER_ATOMIC_XOR_X2 0x5b +#define SQ_BUFFER_ATOMIC_INC_X2 0x5c +#define SQ_BUFFER_ATOMIC_DEC_X2 0x5d +#define SQ_BUFFER_ATOMIC_FCMPSWAP_X2 0x5e +#define SQ_BUFFER_ATOMIC_FMIN_X2 0x5f +#define SQ_BUFFER_ATOMIC_FMAX_X2 0x60 +#define SQ_BUFFER_WBINVL1_VOL 0x70 +#define SQ_BUFFER_WBINVL1 0x71 +#define SQ_DS_ADD_U32 0x0 +#define SQ_DS_SUB_U32 0x1 +#define SQ_DS_RSUB_U32 0x2 +#define SQ_DS_INC_U32 0x3 +#define SQ_DS_DEC_U32 0x4 +#define SQ_DS_MIN_I32 0x5 +#define SQ_DS_MAX_I32 0x6 +#define SQ_DS_MIN_U32 0x7 +#define SQ_DS_MAX_U32 0x8 +#define SQ_DS_AND_B32 0x9 +#define SQ_DS_OR_B32 0xa +#define SQ_DS_XOR_B32 0xb +#define SQ_DS_MSKOR_B32 0xc +#define SQ_DS_WRITE_B32 0xd +#define SQ_DS_WRITE2_B32 0xe +#define SQ_DS_WRITE2ST64_B32 0xf +#define SQ_DS_CMPST_B32 0x10 +#define SQ_DS_CMPST_F32 0x11 +#define SQ_DS_MIN_F32 0x12 +#define SQ_DS_MAX_F32 0x13 +#define SQ_DS_NOP 0x14 +#define SQ_DS_GWS_SEMA_RELEASE_ALL 0x18 +#define SQ_DS_GWS_INIT 0x19 +#define SQ_DS_GWS_SEMA_V 0x1a +#define SQ_DS_GWS_SEMA_BR 0x1b +#define SQ_DS_GWS_SEMA_P 0x1c +#define SQ_DS_GWS_BARRIER 0x1d +#define SQ_DS_WRITE_B8 0x1e +#define SQ_DS_WRITE_B16 0x1f +#define SQ_DS_ADD_RTN_U32 0x20 +#define SQ_DS_SUB_RTN_U32 0x21 +#define SQ_DS_RSUB_RTN_U32 0x22 +#define SQ_DS_INC_RTN_U32 0x23 +#define SQ_DS_DEC_RTN_U32 0x24 +#define SQ_DS_MIN_RTN_I32 0x25 +#define SQ_DS_MAX_RTN_I32 0x26 +#define SQ_DS_MIN_RTN_U32 0x27 +#define SQ_DS_MAX_RTN_U32 0x28 +#define SQ_DS_AND_RTN_B32 0x29 +#define SQ_DS_OR_RTN_B32 0x2a +#define SQ_DS_XOR_RTN_B32 0x2b +#define SQ_DS_MSKOR_RTN_B32 0x2c +#define SQ_DS_WRXCHG_RTN_B32 0x2d +#define SQ_DS_WRXCHG2_RTN_B32 0x2e +#define SQ_DS_WRXCHG2ST64_RTN_B32 0x2f +#define SQ_DS_CMPST_RTN_B32 0x30 +#define SQ_DS_CMPST_RTN_F32 0x31 +#define SQ_DS_MIN_RTN_F32 0x32 +#define SQ_DS_MAX_RTN_F32 0x33 +#define SQ_DS_WRAP_RTN_B32 0x34 +#define SQ_DS_SWIZZLE_B32 0x35 +#define SQ_DS_READ_B32 0x36 +#define SQ_DS_READ2_B32 0x37 +#define SQ_DS_READ2ST64_B32 0x38 +#define SQ_DS_READ_I8 0x39 +#define SQ_DS_READ_U8 0x3a +#define SQ_DS_READ_I16 0x3b +#define SQ_DS_READ_U16 0x3c +#define SQ_DS_CONSUME 0x3d +#define SQ_DS_APPEND 0x3e +#define SQ_DS_ORDERED_COUNT 0x3f +#define SQ_DS_ADD_U64 0x40 +#define SQ_DS_SUB_U64 0x41 +#define SQ_DS_RSUB_U64 0x42 +#define SQ_DS_INC_U64 0x43 +#define SQ_DS_DEC_U64 0x44 +#define SQ_DS_MIN_I64 0x45 +#define SQ_DS_MAX_I64 0x46 +#define SQ_DS_MIN_U64 0x47 +#define SQ_DS_MAX_U64 0x48 +#define SQ_DS_AND_B64 0x49 +#define SQ_DS_OR_B64 0x4a +#define SQ_DS_XOR_B64 0x4b +#define SQ_DS_MSKOR_B64 0x4c +#define SQ_DS_WRITE_B64 0x4d +#define SQ_DS_WRITE2_B64 0x4e +#define SQ_DS_WRITE2ST64_B64 0x4f +#define SQ_DS_CMPST_B64 0x50 +#define SQ_DS_CMPST_F64 0x51 +#define SQ_DS_MIN_F64 0x52 +#define SQ_DS_MAX_F64 0x53 +#define SQ_DS_ADD_RTN_U64 0x60 +#define SQ_DS_SUB_RTN_U64 0x61 +#define SQ_DS_RSUB_RTN_U64 0x62 +#define SQ_DS_INC_RTN_U64 0x63 +#define SQ_DS_DEC_RTN_U64 0x64 +#define SQ_DS_MIN_RTN_I64 0x65 +#define SQ_DS_MAX_RTN_I64 0x66 +#define SQ_DS_MIN_RTN_U64 0x67 +#define SQ_DS_MAX_RTN_U64 0x68 +#define SQ_DS_AND_RTN_B64 0x69 +#define SQ_DS_OR_RTN_B64 0x6a +#define SQ_DS_XOR_RTN_B64 0x6b +#define SQ_DS_MSKOR_RTN_B64 0x6c +#define SQ_DS_WRXCHG_RTN_B64 0x6d +#define SQ_DS_WRXCHG2_RTN_B64 0x6e +#define SQ_DS_WRXCHG2ST64_RTN_B64 0x6f +#define SQ_DS_CMPST_RTN_B64 0x70 +#define SQ_DS_CMPST_RTN_F64 0x71 +#define SQ_DS_MIN_RTN_F64 0x72 +#define SQ_DS_MAX_RTN_F64 0x73 +#define SQ_DS_READ_B64 0x76 +#define SQ_DS_READ2_B64 0x77 +#define SQ_DS_READ2ST64_B64 0x78 +#define SQ_DS_CONDXCHG32_RTN_B64 0x7e +#define SQ_DS_ADD_SRC2_U32 0x80 +#define SQ_DS_SUB_SRC2_U32 0x81 +#define SQ_DS_RSUB_SRC2_U32 0x82 +#define SQ_DS_INC_SRC2_U32 0x83 +#define SQ_DS_DEC_SRC2_U32 0x84 +#define SQ_DS_MIN_SRC2_I32 0x85 +#define SQ_DS_MAX_SRC2_I32 0x86 +#define SQ_DS_MIN_SRC2_U32 0x87 +#define SQ_DS_MAX_SRC2_U32 0x88 +#define SQ_DS_AND_SRC2_B32 0x89 +#define SQ_DS_OR_SRC2_B32 0x8a +#define SQ_DS_XOR_SRC2_B32 0x8b +#define SQ_DS_WRITE_SRC2_B32 0x8d +#define SQ_DS_MIN_SRC2_F32 0x92 +#define SQ_DS_MAX_SRC2_F32 0x93 +#define SQ_DS_ADD_SRC2_U64 0xc0 +#define SQ_DS_SUB_SRC2_U64 0xc1 +#define SQ_DS_RSUB_SRC2_U64 0xc2 +#define SQ_DS_INC_SRC2_U64 0xc3 +#define SQ_DS_DEC_SRC2_U64 0xc4 +#define SQ_DS_MIN_SRC2_I64 0xc5 +#define SQ_DS_MAX_SRC2_I64 0xc6 +#define SQ_DS_MIN_SRC2_U64 0xc7 +#define SQ_DS_MAX_SRC2_U64 0xc8 +#define SQ_DS_AND_SRC2_B64 0xc9 +#define SQ_DS_OR_SRC2_B64 0xca +#define SQ_DS_XOR_SRC2_B64 0xcb +#define SQ_DS_WRITE_SRC2_B64 0xcd +#define SQ_DS_MIN_SRC2_F64 0xd2 +#define SQ_DS_MAX_SRC2_F64 0xd3 +#define SQ_DS_WRITE_B96 0xde +#define SQ_DS_WRITE_B128 0xdf +#define SQ_DS_CONDXCHG32_RTN_B128 0xfd +#define SQ_DS_READ_B96 0xfe +#define SQ_DS_READ_B128 0xff +#define SQ_SRC_SCC 0xfd +#define SQ_OMOD_OFF 0x0 +#define SQ_OMOD_M2 0x1 +#define SQ_OMOD_M4 0x2 +#define SQ_OMOD_D2 0x3 +#define SQ_EXP_GDS0 0x18 +#define SQ_GS_OP_NOP 0x0 +#define SQ_GS_OP_CUT 0x1 +#define SQ_GS_OP_EMIT 0x2 +#define SQ_GS_OP_EMIT_CUT 0x3 +#define SQ_IMAGE_LOAD 0x0 +#define SQ_IMAGE_LOAD_MIP 0x1 +#define SQ_IMAGE_LOAD_PCK 0x2 +#define SQ_IMAGE_LOAD_PCK_SGN 0x3 +#define SQ_IMAGE_LOAD_MIP_PCK 0x4 +#define SQ_IMAGE_LOAD_MIP_PCK_SGN 0x5 +#define SQ_IMAGE_STORE 0x8 +#define SQ_IMAGE_STORE_MIP 0x9 +#define SQ_IMAGE_STORE_PCK 0xa +#define SQ_IMAGE_STORE_MIP_PCK 0xb +#define SQ_IMAGE_GET_RESINFO 0xe +#define SQ_IMAGE_ATOMIC_SWAP 0xf +#define SQ_IMAGE_ATOMIC_CMPSWAP 0x10 +#define SQ_IMAGE_ATOMIC_ADD 0x11 +#define SQ_IMAGE_ATOMIC_SUB 0x12 +#define SQ_IMAGE_ATOMIC_SMIN 0x14 +#define SQ_IMAGE_ATOMIC_UMIN 0x15 +#define SQ_IMAGE_ATOMIC_SMAX 0x16 +#define SQ_IMAGE_ATOMIC_UMAX 0x17 +#define SQ_IMAGE_ATOMIC_AND 0x18 +#define SQ_IMAGE_ATOMIC_OR 0x19 +#define SQ_IMAGE_ATOMIC_XOR 0x1a +#define SQ_IMAGE_ATOMIC_INC 0x1b +#define SQ_IMAGE_ATOMIC_DEC 0x1c +#define SQ_IMAGE_ATOMIC_FCMPSWAP 0x1d +#define SQ_IMAGE_ATOMIC_FMIN 0x1e +#define SQ_IMAGE_ATOMIC_FMAX 0x1f +#define SQ_IMAGE_SAMPLE 0x20 +#define SQ_IMAGE_SAMPLE_CL 0x21 +#define SQ_IMAGE_SAMPLE_D 0x22 +#define SQ_IMAGE_SAMPLE_D_CL 0x23 +#define SQ_IMAGE_SAMPLE_L 0x24 +#define SQ_IMAGE_SAMPLE_B 0x25 +#define SQ_IMAGE_SAMPLE_B_CL 0x26 +#define SQ_IMAGE_SAMPLE_LZ 0x27 +#define SQ_IMAGE_SAMPLE_C 0x28 +#define SQ_IMAGE_SAMPLE_C_CL 0x29 +#define SQ_IMAGE_SAMPLE_C_D 0x2a +#define SQ_IMAGE_SAMPLE_C_D_CL 0x2b +#define SQ_IMAGE_SAMPLE_C_L 0x2c +#define SQ_IMAGE_SAMPLE_C_B 0x2d +#define SQ_IMAGE_SAMPLE_C_B_CL 0x2e +#define SQ_IMAGE_SAMPLE_C_LZ 0x2f +#define SQ_IMAGE_SAMPLE_O 0x30 +#define SQ_IMAGE_SAMPLE_CL_O 0x31 +#define SQ_IMAGE_SAMPLE_D_O 0x32 +#define SQ_IMAGE_SAMPLE_D_CL_O 0x33 +#define SQ_IMAGE_SAMPLE_L_O 0x34 +#define SQ_IMAGE_SAMPLE_B_O 0x35 +#define SQ_IMAGE_SAMPLE_B_CL_O 0x36 +#define SQ_IMAGE_SAMPLE_LZ_O 0x37 +#define SQ_IMAGE_SAMPLE_C_O 0x38 +#define SQ_IMAGE_SAMPLE_C_CL_O 0x39 +#define SQ_IMAGE_SAMPLE_C_D_O 0x3a +#define SQ_IMAGE_SAMPLE_C_D_CL_O 0x3b +#define SQ_IMAGE_SAMPLE_C_L_O 0x3c +#define SQ_IMAGE_SAMPLE_C_B_O 0x3d +#define SQ_IMAGE_SAMPLE_C_B_CL_O 0x3e +#define SQ_IMAGE_SAMPLE_C_LZ_O 0x3f +#define SQ_IMAGE_GATHER4 0x40 +#define SQ_IMAGE_GATHER4_CL 0x41 +#define SQ_IMAGE_GATHER4_L 0x44 +#define SQ_IMAGE_GATHER4_B 0x45 +#define SQ_IMAGE_GATHER4_B_CL 0x46 +#define SQ_IMAGE_GATHER4_LZ 0x47 +#define SQ_IMAGE_GATHER4_C 0x48 +#define SQ_IMAGE_GATHER4_C_CL 0x49 +#define SQ_IMAGE_GATHER4_C_L 0x4c +#define SQ_IMAGE_GATHER4_C_B 0x4d +#define SQ_IMAGE_GATHER4_C_B_CL 0x4e +#define SQ_IMAGE_GATHER4_C_LZ 0x4f +#define SQ_IMAGE_GATHER4_O 0x50 +#define SQ_IMAGE_GATHER4_CL_O 0x51 +#define SQ_IMAGE_GATHER4_L_O 0x54 +#define SQ_IMAGE_GATHER4_B_O 0x55 +#define SQ_IMAGE_GATHER4_B_CL_O 0x56 +#define SQ_IMAGE_GATHER4_LZ_O 0x57 +#define SQ_IMAGE_GATHER4_C_O 0x58 +#define SQ_IMAGE_GATHER4_C_CL_O 0x59 +#define SQ_IMAGE_GATHER4_C_L_O 0x5c +#define SQ_IMAGE_GATHER4_C_B_O 0x5d +#define SQ_IMAGE_GATHER4_C_B_CL_O 0x5e +#define SQ_IMAGE_GATHER4_C_LZ_O 0x5f +#define SQ_IMAGE_GET_LOD 0x60 +#define SQ_IMAGE_SAMPLE_CD 0x68 +#define SQ_IMAGE_SAMPLE_CD_CL 0x69 +#define SQ_IMAGE_SAMPLE_C_CD 0x6a +#define SQ_IMAGE_SAMPLE_C_CD_CL 0x6b +#define SQ_IMAGE_SAMPLE_CD_O 0x6c +#define SQ_IMAGE_SAMPLE_CD_CL_O 0x6d +#define SQ_IMAGE_SAMPLE_C_CD_O 0x6e +#define SQ_IMAGE_SAMPLE_C_CD_CL_O 0x6f +#define SQ_IMAGE_RSRC256 0x7e +#define SQ_IMAGE_SAMPLER 0x7f +#define SQ_SRC_VCCZ 0xfb +#define SQ_SRC_VGPR0 0x100 +#define SQ_DFMT_INVALID 0x0 +#define SQ_DFMT_8 0x1 +#define SQ_DFMT_16 0x2 +#define SQ_DFMT_8_8 0x3 +#define SQ_DFMT_32 0x4 +#define SQ_DFMT_16_16 0x5 +#define SQ_DFMT_10_11_11 0x6 +#define SQ_DFMT_11_11_10 0x7 +#define SQ_DFMT_10_10_10_2 0x8 +#define SQ_DFMT_2_10_10_10 0x9 +#define SQ_DFMT_8_8_8_8 0xa +#define SQ_DFMT_32_32 0xb +#define SQ_DFMT_16_16_16_16 0xc +#define SQ_DFMT_32_32_32 0xd +#define SQ_DFMT_32_32_32_32 0xe +#define SQ_TBUFFER_LOAD_FORMAT_X 0x0 +#define SQ_TBUFFER_LOAD_FORMAT_XY 0x1 +#define SQ_TBUFFER_LOAD_FORMAT_XYZ 0x2 +#define SQ_TBUFFER_LOAD_FORMAT_XYZW 0x3 +#define SQ_TBUFFER_STORE_FORMAT_X 0x4 +#define SQ_TBUFFER_STORE_FORMAT_XY 0x5 +#define SQ_TBUFFER_STORE_FORMAT_XYZ 0x6 +#define SQ_TBUFFER_STORE_FORMAT_XYZW 0x7 +#define SQ_CHAN_X 0x0 +#define SQ_CHAN_Y 0x1 +#define SQ_CHAN_Z 0x2 +#define SQ_CHAN_W 0x3 +#define SQ_EXEC_LO 0x7e +#define SQ_EXEC_HI 0x7f +#define SQ_S_LOAD_DWORD 0x0 +#define SQ_S_LOAD_DWORDX2 0x1 +#define SQ_S_LOAD_DWORDX4 0x2 +#define SQ_S_LOAD_DWORDX8 0x3 +#define SQ_S_LOAD_DWORDX16 0x4 +#define SQ_S_BUFFER_LOAD_DWORD 0x8 +#define SQ_S_BUFFER_LOAD_DWORDX2 0x9 +#define SQ_S_BUFFER_LOAD_DWORDX4 0xa +#define SQ_S_BUFFER_LOAD_DWORDX8 0xb +#define SQ_S_BUFFER_LOAD_DWORDX16 0xc +#define SQ_S_DCACHE_INV_VOL 0x1d +#define SQ_S_MEMTIME 0x1e +#define SQ_S_DCACHE_INV 0x1f +#define SQ_V_NOP 0x0 +#define SQ_V_MOV_B32 0x1 +#define SQ_V_READFIRSTLANE_B32 0x2 +#define SQ_V_CVT_I32_F64 0x3 +#define SQ_V_CVT_F64_I32 0x4 +#define SQ_V_CVT_F32_I32 0x5 +#define SQ_V_CVT_F32_U32 0x6 +#define SQ_V_CVT_U32_F32 0x7 +#define SQ_V_CVT_I32_F32 0x8 +#define SQ_V_MOV_FED_B32 0x9 +#define SQ_V_CVT_F16_F32 0xa +#define SQ_V_CVT_F32_F16 0xb +#define SQ_V_CVT_RPI_I32_F32 0xc +#define SQ_V_CVT_FLR_I32_F32 0xd +#define SQ_V_CVT_OFF_F32_I4 0xe +#define SQ_V_CVT_F32_F64 0xf +#define SQ_V_CVT_F64_F32 0x10 +#define SQ_V_CVT_F32_UBYTE0 0x11 +#define SQ_V_CVT_F32_UBYTE1 0x12 +#define SQ_V_CVT_F32_UBYTE2 0x13 +#define SQ_V_CVT_F32_UBYTE3 0x14 +#define SQ_V_CVT_U32_F64 0x15 +#define SQ_V_CVT_F64_U32 0x16 +#define SQ_V_TRUNC_F64 0x17 +#define SQ_V_CEIL_F64 0x18 +#define SQ_V_RNDNE_F64 0x19 +#define SQ_V_FLOOR_F64 0x1a +#define SQ_V_FRACT_F32 0x20 +#define SQ_V_TRUNC_F32 0x21 +#define SQ_V_CEIL_F32 0x22 +#define SQ_V_RNDNE_F32 0x23 +#define SQ_V_FLOOR_F32 0x24 +#define SQ_V_EXP_F32 0x25 +#define SQ_V_LOG_CLAMP_F32 0x26 +#define SQ_V_LOG_F32 0x27 +#define SQ_V_RCP_CLAMP_F32 0x28 +#define SQ_V_RCP_LEGACY_F32 0x29 +#define SQ_V_RCP_F32 0x2a +#define SQ_V_RCP_IFLAG_F32 0x2b +#define SQ_V_RSQ_CLAMP_F32 0x2c +#define SQ_V_RSQ_LEGACY_F32 0x2d +#define SQ_V_RSQ_F32 0x2e +#define SQ_V_RCP_F64 0x2f +#define SQ_V_RCP_CLAMP_F64 0x30 +#define SQ_V_RSQ_F64 0x31 +#define SQ_V_RSQ_CLAMP_F64 0x32 +#define SQ_V_SQRT_F32 0x33 +#define SQ_V_SQRT_F64 0x34 +#define SQ_V_SIN_F32 0x35 +#define SQ_V_COS_F32 0x36 +#define SQ_V_NOT_B32 0x37 +#define SQ_V_BFREV_B32 0x38 +#define SQ_V_FFBH_U32 0x39 +#define SQ_V_FFBL_B32 0x3a +#define SQ_V_FFBH_I32 0x3b +#define SQ_V_FREXP_EXP_I32_F64 0x3c +#define SQ_V_FREXP_MANT_F64 0x3d +#define SQ_V_FRACT_F64 0x3e +#define SQ_V_FREXP_EXP_I32_F32 0x3f +#define SQ_V_FREXP_MANT_F32 0x40 +#define SQ_V_CLREXCP 0x41 +#define SQ_V_MOVRELD_B32 0x42 +#define SQ_V_MOVRELS_B32 0x43 +#define SQ_V_MOVRELSD_B32 0x44 +#define SQ_V_LOG_LEGACY_F32 0x45 +#define SQ_V_EXP_LEGACY_F32 0x46 +#define SQ_NFMT_UNORM 0x0 +#define SQ_NFMT_SNORM 0x1 +#define SQ_NFMT_USCALED 0x2 +#define SQ_NFMT_SSCALED 0x3 +#define SQ_NFMT_UINT 0x4 +#define SQ_NFMT_SINT 0x5 +#define SQ_NFMT_SNORM_OGL 0x6 +#define SQ_NFMT_FLOAT 0x7 +#define SQ_V_OP1_OFFSET 0x180 +#define SQ_V_OP2_OFFSET 0x100 +#define SQ_V_OPC_OFFSET 0x0 +#define SQ_V_INTERP_P1_F32 0x0 +#define SQ_V_INTERP_P2_F32 0x1 +#define SQ_V_INTERP_MOV_F32 0x2 +#define SQ_S_NOP 0x0 +#define SQ_S_ENDPGM 0x1 +#define SQ_S_BRANCH 0x2 +#define SQ_S_CBRANCH_SCC0 0x4 +#define SQ_S_CBRANCH_SCC1 0x5 +#define SQ_S_CBRANCH_VCCZ 0x6 +#define SQ_S_CBRANCH_VCCNZ 0x7 +#define SQ_S_CBRANCH_EXECZ 0x8 +#define SQ_S_CBRANCH_EXECNZ 0x9 +#define SQ_S_BARRIER 0xa +#define SQ_S_SETKILL 0xb +#define SQ_S_WAITCNT 0xc +#define SQ_S_SETHALT 0xd +#define SQ_S_SLEEP 0xe +#define SQ_S_SETPRIO 0xf +#define SQ_S_SENDMSG 0x10 +#define SQ_S_SENDMSGHALT 0x11 +#define SQ_S_TRAP 0x12 +#define SQ_S_ICACHE_INV 0x13 +#define SQ_S_INCPERFLEVEL 0x14 +#define SQ_S_DECPERFLEVEL 0x15 +#define SQ_S_TTRACEDATA 0x16 +#define SQ_S_CBRANCH_CDBGSYS 0x17 +#define SQ_S_CBRANCH_CDBGUSER 0x18 +#define SQ_S_CBRANCH_CDBGSYS_OR_USER 0x19 +#define SQ_S_CBRANCH_CDBGSYS_AND_USER 0x1a +#define SQ_SRC_LITERAL 0xff +#define SQ_VCC_LO 0x6a +#define SQ_VCC_HI 0x6b +#define SQ_PARAM_P10 0x0 +#define SQ_PARAM_P20 0x1 +#define SQ_PARAM_P0 0x2 +#define SQ_SRC_LDS_DIRECT 0xfe +#define SQ_FLAT_SCRATCH_LO 0x68 +#define SQ_FLAT_SCRATCH_HI 0x69 +#define SQ_V_CNDMASK_B32 0x0 +#define SQ_V_READLANE_B32 0x1 +#define SQ_V_WRITELANE_B32 0x2 +#define SQ_V_ADD_F32 0x3 +#define SQ_V_SUB_F32 0x4 +#define SQ_V_SUBREV_F32 0x5 +#define SQ_V_MAC_LEGACY_F32 0x6 +#define SQ_V_MUL_LEGACY_F32 0x7 +#define SQ_V_MUL_F32 0x8 +#define SQ_V_MUL_I32_I24 0x9 +#define SQ_V_MUL_HI_I32_I24 0xa +#define SQ_V_MUL_U32_U24 0xb +#define SQ_V_MUL_HI_U32_U24 0xc +#define SQ_V_MIN_LEGACY_F32 0xd +#define SQ_V_MAX_LEGACY_F32 0xe +#define SQ_V_MIN_F32 0xf +#define SQ_V_MAX_F32 0x10 +#define SQ_V_MIN_I32 0x11 +#define SQ_V_MAX_I32 0x12 +#define SQ_V_MIN_U32 0x13 +#define SQ_V_MAX_U32 0x14 +#define SQ_V_LSHR_B32 0x15 +#define SQ_V_LSHRREV_B32 0x16 +#define SQ_V_ASHR_I32 0x17 +#define SQ_V_ASHRREV_I32 0x18 +#define SQ_V_LSHL_B32 0x19 +#define SQ_V_LSHLREV_B32 0x1a +#define SQ_V_AND_B32 0x1b +#define SQ_V_OR_B32 0x1c +#define SQ_V_XOR_B32 0x1d +#define SQ_V_BFM_B32 0x1e +#define SQ_V_MAC_F32 0x1f +#define SQ_V_MADMK_F32 0x20 +#define SQ_V_MADAK_F32 0x21 +#define SQ_V_BCNT_U32_B32 0x22 +#define SQ_V_MBCNT_LO_U32_B32 0x23 +#define SQ_V_MBCNT_HI_U32_B32 0x24 +#define SQ_V_ADD_I32 0x25 +#define SQ_V_SUB_I32 0x26 +#define SQ_V_SUBREV_I32 0x27 +#define SQ_V_ADDC_U32 0x28 +#define SQ_V_SUBB_U32 0x29 +#define SQ_V_SUBBREV_U32 0x2a +#define SQ_V_LDEXP_F32 0x2b +#define SQ_V_CVT_PKACCUM_U8_F32 0x2c +#define SQ_V_CVT_PKNORM_I16_F32 0x2d +#define SQ_V_CVT_PKNORM_U16_F32 0x2e +#define SQ_V_CVT_PKRTZ_F16_F32 0x2f +#define SQ_V_CVT_PK_U16_U32 0x30 +#define SQ_V_CVT_PK_I16_I32 0x31 +#define SQ_FLAT_LOAD_UBYTE 0x8 +#define SQ_FLAT_LOAD_SBYTE 0x9 +#define SQ_FLAT_LOAD_USHORT 0xa +#define SQ_FLAT_LOAD_SSHORT 0xb +#define SQ_FLAT_LOAD_DWORD 0xc +#define SQ_FLAT_LOAD_DWORDX2 0xd +#define SQ_FLAT_LOAD_DWORDX4 0xe +#define SQ_FLAT_LOAD_DWORDX3 0xf +#define SQ_FLAT_STORE_BYTE 0x18 +#define SQ_FLAT_STORE_SHORT 0x1a +#define SQ_FLAT_STORE_DWORD 0x1c +#define SQ_FLAT_STORE_DWORDX2 0x1d +#define SQ_FLAT_STORE_DWORDX4 0x1e +#define SQ_FLAT_STORE_DWORDX3 0x1f +#define SQ_FLAT_ATOMIC_SWAP 0x30 +#define SQ_FLAT_ATOMIC_CMPSWAP 0x31 +#define SQ_FLAT_ATOMIC_ADD 0x32 +#define SQ_FLAT_ATOMIC_SUB 0x33 +#define SQ_FLAT_ATOMIC_SMIN 0x35 +#define SQ_FLAT_ATOMIC_UMIN 0x36 +#define SQ_FLAT_ATOMIC_SMAX 0x37 +#define SQ_FLAT_ATOMIC_UMAX 0x38 +#define SQ_FLAT_ATOMIC_AND 0x39 +#define SQ_FLAT_ATOMIC_OR 0x3a +#define SQ_FLAT_ATOMIC_XOR 0x3b +#define SQ_FLAT_ATOMIC_INC 0x3c +#define SQ_FLAT_ATOMIC_DEC 0x3d +#define SQ_FLAT_ATOMIC_FCMPSWAP 0x3e +#define SQ_FLAT_ATOMIC_FMIN 0x3f +#define SQ_FLAT_ATOMIC_FMAX 0x40 +#define SQ_FLAT_ATOMIC_SWAP_X2 0x50 +#define SQ_FLAT_ATOMIC_CMPSWAP_X2 0x51 +#define SQ_FLAT_ATOMIC_ADD_X2 0x52 +#define SQ_FLAT_ATOMIC_SUB_X2 0x53 +#define SQ_FLAT_ATOMIC_SMIN_X2 0x55 +#define SQ_FLAT_ATOMIC_UMIN_X2 0x56 +#define SQ_FLAT_ATOMIC_SMAX_X2 0x57 +#define SQ_FLAT_ATOMIC_UMAX_X2 0x58 +#define SQ_FLAT_ATOMIC_AND_X2 0x59 +#define SQ_FLAT_ATOMIC_OR_X2 0x5a +#define SQ_FLAT_ATOMIC_XOR_X2 0x5b +#define SQ_FLAT_ATOMIC_INC_X2 0x5c +#define SQ_FLAT_ATOMIC_DEC_X2 0x5d +#define SQ_FLAT_ATOMIC_FCMPSWAP_X2 0x5e +#define SQ_FLAT_ATOMIC_FMIN_X2 0x5f +#define SQ_FLAT_ATOMIC_FMAX_X2 0x60 +#define SQ_S_CMP_EQ_I32 0x0 +#define SQ_S_CMP_LG_I32 0x1 +#define SQ_S_CMP_GT_I32 0x2 +#define SQ_S_CMP_GE_I32 0x3 +#define SQ_S_CMP_LT_I32 0x4 +#define SQ_S_CMP_LE_I32 0x5 +#define SQ_S_CMP_EQ_U32 0x6 +#define SQ_S_CMP_LG_U32 0x7 +#define SQ_S_CMP_GT_U32 0x8 +#define SQ_S_CMP_GE_U32 0x9 +#define SQ_S_CMP_LT_U32 0xa +#define SQ_S_CMP_LE_U32 0xb +#define SQ_S_BITCMP0_B32 0xc +#define SQ_S_BITCMP1_B32 0xd +#define SQ_S_BITCMP0_B64 0xe +#define SQ_S_BITCMP1_B64 0xf +#define SQ_S_SETVSKIP 0x10 +#define SQ_M0 0x7c +#define SQ_V_MAD_LEGACY_F32 0x140 +#define SQ_V_MAD_F32 0x141 +#define SQ_V_MAD_I32_I24 0x142 +#define SQ_V_MAD_U32_U24 0x143 +#define SQ_V_CUBEID_F32 0x144 +#define SQ_V_CUBESC_F32 0x145 +#define SQ_V_CUBETC_F32 0x146 +#define SQ_V_CUBEMA_F32 0x147 +#define SQ_V_BFE_U32 0x148 +#define SQ_V_BFE_I32 0x149 +#define SQ_V_BFI_B32 0x14a +#define SQ_V_FMA_F32 0x14b +#define SQ_V_FMA_F64 0x14c +#define SQ_V_LERP_U8 0x14d +#define SQ_V_ALIGNBIT_B32 0x14e +#define SQ_V_ALIGNBYTE_B32 0x14f +#define SQ_V_MULLIT_F32 0x150 +#define SQ_V_MIN3_F32 0x151 +#define SQ_V_MIN3_I32 0x152 +#define SQ_V_MIN3_U32 0x153 +#define SQ_V_MAX3_F32 0x154 +#define SQ_V_MAX3_I32 0x155 +#define SQ_V_MAX3_U32 0x156 +#define SQ_V_MED3_F32 0x157 +#define SQ_V_MED3_I32 0x158 +#define SQ_V_MED3_U32 0x159 +#define SQ_V_SAD_U8 0x15a +#define SQ_V_SAD_HI_U8 0x15b +#define SQ_V_SAD_U16 0x15c +#define SQ_V_SAD_U32 0x15d +#define SQ_V_CVT_PK_U8_F32 0x15e +#define SQ_V_DIV_FIXUP_F32 0x15f +#define SQ_V_DIV_FIXUP_F64 0x160 +#define SQ_V_LSHL_B64 0x161 +#define SQ_V_LSHR_B64 0x162 +#define SQ_V_ASHR_I64 0x163 +#define SQ_V_ADD_F64 0x164 +#define SQ_V_MUL_F64 0x165 +#define SQ_V_MIN_F64 0x166 +#define SQ_V_MAX_F64 0x167 +#define SQ_V_LDEXP_F64 0x168 +#define SQ_V_MUL_LO_U32 0x169 +#define SQ_V_MUL_HI_U32 0x16a +#define SQ_V_MUL_LO_I32 0x16b +#define SQ_V_MUL_HI_I32 0x16c +#define SQ_V_DIV_SCALE_F32 0x16d +#define SQ_V_DIV_SCALE_F64 0x16e +#define SQ_V_DIV_FMAS_F32 0x16f +#define SQ_V_DIV_FMAS_F64 0x170 +#define SQ_V_MSAD_U8 0x171 +#define SQ_V_QSAD_PK_U16_U8 0x172 +#define SQ_V_MQSAD_PK_U16_U8 0x173 +#define SQ_V_TRIG_PREOP_F64 0x174 +#define SQ_V_MQSAD_U32_U8 0x175 +#define SQ_V_MAD_U64_U32 0x176 +#define SQ_V_MAD_I64_I32 0x177 +#define SQ_VCC_ALL 0x0 +#define SQ_SRC_EXECZ 0xfc +#define SQ_SYSMSG_OP_ECC_ERR_INTERRUPT 0x1 +#define SQ_SYSMSG_OP_REG_RD 0x2 +#define SQ_SYSMSG_OP_HOST_TRAP_ACK 0x3 +#define SQ_SYSMSG_OP_TTRACE_PC 0x4 +#define SQ_HW_REG_MODE 0x1 +#define SQ_HW_REG_STATUS 0x2 +#define SQ_HW_REG_TRAPSTS 0x3 +#define SQ_HW_REG_HW_ID 0x4 +#define SQ_HW_REG_GPR_ALLOC 0x5 +#define SQ_HW_REG_LDS_ALLOC 0x6 +#define SQ_HW_REG_IB_STS 0x7 +#define SQ_HW_REG_PC_LO 0x8 +#define SQ_HW_REG_PC_HI 0x9 +#define SQ_HW_REG_INST_DW0 0xa +#define SQ_HW_REG_INST_DW1 0xb +#define SQ_HW_REG_IB_DBG0 0xc +#define SQ_S_ADD_U32 0x0 +#define SQ_S_SUB_U32 0x1 +#define SQ_S_ADD_I32 0x2 +#define SQ_S_SUB_I32 0x3 +#define SQ_S_ADDC_U32 0x4 +#define SQ_S_SUBB_U32 0x5 +#define SQ_S_MIN_I32 0x6 +#define SQ_S_MIN_U32 0x7 +#define SQ_S_MAX_I32 0x8 +#define SQ_S_MAX_U32 0x9 +#define SQ_S_CSELECT_B32 0xa +#define SQ_S_CSELECT_B64 0xb +#define SQ_S_AND_B32 0xe +#define SQ_S_AND_B64 0xf +#define SQ_S_OR_B32 0x10 +#define SQ_S_OR_B64 0x11 +#define SQ_S_XOR_B32 0x12 +#define SQ_S_XOR_B64 0x13 +#define SQ_S_ANDN2_B32 0x14 +#define SQ_S_ANDN2_B64 0x15 +#define SQ_S_ORN2_B32 0x16 +#define SQ_S_ORN2_B64 0x17 +#define SQ_S_NAND_B32 0x18 +#define SQ_S_NAND_B64 0x19 +#define SQ_S_NOR_B32 0x1a +#define SQ_S_NOR_B64 0x1b +#define SQ_S_XNOR_B32 0x1c +#define SQ_S_XNOR_B64 0x1d +#define SQ_S_LSHL_B32 0x1e +#define SQ_S_LSHL_B64 0x1f +#define SQ_S_LSHR_B32 0x20 +#define SQ_S_LSHR_B64 0x21 +#define SQ_S_ASHR_I32 0x22 +#define SQ_S_ASHR_I64 0x23 +#define SQ_S_BFM_B32 0x24 +#define SQ_S_BFM_B64 0x25 +#define SQ_S_MUL_I32 0x26 +#define SQ_S_BFE_U32 0x27 +#define SQ_S_BFE_I32 0x28 +#define SQ_S_BFE_U64 0x29 +#define SQ_S_BFE_I64 0x2a +#define SQ_S_CBRANCH_G_FORK 0x2b +#define SQ_S_ABSDIFF_I32 0x2c +#define SQ_MSG_INTERRUPT 0x1 +#define SQ_MSG_GS 0x2 +#define SQ_MSG_GS_DONE 0x3 +#define SQ_MSG_SYSMSG 0xf +typedef enum TEX_BORDER_COLOR_TYPE { + TEX_BorderColor_TransparentBlack = 0x0, + TEX_BorderColor_OpaqueBlack = 0x1, + TEX_BorderColor_OpaqueWhite = 0x2, + TEX_BorderColor_Register = 0x3, +} TEX_BORDER_COLOR_TYPE; +typedef enum TEX_CHROMA_KEY { + TEX_ChromaKey_Disabled = 0x0, + TEX_ChromaKey_Kill = 0x1, + TEX_ChromaKey_Blend = 0x2, + TEX_ChromaKey_RESERVED_3 = 0x3, +} TEX_CHROMA_KEY; +typedef enum TEX_CLAMP { + TEX_Clamp_Repeat = 0x0, + TEX_Clamp_Mirror = 0x1, + TEX_Clamp_ClampToLast = 0x2, + TEX_Clamp_MirrorOnceToLast = 0x3, + TEX_Clamp_ClampHalfToBorder = 0x4, + TEX_Clamp_MirrorOnceHalfToBorder = 0x5, + TEX_Clamp_ClampToBorder = 0x6, + TEX_Clamp_MirrorOnceToBorder = 0x7, +} TEX_CLAMP; +typedef enum TEX_COORD_TYPE { + TEX_CoordType_Unnormalized = 0x0, + TEX_CoordType_Normalized = 0x1, +} TEX_COORD_TYPE; +typedef enum TEX_DEPTH_COMPARE_FUNCTION { + TEX_DepthCompareFunction_Never = 0x0, + TEX_DepthCompareFunction_Less = 0x1, + TEX_DepthCompareFunction_Equal = 0x2, + TEX_DepthCompareFunction_LessEqual = 0x3, + TEX_DepthCompareFunction_Greater = 0x4, + TEX_DepthCompareFunction_NotEqual = 0x5, + TEX_DepthCompareFunction_GreaterEqual = 0x6, + TEX_DepthCompareFunction_Always = 0x7, +} TEX_DEPTH_COMPARE_FUNCTION; +typedef enum TEX_DIM { + TEX_Dim_1D = 0x0, + TEX_Dim_2D = 0x1, + TEX_Dim_3D = 0x2, + TEX_Dim_CubeMap = 0x3, + TEX_Dim_1DArray = 0x4, + TEX_Dim_2DArray = 0x5, + TEX_Dim_2D_MSAA = 0x6, + TEX_Dim_2DArray_MSAA = 0x7, +} TEX_DIM; +typedef enum TEX_FORMAT_COMP { + TEX_FormatComp_Unsigned = 0x0, + TEX_FormatComp_Signed = 0x1, + TEX_FormatComp_UnsignedBiased = 0x2, + TEX_FormatComp_RESERVED_3 = 0x3, +} TEX_FORMAT_COMP; +typedef enum TEX_MAX_ANISO_RATIO { + TEX_MaxAnisoRatio_1to1 = 0x0, + TEX_MaxAnisoRatio_2to1 = 0x1, + TEX_MaxAnisoRatio_4to1 = 0x2, + TEX_MaxAnisoRatio_8to1 = 0x3, + TEX_MaxAnisoRatio_16to1 = 0x4, + TEX_MaxAnisoRatio_RESERVED_5 = 0x5, + TEX_MaxAnisoRatio_RESERVED_6 = 0x6, + TEX_MaxAnisoRatio_RESERVED_7 = 0x7, +} TEX_MAX_ANISO_RATIO; +typedef enum TEX_MIP_FILTER { + TEX_MipFilter_None = 0x0, + TEX_MipFilter_Point = 0x1, + TEX_MipFilter_Linear = 0x2, + TEX_MipFilter_RESERVED_3 = 0x3, +} TEX_MIP_FILTER; +typedef enum TEX_REQUEST_SIZE { + TEX_RequestSize_32B = 0x0, + TEX_RequestSize_64B = 0x1, + TEX_RequestSize_128B = 0x2, + TEX_RequestSize_2X64B = 0x3, +} TEX_REQUEST_SIZE; +typedef enum TEX_SAMPLER_TYPE { + TEX_SamplerType_Invalid = 0x0, + TEX_SamplerType_Valid = 0x1, +} TEX_SAMPLER_TYPE; +typedef enum TEX_XY_FILTER { + TEX_XYFilter_Point = 0x0, + TEX_XYFilter_Linear = 0x1, + TEX_XYFilter_AnisoPoint = 0x2, + TEX_XYFilter_AnisoLinear = 0x3, +} TEX_XY_FILTER; +typedef enum TEX_Z_FILTER { + TEX_ZFilter_None = 0x0, + TEX_ZFilter_Point = 0x1, + TEX_ZFilter_Linear = 0x2, + TEX_ZFilter_RESERVED_3 = 0x3, +} TEX_Z_FILTER; +typedef enum VTX_CLAMP { + VTX_Clamp_ClampToZero = 0x0, + VTX_Clamp_ClampToNAN = 0x1, +} VTX_CLAMP; +typedef enum VTX_FETCH_TYPE { + VTX_FetchType_VertexData = 0x0, + VTX_FetchType_InstanceData = 0x1, + VTX_FetchType_NoIndexOffset = 0x2, + VTX_FetchType_RESERVED_3 = 0x3, +} VTX_FETCH_TYPE; +typedef enum VTX_FORMAT_COMP_ALL { + VTX_FormatCompAll_Unsigned = 0x0, + VTX_FormatCompAll_Signed = 0x1, +} VTX_FORMAT_COMP_ALL; +typedef enum VTX_MEM_REQUEST_SIZE { + VTX_MemRequestSize_32B = 0x0, + VTX_MemRequestSize_64B = 0x1, +} VTX_MEM_REQUEST_SIZE; +typedef enum TVX_DATA_FORMAT { + TVX_FMT_INVALID = 0x0, + TVX_FMT_8 = 0x1, + TVX_FMT_4_4 = 0x2, + TVX_FMT_3_3_2 = 0x3, + TVX_FMT_RESERVED_4 = 0x4, + TVX_FMT_16 = 0x5, + TVX_FMT_16_FLOAT = 0x6, + TVX_FMT_8_8 = 0x7, + TVX_FMT_5_6_5 = 0x8, + TVX_FMT_6_5_5 = 0x9, + TVX_FMT_1_5_5_5 = 0xa, + TVX_FMT_4_4_4_4 = 0xb, + TVX_FMT_5_5_5_1 = 0xc, + TVX_FMT_32 = 0xd, + TVX_FMT_32_FLOAT = 0xe, + TVX_FMT_16_16 = 0xf, + TVX_FMT_16_16_FLOAT = 0x10, + TVX_FMT_8_24 = 0x11, + TVX_FMT_8_24_FLOAT = 0x12, + TVX_FMT_24_8 = 0x13, + TVX_FMT_24_8_FLOAT = 0x14, + TVX_FMT_10_11_11 = 0x15, + TVX_FMT_10_11_11_FLOAT = 0x16, + TVX_FMT_11_11_10 = 0x17, + TVX_FMT_11_11_10_FLOAT = 0x18, + TVX_FMT_2_10_10_10 = 0x19, + TVX_FMT_8_8_8_8 = 0x1a, + TVX_FMT_10_10_10_2 = 0x1b, + TVX_FMT_X24_8_32_FLOAT = 0x1c, + TVX_FMT_32_32 = 0x1d, + TVX_FMT_32_32_FLOAT = 0x1e, + TVX_FMT_16_16_16_16 = 0x1f, + TVX_FMT_16_16_16_16_FLOAT = 0x20, + TVX_FMT_RESERVED_33 = 0x21, + TVX_FMT_32_32_32_32 = 0x22, + TVX_FMT_32_32_32_32_FLOAT = 0x23, + TVX_FMT_RESERVED_36 = 0x24, + TVX_FMT_1 = 0x25, + TVX_FMT_1_REVERSED = 0x26, + TVX_FMT_GB_GR = 0x27, + TVX_FMT_BG_RG = 0x28, + TVX_FMT_32_AS_8 = 0x29, + TVX_FMT_32_AS_8_8 = 0x2a, + TVX_FMT_5_9_9_9_SHAREDEXP = 0x2b, + TVX_FMT_8_8_8 = 0x2c, + TVX_FMT_16_16_16 = 0x2d, + TVX_FMT_16_16_16_FLOAT = 0x2e, + TVX_FMT_32_32_32 = 0x2f, + TVX_FMT_32_32_32_FLOAT = 0x30, + TVX_FMT_BC1 = 0x31, + TVX_FMT_BC2 = 0x32, + TVX_FMT_BC3 = 0x33, + TVX_FMT_BC4 = 0x34, + TVX_FMT_BC5 = 0x35, + TVX_FMT_APC0 = 0x36, + TVX_FMT_APC1 = 0x37, + TVX_FMT_APC2 = 0x38, + TVX_FMT_APC3 = 0x39, + TVX_FMT_APC4 = 0x3a, + TVX_FMT_APC5 = 0x3b, + TVX_FMT_APC6 = 0x3c, + TVX_FMT_APC7 = 0x3d, + TVX_FMT_CTX1 = 0x3e, + TVX_FMT_RESERVED_63 = 0x3f, +} TVX_DATA_FORMAT; +typedef enum TVX_DST_SEL { + TVX_DstSel_X = 0x0, + TVX_DstSel_Y = 0x1, + TVX_DstSel_Z = 0x2, + TVX_DstSel_W = 0x3, + TVX_DstSel_0f = 0x4, + TVX_DstSel_1f = 0x5, + TVX_DstSel_RESERVED_6 = 0x6, + TVX_DstSel_Mask = 0x7, +} TVX_DST_SEL; +typedef enum TVX_ENDIAN_SWAP { + TVX_EndianSwap_None = 0x0, + TVX_EndianSwap_8in16 = 0x1, + TVX_EndianSwap_8in32 = 0x2, + TVX_EndianSwap_8in64 = 0x3, +} TVX_ENDIAN_SWAP; +typedef enum TVX_INST { + TVX_Inst_NormalVertexFetch = 0x0, + TVX_Inst_SemanticVertexFetch = 0x1, + TVX_Inst_RESERVED_2 = 0x2, + TVX_Inst_LD = 0x3, + TVX_Inst_GetTextureResInfo = 0x4, + TVX_Inst_GetNumberOfSamples = 0x5, + TVX_Inst_GetLOD = 0x6, + TVX_Inst_GetGradientsH = 0x7, + TVX_Inst_GetGradientsV = 0x8, + TVX_Inst_SetTextureOffsets = 0x9, + TVX_Inst_KeepGradients = 0xa, + TVX_Inst_SetGradientsH = 0xb, + TVX_Inst_SetGradientsV = 0xc, + TVX_Inst_Pass = 0xd, + TVX_Inst_GetBufferResInfo = 0xe, + TVX_Inst_RESERVED_15 = 0xf, + TVX_Inst_Sample = 0x10, + TVX_Inst_Sample_L = 0x11, + TVX_Inst_Sample_LB = 0x12, + TVX_Inst_Sample_LZ = 0x13, + TVX_Inst_Sample_G = 0x14, + TVX_Inst_Gather4 = 0x15, + TVX_Inst_Sample_G_LB = 0x16, + TVX_Inst_Gather4_O = 0x17, + TVX_Inst_Sample_C = 0x18, + TVX_Inst_Sample_C_L = 0x19, + TVX_Inst_Sample_C_LB = 0x1a, + TVX_Inst_Sample_C_LZ = 0x1b, + TVX_Inst_Sample_C_G = 0x1c, + TVX_Inst_Gather4_C = 0x1d, + TVX_Inst_Sample_C_G_LB = 0x1e, + TVX_Inst_Gather4_C_O = 0x1f, +} TVX_INST; +typedef enum TVX_NUM_FORMAT_ALL { + TVX_NumFormatAll_Norm = 0x0, + TVX_NumFormatAll_Int = 0x1, + TVX_NumFormatAll_Scaled = 0x2, + TVX_NumFormatAll_RESERVED_3 = 0x3, +} TVX_NUM_FORMAT_ALL; +typedef enum TVX_SRC_SEL { + TVX_SrcSel_X = 0x0, + TVX_SrcSel_Y = 0x1, + TVX_SrcSel_Z = 0x2, + TVX_SrcSel_W = 0x3, + TVX_SrcSel_0f = 0x4, + TVX_SrcSel_1f = 0x5, +} TVX_SRC_SEL; +typedef enum TVX_SRF_MODE_ALL { + TVX_SRFModeAll_ZCMO = 0x0, + TVX_SRFModeAll_NZ = 0x1, +} TVX_SRF_MODE_ALL; +typedef enum TVX_TYPE { + TVX_Type_InvalidTextureResource = 0x0, + TVX_Type_InvalidVertexBuffer = 0x1, + TVX_Type_ValidTextureResource = 0x2, + TVX_Type_ValidVertexBuffer = 0x3, +} TVX_TYPE; +typedef enum TC_OP_MASKS { + TC_OP_MASK_FLUSH_DENROM = 0x8, + TC_OP_MASK_64 = 0x20, + TC_OP_MASK_NO_RTN = 0x40, +} TC_OP_MASKS; +typedef enum TC_OP { + TC_OP_READ = 0x0, + TC_OP_ATOMIC_FCMPSWAP_RTN_32 = 0x1, + TC_OP_ATOMIC_FMIN_RTN_32 = 0x2, + TC_OP_ATOMIC_FMAX_RTN_32 = 0x3, + TC_OP_RESERVED_FOP_RTN_32_0 = 0x4, + TC_OP_RESERVED_FOP_RTN_32_1 = 0x5, + TC_OP_RESERVED_FOP_RTN_32_2 = 0x6, + TC_OP_ATOMIC_SWAP_RTN_32 = 0x7, + TC_OP_ATOMIC_CMPSWAP_RTN_32 = 0x8, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_RTN_32 = 0x9, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_RTN_32 = 0xa, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_RTN_32 = 0xb, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_0 = 0xc, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_1 = 0xd, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_32_2 = 0xe, + TC_OP_ATOMIC_ADD_RTN_32 = 0xf, + TC_OP_ATOMIC_SUB_RTN_32 = 0x10, + TC_OP_ATOMIC_SMIN_RTN_32 = 0x11, + TC_OP_ATOMIC_UMIN_RTN_32 = 0x12, + TC_OP_ATOMIC_SMAX_RTN_32 = 0x13, + TC_OP_ATOMIC_UMAX_RTN_32 = 0x14, + TC_OP_ATOMIC_AND_RTN_32 = 0x15, + TC_OP_ATOMIC_OR_RTN_32 = 0x16, + TC_OP_ATOMIC_XOR_RTN_32 = 0x17, + TC_OP_ATOMIC_INC_RTN_32 = 0x18, + TC_OP_ATOMIC_DEC_RTN_32 = 0x19, + TC_OP_WBINVL1_VOL = 0x1a, + TC_OP_RESERVED_NON_FLOAT_RTN_32_0 = 0x1b, + TC_OP_RESERVED_NON_FLOAT_RTN_32_1 = 0x1c, + TC_OP_RESERVED_NON_FLOAT_RTN_32_2 = 0x1d, + TC_OP_RESERVED_NON_FLOAT_RTN_32_3 = 0x1e, + TC_OP_RESERVED_NON_FLOAT_RTN_32_4 = 0x1f, + TC_OP_WRITE = 0x20, + TC_OP_ATOMIC_FCMPSWAP_RTN_64 = 0x21, + TC_OP_ATOMIC_FMIN_RTN_64 = 0x22, + TC_OP_ATOMIC_FMAX_RTN_64 = 0x23, + TC_OP_RESERVED_FOP_RTN_64_0 = 0x24, + TC_OP_RESERVED_FOP_RTN_64_1 = 0x25, + TC_OP_RESERVED_FOP_RTN_64_2 = 0x26, + TC_OP_ATOMIC_SWAP_RTN_64 = 0x27, + TC_OP_ATOMIC_CMPSWAP_RTN_64 = 0x28, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_RTN_64 = 0x29, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_RTN_64 = 0x2a, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_RTN_64 = 0x2b, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_64_0 = 0x2c, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_64_1 = 0x2d, + TC_OP_RESERVED_FOP_FLUSH_DENORM_RTN_64_2 = 0x2e, + TC_OP_ATOMIC_ADD_RTN_64 = 0x2f, + TC_OP_ATOMIC_SUB_RTN_64 = 0x30, + TC_OP_ATOMIC_SMIN_RTN_64 = 0x31, + TC_OP_ATOMIC_UMIN_RTN_64 = 0x32, + TC_OP_ATOMIC_SMAX_RTN_64 = 0x33, + TC_OP_ATOMIC_UMAX_RTN_64 = 0x34, + TC_OP_ATOMIC_AND_RTN_64 = 0x35, + TC_OP_ATOMIC_OR_RTN_64 = 0x36, + TC_OP_ATOMIC_XOR_RTN_64 = 0x37, + TC_OP_ATOMIC_INC_RTN_64 = 0x38, + TC_OP_ATOMIC_DEC_RTN_64 = 0x39, + TC_OP_WBL2_VOL = 0x3a, + TC_OP_RESERVED_NON_FLOAT_RTN_64_0 = 0x3b, + TC_OP_RESERVED_NON_FLOAT_RTN_64_1 = 0x3c, + TC_OP_RESERVED_NON_FLOAT_RTN_64_2 = 0x3d, + TC_OP_RESERVED_NON_FLOAT_RTN_64_3 = 0x3e, + TC_OP_RESERVED_NON_FLOAT_RTN_64_4 = 0x3f, + TC_OP_WBINVL1 = 0x40, + TC_OP_ATOMIC_FCMPSWAP_32 = 0x41, + TC_OP_ATOMIC_FMIN_32 = 0x42, + TC_OP_ATOMIC_FMAX_32 = 0x43, + TC_OP_RESERVED_FOP_32_0 = 0x44, + TC_OP_RESERVED_FOP_32_1 = 0x45, + TC_OP_RESERVED_FOP_32_2 = 0x46, + TC_OP_ATOMIC_SWAP_32 = 0x47, + TC_OP_ATOMIC_CMPSWAP_32 = 0x48, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_32 = 0x49, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_32 = 0x4a, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_32 = 0x4b, + TC_OP_RESERVED_FOP_FLUSH_DENORM_32_0 = 0x4c, + TC_OP_RESERVED_FOP_FLUSH_DENORM_32_1 = 0x4d, + TC_OP_RESERVED_FOP_FLUSH_DENORM_32_2 = 0x4e, + TC_OP_ATOMIC_ADD_32 = 0x4f, + TC_OP_ATOMIC_SUB_32 = 0x50, + TC_OP_ATOMIC_SMIN_32 = 0x51, + TC_OP_ATOMIC_UMIN_32 = 0x52, + TC_OP_ATOMIC_SMAX_32 = 0x53, + TC_OP_ATOMIC_UMAX_32 = 0x54, + TC_OP_ATOMIC_AND_32 = 0x55, + TC_OP_ATOMIC_OR_32 = 0x56, + TC_OP_ATOMIC_XOR_32 = 0x57, + TC_OP_ATOMIC_INC_32 = 0x58, + TC_OP_ATOMIC_DEC_32 = 0x59, + TC_OP_INVL2_VOL = 0x5a, + TC_OP_RESERVED_NON_FLOAT_32_0 = 0x5b, + TC_OP_RESERVED_NON_FLOAT_32_1 = 0x5c, + TC_OP_RESERVED_NON_FLOAT_32_2 = 0x5d, + TC_OP_RESERVED_NON_FLOAT_32_3 = 0x5e, + TC_OP_RESERVED_NON_FLOAT_32_4 = 0x5f, + TC_OP_WBINVL2 = 0x60, + TC_OP_ATOMIC_FCMPSWAP_64 = 0x61, + TC_OP_ATOMIC_FMIN_64 = 0x62, + TC_OP_ATOMIC_FMAX_64 = 0x63, + TC_OP_RESERVED_FOP_64_0 = 0x64, + TC_OP_RESERVED_FOP_64_1 = 0x65, + TC_OP_RESERVED_FOP_64_2 = 0x66, + TC_OP_ATOMIC_SWAP_64 = 0x67, + TC_OP_ATOMIC_CMPSWAP_64 = 0x68, + TC_OP_ATOMIC_FCMPSWAP_FLUSH_DENORM_64 = 0x69, + TC_OP_ATOMIC_FMIN_FLUSH_DENORM_64 = 0x6a, + TC_OP_ATOMIC_FMAX_FLUSH_DENORM_64 = 0x6b, + TC_OP_RESERVED_FOP_FLUSH_DENORM_64_0 = 0x6c, + TC_OP_RESERVED_FOP_FLUSH_DENORM_64_1 = 0x6d, + TC_OP_RESERVED_FOP_FLUSH_DENORM_64_2 = 0x6e, + TC_OP_ATOMIC_ADD_64 = 0x6f, + TC_OP_ATOMIC_SUB_64 = 0x70, + TC_OP_ATOMIC_SMIN_64 = 0x71, + TC_OP_ATOMIC_UMIN_64 = 0x72, + TC_OP_ATOMIC_SMAX_64 = 0x73, + TC_OP_ATOMIC_UMAX_64 = 0x74, + TC_OP_ATOMIC_AND_64 = 0x75, + TC_OP_ATOMIC_OR_64 = 0x76, + TC_OP_ATOMIC_XOR_64 = 0x77, + TC_OP_ATOMIC_INC_64 = 0x78, + TC_OP_ATOMIC_DEC_64 = 0x79, + TC_OP_INVL1L2_VOL = 0x7a, + TC_OP_RESERVED_NON_FLOAT_64_0 = 0x7b, + TC_OP_RESERVED_NON_FLOAT_64_1 = 0x7c, + TC_OP_RESERVED_NON_FLOAT_64_2 = 0x7d, + TC_OP_RESERVED_NON_FLOAT_64_3 = 0x7e, + TC_OP_RESERVED_NON_FLOAT_64_4 = 0x7f, +} TC_OP; +typedef enum TC_CHUB_REQ_CREDITS_ENUM { + TC_CHUB_REQ_CREDITS = 0x10, +} TC_CHUB_REQ_CREDITS_ENUM; +typedef enum CHUB_TC_RET_CREDITS_ENUM { + CHUB_TC_RET_CREDITS = 0x20, +} CHUB_TC_RET_CREDITS_ENUM; +typedef enum TC_NACKS { + TC_NACK_NO_FAULT = 0x0, + TC_NACK_PAGE_FAULT = 0x1, + TC_NACK_PROTECTION_FAULT = 0x2, + TC_NACK_DATA_ERROR = 0x3, +} TC_NACKS; +typedef enum TCC_PERF_SEL { + TCC_PERF_SEL_NONE = 0x0, + TCC_PERF_SEL_CYCLE = 0x1, + TCC_PERF_SEL_BUSY = 0x2, + TCC_PERF_SEL_REQ = 0x3, + TCC_PERF_SEL_STREAMING_REQ = 0x4, + TCC_PERF_SEL_READ = 0x5, + TCC_PERF_SEL_WRITE = 0x6, + TCC_PERF_SEL_ATOMIC = 0x7, + TCC_PERF_SEL_WBINVL2 = 0x8, + TCC_PERF_SEL_WBINVL2_CYCLE = 0x9, + TCC_PERF_SEL_HIT = 0xa, + TCC_PERF_SEL_MISS = 0xb, + TCC_PERF_SEL_DEWRITE_ALLOCATE_HIT = 0xc, + TCC_PERF_SEL_FULLY_WRITTEN_HIT = 0xd, + TCC_PERF_SEL_WRITEBACK = 0xe, + TCC_PERF_SEL_LATENCY_FIFO_FULL = 0xf, + TCC_PERF_SEL_SRC_FIFO_FULL = 0x10, + TCC_PERF_SEL_HOLE_FIFO_FULL = 0x11, + TCC_PERF_SEL_MC_WRREQ = 0x12, + TCC_PERF_SEL_MC_WRREQ_STALL = 0x13, + TCC_PERF_SEL_MC_WRREQ_CREDIT_STALL = 0x14, + TCC_PERF_SEL_MC_WRREQ_MC_HALT_STALL = 0x15, + TCC_PERF_SEL_TOO_MANY_MC_WRREQS_STALL = 0x16, + TCC_PERF_SEL_MC_WRREQ_LEVEL = 0x17, + TCC_PERF_SEL_MC_RDREQ = 0x18, + TCC_PERF_SEL_MC_RDREQ_CREDIT_STALL = 0x19, + TCC_PERF_SEL_MC_RDREQ_MC_HALT_STALL = 0x1a, + TCC_PERF_SEL_MC_RDREQ_LEVEL = 0x1b, + TCC_PERF_SEL_TAG_STALL = 0x1c, + TCC_PERF_SEL_TAG_WRITEBACK_FIFO_FULL = 0x1d, + TCC_PERF_SEL_TAG_MISS_NOTHING_REPLACEABLE_STALL = 0x1e, + TCC_PERF_SEL_READ_RETURN_TIMEOUT = 0x1f, + TCC_PERF_SEL_WRITEBACK_READ_TIMEOUT = 0x20, + TCC_PERF_SEL_READ_RETURN_FULL_BUBBLE = 0x21, + TCC_PERF_SEL_BUBBLE = 0x22, + TCC_PERF_SEL_RETURN_ACK = 0x23, + TCC_PERF_SEL_RETURN_DATA = 0x24, + TCC_PERF_SEL_RETURN_HOLE = 0x25, + TCC_PERF_SEL_RETURN_ACK_HOLE = 0x26, + TCC_PERF_SEL_IB_STALL = 0x27, + TCC_PERF_SEL_TCA_LEVEL = 0x28, + TCC_PERF_SEL_HOLE_LEVEL = 0x29, + TCC_PERF_SEL_MC_RDRET_NACK = 0x2a, + TCC_PERF_SEL_MC_WRRET_NACK = 0x2b, + TCC_PERF_SEL_EXE_REQ = 0x2c, + TCC_PERF_SEL_CLIENT0_REQ = 0x40, + TCC_PERF_SEL_CLIENT1_REQ = 0x41, + TCC_PERF_SEL_CLIENT2_REQ = 0x42, + TCC_PERF_SEL_CLIENT3_REQ = 0x43, + TCC_PERF_SEL_CLIENT4_REQ = 0x44, + TCC_PERF_SEL_CLIENT5_REQ = 0x45, + TCC_PERF_SEL_CLIENT6_REQ = 0x46, + TCC_PERF_SEL_CLIENT7_REQ = 0x47, + TCC_PERF_SEL_CLIENT8_REQ = 0x48, + TCC_PERF_SEL_CLIENT9_REQ = 0x49, + TCC_PERF_SEL_CLIENT10_REQ = 0x4a, + TCC_PERF_SEL_CLIENT11_REQ = 0x4b, + TCC_PERF_SEL_CLIENT12_REQ = 0x4c, + TCC_PERF_SEL_CLIENT13_REQ = 0x4d, + TCC_PERF_SEL_CLIENT14_REQ = 0x4e, + TCC_PERF_SEL_CLIENT15_REQ = 0x4f, + TCC_PERF_SEL_CLIENT16_REQ = 0x50, + TCC_PERF_SEL_CLIENT17_REQ = 0x51, + TCC_PERF_SEL_CLIENT18_REQ = 0x52, + TCC_PERF_SEL_CLIENT19_REQ = 0x53, + TCC_PERF_SEL_CLIENT20_REQ = 0x54, + TCC_PERF_SEL_CLIENT21_REQ = 0x55, + TCC_PERF_SEL_CLIENT22_REQ = 0x56, + TCC_PERF_SEL_CLIENT23_REQ = 0x57, + TCC_PERF_SEL_CLIENT24_REQ = 0x58, + TCC_PERF_SEL_CLIENT25_REQ = 0x59, + TCC_PERF_SEL_CLIENT26_REQ = 0x5a, + TCC_PERF_SEL_CLIENT27_REQ = 0x5b, + TCC_PERF_SEL_CLIENT28_REQ = 0x5c, + TCC_PERF_SEL_CLIENT29_REQ = 0x5d, + TCC_PERF_SEL_CLIENT30_REQ = 0x5e, + TCC_PERF_SEL_CLIENT31_REQ = 0x5f, + TCC_PERF_SEL_CLIENT32_REQ = 0x60, + TCC_PERF_SEL_CLIENT33_REQ = 0x61, + TCC_PERF_SEL_CLIENT34_REQ = 0x62, + TCC_PERF_SEL_CLIENT35_REQ = 0x63, + TCC_PERF_SEL_CLIENT36_REQ = 0x64, + TCC_PERF_SEL_CLIENT37_REQ = 0x65, + TCC_PERF_SEL_CLIENT38_REQ = 0x66, + TCC_PERF_SEL_CLIENT39_REQ = 0x67, + TCC_PERF_SEL_CLIENT40_REQ = 0x68, + TCC_PERF_SEL_CLIENT41_REQ = 0x69, + TCC_PERF_SEL_CLIENT42_REQ = 0x6a, + TCC_PERF_SEL_CLIENT43_REQ = 0x6b, + TCC_PERF_SEL_CLIENT44_REQ = 0x6c, + TCC_PERF_SEL_CLIENT45_REQ = 0x6d, + TCC_PERF_SEL_CLIENT46_REQ = 0x6e, + TCC_PERF_SEL_CLIENT47_REQ = 0x6f, + TCC_PERF_SEL_CLIENT48_REQ = 0x70, + TCC_PERF_SEL_CLIENT49_REQ = 0x71, + TCC_PERF_SEL_CLIENT50_REQ = 0x72, + TCC_PERF_SEL_CLIENT51_REQ = 0x73, + TCC_PERF_SEL_CLIENT52_REQ = 0x74, + TCC_PERF_SEL_CLIENT53_REQ = 0x75, + TCC_PERF_SEL_CLIENT54_REQ = 0x76, + TCC_PERF_SEL_CLIENT55_REQ = 0x77, + TCC_PERF_SEL_CLIENT56_REQ = 0x78, + TCC_PERF_SEL_CLIENT57_REQ = 0x79, + TCC_PERF_SEL_CLIENT58_REQ = 0x7a, + TCC_PERF_SEL_CLIENT59_REQ = 0x7b, + TCC_PERF_SEL_CLIENT60_REQ = 0x7c, + TCC_PERF_SEL_CLIENT61_REQ = 0x7d, + TCC_PERF_SEL_CLIENT62_REQ = 0x7e, + TCC_PERF_SEL_CLIENT63_REQ = 0x7f, + TCC_PERF_SEL_NORMAL_WRITEBACK = 0x80, + TCC_PERF_SEL_TC_OP_WBL2_VOL_WRITEBACK = 0x81, + TCC_PERF_SEL_TC_OP_WBINVL2_WRITEBACK = 0x82, + TCC_PERF_SEL_ALL_TC_OP_WB_WRITEBACK = 0x83, + TCC_PERF_SEL_NORMAL_EVICT = 0x84, + TCC_PERF_SEL_TC_OP_INVL2_VOL_EVICT = 0x85, + TCC_PERF_SEL_TC_OP_INVL1L2_VOL_EVICT = 0x86, + TCC_PERF_SEL_TC_OP_WBL2_VOL_EVICT = 0x87, + TCC_PERF_SEL_TC_OP_WBINVL2_EVICT = 0x88, + TCC_PERF_SEL_ALL_TC_OP_INV_EVICT = 0x89, + TCC_PERF_SEL_ALL_TC_OP_INV_VOL_EVICT = 0x8a, + TCC_PERF_SEL_TC_OP_WBL2_VOL_CYCLE = 0x8b, + TCC_PERF_SEL_TC_OP_INVL2_VOL_CYCLE = 0x8c, + TCC_PERF_SEL_TC_OP_INVL1L2_VOL_CYCLE = 0x8d, + TCC_PERF_SEL_TC_OP_WBINVL2_CYCLE = 0x8e, + TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_CYCLE = 0x8f, + TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_VOL_CYCLE = 0x90, + TCC_PERF_SEL_TC_OP_WBL2_VOL_START = 0x91, + TCC_PERF_SEL_TC_OP_INVL2_VOL_START = 0x92, + TCC_PERF_SEL_TC_OP_INVL1L2_VOL_START = 0x93, + TCC_PERF_SEL_TC_OP_WBINVL2_START = 0x94, + TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_START = 0x95, + TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_VOL_START = 0x96, + TCC_PERF_SEL_TC_OP_WBL2_VOL_FINISH = 0x97, + TCC_PERF_SEL_TC_OP_INVL2_VOL_FINISH = 0x98, + TCC_PERF_SEL_TC_OP_INVL1L2_VOL_FINISH = 0x99, + TCC_PERF_SEL_TC_OP_WBINVL2_FINISH = 0x9a, + TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_FINISH = 0x9b, + TCC_PERF_SEL_ALL_TC_OP_WB_OR_INV_VOL_FINISH = 0x9c, + TCC_PERF_SEL_VOL_MC_WRREQ = 0x9d, + TCC_PERF_SEL_VOL_MC_RDREQ = 0x9e, + TCC_PERF_SEL_VOL_REQ = 0x9f, +} TCC_PERF_SEL; +typedef enum TCA_PERF_SEL { + TCA_PERF_SEL_NONE = 0x0, + TCA_PERF_SEL_CYCLE = 0x1, + TCA_PERF_SEL_BUSY = 0x2, + TCA_PERF_SEL_FORCED_HOLE_TCC0 = 0x3, + TCA_PERF_SEL_FORCED_HOLE_TCC1 = 0x4, + TCA_PERF_SEL_FORCED_HOLE_TCC2 = 0x5, + TCA_PERF_SEL_FORCED_HOLE_TCC3 = 0x6, + TCA_PERF_SEL_FORCED_HOLE_TCC4 = 0x7, + TCA_PERF_SEL_FORCED_HOLE_TCC5 = 0x8, + TCA_PERF_SEL_FORCED_HOLE_TCC6 = 0x9, + TCA_PERF_SEL_FORCED_HOLE_TCC7 = 0xa, + TCA_PERF_SEL_REQ_TCC0 = 0xb, + TCA_PERF_SEL_REQ_TCC1 = 0xc, + TCA_PERF_SEL_REQ_TCC2 = 0xd, + TCA_PERF_SEL_REQ_TCC3 = 0xe, + TCA_PERF_SEL_REQ_TCC4 = 0xf, + TCA_PERF_SEL_REQ_TCC5 = 0x10, + TCA_PERF_SEL_REQ_TCC6 = 0x11, + TCA_PERF_SEL_REQ_TCC7 = 0x12, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC0 = 0x13, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC1 = 0x14, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC2 = 0x15, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC3 = 0x16, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC4 = 0x17, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC5 = 0x18, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC6 = 0x19, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCC7 = 0x1a, + TCA_PERF_SEL_CROSSBAR_STALL_TCC0 = 0x1b, + TCA_PERF_SEL_CROSSBAR_STALL_TCC1 = 0x1c, + TCA_PERF_SEL_CROSSBAR_STALL_TCC2 = 0x1d, + TCA_PERF_SEL_CROSSBAR_STALL_TCC3 = 0x1e, + TCA_PERF_SEL_CROSSBAR_STALL_TCC4 = 0x1f, + TCA_PERF_SEL_CROSSBAR_STALL_TCC5 = 0x20, + TCA_PERF_SEL_CROSSBAR_STALL_TCC6 = 0x21, + TCA_PERF_SEL_CROSSBAR_STALL_TCC7 = 0x22, + TCA_PERF_SEL_FORCED_HOLE_TCS = 0x23, + TCA_PERF_SEL_REQ_TCS = 0x24, + TCA_PERF_SEL_CROSSBAR_DOUBLE_ARB_TCS = 0x25, + TCA_PERF_SEL_CROSSBAR_STALL_TCS = 0x26, +} TCA_PERF_SEL; +typedef enum TCS_PERF_SEL { + TCS_PERF_SEL_NONE = 0x0, + TCS_PERF_SEL_CYCLE = 0x1, + TCS_PERF_SEL_BUSY = 0x2, + TCS_PERF_SEL_REQ = 0x3, + TCS_PERF_SEL_READ = 0x4, + TCS_PERF_SEL_WRITE = 0x5, + TCS_PERF_SEL_ATOMIC = 0x6, + TCS_PERF_SEL_HOLE_FIFO_FULL = 0x7, + TCS_PERF_SEL_REQ_FIFO_FULL = 0x8, + TCS_PERF_SEL_REQ_CREDIT_STALL = 0x9, + TCS_PERF_SEL_REQ_NO_SRC_DATA_STALL = 0xa, + TCS_PERF_SEL_REQ_STALL = 0xb, + TCS_PERF_SEL_TCS_CHUB_REQ_SEND = 0xc, + TCS_PERF_SEL_CHUB_TCS_RET_SEND = 0xd, + TCS_PERF_SEL_RETURN_ACK = 0xe, + TCS_PERF_SEL_RETURN_DATA = 0xf, + TCS_PERF_SEL_IB_TOTAL_REQUESTS_STALL = 0x10, + TCS_PERF_SEL_IB_STALL = 0x11, + TCS_PERF_SEL_TCA_LEVEL = 0x12, + TCS_PERF_SEL_HOLE_LEVEL = 0x13, + TCS_PERF_SEL_CHUB_LEVEL = 0x14, + TCS_PERF_SEL_CLIENT0_REQ = 0x40, + TCS_PERF_SEL_CLIENT1_REQ = 0x41, + TCS_PERF_SEL_CLIENT2_REQ = 0x42, + TCS_PERF_SEL_CLIENT3_REQ = 0x43, + TCS_PERF_SEL_CLIENT4_REQ = 0x44, + TCS_PERF_SEL_CLIENT5_REQ = 0x45, + TCS_PERF_SEL_CLIENT6_REQ = 0x46, + TCS_PERF_SEL_CLIENT7_REQ = 0x47, + TCS_PERF_SEL_CLIENT8_REQ = 0x48, + TCS_PERF_SEL_CLIENT9_REQ = 0x49, + TCS_PERF_SEL_CLIENT10_REQ = 0x4a, + TCS_PERF_SEL_CLIENT11_REQ = 0x4b, + TCS_PERF_SEL_CLIENT12_REQ = 0x4c, + TCS_PERF_SEL_CLIENT13_REQ = 0x4d, + TCS_PERF_SEL_CLIENT14_REQ = 0x4e, + TCS_PERF_SEL_CLIENT15_REQ = 0x4f, + TCS_PERF_SEL_CLIENT16_REQ = 0x50, + TCS_PERF_SEL_CLIENT17_REQ = 0x51, + TCS_PERF_SEL_CLIENT18_REQ = 0x52, + TCS_PERF_SEL_CLIENT19_REQ = 0x53, + TCS_PERF_SEL_CLIENT20_REQ = 0x54, + TCS_PERF_SEL_CLIENT21_REQ = 0x55, + TCS_PERF_SEL_CLIENT22_REQ = 0x56, + TCS_PERF_SEL_CLIENT23_REQ = 0x57, + TCS_PERF_SEL_CLIENT24_REQ = 0x58, + TCS_PERF_SEL_CLIENT25_REQ = 0x59, + TCS_PERF_SEL_CLIENT26_REQ = 0x5a, + TCS_PERF_SEL_CLIENT27_REQ = 0x5b, + TCS_PERF_SEL_CLIENT28_REQ = 0x5c, + TCS_PERF_SEL_CLIENT29_REQ = 0x5d, + TCS_PERF_SEL_CLIENT30_REQ = 0x5e, + TCS_PERF_SEL_CLIENT31_REQ = 0x5f, + TCS_PERF_SEL_CLIENT32_REQ = 0x60, + TCS_PERF_SEL_CLIENT33_REQ = 0x61, + TCS_PERF_SEL_CLIENT34_REQ = 0x62, + TCS_PERF_SEL_CLIENT35_REQ = 0x63, + TCS_PERF_SEL_CLIENT36_REQ = 0x64, + TCS_PERF_SEL_CLIENT37_REQ = 0x65, + TCS_PERF_SEL_CLIENT38_REQ = 0x66, + TCS_PERF_SEL_CLIENT39_REQ = 0x67, + TCS_PERF_SEL_CLIENT40_REQ = 0x68, + TCS_PERF_SEL_CLIENT41_REQ = 0x69, + TCS_PERF_SEL_CLIENT42_REQ = 0x6a, + TCS_PERF_SEL_CLIENT43_REQ = 0x6b, + TCS_PERF_SEL_CLIENT44_REQ = 0x6c, + TCS_PERF_SEL_CLIENT45_REQ = 0x6d, + TCS_PERF_SEL_CLIENT46_REQ = 0x6e, + TCS_PERF_SEL_CLIENT47_REQ = 0x6f, + TCS_PERF_SEL_CLIENT48_REQ = 0x70, + TCS_PERF_SEL_CLIENT49_REQ = 0x71, + TCS_PERF_SEL_CLIENT50_REQ = 0x72, + TCS_PERF_SEL_CLIENT51_REQ = 0x73, + TCS_PERF_SEL_CLIENT52_REQ = 0x74, + TCS_PERF_SEL_CLIENT53_REQ = 0x75, + TCS_PERF_SEL_CLIENT54_REQ = 0x76, + TCS_PERF_SEL_CLIENT55_REQ = 0x77, + TCS_PERF_SEL_CLIENT56_REQ = 0x78, + TCS_PERF_SEL_CLIENT57_REQ = 0x79, + TCS_PERF_SEL_CLIENT58_REQ = 0x7a, + TCS_PERF_SEL_CLIENT59_REQ = 0x7b, + TCS_PERF_SEL_CLIENT60_REQ = 0x7c, + TCS_PERF_SEL_CLIENT61_REQ = 0x7d, + TCS_PERF_SEL_CLIENT62_REQ = 0x7e, + TCS_PERF_SEL_CLIENT63_REQ = 0x7f, +} TCS_PERF_SEL; +typedef enum TA_TC_ADDR_MODES { + TA_TC_ADDR_MODE_DEFAULT = 0x0, + TA_TC_ADDR_MODE_COMP0 = 0x1, + TA_TC_ADDR_MODE_COMP1 = 0x2, + TA_TC_ADDR_MODE_COMP2 = 0x3, + TA_TC_ADDR_MODE_COMP3 = 0x4, + TA_TC_ADDR_MODE_UNALIGNED = 0x5, + TA_TC_ADDR_MODE_BORDER_COLOR = 0x6, +} TA_TC_ADDR_MODES; +typedef enum TA_PERFCOUNT_SEL { + TA_PERF_SEL_ta_busy = 0x0, + TA_PERF_SEL_sh_fifo_busy = 0x1, + TA_PERF_SEL_sh_fifo_cmd_busy = 0x2, + TA_PERF_SEL_sh_fifo_addr_busy = 0x3, + TA_PERF_SEL_sh_fifo_data_busy = 0x4, + TA_PERF_SEL_sh_fifo_data_sfifo_busy = 0x5, + TA_PERF_SEL_sh_fifo_data_tfifo_busy = 0x6, + TA_PERF_SEL_gradient_busy = 0x7, + TA_PERF_SEL_gradient_fifo_busy = 0x8, + TA_PERF_SEL_lod_busy = 0x9, + TA_PERF_SEL_lod_fifo_busy = 0xa, + TA_PERF_SEL_addresser_busy = 0xb, + TA_PERF_SEL_addresser_fifo_busy = 0xc, + TA_PERF_SEL_aligner_busy = 0xd, + TA_PERF_SEL_write_path_busy = 0xe, + TA_PERF_SEL_RESERVED_15 = 0xf, + TA_PERF_SEL_sq_ta_cmd_cycles = 0x10, + TA_PERF_SEL_sp_ta_addr_cycles = 0x11, + TA_PERF_SEL_sp_ta_data_cycles = 0x12, + TA_PERF_SEL_ta_fa_data_state_cycles = 0x13, + TA_PERF_SEL_sh_fifo_addr_waiting_on_cmd_cycles = 0x14, + TA_PERF_SEL_sh_fifo_cmd_waiting_on_addr_cycles = 0x15, + TA_PERF_SEL_sh_fifo_addr_starved_while_busy_cycles= 0x16, + TA_PERF_SEL_sh_fifo_cmd_starved_while_busy_cycles= 0x17, + TA_PERF_SEL_sh_fifo_data_waiting_on_data_state_cycles= 0x18, + TA_PERF_SEL_sh_fifo_data_state_waiting_on_data_cycles= 0x19, + TA_PERF_SEL_sh_fifo_data_starved_while_busy_cycles= 0x1a, + TA_PERF_SEL_sh_fifo_data_state_starved_while_busy_cycles= 0x1b, + TA_PERF_SEL_RESERVED_28 = 0x1c, + TA_PERF_SEL_RESERVED_29 = 0x1d, + TA_PERF_SEL_sh_fifo_addr_cycles = 0x1e, + TA_PERF_SEL_sh_fifo_data_cycles = 0x1f, + TA_PERF_SEL_total_wavefronts = 0x20, + TA_PERF_SEL_gradient_cycles = 0x21, + TA_PERF_SEL_walker_cycles = 0x22, + TA_PERF_SEL_aligner_cycles = 0x23, + TA_PERF_SEL_image_wavefronts = 0x24, + TA_PERF_SEL_image_read_wavefronts = 0x25, + TA_PERF_SEL_image_write_wavefronts = 0x26, + TA_PERF_SEL_image_atomic_wavefronts = 0x27, + TA_PERF_SEL_image_total_cycles = 0x28, + TA_PERF_SEL_RESERVED_41 = 0x29, + TA_PERF_SEL_RESERVED_42 = 0x2a, + TA_PERF_SEL_RESERVED_43 = 0x2b, + TA_PERF_SEL_buffer_wavefronts = 0x2c, + TA_PERF_SEL_buffer_read_wavefronts = 0x2d, + TA_PERF_SEL_buffer_write_wavefronts = 0x2e, + TA_PERF_SEL_buffer_atomic_wavefronts = 0x2f, + TA_PERF_SEL_buffer_coalescable_wavefronts = 0x30, + TA_PERF_SEL_buffer_total_cycles = 0x31, + TA_PERF_SEL_buffer_coalescable_addr_multicycled_cycles= 0x32, + TA_PERF_SEL_buffer_coalescable_clamp_16kdword_multicycled_cycles= 0x33, + TA_PERF_SEL_buffer_coalesced_read_cycles = 0x34, + TA_PERF_SEL_buffer_coalesced_write_cycles = 0x35, + TA_PERF_SEL_addr_stalled_by_tc_cycles = 0x36, + TA_PERF_SEL_addr_stalled_by_td_cycles = 0x37, + TA_PERF_SEL_data_stalled_by_tc_cycles = 0x38, + TA_PERF_SEL_addresser_stalled_by_aligner_only_cycles= 0x39, + TA_PERF_SEL_addresser_stalled_cycles = 0x3a, + TA_PERF_SEL_aniso_stalled_by_addresser_only_cycles= 0x3b, + TA_PERF_SEL_aniso_stalled_cycles = 0x3c, + TA_PERF_SEL_deriv_stalled_by_aniso_only_cycles = 0x3d, + TA_PERF_SEL_deriv_stalled_cycles = 0x3e, + TA_PERF_SEL_aniso_gt1_cycle_quads = 0x3f, + TA_PERF_SEL_color_1_cycle_pixels = 0x40, + TA_PERF_SEL_color_2_cycle_pixels = 0x41, + TA_PERF_SEL_color_3_cycle_pixels = 0x42, + TA_PERF_SEL_color_4_cycle_pixels = 0x43, + TA_PERF_SEL_mip_1_cycle_pixels = 0x44, + TA_PERF_SEL_mip_2_cycle_pixels = 0x45, + TA_PERF_SEL_vol_1_cycle_pixels = 0x46, + TA_PERF_SEL_vol_2_cycle_pixels = 0x47, + TA_PERF_SEL_bilin_point_1_cycle_pixels = 0x48, + TA_PERF_SEL_mipmap_lod_0_samples = 0x49, + TA_PERF_SEL_mipmap_lod_1_samples = 0x4a, + TA_PERF_SEL_mipmap_lod_2_samples = 0x4b, + TA_PERF_SEL_mipmap_lod_3_samples = 0x4c, + TA_PERF_SEL_mipmap_lod_4_samples = 0x4d, + TA_PERF_SEL_mipmap_lod_5_samples = 0x4e, + TA_PERF_SEL_mipmap_lod_6_samples = 0x4f, + TA_PERF_SEL_mipmap_lod_7_samples = 0x50, + TA_PERF_SEL_mipmap_lod_8_samples = 0x51, + TA_PERF_SEL_mipmap_lod_9_samples = 0x52, + TA_PERF_SEL_mipmap_lod_10_samples = 0x53, + TA_PERF_SEL_mipmap_lod_11_samples = 0x54, + TA_PERF_SEL_mipmap_lod_12_samples = 0x55, + TA_PERF_SEL_mipmap_lod_13_samples = 0x56, + TA_PERF_SEL_mipmap_lod_14_samples = 0x57, + TA_PERF_SEL_mipmap_invalid_samples = 0x58, + TA_PERF_SEL_aniso_1_cycle_quads = 0x59, + TA_PERF_SEL_aniso_2_cycle_quads = 0x5a, + TA_PERF_SEL_aniso_4_cycle_quads = 0x5b, + TA_PERF_SEL_aniso_6_cycle_quads = 0x5c, + TA_PERF_SEL_aniso_8_cycle_quads = 0x5d, + TA_PERF_SEL_aniso_10_cycle_quads = 0x5e, + TA_PERF_SEL_aniso_12_cycle_quads = 0x5f, + TA_PERF_SEL_aniso_14_cycle_quads = 0x60, + TA_PERF_SEL_aniso_16_cycle_quads = 0x61, + TA_PERF_SEL_write_path_input_cycles = 0x62, + TA_PERF_SEL_write_path_output_cycles = 0x63, + TA_PERF_SEL_flat_wavefronts = 0x64, + TA_PERF_SEL_flat_read_wavefronts = 0x65, + TA_PERF_SEL_flat_write_wavefronts = 0x66, + TA_PERF_SEL_flat_atomic_wavefronts = 0x67, + TA_PERF_SEL_flat_coalesceable_wavefronts = 0x68, + TA_PERF_SEL_reg_sclk_vld = 0x69, + TA_PERF_SEL_local_cg_dyn_sclk_grp0_en = 0x6a, + TA_PERF_SEL_local_cg_dyn_sclk_grp1_en = 0x6b, + TA_PERF_SEL_local_cg_dyn_sclk_grp1_mems_en = 0x6c, + TA_PERF_SEL_local_cg_dyn_sclk_grp4_en = 0x6d, + TA_PERF_SEL_local_cg_dyn_sclk_grp5_en = 0x6e, +} TA_PERFCOUNT_SEL; +typedef enum TD_PERFCOUNT_SEL { + TD_PERF_SEL_td_busy = 0x0, + TD_PERF_SEL_input_busy = 0x1, + TD_PERF_SEL_output_busy = 0x2, + TD_PERF_SEL_lerp_busy = 0x3, + TD_PERF_SEL_RESERVED_4 = 0x4, + TD_PERF_SEL_reg_sclk_vld = 0x5, + TD_PERF_SEL_local_cg_dyn_sclk_grp0_en = 0x6, + TD_PERF_SEL_local_cg_dyn_sclk_grp1_en = 0x7, + TD_PERF_SEL_local_cg_dyn_sclk_grp4_en = 0x8, + TD_PERF_SEL_local_cg_dyn_sclk_grp5_en = 0x9, + TD_PERF_SEL_tc_td_fifo_full = 0xa, + TD_PERF_SEL_constant_state_full = 0xb, + TD_PERF_SEL_sample_state_full = 0xc, + TD_PERF_SEL_output_fifo_full = 0xd, + TD_PERF_SEL_RESERVED_14 = 0xe, + TD_PERF_SEL_tc_stall = 0xf, + TD_PERF_SEL_pc_stall = 0x10, + TD_PERF_SEL_gds_stall = 0x11, + TD_PERF_SEL_RESERVED_18 = 0x12, + TD_PERF_SEL_RESERVED_19 = 0x13, + TD_PERF_SEL_gather4_wavefront = 0x14, + TD_PERF_SEL_sample_c_wavefront = 0x15, + TD_PERF_SEL_load_wavefront = 0x16, + TD_PERF_SEL_atomic_wavefront = 0x17, + TD_PERF_SEL_store_wavefront = 0x18, + TD_PERF_SEL_ldfptr_wavefront = 0x19, + TD_PERF_SEL_RESERVED_26 = 0x1a, + TD_PERF_SEL_RESERVED_27 = 0x1b, + TD_PERF_SEL_RESERVED_28 = 0x1c, + TD_PERF_SEL_RESERVED_29 = 0x1d, + TD_PERF_SEL_bypass_filter_wavefront = 0x1e, + TD_PERF_SEL_min_max_filter_wavefront = 0x1f, + TD_PERF_SEL_coalescable_wavefront = 0x20, + TD_PERF_SEL_coalesced_phase = 0x21, + TD_PERF_SEL_four_phase_wavefront = 0x22, + TD_PERF_SEL_eight_phase_wavefront = 0x23, + TD_PERF_SEL_sixteen_phase_wavefront = 0x24, + TD_PERF_SEL_four_phase_forward_wavefront = 0x25, + TD_PERF_SEL_write_ack_wavefront = 0x26, + TD_PERF_SEL_RESERVED_39 = 0x27, + TD_PERF_SEL_user_defined_border = 0x28, + TD_PERF_SEL_white_border = 0x29, + TD_PERF_SEL_opaque_black_border = 0x2a, + TD_PERF_SEL_RESERVED_43 = 0x2b, + TD_PERF_SEL_RESERVED_44 = 0x2c, + TD_PERF_SEL_nack = 0x2d, + TD_PERF_SEL_td_sp_traffic = 0x2e, + TD_PERF_SEL_consume_gds_traffic = 0x2f, + TD_PERF_SEL_addresscmd_poison = 0x30, + TD_PERF_SEL_data_poison = 0x31, + TD_PERF_SEL_start_cycle_0 = 0x32, + TD_PERF_SEL_start_cycle_1 = 0x33, + TD_PERF_SEL_start_cycle_2 = 0x34, + TD_PERF_SEL_start_cycle_3 = 0x35, + TD_PERF_SEL_null_cycle_output = 0x36, +} TD_PERFCOUNT_SEL; +typedef enum TCP_PERFCOUNT_SELECT { + TCP_PERF_SEL_TA_TCP_ADDR_STARVE_CYCLES = 0x0, + TCP_PERF_SEL_TA_TCP_DATA_STARVE_CYCLES = 0x1, + TCP_PERF_SEL_TCP_TA_ADDR_STALL_CYCLES = 0x2, + TCP_PERF_SEL_TCP_TA_DATA_STALL_CYCLES = 0x3, + TCP_PERF_SEL_TD_TCP_STALL_CYCLES = 0x4, + TCP_PERF_SEL_TCR_TCP_STALL_CYCLES = 0x5, + TCP_PERF_SEL_LOD_STALL_CYCLES = 0x6, + TCP_PERF_SEL_READ_TAGCONFLICT_STALL_CYCLES = 0x7, + TCP_PERF_SEL_WRITE_TAGCONFLICT_STALL_CYCLES = 0x8, + TCP_PERF_SEL_ATOMIC_TAGCONFLICT_STALL_CYCLES = 0x9, + TCP_PERF_SEL_ALLOC_STALL_CYCLES = 0xa, + TCP_PERF_SEL_LFIFO_STALL_CYCLES = 0xb, + TCP_PERF_SEL_RFIFO_STALL_CYCLES = 0xc, + TCP_PERF_SEL_TCR_RDRET_STALL = 0xd, + TCP_PERF_SEL_WRITE_CONFLICT_STALL = 0xe, + TCP_PERF_SEL_HOLE_READ_STALL = 0xf, + TCP_PERF_SEL_READCONFLICT_STALL_CYCLES = 0x10, + TCP_PERF_SEL_PENDING_STALL_CYCLES = 0x11, + TCP_PERF_SEL_READFIFO_STALL_CYCLES = 0x12, + TCP_PERF_SEL_TCP_LATENCY = 0x13, + TCP_PERF_SEL_TCC_READ_REQ_LATENCY = 0x14, + TCP_PERF_SEL_TCC_WRITE_REQ_LATENCY = 0x15, + TCP_PERF_SEL_TCC_WRITE_REQ_HOLE_LATENCY = 0x16, + TCP_PERF_SEL_TCC_READ_REQ = 0x17, + TCP_PERF_SEL_TCC_WRITE_REQ = 0x18, + TCP_PERF_SEL_TCC_ATOMIC_WITH_RET_REQ = 0x19, + TCP_PERF_SEL_TCC_ATOMIC_WITHOUT_RET_REQ = 0x1a, + TCP_PERF_SEL_TOTAL_LOCAL_READ = 0x1b, + TCP_PERF_SEL_TOTAL_GLOBAL_READ = 0x1c, + TCP_PERF_SEL_TOTAL_LOCAL_WRITE = 0x1d, + TCP_PERF_SEL_TOTAL_GLOBAL_WRITE = 0x1e, + TCP_PERF_SEL_TOTAL_ATOMIC_WITH_RET = 0x1f, + TCP_PERF_SEL_TOTAL_ATOMIC_WITHOUT_RET = 0x20, + TCP_PERF_SEL_TOTAL_WBINVL1 = 0x21, + TCP_PERF_SEL_IMG_READ_FMT_1 = 0x22, + TCP_PERF_SEL_IMG_READ_FMT_8 = 0x23, + TCP_PERF_SEL_IMG_READ_FMT_16 = 0x24, + TCP_PERF_SEL_IMG_READ_FMT_32 = 0x25, + TCP_PERF_SEL_IMG_READ_FMT_32_AS_8 = 0x26, + TCP_PERF_SEL_IMG_READ_FMT_32_AS_16 = 0x27, + TCP_PERF_SEL_IMG_READ_FMT_32_AS_128 = 0x28, + TCP_PERF_SEL_IMG_READ_FMT_64_2_CYCLE = 0x29, + TCP_PERF_SEL_IMG_READ_FMT_64_1_CYCLE = 0x2a, + TCP_PERF_SEL_IMG_READ_FMT_96 = 0x2b, + TCP_PERF_SEL_IMG_READ_FMT_128_4_CYCLE = 0x2c, + TCP_PERF_SEL_IMG_READ_FMT_128_1_CYCLE = 0x2d, + TCP_PERF_SEL_IMG_READ_FMT_BC1 = 0x2e, + TCP_PERF_SEL_IMG_READ_FMT_BC2 = 0x2f, + TCP_PERF_SEL_IMG_READ_FMT_BC3 = 0x30, + TCP_PERF_SEL_IMG_READ_FMT_BC4 = 0x31, + TCP_PERF_SEL_IMG_READ_FMT_BC5 = 0x32, + TCP_PERF_SEL_IMG_READ_FMT_BC6 = 0x33, + TCP_PERF_SEL_IMG_READ_FMT_BC7 = 0x34, + TCP_PERF_SEL_IMG_READ_FMT_I8 = 0x35, + TCP_PERF_SEL_IMG_READ_FMT_I16 = 0x36, + TCP_PERF_SEL_IMG_READ_FMT_I32 = 0x37, + TCP_PERF_SEL_IMG_READ_FMT_I32_AS_8 = 0x38, + TCP_PERF_SEL_IMG_READ_FMT_I32_AS_16 = 0x39, + TCP_PERF_SEL_IMG_READ_FMT_D8 = 0x3a, + TCP_PERF_SEL_IMG_READ_FMT_D16 = 0x3b, + TCP_PERF_SEL_IMG_READ_FMT_D32 = 0x3c, + TCP_PERF_SEL_IMG_WRITE_FMT_8 = 0x3d, + TCP_PERF_SEL_IMG_WRITE_FMT_16 = 0x3e, + TCP_PERF_SEL_IMG_WRITE_FMT_32 = 0x3f, + TCP_PERF_SEL_IMG_WRITE_FMT_64 = 0x40, + TCP_PERF_SEL_IMG_WRITE_FMT_128 = 0x41, + TCP_PERF_SEL_IMG_WRITE_FMT_D8 = 0x42, + TCP_PERF_SEL_IMG_WRITE_FMT_D16 = 0x43, + TCP_PERF_SEL_IMG_WRITE_FMT_D32 = 0x44, + TCP_PERF_SEL_IMG_ATOMIC_WITH_RET_FMT_32 = 0x45, + TCP_PERF_SEL_IMG_ATOMIC_WITHOUT_RET_FMT_32 = 0x46, + TCP_PERF_SEL_IMG_ATOMIC_WITH_RET_FMT_64 = 0x47, + TCP_PERF_SEL_IMG_ATOMIC_WITHOUT_RET_FMT_64 = 0x48, + TCP_PERF_SEL_BUF_READ_FMT_8 = 0x49, + TCP_PERF_SEL_BUF_READ_FMT_16 = 0x4a, + TCP_PERF_SEL_BUF_READ_FMT_32 = 0x4b, + TCP_PERF_SEL_BUF_WRITE_FMT_8 = 0x4c, + TCP_PERF_SEL_BUF_WRITE_FMT_16 = 0x4d, + TCP_PERF_SEL_BUF_WRITE_FMT_32 = 0x4e, + TCP_PERF_SEL_BUF_ATOMIC_WITH_RET_FMT_32 = 0x4f, + TCP_PERF_SEL_BUF_ATOMIC_WITHOUT_RET_FMT_32 = 0x50, + TCP_PERF_SEL_BUF_ATOMIC_WITH_RET_FMT_64 = 0x51, + TCP_PERF_SEL_BUF_ATOMIC_WITHOUT_RET_FMT_64 = 0x52, + TCP_PERF_SEL_ARR_LINEAR_GENERAL = 0x53, + TCP_PERF_SEL_ARR_LINEAR_ALIGNED = 0x54, + TCP_PERF_SEL_ARR_1D_THIN1 = 0x55, + TCP_PERF_SEL_ARR_1D_THICK = 0x56, + TCP_PERF_SEL_ARR_2D_THIN1 = 0x57, + TCP_PERF_SEL_ARR_2D_THICK = 0x58, + TCP_PERF_SEL_ARR_2D_XTHICK = 0x59, + TCP_PERF_SEL_ARR_3D_THIN1 = 0x5a, + TCP_PERF_SEL_ARR_3D_THICK = 0x5b, + TCP_PERF_SEL_ARR_3D_XTHICK = 0x5c, + TCP_PERF_SEL_DIM_1D = 0x5d, + TCP_PERF_SEL_DIM_2D = 0x5e, + TCP_PERF_SEL_DIM_3D = 0x5f, + TCP_PERF_SEL_DIM_1D_ARRAY = 0x60, + TCP_PERF_SEL_DIM_2D_ARRAY = 0x61, + TCP_PERF_SEL_DIM_2D_MSAA = 0x62, + TCP_PERF_SEL_DIM_2D_ARRAY_MSAA = 0x63, + TCP_PERF_SEL_DIM_CUBE_ARRAY = 0x64, + TCP_PERF_SEL_CP_TCP_INVALIDATE = 0x65, + TCP_PERF_SEL_TA_TCP_STATE_READ = 0x66, + TCP_PERF_SEL_TAGRAM0_REQ = 0x67, + TCP_PERF_SEL_TAGRAM1_REQ = 0x68, + TCP_PERF_SEL_TAGRAM2_REQ = 0x69, + TCP_PERF_SEL_TAGRAM3_REQ = 0x6a, + TCP_PERF_SEL_GATE_EN1 = 0x6b, + TCP_PERF_SEL_GATE_EN2 = 0x6c, + TCP_PERF_SEL_CORE_REG_SCLK_VLD = 0x6d, + TCP_PERF_SEL_TCC_REQ = 0x6e, + TCP_PERF_SEL_TCC_NON_READ_REQ = 0x6f, + TCP_PERF_SEL_TCC_BYPASS_READ_REQ = 0x70, + TCP_PERF_SEL_TCC_MISS_EVICT_READ_REQ = 0x71, + TCP_PERF_SEL_TCC_VOLATILE_READ_REQ = 0x72, + TCP_PERF_SEL_TCC_VOLATILE_BYPASS_READ_REQ = 0x73, + TCP_PERF_SEL_TCC_VOLATILE_MISS_EVICT_READ_REQ = 0x74, + TCP_PERF_SEL_TCC_BYPASS_WRITE_REQ = 0x75, + TCP_PERF_SEL_TCC_MISS_EVICT_WRITE_REQ = 0x76, + TCP_PERF_SEL_TCC_VOLATILE_BYPASS_WRITE_REQ = 0x77, + TCP_PERF_SEL_TCC_VOLATILE_WRITE_REQ = 0x78, + TCP_PERF_SEL_TCC_VOLATILE_MISS_EVICT_WRITE_REQ = 0x79, + TCP_PERF_SEL_TCC_BYPASS_ATOMIC_REQ = 0x7a, + TCP_PERF_SEL_TCC_ATOMIC_REQ = 0x7b, + TCP_PERF_SEL_TCC_VOLATILE_ATOMIC_REQ = 0x7c, + TCP_PERF_SEL_TCC_DATA_BUS_BUSY = 0x7d, + TCP_PERF_SEL_TOTAL_ACCESSES = 0x7e, + TCP_PERF_SEL_TOTAL_READ = 0x7f, + TCP_PERF_SEL_TOTAL_HIT_LRU_READ = 0x80, + TCP_PERF_SEL_TOTAL_HIT_EVICT_READ = 0x81, + TCP_PERF_SEL_TOTAL_MISS_LRU_READ = 0x82, + TCP_PERF_SEL_TOTAL_MISS_EVICT_READ = 0x83, + TCP_PERF_SEL_TOTAL_NON_READ = 0x84, + TCP_PERF_SEL_TOTAL_WRITE = 0x85, + TCP_PERF_SEL_TOTAL_MISS_LRU_WRITE = 0x86, + TCP_PERF_SEL_TOTAL_MISS_EVICT_WRITE = 0x87, + TCP_PERF_SEL_TOTAL_WBINVL1_VOL = 0x88, + TCP_PERF_SEL_TOTAL_WRITEBACK_INVALIDATES = 0x89, + TCP_PERF_SEL_DISPLAY_MICROTILING = 0x8a, + TCP_PERF_SEL_THIN_MICROTILING = 0x8b, + TCP_PERF_SEL_DEPTH_MICROTILING = 0x8c, + TCP_PERF_SEL_ARR_PRT_THIN1 = 0x8d, + TCP_PERF_SEL_ARR_PRT_2D_THIN1 = 0x8e, + TCP_PERF_SEL_ARR_PRT_3D_THIN1 = 0x8f, + TCP_PERF_SEL_ARR_PRT_THICK = 0x90, + TCP_PERF_SEL_ARR_PRT_2D_THICK = 0x91, + TCP_PERF_SEL_ARR_PRT_3D_THICK = 0x92, + TCP_PERF_SEL_CP_TCP_INVALIDATE_VOL = 0x93, + TCP_PERF_SEL_SQ_TCP_INVALIDATE_VOL = 0x94, + TCP_PERF_SEL_UNALIGNED = 0x95, + TCP_PERF_SEL_ROTATED_MICROTILING = 0x96, + TCP_PERF_SEL_THICK_MICROTILING = 0x97, + TCP_PERF_SEL_ATC = 0x98, + TCP_PERF_SEL_POWER_STALL = 0x99, +} TCP_PERFCOUNT_SELECT; +typedef enum TCP_CACHE_POLICIES { + TCP_CACHE_POLICY_MISS_LRU = 0x0, + TCP_CACHE_POLICY_MISS_EVICT = 0x1, + TCP_CACHE_POLICY_HIT_LRU = 0x2, + TCP_CACHE_POLICY_HIT_EVICT = 0x3, +} TCP_CACHE_POLICIES; +typedef enum TCP_CACHE_STORE_POLICIES { + TCP_CACHE_STORE_POLICY_MISS_LRU = 0x0, + TCP_CACHE_STORE_POLICY_MISS_EVICT = 0x1, +} TCP_CACHE_STORE_POLICIES; +typedef enum TCP_WATCH_MODES { + TCP_WATCH_MODE_READ = 0x0, + TCP_WATCH_MODE_NONREAD = 0x1, + TCP_WATCH_MODE_ATOMIC = 0x2, + TCP_WATCH_MODE_ALL = 0x3, +} TCP_WATCH_MODES; +typedef enum VGT_OUT_PRIM_TYPE { + VGT_OUT_POINT = 0x0, + VGT_OUT_LINE = 0x1, + VGT_OUT_TRI = 0x2, + VGT_OUT_RECT_V0 = 0x3, + VGT_OUT_RECT_V1 = 0x4, + VGT_OUT_RECT_V2 = 0x5, + VGT_OUT_RECT_V3 = 0x6, + VGT_OUT_RESERVED = 0x7, + VGT_TE_QUAD = 0x8, + VGT_TE_PRIM_INDEX_LINE = 0x9, + VGT_TE_PRIM_INDEX_TRI = 0xa, + VGT_TE_PRIM_INDEX_QUAD = 0xb, + VGT_OUT_LINE_ADJ = 0xc, + VGT_OUT_TRI_ADJ = 0xd, + VGT_OUT_PATCH = 0xe, +} VGT_OUT_PRIM_TYPE; +typedef enum VGT_DI_PRIM_TYPE { + DI_PT_NONE = 0x0, + DI_PT_POINTLIST = 0x1, + DI_PT_LINELIST = 0x2, + DI_PT_LINESTRIP = 0x3, + DI_PT_TRILIST = 0x4, + DI_PT_TRIFAN = 0x5, + DI_PT_TRISTRIP = 0x6, + DI_PT_UNUSED_0 = 0x7, + DI_PT_UNUSED_1 = 0x8, + DI_PT_PATCH = 0x9, + DI_PT_LINELIST_ADJ = 0xa, + DI_PT_LINESTRIP_ADJ = 0xb, + DI_PT_TRILIST_ADJ = 0xc, + DI_PT_TRISTRIP_ADJ = 0xd, + DI_PT_UNUSED_3 = 0xe, + DI_PT_UNUSED_4 = 0xf, + DI_PT_TRI_WITH_WFLAGS = 0x10, + DI_PT_RECTLIST = 0x11, + DI_PT_LINELOOP = 0x12, + DI_PT_QUADLIST = 0x13, + DI_PT_QUADSTRIP = 0x14, + DI_PT_POLYGON = 0x15, + DI_PT_2D_COPY_RECT_LIST_V0 = 0x16, + DI_PT_2D_COPY_RECT_LIST_V1 = 0x17, + DI_PT_2D_COPY_RECT_LIST_V2 = 0x18, + DI_PT_2D_COPY_RECT_LIST_V3 = 0x19, + DI_PT_2D_FILL_RECT_LIST = 0x1a, + DI_PT_2D_LINE_STRIP = 0x1b, + DI_PT_2D_TRI_STRIP = 0x1c, +} VGT_DI_PRIM_TYPE; +typedef enum VGT_DI_SOURCE_SELECT { + DI_SRC_SEL_DMA = 0x0, + DI_SRC_SEL_IMMEDIATE = 0x1, + DI_SRC_SEL_AUTO_INDEX = 0x2, + DI_SRC_SEL_RESERVED = 0x3, +} VGT_DI_SOURCE_SELECT; +typedef enum VGT_DI_MAJOR_MODE_SELECT { + DI_MAJOR_MODE_0 = 0x0, + DI_MAJOR_MODE_1 = 0x1, +} VGT_DI_MAJOR_MODE_SELECT; +typedef enum VGT_DI_INDEX_SIZE { + DI_INDEX_SIZE_16_BIT = 0x0, + DI_INDEX_SIZE_32_BIT = 0x1, +} VGT_DI_INDEX_SIZE; +typedef enum VGT_EVENT_TYPE { + Reserved_0x00 = 0x0, + SAMPLE_STREAMOUTSTATS1 = 0x1, + SAMPLE_STREAMOUTSTATS2 = 0x2, + SAMPLE_STREAMOUTSTATS3 = 0x3, + CACHE_FLUSH_TS = 0x4, + CONTEXT_DONE = 0x5, + CACHE_FLUSH = 0x6, + CS_PARTIAL_FLUSH = 0x7, + VGT_STREAMOUT_SYNC = 0x8, + Reserved_0x09 = 0x9, + VGT_STREAMOUT_RESET = 0xa, + END_OF_PIPE_INCR_DE = 0xb, + END_OF_PIPE_IB_END = 0xc, + RST_PIX_CNT = 0xd, + Reserved_0x0E = 0xe, + VS_PARTIAL_FLUSH = 0xf, + PS_PARTIAL_FLUSH = 0x10, + FLUSH_HS_OUTPUT = 0x11, + FLUSH_LS_OUTPUT = 0x12, + Reserved_0x13 = 0x13, + CACHE_FLUSH_AND_INV_TS_EVENT = 0x14, + ZPASS_DONE = 0x15, + CACHE_FLUSH_AND_INV_EVENT = 0x16, + PERFCOUNTER_START = 0x17, + PERFCOUNTER_STOP = 0x18, + PIPELINESTAT_START = 0x19, + PIPELINESTAT_STOP = 0x1a, + PERFCOUNTER_SAMPLE = 0x1b, + FLUSH_ES_OUTPUT = 0x1c, + FLUSH_GS_OUTPUT = 0x1d, + SAMPLE_PIPELINESTAT = 0x1e, + SO_VGTSTREAMOUT_FLUSH = 0x1f, + SAMPLE_STREAMOUTSTATS = 0x20, + RESET_VTX_CNT = 0x21, + BLOCK_CONTEXT_DONE = 0x22, + CS_CONTEXT_DONE = 0x23, + VGT_FLUSH = 0x24, + Reserved_0x25 = 0x25, + SQ_NON_EVENT = 0x26, + SC_SEND_DB_VPZ = 0x27, + BOTTOM_OF_PIPE_TS = 0x28, + FLUSH_SX_TS = 0x29, + DB_CACHE_FLUSH_AND_INV = 0x2a, + FLUSH_AND_INV_DB_DATA_TS = 0x2b, + FLUSH_AND_INV_DB_META = 0x2c, + FLUSH_AND_INV_CB_DATA_TS = 0x2d, + FLUSH_AND_INV_CB_META = 0x2e, + CS_DONE = 0x2f, + PS_DONE = 0x30, + FLUSH_AND_INV_CB_PIXEL_DATA = 0x31, + SX_CB_RAT_ACK_REQUEST = 0x32, + THREAD_TRACE_START = 0x33, + THREAD_TRACE_STOP = 0x34, + THREAD_TRACE_MARKER = 0x35, + THREAD_TRACE_FLUSH = 0x36, + THREAD_TRACE_FINISH = 0x37, + PIXEL_PIPE_STAT_CONTROL = 0x38, + PIXEL_PIPE_STAT_DUMP = 0x39, + PIXEL_PIPE_STAT_RESET = 0x3a, + CONTEXT_SUSPEND = 0x3b, +} VGT_EVENT_TYPE; +typedef enum VGT_DMA_SWAP_MODE { + VGT_DMA_SWAP_NONE = 0x0, + VGT_DMA_SWAP_16_BIT = 0x1, + VGT_DMA_SWAP_32_BIT = 0x2, + VGT_DMA_SWAP_WORD = 0x3, +} VGT_DMA_SWAP_MODE; +typedef enum VGT_INDEX_TYPE_MODE { + VGT_INDEX_16 = 0x0, + VGT_INDEX_32 = 0x1, +} VGT_INDEX_TYPE_MODE; +typedef enum VGT_DMA_BUF_TYPE { + VGT_DMA_BUF_MEM = 0x0, + VGT_DMA_BUF_RING = 0x1, + VGT_DMA_BUF_SETUP = 0x2, +} VGT_DMA_BUF_TYPE; +typedef enum VGT_OUTPATH_SELECT { + VGT_OUTPATH_VTX_REUSE = 0x0, + VGT_OUTPATH_TESS_EN = 0x1, + VGT_OUTPATH_PASSTHRU = 0x2, + VGT_OUTPATH_GS_BLOCK = 0x3, + VGT_OUTPATH_HS_BLOCK = 0x4, +} VGT_OUTPATH_SELECT; +typedef enum VGT_GRP_PRIM_TYPE { + VGT_GRP_3D_POINT = 0x0, + VGT_GRP_3D_LINE = 0x1, + VGT_GRP_3D_TRI = 0x2, + VGT_GRP_3D_RECT = 0x3, + VGT_GRP_3D_QUAD = 0x4, + VGT_GRP_2D_COPY_RECT_V0 = 0x5, + VGT_GRP_2D_COPY_RECT_V1 = 0x6, + VGT_GRP_2D_COPY_RECT_V2 = 0x7, + VGT_GRP_2D_COPY_RECT_V3 = 0x8, + VGT_GRP_2D_FILL_RECT = 0x9, + VGT_GRP_2D_LINE = 0xa, + VGT_GRP_2D_TRI = 0xb, + VGT_GRP_PRIM_INDEX_LINE = 0xc, + VGT_GRP_PRIM_INDEX_TRI = 0xd, + VGT_GRP_PRIM_INDEX_QUAD = 0xe, + VGT_GRP_3D_LINE_ADJ = 0xf, + VGT_GRP_3D_TRI_ADJ = 0x10, + VGT_GRP_3D_PATCH = 0x11, +} VGT_GRP_PRIM_TYPE; +typedef enum VGT_GRP_PRIM_ORDER { + VGT_GRP_LIST = 0x0, + VGT_GRP_STRIP = 0x1, + VGT_GRP_FAN = 0x2, + VGT_GRP_LOOP = 0x3, + VGT_GRP_POLYGON = 0x4, +} VGT_GRP_PRIM_ORDER; +typedef enum VGT_GROUP_CONV_SEL { + VGT_GRP_INDEX_16 = 0x0, + VGT_GRP_INDEX_32 = 0x1, + VGT_GRP_UINT_16 = 0x2, + VGT_GRP_UINT_32 = 0x3, + VGT_GRP_SINT_16 = 0x4, + VGT_GRP_SINT_32 = 0x5, + VGT_GRP_FLOAT_32 = 0x6, + VGT_GRP_AUTO_PRIM = 0x7, + VGT_GRP_FIX_1_23_TO_FLOAT = 0x8, +} VGT_GROUP_CONV_SEL; +typedef enum VGT_GS_MODE_TYPE { + GS_OFF = 0x0, + GS_SCENARIO_A = 0x1, + GS_SCENARIO_B = 0x2, + GS_SCENARIO_G = 0x3, + GS_SCENARIO_C = 0x4, + SPRITE_EN = 0x5, +} VGT_GS_MODE_TYPE; +typedef enum VGT_GS_CUT_MODE { + GS_CUT_1024 = 0x0, + GS_CUT_512 = 0x1, + GS_CUT_256 = 0x2, + GS_CUT_128 = 0x3, +} VGT_GS_CUT_MODE; +typedef enum VGT_GS_OUTPRIM_TYPE { + POINTLIST = 0x0, + LINESTRIP = 0x1, + TRISTRIP = 0x2, +} VGT_GS_OUTPRIM_TYPE; +typedef enum VGT_CACHE_INVALID_MODE { + VC_ONLY = 0x0, + TC_ONLY = 0x1, + VC_AND_TC = 0x2, +} VGT_CACHE_INVALID_MODE; +typedef enum VGT_TESS_TYPE { + TESS_ISOLINE = 0x0, + TESS_TRIANGLE = 0x1, + TESS_QUAD = 0x2, +} VGT_TESS_TYPE; +typedef enum VGT_TESS_PARTITION { + PART_INTEGER = 0x0, + PART_POW2 = 0x1, + PART_FRAC_ODD = 0x2, + PART_FRAC_EVEN = 0x3, +} VGT_TESS_PARTITION; +typedef enum VGT_TESS_TOPOLOGY { + OUTPUT_POINT = 0x0, + OUTPUT_LINE = 0x1, + OUTPUT_TRIANGLE_CW = 0x2, + OUTPUT_TRIANGLE_CCW = 0x3, +} VGT_TESS_TOPOLOGY; +typedef enum VGT_RDREQ_POLICY { + VGT_POLICY_LRU = 0x0, + VGT_POLICY_STREAM = 0x1, + VGT_POLICY_BYPASS = 0x2, + VGT_POLICY_RESERVED = 0x3, +} VGT_RDREQ_POLICY; +typedef enum VGT_STAGES_LS_EN { + LS_STAGE_OFF = 0x0, + LS_STAGE_ON = 0x1, + CS_STAGE_ON = 0x2, + RESERVED_LS = 0x3, +} VGT_STAGES_LS_EN; +typedef enum VGT_STAGES_HS_EN { + HS_STAGE_OFF = 0x0, + HS_STAGE_ON = 0x1, +} VGT_STAGES_HS_EN; +typedef enum VGT_STAGES_ES_EN { + ES_STAGE_OFF = 0x0, + ES_STAGE_DS = 0x1, + ES_STAGE_REAL = 0x2, + RESERVED_ES = 0x3, +} VGT_STAGES_ES_EN; +typedef enum VGT_STAGES_GS_EN { + GS_STAGE_OFF = 0x0, + GS_STAGE_ON = 0x1, +} VGT_STAGES_GS_EN; +typedef enum VGT_STAGES_VS_EN { + VS_STAGE_REAL = 0x0, + VS_STAGE_DS = 0x1, + VS_STAGE_COPY_SHADER = 0x2, + RESERVED_VS = 0x3, +} VGT_STAGES_VS_EN; +typedef enum VGT_PERFCOUNT_SELECT { + vgt_perf_VGT_SPI_ESTHREAD_EVENT_WINDOW_ACTIVE = 0x0, + vgt_perf_VGT_SPI_ESVERT_VALID = 0x1, + vgt_perf_VGT_SPI_ESVERT_EOV = 0x2, + vgt_perf_VGT_SPI_ESVERT_STALLED = 0x3, + vgt_perf_VGT_SPI_ESVERT_STARVED_BUSY = 0x4, + vgt_perf_VGT_SPI_ESVERT_STARVED_IDLE = 0x5, + vgt_perf_VGT_SPI_ESVERT_STATIC = 0x6, + vgt_perf_VGT_SPI_ESTHREAD_IS_EVENT = 0x7, + vgt_perf_VGT_SPI_ESTHREAD_SEND = 0x8, + vgt_perf_VGT_SPI_GSPRIM_VALID = 0x9, + vgt_perf_VGT_SPI_GSPRIM_EOV = 0xa, + vgt_perf_VGT_SPI_GSPRIM_CONT = 0xb, + vgt_perf_VGT_SPI_GSPRIM_STALLED = 0xc, + vgt_perf_VGT_SPI_GSPRIM_STARVED_BUSY = 0xd, + vgt_perf_VGT_SPI_GSPRIM_STARVED_IDLE = 0xe, + vgt_perf_VGT_SPI_GSPRIM_STATIC = 0xf, + vgt_perf_VGT_SPI_GSTHREAD_EVENT_WINDOW_ACTIVE = 0x10, + vgt_perf_VGT_SPI_GSTHREAD_IS_EVENT = 0x11, + vgt_perf_VGT_SPI_GSTHREAD_SEND = 0x12, + vgt_perf_VGT_SPI_VSTHREAD_EVENT_WINDOW_ACTIVE = 0x13, + vgt_perf_VGT_SPI_VSVERT_SEND = 0x14, + vgt_perf_VGT_SPI_VSVERT_EOV = 0x15, + vgt_perf_VGT_SPI_VSVERT_STALLED = 0x16, + vgt_perf_VGT_SPI_VSVERT_STARVED_BUSY = 0x17, + vgt_perf_VGT_SPI_VSVERT_STARVED_IDLE = 0x18, + vgt_perf_VGT_SPI_VSVERT_STATIC = 0x19, + vgt_perf_VGT_SPI_VSTHREAD_IS_EVENT = 0x1a, + vgt_perf_VGT_SPI_VSTHREAD_SEND = 0x1b, + vgt_perf_VGT_PA_EVENT_WINDOW_ACTIVE = 0x1c, + vgt_perf_VGT_PA_CLIPV_SEND = 0x1d, + vgt_perf_VGT_PA_CLIPV_FIRSTVERT = 0x1e, + vgt_perf_VGT_PA_CLIPV_STALLED = 0x1f, + vgt_perf_VGT_PA_CLIPV_STARVED_BUSY = 0x20, + vgt_perf_VGT_PA_CLIPV_STARVED_IDLE = 0x21, + vgt_perf_VGT_PA_CLIPV_STATIC = 0x22, + vgt_perf_VGT_PA_CLIPP_SEND = 0x23, + vgt_perf_VGT_PA_CLIPP_EOP = 0x24, + vgt_perf_VGT_PA_CLIPP_IS_EVENT = 0x25, + vgt_perf_VGT_PA_CLIPP_NULL_PRIM = 0x26, + vgt_perf_VGT_PA_CLIPP_NEW_VTX_VECT = 0x27, + vgt_perf_VGT_PA_CLIPP_STALLED = 0x28, + vgt_perf_VGT_PA_CLIPP_STARVED_BUSY = 0x29, + vgt_perf_VGT_PA_CLIPP_STARVED_IDLE = 0x2a, + vgt_perf_VGT_PA_CLIPP_STATIC = 0x2b, + vgt_perf_VGT_PA_CLIPS_SEND = 0x2c, + vgt_perf_VGT_PA_CLIPS_STALLED = 0x2d, + vgt_perf_VGT_PA_CLIPS_STARVED_BUSY = 0x2e, + vgt_perf_VGT_PA_CLIPS_STARVED_IDLE = 0x2f, + vgt_perf_VGT_PA_CLIPS_STATIC = 0x30, + vgt_perf_vsvert_ds_send = 0x31, + vgt_perf_vsvert_api_send = 0x32, + vgt_perf_hs_tif_stall = 0x33, + vgt_perf_hs_input_stall = 0x34, + vgt_perf_hs_interface_stall = 0x35, + vgt_perf_hs_tfm_stall = 0x36, + vgt_perf_te11_starved = 0x37, + vgt_perf_gs_event_stall = 0x38, + vgt_perf_vgt_pa_clipp_send_not_event = 0x39, + vgt_perf_vgt_pa_clipp_valid_prim = 0x3a, + vgt_perf_reused_es_indices = 0x3b, + vgt_perf_vs_cache_hits = 0x3c, + vgt_perf_gs_cache_hits = 0x3d, + vgt_perf_ds_cache_hits = 0x3e, + vgt_perf_total_cache_hits = 0x3f, + vgt_perf_vgt_busy = 0x40, + vgt_perf_vgt_gs_busy = 0x41, + vgt_perf_esvert_stalled_es_tbl = 0x42, + vgt_perf_esvert_stalled_gs_tbl = 0x43, + vgt_perf_esvert_stalled_gs_event = 0x44, + vgt_perf_esvert_stalled_gsprim = 0x45, + vgt_perf_gsprim_stalled_es_tbl = 0x46, + vgt_perf_gsprim_stalled_gs_tbl = 0x47, + vgt_perf_gsprim_stalled_gs_event = 0x48, + vgt_perf_gsprim_stalled_esvert = 0x49, + vgt_perf_esthread_stalled_es_rb_full = 0x4a, + vgt_perf_esthread_stalled_spi_bp = 0x4b, + vgt_perf_counters_avail_stalled = 0x4c, + vgt_perf_gs_rb_space_avail_stalled = 0x4d, + vgt_perf_gs_issue_rtr_stalled = 0x4e, + vgt_perf_gsthread_stalled = 0x4f, + vgt_perf_strmout_stalled = 0x50, + vgt_perf_wait_for_es_done_stalled = 0x51, + vgt_perf_cm_stalled_by_gog = 0x52, + vgt_perf_cm_reading_stalled = 0x53, + vgt_perf_cm_stalled_by_gsfetch_done = 0x54, + vgt_perf_gog_vs_tbl_stalled = 0x55, + vgt_perf_gog_out_indx_stalled = 0x56, + vgt_perf_gog_out_prim_stalled = 0x57, + vgt_perf_waveid_stalled = 0x58, + vgt_perf_gog_busy = 0x59, + vgt_perf_reused_vs_indices = 0x5a, + vgt_perf_sclk_reg_vld_event = 0x5b, + vgt_perf_RESERVED0 = 0x5c, + vgt_perf_sclk_core_vld_event = 0x5d, + vgt_perf_RESERVED1 = 0x5e, + vgt_perf_sclk_gs_vld_event = 0x5f, + vgt_perf_VGT_SPI_LSVERT_VALID = 0x60, + vgt_perf_VGT_SPI_LSVERT_EOV = 0x61, + vgt_perf_VGT_SPI_LSVERT_STALLED = 0x62, + vgt_perf_VGT_SPI_LSVERT_STARVED_BUSY = 0x63, + vgt_perf_VGT_SPI_LSVERT_STARVED_IDLE = 0x64, + vgt_perf_VGT_SPI_LSVERT_STATIC = 0x65, + vgt_perf_VGT_SPI_LSWAVE_EVENT_WINDOW_ACTIVE = 0x66, + vgt_perf_VGT_SPI_LSWAVE_IS_EVENT = 0x67, + vgt_perf_VGT_SPI_LSWAVE_SEND = 0x68, + vgt_perf_VGT_SPI_HSVERT_VALID = 0x69, + vgt_perf_VGT_SPI_HSVERT_EOV = 0x6a, + vgt_perf_VGT_SPI_HSVERT_STALLED = 0x6b, + vgt_perf_VGT_SPI_HSVERT_STARVED_BUSY = 0x6c, + vgt_perf_VGT_SPI_HSVERT_STARVED_IDLE = 0x6d, + vgt_perf_VGT_SPI_HSVERT_STATIC = 0x6e, + vgt_perf_VGT_SPI_HSWAVE_EVENT_WINDOW_ACTIVE = 0x6f, + vgt_perf_VGT_SPI_HSWAVE_IS_EVENT = 0x70, + vgt_perf_VGT_SPI_HSWAVE_SEND = 0x71, + vgt_perf_ds_prims = 0x72, + vgt_perf_null_tess_patches = 0x73, + vgt_perf_ls_thread_groups = 0x74, + vgt_perf_hs_thread_groups = 0x75, + vgt_perf_es_thread_groups = 0x76, + vgt_perf_vs_thread_groups = 0x77, + vgt_perf_ls_done_latency = 0x78, + vgt_perf_hs_done_latency = 0x79, + vgt_perf_es_done_latency = 0x7a, + vgt_perf_gs_done_latency = 0x7b, + vgt_perf_vgt_hs_busy = 0x7c, + vgt_perf_vgt_te11_busy = 0x7d, + vgt_perf_ls_flush = 0x7e, + vgt_perf_hs_flush = 0x7f, + vgt_perf_es_flush = 0x80, + vgt_perf_gs_flush = 0x81, + vgt_perf_ls_done = 0x82, + vgt_perf_hs_done = 0x83, + vgt_perf_es_done = 0x84, + vgt_perf_gs_done = 0x85, + vgt_perf_vsfetch_done = 0x86, + vgt_perf_RESERVED2 = 0x87, + vgt_perf_es_ring_high_water_mark = 0x88, + vgt_perf_gs_ring_high_water_mark = 0x89, + vgt_perf_vs_table_high_water_mark = 0x8a, + vgt_perf_hs_tgs_active_high_water_mark = 0x8b, +} VGT_PERFCOUNT_SELECT; +typedef enum IA_PERFCOUNT_SELECT { + ia_perf_GRP_INPUT_EVENT_WINDOW_ACTIVE = 0x0, + ia_perf_MC_LAT_BIN_0 = 0x1, + ia_perf_MC_LAT_BIN_1 = 0x2, + ia_perf_MC_LAT_BIN_2 = 0x3, + ia_perf_MC_LAT_BIN_3 = 0x4, + ia_perf_MC_LAT_BIN_4 = 0x5, + ia_perf_MC_LAT_BIN_5 = 0x6, + ia_perf_MC_LAT_BIN_6 = 0x7, + ia_perf_MC_LAT_BIN_7 = 0x8, + ia_perf_ia_busy = 0x9, + ia_perf_ia_sclk_reg_vld_event = 0xa, + ia_perf_RESERVED0 = 0xb, + ia_perf_ia_sclk_core_vld_event = 0xc, + ia_perf_RESERVED1 = 0xd, + ia_perf_ia_dma_return = 0xe, + ia_perf_shift_starved_pipe1_event = 0xf, + ia_perf_shift_starved_pipe0_event = 0x10, + ia_perf_ia_stalled = 0x11, +} IA_PERFCOUNT_SELECT; +typedef enum WD_PERFCOUNT_SELECT { + wd_perf_RBIU_FIFOS_EVENT_WINDOW_ACTIVE = 0x0, + wd_perf_RBIU_DR_FIFO_STARVED = 0x1, + wd_perf_RBIU_DR_FIFO_STALLED = 0x2, + wd_perf_RBIU_DI_FIFO_STARVED = 0x3, + wd_perf_RBIU_DI_FIFO_STALLED = 0x4, + wd_perf_wd_busy = 0x5, + wd_perf_wd_sclk_reg_vld_event = 0x6, + wd_perf_wd_sclk_input_vld_event = 0x7, + wd_perf_wd_sclk_core_vld_event = 0x8, + wd_perf_wd_stalled = 0x9, +} WD_PERFCOUNT_SELECT; +typedef enum WD_IA_DRAW_TYPE { + WD_IA_DRAW_TYPE_DI_MM0 = 0x0, + WD_IA_DRAW_TYPE_DI_MM1 = 0x1, + WD_IA_DRAW_TYPE_EVENT_INIT = 0x2, + WD_IA_DRAW_TYPE_EVENT_ADDR = 0x3, + WD_IA_DRAW_TYPE_MIN_INDX = 0x4, + WD_IA_DRAW_TYPE_MAX_INDX = 0x5, + WD_IA_DRAW_TYPE_INDX_OFF = 0x6, + WD_IA_DRAW_TYPE_IMM_DATA = 0x7, +} WD_IA_DRAW_TYPE; +#define GSTHREADID_SIZE 0x2 +typedef enum SurfaceEndian { + ENDIAN_NONE = 0x0, + ENDIAN_8IN16 = 0x1, + ENDIAN_8IN32 = 0x2, + ENDIAN_8IN64 = 0x3, +} SurfaceEndian; +typedef enum ArrayMode { + ARRAY_LINEAR_GENERAL = 0x0, + ARRAY_LINEAR_ALIGNED = 0x1, + ARRAY_1D_TILED_THIN1 = 0x2, + ARRAY_1D_TILED_THICK = 0x3, + ARRAY_2D_TILED_THIN1 = 0x4, + ARRAY_PRT_TILED_THIN1 = 0x5, + ARRAY_PRT_2D_TILED_THIN1 = 0x6, + ARRAY_2D_TILED_THICK = 0x7, + ARRAY_2D_TILED_XTHICK = 0x8, + ARRAY_PRT_TILED_THICK = 0x9, + ARRAY_PRT_2D_TILED_THICK = 0xa, + ARRAY_PRT_3D_TILED_THIN1 = 0xb, + ARRAY_3D_TILED_THIN1 = 0xc, + ARRAY_3D_TILED_THICK = 0xd, + ARRAY_3D_TILED_XTHICK = 0xe, + ARRAY_PRT_3D_TILED_THICK = 0xf, +} ArrayMode; +typedef enum PipeTiling { + CONFIG_1_PIPE = 0x0, + CONFIG_2_PIPE = 0x1, + CONFIG_4_PIPE = 0x2, + CONFIG_8_PIPE = 0x3, +} PipeTiling; +typedef enum BankTiling { + CONFIG_4_BANK = 0x0, + CONFIG_8_BANK = 0x1, +} BankTiling; +typedef enum GroupInterleave { + CONFIG_256B_GROUP = 0x0, + CONFIG_512B_GROUP = 0x1, +} GroupInterleave; +typedef enum RowTiling { + CONFIG_1KB_ROW = 0x0, + CONFIG_2KB_ROW = 0x1, + CONFIG_4KB_ROW = 0x2, + CONFIG_8KB_ROW = 0x3, + CONFIG_1KB_ROW_OPT = 0x4, + CONFIG_2KB_ROW_OPT = 0x5, + CONFIG_4KB_ROW_OPT = 0x6, + CONFIG_8KB_ROW_OPT = 0x7, +} RowTiling; +typedef enum BankSwapBytes { + CONFIG_128B_SWAPS = 0x0, + CONFIG_256B_SWAPS = 0x1, + CONFIG_512B_SWAPS = 0x2, + CONFIG_1KB_SWAPS = 0x3, +} BankSwapBytes; +typedef enum SampleSplitBytes { + CONFIG_1KB_SPLIT = 0x0, + CONFIG_2KB_SPLIT = 0x1, + CONFIG_4KB_SPLIT = 0x2, + CONFIG_8KB_SPLIT = 0x3, +} SampleSplitBytes; +typedef enum NumPipes { + ADDR_CONFIG_1_PIPE = 0x0, + ADDR_CONFIG_2_PIPE = 0x1, + ADDR_CONFIG_4_PIPE = 0x2, + ADDR_CONFIG_8_PIPE = 0x3, + ADDR_CONFIG_16_PIPE = 0x4, +} NumPipes; +typedef enum PipeInterleaveSize { + ADDR_CONFIG_PIPE_INTERLEAVE_256B = 0x0, + ADDR_CONFIG_PIPE_INTERLEAVE_512B = 0x1, +} PipeInterleaveSize; +typedef enum BankInterleaveSize { + ADDR_CONFIG_BANK_INTERLEAVE_1 = 0x0, + ADDR_CONFIG_BANK_INTERLEAVE_2 = 0x1, + ADDR_CONFIG_BANK_INTERLEAVE_4 = 0x2, + ADDR_CONFIG_BANK_INTERLEAVE_8 = 0x3, +} BankInterleaveSize; +typedef enum NumShaderEngines { + ADDR_CONFIG_1_SHADER_ENGINE = 0x0, + ADDR_CONFIG_2_SHADER_ENGINE = 0x1, +} NumShaderEngines; +typedef enum ShaderEngineTileSize { + ADDR_CONFIG_SE_TILE_16 = 0x0, + ADDR_CONFIG_SE_TILE_32 = 0x1, +} ShaderEngineTileSize; +typedef enum NumGPUs { + ADDR_CONFIG_1_GPU = 0x0, + ADDR_CONFIG_2_GPU = 0x1, + ADDR_CONFIG_4_GPU = 0x2, +} NumGPUs; +typedef enum MultiGPUTileSize { + ADDR_CONFIG_GPU_TILE_16 = 0x0, + ADDR_CONFIG_GPU_TILE_32 = 0x1, + ADDR_CONFIG_GPU_TILE_64 = 0x2, + ADDR_CONFIG_GPU_TILE_128 = 0x3, +} MultiGPUTileSize; +typedef enum RowSize { + ADDR_CONFIG_1KB_ROW = 0x0, + ADDR_CONFIG_2KB_ROW = 0x1, + ADDR_CONFIG_4KB_ROW = 0x2, +} RowSize; +typedef enum NumLowerPipes { + ADDR_CONFIG_1_LOWER_PIPES = 0x0, + ADDR_CONFIG_2_LOWER_PIPES = 0x1, +} NumLowerPipes; +typedef enum DebugBlockId { + DBG_CLIENT_BLKID_RESERVED = 0x0, + DBG_CLIENT_BLKID_dbg = 0x1, + DBG_CLIENT_BLKID_dco0 = 0x2, + DBG_CLIENT_BLKID_wd = 0x3, + DBG_CLIENT_BLKID_vmc = 0x4, + DBG_CLIENT_BLKID_scf2 = 0x5, + DBG_CLIENT_BLKID_spim3 = 0x6, + DBG_CLIENT_BLKID_cb3 = 0x7, + DBG_CLIENT_BLKID_sx0 = 0x8, + DBG_CLIENT_BLKID_cb2 = 0x9, + DBG_CLIENT_BLKID_bci1 = 0xa, + DBG_CLIENT_BLKID_xdma = 0xb, + DBG_CLIENT_BLKID_bci0 = 0xc, + DBG_CLIENT_BLKID_spim0 = 0xd, + DBG_CLIENT_BLKID_mcd0 = 0xe, + DBG_CLIENT_BLKID_mcc0 = 0xf, + DBG_CLIENT_BLKID_cb0 = 0x10, + DBG_CLIENT_BLKID_cb1 = 0x11, + DBG_CLIENT_BLKID_cpc_0 = 0x12, + DBG_CLIENT_BLKID_cpc_1 = 0x13, + DBG_CLIENT_BLKID_cpf = 0x14, + DBG_CLIENT_BLKID_rlc = 0x15, + DBG_CLIENT_BLKID_grbm = 0x16, + DBG_CLIENT_BLKID_bif = 0x17, + DBG_CLIENT_BLKID_scf1 = 0x18, + DBG_CLIENT_BLKID_sam = 0x19, + DBG_CLIENT_BLKID_mcd4 = 0x1a, + DBG_CLIENT_BLKID_mcc4 = 0x1b, + DBG_CLIENT_BLKID_gmcon = 0x1c, + DBG_CLIENT_BLKID_mcb = 0x1d, + DBG_CLIENT_BLKID_vgt0 = 0x1e, + DBG_CLIENT_BLKID_pc0 = 0x1f, + DBG_CLIENT_BLKID_spim1 = 0x20, + DBG_CLIENT_BLKID_bci2 = 0x21, + DBG_CLIENT_BLKID_mcd6 = 0x22, + DBG_CLIENT_BLKID_mcc6 = 0x23, + DBG_CLIENT_BLKID_mcd3 = 0x24, + DBG_CLIENT_BLKID_mcc3 = 0x25, + DBG_CLIENT_BLKID_uvdm_0 = 0x26, + DBG_CLIENT_BLKID_uvdm_1 = 0x27, + DBG_CLIENT_BLKID_uvdm_2 = 0x28, + DBG_CLIENT_BLKID_uvdm_3 = 0x29, + DBG_CLIENT_BLKID_spim2 = 0x2a, + DBG_CLIENT_BLKID_ds = 0x2b, + DBG_CLIENT_BLKID_srbm = 0x2c, + DBG_CLIENT_BLKID_ih = 0x2d, + DBG_CLIENT_BLKID_sem = 0x2e, + DBG_CLIENT_BLKID_sdma_0 = 0x2f, + DBG_CLIENT_BLKID_sdma_1 = 0x30, + DBG_CLIENT_BLKID_hdp = 0x31, + DBG_CLIENT_BLKID_acp_0 = 0x32, + DBG_CLIENT_BLKID_acp_1 = 0x33, + DBG_CLIENT_BLKID_vceb_0 = 0x34, + DBG_CLIENT_BLKID_vceb_1 = 0x35, + DBG_CLIENT_BLKID_vceb_2 = 0x36, + DBG_CLIENT_BLKID_mcd2 = 0x37, + DBG_CLIENT_BLKID_mcc2 = 0x38, + DBG_CLIENT_BLKID_scf3 = 0x39, + DBG_CLIENT_BLKID_bci3 = 0x3a, + DBG_CLIENT_BLKID_mcd5 = 0x3b, + DBG_CLIENT_BLKID_mcc5 = 0x3c, + DBG_CLIENT_BLKID_vgt2 = 0x3d, + DBG_CLIENT_BLKID_pc2 = 0x3e, + DBG_CLIENT_BLKID_smu_0 = 0x3f, + DBG_CLIENT_BLKID_smu_1 = 0x40, + DBG_CLIENT_BLKID_smu_2 = 0x41, + DBG_CLIENT_BLKID_vcea_0 = 0x42, + DBG_CLIENT_BLKID_vcea_1 = 0x43, + DBG_CLIENT_BLKID_vcea_2 = 0x44, + DBG_CLIENT_BLKID_vcea_3 = 0x45, + DBG_CLIENT_BLKID_vcea_4 = 0x46, + DBG_CLIENT_BLKID_vcea_5 = 0x47, + DBG_CLIENT_BLKID_vcea_6 = 0x48, + DBG_CLIENT_BLKID_scf0 = 0x49, + DBG_CLIENT_BLKID_vgt1 = 0x4a, + DBG_CLIENT_BLKID_pc1 = 0x4b, + DBG_CLIENT_BLKID_gdc_0 = 0x4c, + DBG_CLIENT_BLKID_gdc_1 = 0x4d, + DBG_CLIENT_BLKID_gdc_2 = 0x4e, + DBG_CLIENT_BLKID_gdc_3 = 0x4f, + DBG_CLIENT_BLKID_gdc_4 = 0x50, + DBG_CLIENT_BLKID_gdc_5 = 0x51, + DBG_CLIENT_BLKID_gdc_6 = 0x52, + DBG_CLIENT_BLKID_gdc_7 = 0x53, + DBG_CLIENT_BLKID_gdc_8 = 0x54, + DBG_CLIENT_BLKID_gdc_9 = 0x55, + DBG_CLIENT_BLKID_gdc_10 = 0x56, + DBG_CLIENT_BLKID_gdc_11 = 0x57, + DBG_CLIENT_BLKID_gdc_12 = 0x58, + DBG_CLIENT_BLKID_gdc_13 = 0x59, + DBG_CLIENT_BLKID_gdc_14 = 0x5a, + DBG_CLIENT_BLKID_gdc_15 = 0x5b, + DBG_CLIENT_BLKID_gdc_16 = 0x5c, + DBG_CLIENT_BLKID_gdc_17 = 0x5d, + DBG_CLIENT_BLKID_gdc_18 = 0x5e, + DBG_CLIENT_BLKID_gdc_19 = 0x5f, + DBG_CLIENT_BLKID_gdc_20 = 0x60, + DBG_CLIENT_BLKID_gdc_21 = 0x61, + DBG_CLIENT_BLKID_gdc_22 = 0x62, + DBG_CLIENT_BLKID_vgt3 = 0x63, + DBG_CLIENT_BLKID_pc3 = 0x64, + DBG_CLIENT_BLKID_uvdu_0 = 0x65, + DBG_CLIENT_BLKID_uvdu_1 = 0x66, + DBG_CLIENT_BLKID_uvdu_2 = 0x67, + DBG_CLIENT_BLKID_uvdu_3 = 0x68, + DBG_CLIENT_BLKID_uvdu_4 = 0x69, + DBG_CLIENT_BLKID_uvdu_5 = 0x6a, + DBG_CLIENT_BLKID_uvdu_6 = 0x6b, + DBG_CLIENT_BLKID_mcd7 = 0x6c, + DBG_CLIENT_BLKID_mcc7 = 0x6d, + DBG_CLIENT_BLKID_cpg_0 = 0x6e, + DBG_CLIENT_BLKID_cpg_1 = 0x6f, + DBG_CLIENT_BLKID_gck = 0x70, + DBG_CLIENT_BLKID_mcd1 = 0x71, + DBG_CLIENT_BLKID_mcc1 = 0x72, + DBG_CLIENT_BLKID_cb101 = 0x73, + DBG_CLIENT_BLKID_cb103 = 0x74, + DBG_CLIENT_BLKID_sx10 = 0x75, + DBG_CLIENT_BLKID_cb102 = 0x76, + DBG_CLIENT_BLKID_cb002 = 0x77, + DBG_CLIENT_BLKID_cb100 = 0x78, + DBG_CLIENT_BLKID_cb000 = 0x79, + DBG_CLIENT_BLKID_pa00 = 0x7a, + DBG_CLIENT_BLKID_pa10 = 0x7b, + DBG_CLIENT_BLKID_ia0 = 0x7c, + DBG_CLIENT_BLKID_ia1 = 0x7d, + DBG_CLIENT_BLKID_tmonw00 = 0x7e, + DBG_CLIENT_BLKID_cb001 = 0x7f, + DBG_CLIENT_BLKID_cb003 = 0x80, + DBG_CLIENT_BLKID_sx00 = 0x81, + DBG_CLIENT_BLKID_sx20 = 0x82, + DBG_CLIENT_BLKID_cb203 = 0x83, + DBG_CLIENT_BLKID_cb201 = 0x84, + DBG_CLIENT_BLKID_cb302 = 0x85, + DBG_CLIENT_BLKID_cb202 = 0x86, + DBG_CLIENT_BLKID_cb300 = 0x87, + DBG_CLIENT_BLKID_cb200 = 0x88, + DBG_CLIENT_BLKID_pa01 = 0x89, + DBG_CLIENT_BLKID_pa11 = 0x8a, + DBG_CLIENT_BLKID_sx30 = 0x8b, + DBG_CLIENT_BLKID_cb303 = 0x8c, + DBG_CLIENT_BLKID_cb301 = 0x8d, + DBG_CLIENT_BLKID_dco = 0x8e, + DBG_CLIENT_BLKID_scb0 = 0x8f, + DBG_CLIENT_BLKID_scb1 = 0x90, + DBG_CLIENT_BLKID_scb2 = 0x91, + DBG_CLIENT_BLKID_scb3 = 0x92, + DBG_CLIENT_BLKID_tmonw01 = 0x93, + DBG_CLIENT_BLKID_RESERVED_LAST = 0x94, +} DebugBlockId; +typedef enum DebugBlockId_OLD { + DBG_BLOCK_ID_RESERVED = 0x0, + DBG_BLOCK_ID_DBG = 0x1, + DBG_BLOCK_ID_VMC = 0x2, + DBG_BLOCK_ID_PDMA = 0x3, + DBG_BLOCK_ID_CG = 0x4, + DBG_BLOCK_ID_SRBM = 0x5, + DBG_BLOCK_ID_GRBM = 0x6, + DBG_BLOCK_ID_RLC = 0x7, + DBG_BLOCK_ID_CSC = 0x8, + DBG_BLOCK_ID_SEM = 0x9, + DBG_BLOCK_ID_IH = 0xa, + DBG_BLOCK_ID_SC = 0xb, + DBG_BLOCK_ID_SQ = 0xc, + DBG_BLOCK_ID_AVP = 0xd, + DBG_BLOCK_ID_GMCON = 0xe, + DBG_BLOCK_ID_SMU = 0xf, + DBG_BLOCK_ID_DMA0 = 0x10, + DBG_BLOCK_ID_DMA1 = 0x11, + DBG_BLOCK_ID_SPIM = 0x12, + DBG_BLOCK_ID_GDS = 0x13, + DBG_BLOCK_ID_SPIS = 0x14, + DBG_BLOCK_ID_UNUSED0 = 0x15, + DBG_BLOCK_ID_PA0 = 0x16, + DBG_BLOCK_ID_PA1 = 0x17, + DBG_BLOCK_ID_CP0 = 0x18, + DBG_BLOCK_ID_CP1 = 0x19, + DBG_BLOCK_ID_CP2 = 0x1a, + DBG_BLOCK_ID_UNUSED1 = 0x1b, + DBG_BLOCK_ID_UVDU = 0x1c, + DBG_BLOCK_ID_UVDM = 0x1d, + DBG_BLOCK_ID_VCE = 0x1e, + DBG_BLOCK_ID_UNUSED2 = 0x1f, + DBG_BLOCK_ID_VGT0 = 0x20, + DBG_BLOCK_ID_VGT1 = 0x21, + DBG_BLOCK_ID_IA = 0x22, + DBG_BLOCK_ID_UNUSED3 = 0x23, + DBG_BLOCK_ID_SCT0 = 0x24, + DBG_BLOCK_ID_SCT1 = 0x25, + DBG_BLOCK_ID_SPM0 = 0x26, + DBG_BLOCK_ID_SPM1 = 0x27, + DBG_BLOCK_ID_TCAA = 0x28, + DBG_BLOCK_ID_TCAB = 0x29, + DBG_BLOCK_ID_TCCA = 0x2a, + DBG_BLOCK_ID_TCCB = 0x2b, + DBG_BLOCK_ID_MCC0 = 0x2c, + DBG_BLOCK_ID_MCC1 = 0x2d, + DBG_BLOCK_ID_MCC2 = 0x2e, + DBG_BLOCK_ID_MCC3 = 0x2f, + DBG_BLOCK_ID_SX0 = 0x30, + DBG_BLOCK_ID_SX1 = 0x31, + DBG_BLOCK_ID_SX2 = 0x32, + DBG_BLOCK_ID_SX3 = 0x33, + DBG_BLOCK_ID_UNUSED4 = 0x34, + DBG_BLOCK_ID_UNUSED5 = 0x35, + DBG_BLOCK_ID_UNUSED6 = 0x36, + DBG_BLOCK_ID_UNUSED7 = 0x37, + DBG_BLOCK_ID_PC0 = 0x38, + DBG_BLOCK_ID_PC1 = 0x39, + DBG_BLOCK_ID_UNUSED8 = 0x3a, + DBG_BLOCK_ID_UNUSED9 = 0x3b, + DBG_BLOCK_ID_UNUSED10 = 0x3c, + DBG_BLOCK_ID_UNUSED11 = 0x3d, + DBG_BLOCK_ID_MCB = 0x3e, + DBG_BLOCK_ID_UNUSED12 = 0x3f, + DBG_BLOCK_ID_SCB0 = 0x40, + DBG_BLOCK_ID_SCB1 = 0x41, + DBG_BLOCK_ID_UNUSED13 = 0x42, + DBG_BLOCK_ID_UNUSED14 = 0x43, + DBG_BLOCK_ID_SCF0 = 0x44, + DBG_BLOCK_ID_SCF1 = 0x45, + DBG_BLOCK_ID_UNUSED15 = 0x46, + DBG_BLOCK_ID_UNUSED16 = 0x47, + DBG_BLOCK_ID_BCI0 = 0x48, + DBG_BLOCK_ID_BCI1 = 0x49, + DBG_BLOCK_ID_BCI2 = 0x4a, + DBG_BLOCK_ID_BCI3 = 0x4b, + DBG_BLOCK_ID_UNUSED17 = 0x4c, + DBG_BLOCK_ID_UNUSED18 = 0x4d, + DBG_BLOCK_ID_UNUSED19 = 0x4e, + DBG_BLOCK_ID_UNUSED20 = 0x4f, + DBG_BLOCK_ID_CB00 = 0x50, + DBG_BLOCK_ID_CB01 = 0x51, + DBG_BLOCK_ID_CB02 = 0x52, + DBG_BLOCK_ID_CB03 = 0x53, + DBG_BLOCK_ID_CB04 = 0x54, + DBG_BLOCK_ID_UNUSED21 = 0x55, + DBG_BLOCK_ID_UNUSED22 = 0x56, + DBG_BLOCK_ID_UNUSED23 = 0x57, + DBG_BLOCK_ID_CB10 = 0x58, + DBG_BLOCK_ID_CB11 = 0x59, + DBG_BLOCK_ID_CB12 = 0x5a, + DBG_BLOCK_ID_CB13 = 0x5b, + DBG_BLOCK_ID_CB14 = 0x5c, + DBG_BLOCK_ID_UNUSED24 = 0x5d, + DBG_BLOCK_ID_UNUSED25 = 0x5e, + DBG_BLOCK_ID_UNUSED26 = 0x5f, + DBG_BLOCK_ID_TCP0 = 0x60, + DBG_BLOCK_ID_TCP1 = 0x61, + DBG_BLOCK_ID_TCP2 = 0x62, + DBG_BLOCK_ID_TCP3 = 0x63, + DBG_BLOCK_ID_TCP4 = 0x64, + DBG_BLOCK_ID_TCP5 = 0x65, + DBG_BLOCK_ID_TCP6 = 0x66, + DBG_BLOCK_ID_TCP7 = 0x67, + DBG_BLOCK_ID_TCP8 = 0x68, + DBG_BLOCK_ID_TCP9 = 0x69, + DBG_BLOCK_ID_TCP10 = 0x6a, + DBG_BLOCK_ID_TCP11 = 0x6b, + DBG_BLOCK_ID_TCP12 = 0x6c, + DBG_BLOCK_ID_TCP13 = 0x6d, + DBG_BLOCK_ID_TCP14 = 0x6e, + DBG_BLOCK_ID_TCP15 = 0x6f, + DBG_BLOCK_ID_TCP16 = 0x70, + DBG_BLOCK_ID_TCP17 = 0x71, + DBG_BLOCK_ID_TCP18 = 0x72, + DBG_BLOCK_ID_TCP19 = 0x73, + DBG_BLOCK_ID_TCP20 = 0x74, + DBG_BLOCK_ID_TCP21 = 0x75, + DBG_BLOCK_ID_TCP22 = 0x76, + DBG_BLOCK_ID_TCP23 = 0x77, + DBG_BLOCK_ID_TCP_RESERVED0 = 0x78, + DBG_BLOCK_ID_TCP_RESERVED1 = 0x79, + DBG_BLOCK_ID_TCP_RESERVED2 = 0x7a, + DBG_BLOCK_ID_TCP_RESERVED3 = 0x7b, + DBG_BLOCK_ID_TCP_RESERVED4 = 0x7c, + DBG_BLOCK_ID_TCP_RESERVED5 = 0x7d, + DBG_BLOCK_ID_TCP_RESERVED6 = 0x7e, + DBG_BLOCK_ID_TCP_RESERVED7 = 0x7f, + DBG_BLOCK_ID_DB00 = 0x80, + DBG_BLOCK_ID_DB01 = 0x81, + DBG_BLOCK_ID_DB02 = 0x82, + DBG_BLOCK_ID_DB03 = 0x83, + DBG_BLOCK_ID_DB04 = 0x84, + DBG_BLOCK_ID_UNUSED27 = 0x85, + DBG_BLOCK_ID_UNUSED28 = 0x86, + DBG_BLOCK_ID_UNUSED29 = 0x87, + DBG_BLOCK_ID_DB10 = 0x88, + DBG_BLOCK_ID_DB11 = 0x89, + DBG_BLOCK_ID_DB12 = 0x8a, + DBG_BLOCK_ID_DB13 = 0x8b, + DBG_BLOCK_ID_DB14 = 0x8c, + DBG_BLOCK_ID_UNUSED30 = 0x8d, + DBG_BLOCK_ID_UNUSED31 = 0x8e, + DBG_BLOCK_ID_UNUSED32 = 0x8f, + DBG_BLOCK_ID_TCC0 = 0x90, + DBG_BLOCK_ID_TCC1 = 0x91, + DBG_BLOCK_ID_TCC2 = 0x92, + DBG_BLOCK_ID_TCC3 = 0x93, + DBG_BLOCK_ID_TCC4 = 0x94, + DBG_BLOCK_ID_TCC5 = 0x95, + DBG_BLOCK_ID_TCC6 = 0x96, + DBG_BLOCK_ID_TCC7 = 0x97, + DBG_BLOCK_ID_SPS00 = 0x98, + DBG_BLOCK_ID_SPS01 = 0x99, + DBG_BLOCK_ID_SPS02 = 0x9a, + DBG_BLOCK_ID_SPS10 = 0x9b, + DBG_BLOCK_ID_SPS11 = 0x9c, + DBG_BLOCK_ID_SPS12 = 0x9d, + DBG_BLOCK_ID_UNUSED33 = 0x9e, + DBG_BLOCK_ID_UNUSED34 = 0x9f, + DBG_BLOCK_ID_TA00 = 0xa0, + DBG_BLOCK_ID_TA01 = 0xa1, + DBG_BLOCK_ID_TA02 = 0xa2, + DBG_BLOCK_ID_TA03 = 0xa3, + DBG_BLOCK_ID_TA04 = 0xa4, + DBG_BLOCK_ID_TA05 = 0xa5, + DBG_BLOCK_ID_TA06 = 0xa6, + DBG_BLOCK_ID_TA07 = 0xa7, + DBG_BLOCK_ID_TA08 = 0xa8, + DBG_BLOCK_ID_TA09 = 0xa9, + DBG_BLOCK_ID_TA0A = 0xaa, + DBG_BLOCK_ID_TA0B = 0xab, + DBG_BLOCK_ID_UNUSED35 = 0xac, + DBG_BLOCK_ID_UNUSED36 = 0xad, + DBG_BLOCK_ID_UNUSED37 = 0xae, + DBG_BLOCK_ID_UNUSED38 = 0xaf, + DBG_BLOCK_ID_TA10 = 0xb0, + DBG_BLOCK_ID_TA11 = 0xb1, + DBG_BLOCK_ID_TA12 = 0xb2, + DBG_BLOCK_ID_TA13 = 0xb3, + DBG_BLOCK_ID_TA14 = 0xb4, + DBG_BLOCK_ID_TA15 = 0xb5, + DBG_BLOCK_ID_TA16 = 0xb6, + DBG_BLOCK_ID_TA17 = 0xb7, + DBG_BLOCK_ID_TA18 = 0xb8, + DBG_BLOCK_ID_TA19 = 0xb9, + DBG_BLOCK_ID_TA1A = 0xba, + DBG_BLOCK_ID_TA1B = 0xbb, + DBG_BLOCK_ID_UNUSED39 = 0xbc, + DBG_BLOCK_ID_UNUSED40 = 0xbd, + DBG_BLOCK_ID_UNUSED41 = 0xbe, + DBG_BLOCK_ID_UNUSED42 = 0xbf, + DBG_BLOCK_ID_TD00 = 0xc0, + DBG_BLOCK_ID_TD01 = 0xc1, + DBG_BLOCK_ID_TD02 = 0xc2, + DBG_BLOCK_ID_TD03 = 0xc3, + DBG_BLOCK_ID_TD04 = 0xc4, + DBG_BLOCK_ID_TD05 = 0xc5, + DBG_BLOCK_ID_TD06 = 0xc6, + DBG_BLOCK_ID_TD07 = 0xc7, + DBG_BLOCK_ID_TD08 = 0xc8, + DBG_BLOCK_ID_TD09 = 0xc9, + DBG_BLOCK_ID_TD0A = 0xca, + DBG_BLOCK_ID_TD0B = 0xcb, + DBG_BLOCK_ID_UNUSED43 = 0xcc, + DBG_BLOCK_ID_UNUSED44 = 0xcd, + DBG_BLOCK_ID_UNUSED45 = 0xce, + DBG_BLOCK_ID_UNUSED46 = 0xcf, + DBG_BLOCK_ID_TD10 = 0xd0, + DBG_BLOCK_ID_TD11 = 0xd1, + DBG_BLOCK_ID_TD12 = 0xd2, + DBG_BLOCK_ID_TD13 = 0xd3, + DBG_BLOCK_ID_TD14 = 0xd4, + DBG_BLOCK_ID_TD15 = 0xd5, + DBG_BLOCK_ID_TD16 = 0xd6, + DBG_BLOCK_ID_TD17 = 0xd7, + DBG_BLOCK_ID_TD18 = 0xd8, + DBG_BLOCK_ID_TD19 = 0xd9, + DBG_BLOCK_ID_TD1A = 0xda, + DBG_BLOCK_ID_TD1B = 0xdb, + DBG_BLOCK_ID_UNUSED47 = 0xdc, + DBG_BLOCK_ID_UNUSED48 = 0xdd, + DBG_BLOCK_ID_UNUSED49 = 0xde, + DBG_BLOCK_ID_UNUSED50 = 0xdf, + DBG_BLOCK_ID_MCD0 = 0xe0, + DBG_BLOCK_ID_MCD1 = 0xe1, + DBG_BLOCK_ID_MCD2 = 0xe2, + DBG_BLOCK_ID_MCD3 = 0xe3, + DBG_BLOCK_ID_MCD4 = 0xe4, + DBG_BLOCK_ID_MCD5 = 0xe5, + DBG_BLOCK_ID_UNUSED51 = 0xe6, + DBG_BLOCK_ID_UNUSED52 = 0xe7, +} DebugBlockId_OLD; +typedef enum DebugBlockId_BY2 { + DBG_BLOCK_ID_RESERVED_BY2 = 0x0, + DBG_BLOCK_ID_VMC_BY2 = 0x1, + DBG_BLOCK_ID_CG_BY2 = 0x2, + DBG_BLOCK_ID_GRBM_BY2 = 0x3, + DBG_BLOCK_ID_CSC_BY2 = 0x4, + DBG_BLOCK_ID_IH_BY2 = 0x5, + DBG_BLOCK_ID_SQ_BY2 = 0x6, + DBG_BLOCK_ID_GMCON_BY2 = 0x7, + DBG_BLOCK_ID_DMA0_BY2 = 0x8, + DBG_BLOCK_ID_SPIM_BY2 = 0x9, + DBG_BLOCK_ID_SPIS_BY2 = 0xa, + DBG_BLOCK_ID_PA0_BY2 = 0xb, + DBG_BLOCK_ID_CP0_BY2 = 0xc, + DBG_BLOCK_ID_CP2_BY2 = 0xd, + DBG_BLOCK_ID_UVDU_BY2 = 0xe, + DBG_BLOCK_ID_VCE_BY2 = 0xf, + DBG_BLOCK_ID_VGT0_BY2 = 0x10, + DBG_BLOCK_ID_IA_BY2 = 0x11, + DBG_BLOCK_ID_SCT0_BY2 = 0x12, + DBG_BLOCK_ID_SPM0_BY2 = 0x13, + DBG_BLOCK_ID_TCAA_BY2 = 0x14, + DBG_BLOCK_ID_TCCA_BY2 = 0x15, + DBG_BLOCK_ID_MCC0_BY2 = 0x16, + DBG_BLOCK_ID_MCC2_BY2 = 0x17, + DBG_BLOCK_ID_SX0_BY2 = 0x18, + DBG_BLOCK_ID_SX2_BY2 = 0x19, + DBG_BLOCK_ID_UNUSED4_BY2 = 0x1a, + DBG_BLOCK_ID_UNUSED6_BY2 = 0x1b, + DBG_BLOCK_ID_PC0_BY2 = 0x1c, + DBG_BLOCK_ID_UNUSED8_BY2 = 0x1d, + DBG_BLOCK_ID_UNUSED10_BY2 = 0x1e, + DBG_BLOCK_ID_MCB_BY2 = 0x1f, + DBG_BLOCK_ID_SCB0_BY2 = 0x20, + DBG_BLOCK_ID_UNUSED13_BY2 = 0x21, + DBG_BLOCK_ID_SCF0_BY2 = 0x22, + DBG_BLOCK_ID_UNUSED15_BY2 = 0x23, + DBG_BLOCK_ID_BCI0_BY2 = 0x24, + DBG_BLOCK_ID_BCI2_BY2 = 0x25, + DBG_BLOCK_ID_UNUSED17_BY2 = 0x26, + DBG_BLOCK_ID_UNUSED19_BY2 = 0x27, + DBG_BLOCK_ID_CB00_BY2 = 0x28, + DBG_BLOCK_ID_CB02_BY2 = 0x29, + DBG_BLOCK_ID_CB04_BY2 = 0x2a, + DBG_BLOCK_ID_UNUSED22_BY2 = 0x2b, + DBG_BLOCK_ID_CB10_BY2 = 0x2c, + DBG_BLOCK_ID_CB12_BY2 = 0x2d, + DBG_BLOCK_ID_CB14_BY2 = 0x2e, + DBG_BLOCK_ID_UNUSED25_BY2 = 0x2f, + DBG_BLOCK_ID_TCP0_BY2 = 0x30, + DBG_BLOCK_ID_TCP2_BY2 = 0x31, + DBG_BLOCK_ID_TCP4_BY2 = 0x32, + DBG_BLOCK_ID_TCP6_BY2 = 0x33, + DBG_BLOCK_ID_TCP8_BY2 = 0x34, + DBG_BLOCK_ID_TCP10_BY2 = 0x35, + DBG_BLOCK_ID_TCP12_BY2 = 0x36, + DBG_BLOCK_ID_TCP14_BY2 = 0x37, + DBG_BLOCK_ID_TCP16_BY2 = 0x38, + DBG_BLOCK_ID_TCP18_BY2 = 0x39, + DBG_BLOCK_ID_TCP20_BY2 = 0x3a, + DBG_BLOCK_ID_TCP22_BY2 = 0x3b, + DBG_BLOCK_ID_TCP_RESERVED0_BY2 = 0x3c, + DBG_BLOCK_ID_TCP_RESERVED2_BY2 = 0x3d, + DBG_BLOCK_ID_TCP_RESERVED4_BY2 = 0x3e, + DBG_BLOCK_ID_TCP_RESERVED6_BY2 = 0x3f, + DBG_BLOCK_ID_DB00_BY2 = 0x40, + DBG_BLOCK_ID_DB02_BY2 = 0x41, + DBG_BLOCK_ID_DB04_BY2 = 0x42, + DBG_BLOCK_ID_UNUSED28_BY2 = 0x43, + DBG_BLOCK_ID_DB10_BY2 = 0x44, + DBG_BLOCK_ID_DB12_BY2 = 0x45, + DBG_BLOCK_ID_DB14_BY2 = 0x46, + DBG_BLOCK_ID_UNUSED31_BY2 = 0x47, + DBG_BLOCK_ID_TCC0_BY2 = 0x48, + DBG_BLOCK_ID_TCC2_BY2 = 0x49, + DBG_BLOCK_ID_TCC4_BY2 = 0x4a, + DBG_BLOCK_ID_TCC6_BY2 = 0x4b, + DBG_BLOCK_ID_SPS00_BY2 = 0x4c, + DBG_BLOCK_ID_SPS02_BY2 = 0x4d, + DBG_BLOCK_ID_SPS11_BY2 = 0x4e, + DBG_BLOCK_ID_UNUSED33_BY2 = 0x4f, + DBG_BLOCK_ID_TA00_BY2 = 0x50, + DBG_BLOCK_ID_TA02_BY2 = 0x51, + DBG_BLOCK_ID_TA04_BY2 = 0x52, + DBG_BLOCK_ID_TA06_BY2 = 0x53, + DBG_BLOCK_ID_TA08_BY2 = 0x54, + DBG_BLOCK_ID_TA0A_BY2 = 0x55, + DBG_BLOCK_ID_UNUSED35_BY2 = 0x56, + DBG_BLOCK_ID_UNUSED37_BY2 = 0x57, + DBG_BLOCK_ID_TA10_BY2 = 0x58, + DBG_BLOCK_ID_TA12_BY2 = 0x59, + DBG_BLOCK_ID_TA14_BY2 = 0x5a, + DBG_BLOCK_ID_TA16_BY2 = 0x5b, + DBG_BLOCK_ID_TA18_BY2 = 0x5c, + DBG_BLOCK_ID_TA1A_BY2 = 0x5d, + DBG_BLOCK_ID_UNUSED39_BY2 = 0x5e, + DBG_BLOCK_ID_UNUSED41_BY2 = 0x5f, + DBG_BLOCK_ID_TD00_BY2 = 0x60, + DBG_BLOCK_ID_TD02_BY2 = 0x61, + DBG_BLOCK_ID_TD04_BY2 = 0x62, + DBG_BLOCK_ID_TD06_BY2 = 0x63, + DBG_BLOCK_ID_TD08_BY2 = 0x64, + DBG_BLOCK_ID_TD0A_BY2 = 0x65, + DBG_BLOCK_ID_UNUSED43_BY2 = 0x66, + DBG_BLOCK_ID_UNUSED45_BY2 = 0x67, + DBG_BLOCK_ID_TD10_BY2 = 0x68, + DBG_BLOCK_ID_TD12_BY2 = 0x69, + DBG_BLOCK_ID_TD14_BY2 = 0x6a, + DBG_BLOCK_ID_TD16_BY2 = 0x6b, + DBG_BLOCK_ID_TD18_BY2 = 0x6c, + DBG_BLOCK_ID_TD1A_BY2 = 0x6d, + DBG_BLOCK_ID_UNUSED47_BY2 = 0x6e, + DBG_BLOCK_ID_UNUSED49_BY2 = 0x6f, + DBG_BLOCK_ID_MCD0_BY2 = 0x70, + DBG_BLOCK_ID_MCD2_BY2 = 0x71, + DBG_BLOCK_ID_MCD4_BY2 = 0x72, + DBG_BLOCK_ID_UNUSED51_BY2 = 0x73, +} DebugBlockId_BY2; +typedef enum DebugBlockId_BY4 { + DBG_BLOCK_ID_RESERVED_BY4 = 0x0, + DBG_BLOCK_ID_CG_BY4 = 0x1, + DBG_BLOCK_ID_CSC_BY4 = 0x2, + DBG_BLOCK_ID_SQ_BY4 = 0x3, + DBG_BLOCK_ID_DMA0_BY4 = 0x4, + DBG_BLOCK_ID_SPIS_BY4 = 0x5, + DBG_BLOCK_ID_CP0_BY4 = 0x6, + DBG_BLOCK_ID_UVDU_BY4 = 0x7, + DBG_BLOCK_ID_VGT0_BY4 = 0x8, + DBG_BLOCK_ID_SCT0_BY4 = 0x9, + DBG_BLOCK_ID_TCAA_BY4 = 0xa, + DBG_BLOCK_ID_MCC0_BY4 = 0xb, + DBG_BLOCK_ID_SX0_BY4 = 0xc, + DBG_BLOCK_ID_UNUSED4_BY4 = 0xd, + DBG_BLOCK_ID_PC0_BY4 = 0xe, + DBG_BLOCK_ID_UNUSED10_BY4 = 0xf, + DBG_BLOCK_ID_SCB0_BY4 = 0x10, + DBG_BLOCK_ID_SCF0_BY4 = 0x11, + DBG_BLOCK_ID_BCI0_BY4 = 0x12, + DBG_BLOCK_ID_UNUSED17_BY4 = 0x13, + DBG_BLOCK_ID_CB00_BY4 = 0x14, + DBG_BLOCK_ID_CB04_BY4 = 0x15, + DBG_BLOCK_ID_CB10_BY4 = 0x16, + DBG_BLOCK_ID_CB14_BY4 = 0x17, + DBG_BLOCK_ID_TCP0_BY4 = 0x18, + DBG_BLOCK_ID_TCP4_BY4 = 0x19, + DBG_BLOCK_ID_TCP8_BY4 = 0x1a, + DBG_BLOCK_ID_TCP12_BY4 = 0x1b, + DBG_BLOCK_ID_TCP16_BY4 = 0x1c, + DBG_BLOCK_ID_TCP20_BY4 = 0x1d, + DBG_BLOCK_ID_TCP_RESERVED0_BY4 = 0x1e, + DBG_BLOCK_ID_TCP_RESERVED4_BY4 = 0x1f, + DBG_BLOCK_ID_DB_BY4 = 0x20, + DBG_BLOCK_ID_DB04_BY4 = 0x21, + DBG_BLOCK_ID_DB10_BY4 = 0x22, + DBG_BLOCK_ID_DB14_BY4 = 0x23, + DBG_BLOCK_ID_TCC0_BY4 = 0x24, + DBG_BLOCK_ID_TCC4_BY4 = 0x25, + DBG_BLOCK_ID_SPS00_BY4 = 0x26, + DBG_BLOCK_ID_SPS11_BY4 = 0x27, + DBG_BLOCK_ID_TA00_BY4 = 0x28, + DBG_BLOCK_ID_TA04_BY4 = 0x29, + DBG_BLOCK_ID_TA08_BY4 = 0x2a, + DBG_BLOCK_ID_UNUSED35_BY4 = 0x2b, + DBG_BLOCK_ID_TA10_BY4 = 0x2c, + DBG_BLOCK_ID_TA14_BY4 = 0x2d, + DBG_BLOCK_ID_TA18_BY4 = 0x2e, + DBG_BLOCK_ID_UNUSED39_BY4 = 0x2f, + DBG_BLOCK_ID_TD00_BY4 = 0x30, + DBG_BLOCK_ID_TD04_BY4 = 0x31, + DBG_BLOCK_ID_TD08_BY4 = 0x32, + DBG_BLOCK_ID_UNUSED43_BY4 = 0x33, + DBG_BLOCK_ID_TD10_BY4 = 0x34, + DBG_BLOCK_ID_TD14_BY4 = 0x35, + DBG_BLOCK_ID_TD18_BY4 = 0x36, + DBG_BLOCK_ID_UNUSED47_BY4 = 0x37, + DBG_BLOCK_ID_MCD0_BY4 = 0x38, + DBG_BLOCK_ID_MCD4_BY4 = 0x39, +} DebugBlockId_BY4; +typedef enum DebugBlockId_BY8 { + DBG_BLOCK_ID_RESERVED_BY8 = 0x0, + DBG_BLOCK_ID_CSC_BY8 = 0x1, + DBG_BLOCK_ID_DMA0_BY8 = 0x2, + DBG_BLOCK_ID_CP0_BY8 = 0x3, + DBG_BLOCK_ID_VGT0_BY8 = 0x4, + DBG_BLOCK_ID_TCAA_BY8 = 0x5, + DBG_BLOCK_ID_SX0_BY8 = 0x6, + DBG_BLOCK_ID_PC0_BY8 = 0x7, + DBG_BLOCK_ID_SCB0_BY8 = 0x8, + DBG_BLOCK_ID_BCI0_BY8 = 0x9, + DBG_BLOCK_ID_CB00_BY8 = 0xa, + DBG_BLOCK_ID_CB10_BY8 = 0xb, + DBG_BLOCK_ID_TCP0_BY8 = 0xc, + DBG_BLOCK_ID_TCP8_BY8 = 0xd, + DBG_BLOCK_ID_TCP16_BY8 = 0xe, + DBG_BLOCK_ID_TCP_RESERVED0_BY8 = 0xf, + DBG_BLOCK_ID_DB00_BY8 = 0x10, + DBG_BLOCK_ID_DB10_BY8 = 0x11, + DBG_BLOCK_ID_TCC0_BY8 = 0x12, + DBG_BLOCK_ID_SPS00_BY8 = 0x13, + DBG_BLOCK_ID_TA00_BY8 = 0x14, + DBG_BLOCK_ID_TA08_BY8 = 0x15, + DBG_BLOCK_ID_TA10_BY8 = 0x16, + DBG_BLOCK_ID_TA18_BY8 = 0x17, + DBG_BLOCK_ID_TD00_BY8 = 0x18, + DBG_BLOCK_ID_TD08_BY8 = 0x19, + DBG_BLOCK_ID_TD10_BY8 = 0x1a, + DBG_BLOCK_ID_TD18_BY8 = 0x1b, + DBG_BLOCK_ID_MCD0_BY8 = 0x1c, +} DebugBlockId_BY8; +typedef enum DebugBlockId_BY16 { + DBG_BLOCK_ID_RESERVED_BY16 = 0x0, + DBG_BLOCK_ID_DMA0_BY16 = 0x1, + DBG_BLOCK_ID_VGT0_BY16 = 0x2, + DBG_BLOCK_ID_SX0_BY16 = 0x3, + DBG_BLOCK_ID_SCB0_BY16 = 0x4, + DBG_BLOCK_ID_CB00_BY16 = 0x5, + DBG_BLOCK_ID_TCP0_BY16 = 0x6, + DBG_BLOCK_ID_TCP16_BY16 = 0x7, + DBG_BLOCK_ID_DB00_BY16 = 0x8, + DBG_BLOCK_ID_TCC0_BY16 = 0x9, + DBG_BLOCK_ID_TA00_BY16 = 0xa, + DBG_BLOCK_ID_TA10_BY16 = 0xb, + DBG_BLOCK_ID_TD00_BY16 = 0xc, + DBG_BLOCK_ID_TD10_BY16 = 0xd, + DBG_BLOCK_ID_MCD0_BY16 = 0xe, +} DebugBlockId_BY16; +typedef enum CompareRef { + REF_NEVER = 0x0, + REF_LESS = 0x1, + REF_EQUAL = 0x2, + REF_LEQUAL = 0x3, + REF_GREATER = 0x4, + REF_NOTEQUAL = 0x5, + REF_GEQUAL = 0x6, + REF_ALWAYS = 0x7, +} CompareRef; +typedef enum ReadSize { + READ_256_BITS = 0x0, + READ_512_BITS = 0x1, +} ReadSize; +typedef enum DepthFormat { + DEPTH_INVALID = 0x0, + DEPTH_16 = 0x1, + DEPTH_X8_24 = 0x2, + DEPTH_8_24 = 0x3, + DEPTH_X8_24_FLOAT = 0x4, + DEPTH_8_24_FLOAT = 0x5, + DEPTH_32_FLOAT = 0x6, + DEPTH_X24_8_32_FLOAT = 0x7, +} DepthFormat; +typedef enum ZFormat { + Z_INVALID = 0x0, + Z_16 = 0x1, + Z_24 = 0x2, + Z_32_FLOAT = 0x3, +} ZFormat; +typedef enum StencilFormat { + STENCIL_INVALID = 0x0, + STENCIL_8 = 0x1, +} StencilFormat; +typedef enum CmaskMode { + CMASK_CLEAR_NONE = 0x0, + CMASK_CLEAR_ONE = 0x1, + CMASK_CLEAR_ALL = 0x2, + CMASK_ANY_EXPANDED = 0x3, + CMASK_ALPHA0_FRAG1 = 0x4, + CMASK_ALPHA0_FRAG2 = 0x5, + CMASK_ALPHA0_FRAG4 = 0x6, + CMASK_ALPHA0_FRAGS = 0x7, + CMASK_ALPHA1_FRAG1 = 0x8, + CMASK_ALPHA1_FRAG2 = 0x9, + CMASK_ALPHA1_FRAG4 = 0xa, + CMASK_ALPHA1_FRAGS = 0xb, + CMASK_ALPHAX_FRAG1 = 0xc, + CMASK_ALPHAX_FRAG2 = 0xd, + CMASK_ALPHAX_FRAG4 = 0xe, + CMASK_ALPHAX_FRAGS = 0xf, +} CmaskMode; +typedef enum QuadExportFormat { + EXPORT_UNUSED = 0x0, + EXPORT_32_R = 0x1, + EXPORT_32_GR = 0x2, + EXPORT_32_AR = 0x3, + EXPORT_FP16_ABGR = 0x4, + EXPORT_UNSIGNED16_ABGR = 0x5, + EXPORT_SIGNED16_ABGR = 0x6, + EXPORT_32_ABGR = 0x7, +} QuadExportFormat; +typedef enum QuadExportFormatOld { + EXPORT_4P_32BPC_ABGR = 0x0, + EXPORT_4P_16BPC_ABGR = 0x1, + EXPORT_4P_32BPC_GR = 0x2, + EXPORT_4P_32BPC_AR = 0x3, + EXPORT_2P_32BPC_ABGR = 0x4, + EXPORT_8P_32BPC_R = 0x5, +} QuadExportFormatOld; +typedef enum ColorFormat { + COLOR_INVALID = 0x0, + COLOR_8 = 0x1, + COLOR_16 = 0x2, + COLOR_8_8 = 0x3, + COLOR_32 = 0x4, + COLOR_16_16 = 0x5, + COLOR_10_11_11 = 0x6, + COLOR_11_11_10 = 0x7, + COLOR_10_10_10_2 = 0x8, + COLOR_2_10_10_10 = 0x9, + COLOR_8_8_8_8 = 0xa, + COLOR_32_32 = 0xb, + COLOR_16_16_16_16 = 0xc, + COLOR_RESERVED_13 = 0xd, + COLOR_32_32_32_32 = 0xe, + COLOR_RESERVED_15 = 0xf, + COLOR_5_6_5 = 0x10, + COLOR_1_5_5_5 = 0x11, + COLOR_5_5_5_1 = 0x12, + COLOR_4_4_4_4 = 0x13, + COLOR_8_24 = 0x14, + COLOR_24_8 = 0x15, + COLOR_X24_8_32_FLOAT = 0x16, + COLOR_RESERVED_23 = 0x17, +} ColorFormat; +typedef enum SurfaceFormat { + FMT_INVALID = 0x0, + FMT_8 = 0x1, + FMT_16 = 0x2, + FMT_8_8 = 0x3, + FMT_32 = 0x4, + FMT_16_16 = 0x5, + FMT_10_11_11 = 0x6, + FMT_11_11_10 = 0x7, + FMT_10_10_10_2 = 0x8, + FMT_2_10_10_10 = 0x9, + FMT_8_8_8_8 = 0xa, + FMT_32_32 = 0xb, + FMT_16_16_16_16 = 0xc, + FMT_32_32_32 = 0xd, + FMT_32_32_32_32 = 0xe, + FMT_RESERVED_4 = 0xf, + FMT_5_6_5 = 0x10, + FMT_1_5_5_5 = 0x11, + FMT_5_5_5_1 = 0x12, + FMT_4_4_4_4 = 0x13, + FMT_8_24 = 0x14, + FMT_24_8 = 0x15, + FMT_X24_8_32_FLOAT = 0x16, + FMT_RESERVED_33 = 0x17, + FMT_11_11_10_FLOAT = 0x18, + FMT_16_FLOAT = 0x19, + FMT_32_FLOAT = 0x1a, + FMT_16_16_FLOAT = 0x1b, + FMT_8_24_FLOAT = 0x1c, + FMT_24_8_FLOAT = 0x1d, + FMT_32_32_FLOAT = 0x1e, + FMT_10_11_11_FLOAT = 0x1f, + FMT_16_16_16_16_FLOAT = 0x20, + FMT_3_3_2 = 0x21, + FMT_6_5_5 = 0x22, + FMT_32_32_32_32_FLOAT = 0x23, + FMT_RESERVED_36 = 0x24, + FMT_1 = 0x25, + FMT_1_REVERSED = 0x26, + FMT_GB_GR = 0x27, + FMT_BG_RG = 0x28, + FMT_32_AS_8 = 0x29, + FMT_32_AS_8_8 = 0x2a, + FMT_5_9_9_9_SHAREDEXP = 0x2b, + FMT_8_8_8 = 0x2c, + FMT_16_16_16 = 0x2d, + FMT_16_16_16_FLOAT = 0x2e, + FMT_4_4 = 0x2f, + FMT_32_32_32_FLOAT = 0x30, + FMT_BC1 = 0x31, + FMT_BC2 = 0x32, + FMT_BC3 = 0x33, + FMT_BC4 = 0x34, + FMT_BC5 = 0x35, + FMT_BC6 = 0x36, + FMT_BC7 = 0x37, + FMT_32_AS_32_32_32_32 = 0x38, + FMT_APC3 = 0x39, + FMT_APC4 = 0x3a, + FMT_APC5 = 0x3b, + FMT_APC6 = 0x3c, + FMT_APC7 = 0x3d, + FMT_CTX1 = 0x3e, + FMT_RESERVED_63 = 0x3f, +} SurfaceFormat; +typedef enum BUF_DATA_FORMAT { + BUF_DATA_FORMAT_INVALID = 0x0, + BUF_DATA_FORMAT_8 = 0x1, + BUF_DATA_FORMAT_16 = 0x2, + BUF_DATA_FORMAT_8_8 = 0x3, + BUF_DATA_FORMAT_32 = 0x4, + BUF_DATA_FORMAT_16_16 = 0x5, + BUF_DATA_FORMAT_10_11_11 = 0x6, + BUF_DATA_FORMAT_11_11_10 = 0x7, + BUF_DATA_FORMAT_10_10_10_2 = 0x8, + BUF_DATA_FORMAT_2_10_10_10 = 0x9, + BUF_DATA_FORMAT_8_8_8_8 = 0xa, + BUF_DATA_FORMAT_32_32 = 0xb, + BUF_DATA_FORMAT_16_16_16_16 = 0xc, + BUF_DATA_FORMAT_32_32_32 = 0xd, + BUF_DATA_FORMAT_32_32_32_32 = 0xe, + BUF_DATA_FORMAT_RESERVED_15 = 0xf, +} BUF_DATA_FORMAT; +typedef enum IMG_DATA_FORMAT { + IMG_DATA_FORMAT_INVALID = 0x0, + IMG_DATA_FORMAT_8 = 0x1, + IMG_DATA_FORMAT_16 = 0x2, + IMG_DATA_FORMAT_8_8 = 0x3, + IMG_DATA_FORMAT_32 = 0x4, + IMG_DATA_FORMAT_16_16 = 0x5, + IMG_DATA_FORMAT_10_11_11 = 0x6, + IMG_DATA_FORMAT_11_11_10 = 0x7, + IMG_DATA_FORMAT_10_10_10_2 = 0x8, + IMG_DATA_FORMAT_2_10_10_10 = 0x9, + IMG_DATA_FORMAT_8_8_8_8 = 0xa, + IMG_DATA_FORMAT_32_32 = 0xb, + IMG_DATA_FORMAT_16_16_16_16 = 0xc, + IMG_DATA_FORMAT_32_32_32 = 0xd, + IMG_DATA_FORMAT_32_32_32_32 = 0xe, + IMG_DATA_FORMAT_RESERVED_15 = 0xf, + IMG_DATA_FORMAT_5_6_5 = 0x10, + IMG_DATA_FORMAT_1_5_5_5 = 0x11, + IMG_DATA_FORMAT_5_5_5_1 = 0x12, + IMG_DATA_FORMAT_4_4_4_4 = 0x13, + IMG_DATA_FORMAT_8_24 = 0x14, + IMG_DATA_FORMAT_24_8 = 0x15, + IMG_DATA_FORMAT_X24_8_32 = 0x16, + IMG_DATA_FORMAT_RESERVED_23 = 0x17, + IMG_DATA_FORMAT_RESERVED_24 = 0x18, + IMG_DATA_FORMAT_RESERVED_25 = 0x19, + IMG_DATA_FORMAT_RESERVED_26 = 0x1a, + IMG_DATA_FORMAT_RESERVED_27 = 0x1b, + IMG_DATA_FORMAT_RESERVED_28 = 0x1c, + IMG_DATA_FORMAT_RESERVED_29 = 0x1d, + IMG_DATA_FORMAT_RESERVED_30 = 0x1e, + IMG_DATA_FORMAT_RESERVED_31 = 0x1f, + IMG_DATA_FORMAT_GB_GR = 0x20, + IMG_DATA_FORMAT_BG_RG = 0x21, + IMG_DATA_FORMAT_5_9_9_9 = 0x22, + IMG_DATA_FORMAT_BC1 = 0x23, + IMG_DATA_FORMAT_BC2 = 0x24, + IMG_DATA_FORMAT_BC3 = 0x25, + IMG_DATA_FORMAT_BC4 = 0x26, + IMG_DATA_FORMAT_BC5 = 0x27, + IMG_DATA_FORMAT_BC6 = 0x28, + IMG_DATA_FORMAT_BC7 = 0x29, + IMG_DATA_FORMAT_RESERVED_42 = 0x2a, + IMG_DATA_FORMAT_RESERVED_43 = 0x2b, + IMG_DATA_FORMAT_FMASK8_S2_F1 = 0x2c, + IMG_DATA_FORMAT_FMASK8_S4_F1 = 0x2d, + IMG_DATA_FORMAT_FMASK8_S8_F1 = 0x2e, + IMG_DATA_FORMAT_FMASK8_S2_F2 = 0x2f, + IMG_DATA_FORMAT_FMASK8_S4_F2 = 0x30, + IMG_DATA_FORMAT_FMASK8_S4_F4 = 0x31, + IMG_DATA_FORMAT_FMASK16_S16_F1 = 0x32, + IMG_DATA_FORMAT_FMASK16_S8_F2 = 0x33, + IMG_DATA_FORMAT_FMASK32_S16_F2 = 0x34, + IMG_DATA_FORMAT_FMASK32_S8_F4 = 0x35, + IMG_DATA_FORMAT_FMASK32_S8_F8 = 0x36, + IMG_DATA_FORMAT_FMASK64_S16_F4 = 0x37, + IMG_DATA_FORMAT_FMASK64_S16_F8 = 0x38, + IMG_DATA_FORMAT_4_4 = 0x39, + IMG_DATA_FORMAT_6_5_5 = 0x3a, + IMG_DATA_FORMAT_1 = 0x3b, + IMG_DATA_FORMAT_1_REVERSED = 0x3c, + IMG_DATA_FORMAT_32_AS_8 = 0x3d, + IMG_DATA_FORMAT_32_AS_8_8 = 0x3e, + IMG_DATA_FORMAT_32_AS_32_32_32_32 = 0x3f, +} IMG_DATA_FORMAT; +typedef enum BUF_NUM_FORMAT { + BUF_NUM_FORMAT_UNORM = 0x0, + BUF_NUM_FORMAT_SNORM = 0x1, + BUF_NUM_FORMAT_USCALED = 0x2, + BUF_NUM_FORMAT_SSCALED = 0x3, + BUF_NUM_FORMAT_UINT = 0x4, + BUF_NUM_FORMAT_SINT = 0x5, + BUF_NUM_FORMAT_SNORM_OGL = 0x6, + BUF_NUM_FORMAT_FLOAT = 0x7, +} BUF_NUM_FORMAT; +typedef enum IMG_NUM_FORMAT { + IMG_NUM_FORMAT_UNORM = 0x0, + IMG_NUM_FORMAT_SNORM = 0x1, + IMG_NUM_FORMAT_USCALED = 0x2, + IMG_NUM_FORMAT_SSCALED = 0x3, + IMG_NUM_FORMAT_UINT = 0x4, + IMG_NUM_FORMAT_SINT = 0x5, + IMG_NUM_FORMAT_SNORM_OGL = 0x6, + IMG_NUM_FORMAT_FLOAT = 0x7, + IMG_NUM_FORMAT_RESERVED_8 = 0x8, + IMG_NUM_FORMAT_SRGB = 0x9, + IMG_NUM_FORMAT_UBNORM = 0xa, + IMG_NUM_FORMAT_UBNORM_OGL = 0xb, + IMG_NUM_FORMAT_UBINT = 0xc, + IMG_NUM_FORMAT_UBSCALED = 0xd, + IMG_NUM_FORMAT_RESERVED_14 = 0xe, + IMG_NUM_FORMAT_RESERVED_15 = 0xf, +} IMG_NUM_FORMAT; +typedef enum TileType { + ARRAY_COLOR_TILE = 0x0, + ARRAY_DEPTH_TILE = 0x1, +} TileType; +typedef enum NonDispTilingOrder { + ADDR_SURF_MICRO_TILING_DISPLAY = 0x0, + ADDR_SURF_MICRO_TILING_NON_DISPLAY = 0x1, +} NonDispTilingOrder; +typedef enum MicroTileMode { + ADDR_SURF_DISPLAY_MICRO_TILING = 0x0, + ADDR_SURF_THIN_MICRO_TILING = 0x1, + ADDR_SURF_DEPTH_MICRO_TILING = 0x2, + ADDR_SURF_ROTATED_MICRO_TILING = 0x3, + ADDR_SURF_THICK_MICRO_TILING = 0x4, +} MicroTileMode; +typedef enum TileSplit { + ADDR_SURF_TILE_SPLIT_64B = 0x0, + ADDR_SURF_TILE_SPLIT_128B = 0x1, + ADDR_SURF_TILE_SPLIT_256B = 0x2, + ADDR_SURF_TILE_SPLIT_512B = 0x3, + ADDR_SURF_TILE_SPLIT_1KB = 0x4, + ADDR_SURF_TILE_SPLIT_2KB = 0x5, + ADDR_SURF_TILE_SPLIT_4KB = 0x6, +} TileSplit; +typedef enum SampleSplit { + ADDR_SURF_SAMPLE_SPLIT_1 = 0x0, + ADDR_SURF_SAMPLE_SPLIT_2 = 0x1, + ADDR_SURF_SAMPLE_SPLIT_4 = 0x2, + ADDR_SURF_SAMPLE_SPLIT_8 = 0x3, +} SampleSplit; +typedef enum PipeConfig { + ADDR_SURF_P2 = 0x0, + ADDR_SURF_P2_RESERVED0 = 0x1, + ADDR_SURF_P2_RESERVED1 = 0x2, + ADDR_SURF_P2_RESERVED2 = 0x3, + ADDR_SURF_P4_8x16 = 0x4, + ADDR_SURF_P4_16x16 = 0x5, + ADDR_SURF_P4_16x32 = 0x6, + ADDR_SURF_P4_32x32 = 0x7, + ADDR_SURF_P8_16x16_8x16 = 0x8, + ADDR_SURF_P8_16x32_8x16 = 0x9, + ADDR_SURF_P8_32x32_8x16 = 0xa, + ADDR_SURF_P8_16x32_16x16 = 0xb, + ADDR_SURF_P8_32x32_16x16 = 0xc, + ADDR_SURF_P8_32x32_16x32 = 0xd, + ADDR_SURF_P8_32x64_32x32 = 0xe, + ADDR_SURF_P8_RESERVED0 = 0xf, + ADDR_SURF_P16_32x32_8x16 = 0x10, + ADDR_SURF_P16_32x32_16x16 = 0x11, +} PipeConfig; +typedef enum NumBanks { + ADDR_SURF_2_BANK = 0x0, + ADDR_SURF_4_BANK = 0x1, + ADDR_SURF_8_BANK = 0x2, + ADDR_SURF_16_BANK = 0x3, +} NumBanks; +typedef enum BankWidth { + ADDR_SURF_BANK_WIDTH_1 = 0x0, + ADDR_SURF_BANK_WIDTH_2 = 0x1, + ADDR_SURF_BANK_WIDTH_4 = 0x2, + ADDR_SURF_BANK_WIDTH_8 = 0x3, +} BankWidth; +typedef enum BankHeight { + ADDR_SURF_BANK_HEIGHT_1 = 0x0, + ADDR_SURF_BANK_HEIGHT_2 = 0x1, + ADDR_SURF_BANK_HEIGHT_4 = 0x2, + ADDR_SURF_BANK_HEIGHT_8 = 0x3, +} BankHeight; +typedef enum BankWidthHeight { + ADDR_SURF_BANK_WH_1 = 0x0, + ADDR_SURF_BANK_WH_2 = 0x1, + ADDR_SURF_BANK_WH_4 = 0x2, + ADDR_SURF_BANK_WH_8 = 0x3, +} BankWidthHeight; +typedef enum MacroTileAspect { + ADDR_SURF_MACRO_ASPECT_1 = 0x0, + ADDR_SURF_MACRO_ASPECT_2 = 0x1, + ADDR_SURF_MACRO_ASPECT_4 = 0x2, + ADDR_SURF_MACRO_ASPECT_8 = 0x3, +} MacroTileAspect; +typedef enum TCC_CACHE_POLICIES { + TCC_CACHE_POLICY_LRU = 0x0, + TCC_CACHE_POLICY_STREAM = 0x1, + TCC_CACHE_POLICY_BYPASS = 0x2, +} TCC_CACHE_POLICIES; +typedef enum MTYPE { + MTYPE_NC_NV = 0x0, + MTYPE_NC = 0x1, + MTYPE_CC = 0x2, + MTYPE_UC = 0x3, +} MTYPE; +typedef enum PERFMON_COUNTER_MODE { + PERFMON_COUNTER_MODE_ACCUM = 0x0, + PERFMON_COUNTER_MODE_ACTIVE_CYCLES = 0x1, + PERFMON_COUNTER_MODE_MAX = 0x2, + PERFMON_COUNTER_MODE_DIRTY = 0x3, + PERFMON_COUNTER_MODE_SAMPLE = 0x4, + PERFMON_COUNTER_MODE_CYCLES_SINCE_FIRST_EVENT = 0x5, + PERFMON_COUNTER_MODE_CYCLES_SINCE_LAST_EVENT = 0x6, + PERFMON_COUNTER_MODE_CYCLES_GE_HI = 0x7, + PERFMON_COUNTER_MODE_CYCLES_EQ_HI = 0x8, + PERFMON_COUNTER_MODE_INACTIVE_CYCLES = 0x9, + PERFMON_COUNTER_MODE_RESERVED = 0xf, +} PERFMON_COUNTER_MODE; +typedef enum PERFMON_SPM_MODE { + PERFMON_SPM_MODE_OFF = 0x0, + PERFMON_SPM_MODE_16BIT_CLAMP = 0x1, + PERFMON_SPM_MODE_16BIT_NO_CLAMP = 0x2, + PERFMON_SPM_MODE_32BIT_CLAMP = 0x3, + PERFMON_SPM_MODE_32BIT_NO_CLAMP = 0x4, + PERFMON_SPM_MODE_RESERVED_5 = 0x5, + PERFMON_SPM_MODE_RESERVED_6 = 0x6, + PERFMON_SPM_MODE_RESERVED_7 = 0x7, + PERFMON_SPM_MODE_TEST_MODE_0 = 0x8, + PERFMON_SPM_MODE_TEST_MODE_1 = 0x9, + PERFMON_SPM_MODE_TEST_MODE_2 = 0xa, +} PERFMON_SPM_MODE; +typedef enum SurfaceTiling { + ARRAY_LINEAR = 0x0, + ARRAY_TILED = 0x1, +} SurfaceTiling; +typedef enum SurfaceArray { + ARRAY_1D = 0x0, + ARRAY_2D = 0x1, + ARRAY_3D = 0x2, + ARRAY_3D_SLICE = 0x3, +} SurfaceArray; +typedef enum ColorArray { + ARRAY_2D_ALT_COLOR = 0x0, + ARRAY_2D_COLOR = 0x1, + ARRAY_3D_SLICE_COLOR = 0x3, +} ColorArray; +typedef enum DepthArray { + ARRAY_2D_ALT_DEPTH = 0x0, + ARRAY_2D_DEPTH = 0x1, +} DepthArray; + +#endif /* GFX_7_2_ENUM_H */ diff --git a/tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h b/tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h new file mode 100644 index 0000000000..267eb09692 --- /dev/null +++ b/tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h @@ -0,0 +1,18444 @@ +/* + * Copyright (C) 2014 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef GFX_7_2_SH_MASK_H +#define GFX_7_2_SH_MASK_H + +#define CB_BLEND_RED__BLEND_RED_MASK 0xffffffff +#define CB_BLEND_RED__BLEND_RED__SHIFT 0x0 +#define CB_BLEND_GREEN__BLEND_GREEN_MASK 0xffffffff +#define CB_BLEND_GREEN__BLEND_GREEN__SHIFT 0x0 +#define CB_BLEND_BLUE__BLEND_BLUE_MASK 0xffffffff +#define CB_BLEND_BLUE__BLEND_BLUE__SHIFT 0x0 +#define CB_BLEND_ALPHA__BLEND_ALPHA_MASK 0xffffffff +#define CB_BLEND_ALPHA__BLEND_ALPHA__SHIFT 0x0 +#define CB_COLOR_CONTROL__DEGAMMA_ENABLE_MASK 0x8 +#define CB_COLOR_CONTROL__DEGAMMA_ENABLE__SHIFT 0x3 +#define CB_COLOR_CONTROL__MODE_MASK 0x70 +#define CB_COLOR_CONTROL__MODE__SHIFT 0x4 +#define CB_COLOR_CONTROL__ROP3_MASK 0xff0000 +#define CB_COLOR_CONTROL__ROP3__SHIFT 0x10 +#define CB_BLEND0_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND0_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND0_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND0_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND0_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND0_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND0_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND0_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND0_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND0_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_BLEND1_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND1_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND1_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND1_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND1_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND1_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND1_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND1_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND1_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND1_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_BLEND2_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND2_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND2_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND2_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND2_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND2_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND2_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND2_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND2_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND2_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_BLEND3_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND3_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND3_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND3_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND3_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND3_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND3_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND3_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND3_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND3_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_BLEND4_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND4_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND4_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND4_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND4_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND4_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND4_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND4_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND4_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND4_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_BLEND5_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND5_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND5_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND5_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND5_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND5_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND5_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND5_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND5_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND5_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_BLEND6_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND6_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND6_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND6_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND6_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND6_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND6_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND6_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND6_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND6_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_BLEND7_CONTROL__COLOR_SRCBLEND_MASK 0x1f +#define CB_BLEND7_CONTROL__COLOR_SRCBLEND__SHIFT 0x0 +#define CB_BLEND7_CONTROL__COLOR_COMB_FCN_MASK 0xe0 +#define CB_BLEND7_CONTROL__COLOR_COMB_FCN__SHIFT 0x5 +#define CB_BLEND7_CONTROL__COLOR_DESTBLEND_MASK 0x1f00 +#define CB_BLEND7_CONTROL__COLOR_DESTBLEND__SHIFT 0x8 +#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND_MASK 0x1f0000 +#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND__SHIFT 0x10 +#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN_MASK 0xe00000 +#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN__SHIFT 0x15 +#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND_MASK 0x1f000000 +#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND__SHIFT 0x18 +#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND_MASK 0x20000000 +#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT 0x1d +#define CB_BLEND7_CONTROL__ENABLE_MASK 0x40000000 +#define CB_BLEND7_CONTROL__ENABLE__SHIFT 0x1e +#define CB_BLEND7_CONTROL__DISABLE_ROP3_MASK 0x80000000 +#define CB_BLEND7_CONTROL__DISABLE_ROP3__SHIFT 0x1f +#define CB_COLOR0_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR0_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR1_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR1_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR2_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR2_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR3_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR3_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR4_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR4_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR5_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR5_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR6_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR6_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR7_BASE__BASE_256B_MASK 0xffffffff +#define CB_COLOR7_BASE__BASE_256B__SHIFT 0x0 +#define CB_COLOR0_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR0_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR0_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR0_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR1_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR1_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR1_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR1_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR2_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR2_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR2_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR2_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR3_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR3_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR3_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR3_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR4_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR4_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR4_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR4_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR5_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR5_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR5_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR5_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR6_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR6_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR6_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR6_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR7_PITCH__TILE_MAX_MASK 0x7ff +#define CB_COLOR7_PITCH__TILE_MAX__SHIFT 0x0 +#define CB_COLOR7_PITCH__FMASK_TILE_MAX_MASK 0x7ff00000 +#define CB_COLOR7_PITCH__FMASK_TILE_MAX__SHIFT 0x14 +#define CB_COLOR0_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR0_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR1_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR1_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR2_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR2_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR3_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR3_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR4_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR4_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR5_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR5_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR6_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR6_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR7_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR7_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR0_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR0_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR0_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR0_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR1_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR1_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR1_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR1_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR2_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR2_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR2_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR2_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR3_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR3_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR3_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR3_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR4_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR4_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR4_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR4_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR5_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR5_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR5_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR5_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR6_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR6_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR6_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR6_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR7_VIEW__SLICE_START_MASK 0x7ff +#define CB_COLOR7_VIEW__SLICE_START__SHIFT 0x0 +#define CB_COLOR7_VIEW__SLICE_MAX_MASK 0xffe000 +#define CB_COLOR7_VIEW__SLICE_MAX__SHIFT 0xd +#define CB_COLOR0_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR0_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR0_INFO__FORMAT_MASK 0x7c +#define CB_COLOR0_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR0_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR0_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR0_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR0_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR0_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR0_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR0_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR0_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR0_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR0_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR0_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR0_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR0_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR0_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR0_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR0_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR0_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR0_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR0_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR0_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR0_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR1_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR1_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR1_INFO__FORMAT_MASK 0x7c +#define CB_COLOR1_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR1_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR1_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR1_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR1_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR1_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR1_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR1_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR1_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR1_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR1_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR1_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR1_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR1_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR1_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR1_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR1_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR1_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR1_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR1_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR1_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR1_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR2_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR2_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR2_INFO__FORMAT_MASK 0x7c +#define CB_COLOR2_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR2_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR2_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR2_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR2_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR2_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR2_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR2_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR2_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR2_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR2_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR2_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR2_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR2_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR2_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR2_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR2_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR2_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR2_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR2_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR2_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR2_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR3_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR3_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR3_INFO__FORMAT_MASK 0x7c +#define CB_COLOR3_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR3_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR3_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR3_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR3_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR3_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR3_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR3_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR3_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR3_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR3_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR3_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR3_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR3_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR3_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR3_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR3_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR3_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR3_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR3_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR3_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR3_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR4_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR4_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR4_INFO__FORMAT_MASK 0x7c +#define CB_COLOR4_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR4_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR4_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR4_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR4_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR4_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR4_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR4_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR4_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR4_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR4_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR4_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR4_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR4_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR4_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR4_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR4_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR4_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR4_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR4_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR4_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR4_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR5_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR5_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR5_INFO__FORMAT_MASK 0x7c +#define CB_COLOR5_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR5_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR5_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR5_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR5_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR5_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR5_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR5_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR5_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR5_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR5_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR5_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR5_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR5_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR5_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR5_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR5_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR5_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR5_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR5_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR5_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR5_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR6_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR6_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR6_INFO__FORMAT_MASK 0x7c +#define CB_COLOR6_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR6_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR6_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR6_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR6_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR6_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR6_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR6_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR6_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR6_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR6_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR6_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR6_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR6_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR6_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR6_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR6_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR6_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR6_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR6_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR6_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR6_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR7_INFO__ENDIAN_MASK 0x3 +#define CB_COLOR7_INFO__ENDIAN__SHIFT 0x0 +#define CB_COLOR7_INFO__FORMAT_MASK 0x7c +#define CB_COLOR7_INFO__FORMAT__SHIFT 0x2 +#define CB_COLOR7_INFO__LINEAR_GENERAL_MASK 0x80 +#define CB_COLOR7_INFO__LINEAR_GENERAL__SHIFT 0x7 +#define CB_COLOR7_INFO__NUMBER_TYPE_MASK 0x700 +#define CB_COLOR7_INFO__NUMBER_TYPE__SHIFT 0x8 +#define CB_COLOR7_INFO__COMP_SWAP_MASK 0x1800 +#define CB_COLOR7_INFO__COMP_SWAP__SHIFT 0xb +#define CB_COLOR7_INFO__FAST_CLEAR_MASK 0x2000 +#define CB_COLOR7_INFO__FAST_CLEAR__SHIFT 0xd +#define CB_COLOR7_INFO__COMPRESSION_MASK 0x4000 +#define CB_COLOR7_INFO__COMPRESSION__SHIFT 0xe +#define CB_COLOR7_INFO__BLEND_CLAMP_MASK 0x8000 +#define CB_COLOR7_INFO__BLEND_CLAMP__SHIFT 0xf +#define CB_COLOR7_INFO__BLEND_BYPASS_MASK 0x10000 +#define CB_COLOR7_INFO__BLEND_BYPASS__SHIFT 0x10 +#define CB_COLOR7_INFO__SIMPLE_FLOAT_MASK 0x20000 +#define CB_COLOR7_INFO__SIMPLE_FLOAT__SHIFT 0x11 +#define CB_COLOR7_INFO__ROUND_MODE_MASK 0x40000 +#define CB_COLOR7_INFO__ROUND_MODE__SHIFT 0x12 +#define CB_COLOR7_INFO__CMASK_IS_LINEAR_MASK 0x80000 +#define CB_COLOR7_INFO__CMASK_IS_LINEAR__SHIFT 0x13 +#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST_MASK 0x700000 +#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST__SHIFT 0x14 +#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL_MASK 0x3800000 +#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT 0x17 +#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE_MASK 0x4000000 +#define CB_COLOR7_INFO__FMASK_COMPRESSION_DISABLE__SHIFT 0x1a +#define CB_COLOR0_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR0_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR0_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR0_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR0_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR0_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR0_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR0_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR1_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR1_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR1_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR1_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR1_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR1_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR1_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR1_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR2_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR2_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR2_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR2_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR2_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR2_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR2_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR2_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR3_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR3_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR3_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR3_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR3_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR3_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR3_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR3_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR4_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR4_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR4_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR4_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR4_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR4_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR4_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR4_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR5_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR5_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR5_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR5_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR5_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR5_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR5_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR5_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR6_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR6_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR6_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR6_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR6_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR6_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR6_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR6_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR7_ATTRIB__TILE_MODE_INDEX_MASK 0x1f +#define CB_COLOR7_ATTRIB__TILE_MODE_INDEX__SHIFT 0x0 +#define CB_COLOR7_ATTRIB__FMASK_TILE_MODE_INDEX_MASK 0x3e0 +#define CB_COLOR7_ATTRIB__FMASK_TILE_MODE_INDEX__SHIFT 0x5 +#define CB_COLOR7_ATTRIB__FMASK_BANK_HEIGHT_MASK 0xc00 +#define CB_COLOR7_ATTRIB__FMASK_BANK_HEIGHT__SHIFT 0xa +#define CB_COLOR7_ATTRIB__NUM_SAMPLES_MASK 0x7000 +#define CB_COLOR7_ATTRIB__NUM_SAMPLES__SHIFT 0xc +#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS_MASK 0x18000 +#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS__SHIFT 0xf +#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1_MASK 0x20000 +#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1__SHIFT 0x11 +#define CB_COLOR0_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR0_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR1_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR1_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR2_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR2_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR3_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR3_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR4_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR4_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR5_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR5_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR6_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR6_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR7_CMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR7_CMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR0_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR0_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR1_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR1_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR2_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR2_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR3_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR3_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR4_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR4_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR5_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR5_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR6_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR6_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR7_CMASK_SLICE__TILE_MAX_MASK 0x3fff +#define CB_COLOR7_CMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR0_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR0_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR1_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR1_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR2_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR2_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR3_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR3_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR4_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR4_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR5_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR5_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR6_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR6_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR7_FMASK__BASE_256B_MASK 0xffffffff +#define CB_COLOR7_FMASK__BASE_256B__SHIFT 0x0 +#define CB_COLOR0_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR0_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR1_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR1_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR2_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR2_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR3_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR3_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR4_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR4_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR5_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR5_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR6_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR6_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR7_FMASK_SLICE__TILE_MAX_MASK 0x3fffff +#define CB_COLOR7_FMASK_SLICE__TILE_MAX__SHIFT 0x0 +#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR0_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR1_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR2_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR3_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR4_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR5_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR6_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0_MASK 0xffffffff +#define CB_COLOR7_CLEAR_WORD0__CLEAR_WORD0__SHIFT 0x0 +#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR0_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR1_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR2_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR3_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR4_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR5_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR6_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1_MASK 0xffffffff +#define CB_COLOR7_CLEAR_WORD1__CLEAR_WORD1__SHIFT 0x0 +#define CB_TARGET_MASK__TARGET0_ENABLE_MASK 0xf +#define CB_TARGET_MASK__TARGET0_ENABLE__SHIFT 0x0 +#define CB_TARGET_MASK__TARGET1_ENABLE_MASK 0xf0 +#define CB_TARGET_MASK__TARGET1_ENABLE__SHIFT 0x4 +#define CB_TARGET_MASK__TARGET2_ENABLE_MASK 0xf00 +#define CB_TARGET_MASK__TARGET2_ENABLE__SHIFT 0x8 +#define CB_TARGET_MASK__TARGET3_ENABLE_MASK 0xf000 +#define CB_TARGET_MASK__TARGET3_ENABLE__SHIFT 0xc +#define CB_TARGET_MASK__TARGET4_ENABLE_MASK 0xf0000 +#define CB_TARGET_MASK__TARGET4_ENABLE__SHIFT 0x10 +#define CB_TARGET_MASK__TARGET5_ENABLE_MASK 0xf00000 +#define CB_TARGET_MASK__TARGET5_ENABLE__SHIFT 0x14 +#define CB_TARGET_MASK__TARGET6_ENABLE_MASK 0xf000000 +#define CB_TARGET_MASK__TARGET6_ENABLE__SHIFT 0x18 +#define CB_TARGET_MASK__TARGET7_ENABLE_MASK 0xf0000000 +#define CB_TARGET_MASK__TARGET7_ENABLE__SHIFT 0x1c +#define CB_SHADER_MASK__OUTPUT0_ENABLE_MASK 0xf +#define CB_SHADER_MASK__OUTPUT0_ENABLE__SHIFT 0x0 +#define CB_SHADER_MASK__OUTPUT1_ENABLE_MASK 0xf0 +#define CB_SHADER_MASK__OUTPUT1_ENABLE__SHIFT 0x4 +#define CB_SHADER_MASK__OUTPUT2_ENABLE_MASK 0xf00 +#define CB_SHADER_MASK__OUTPUT2_ENABLE__SHIFT 0x8 +#define CB_SHADER_MASK__OUTPUT3_ENABLE_MASK 0xf000 +#define CB_SHADER_MASK__OUTPUT3_ENABLE__SHIFT 0xc +#define CB_SHADER_MASK__OUTPUT4_ENABLE_MASK 0xf0000 +#define CB_SHADER_MASK__OUTPUT4_ENABLE__SHIFT 0x10 +#define CB_SHADER_MASK__OUTPUT5_ENABLE_MASK 0xf00000 +#define CB_SHADER_MASK__OUTPUT5_ENABLE__SHIFT 0x14 +#define CB_SHADER_MASK__OUTPUT6_ENABLE_MASK 0xf000000 +#define CB_SHADER_MASK__OUTPUT6_ENABLE__SHIFT 0x18 +#define CB_SHADER_MASK__OUTPUT7_ENABLE_MASK 0xf0000000 +#define CB_SHADER_MASK__OUTPUT7_ENABLE__SHIFT 0x1c +#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT_MASK 0xf +#define CB_HW_CONTROL__CM_CACHE_EVICT_POINT__SHIFT 0x0 +#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT_MASK 0x3c0 +#define CB_HW_CONTROL__FC_CACHE_EVICT_POINT__SHIFT 0x6 +#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT_MASK 0xf000 +#define CB_HW_CONTROL__CC_CACHE_EVICT_POINT__SHIFT 0xc +#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE_MASK 0x10000 +#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE__SHIFT 0x10 +#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING_MASK 0x40000 +#define CB_HW_CONTROL__DISABLE_INTNORM_LE11BPC_CLAMPING__SHIFT 0x12 +#define CB_HW_CONTROL__FORCE_NEEDS_DST_MASK 0x80000 +#define CB_HW_CONTROL__FORCE_NEEDS_DST__SHIFT 0x13 +#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE_MASK 0x100000 +#define CB_HW_CONTROL__FORCE_ALWAYS_TOGGLE__SHIFT 0x14 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST_MASK 0x200000 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST__SHIFT 0x15 +#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK_MASK 0x400000 +#define CB_HW_CONTROL__DISABLE_FULL_WRITE_MASK__SHIFT 0x16 +#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG_MASK 0x800000 +#define CB_HW_CONTROL__DISABLE_RESOLVE_OPT_FOR_SINGLE_FRAG__SHIFT 0x17 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST_MASK 0x1000000 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT 0x18 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS_MASK 0x2000000 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS__SHIFT 0x19 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK 0x4000000 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT 0x1a +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED_MASK 0x8000000 +#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED__SHIFT 0x1b +#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT_MASK 0x10000000 +#define CB_HW_CONTROL__PRIORITIZE_FC_WR_OVER_FC_RD_ON_CMASK_CONFLICT__SHIFT 0x1c +#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT_MASK 0x20000000 +#define CB_HW_CONTROL__PRIORITIZE_FC_EVICT_OVER_FOP_RD_ON_BANK_CONFLICT__SHIFT 0x1d +#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT_MASK 0x40000000 +#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT__SHIFT 0x1e +#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE_MASK 0x80000000 +#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE__SHIFT 0x1f +#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS_MASK 0x1f +#define CB_HW_CONTROL_1__CM_CACHE_NUM_TAGS__SHIFT 0x0 +#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS_MASK 0x7e0 +#define CB_HW_CONTROL_1__FC_CACHE_NUM_TAGS__SHIFT 0x5 +#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS_MASK 0x1f800 +#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS__SHIFT 0xb +#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH_MASK 0x3fe0000 +#define CB_HW_CONTROL_1__CM_TILE_FIFO_DEPTH__SHIFT 0x11 +#define CB_HW_CONTROL_1__CHICKEN_BITS_MASK 0xfc000000 +#define CB_HW_CONTROL_1__CHICKEN_BITS__SHIFT 0x1a +#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH_MASK 0xff +#define CB_HW_CONTROL_2__CC_EVEN_ODD_FIFO_DEPTH__SHIFT 0x0 +#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH_MASK 0x7f00 +#define CB_HW_CONTROL_2__FC_RDLAT_TILE_FIFO_DEPTH__SHIFT 0x8 +#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH_MASK 0x7f8000 +#define CB_HW_CONTROL_2__FC_RDLAT_QUAD_FIFO_DEPTH__SHIFT 0xf +#define CB_HW_CONTROL_2__CHICKEN_BITS_MASK 0xff000000 +#define CB_HW_CONTROL_2__CHICKEN_BITS__SHIFT 0x18 +#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL_MASK 0x1 +#define CB_HW_CONTROL_3__DISABLE_SLOW_MODE_EMPTY_HALF_QUAD_KILL__SHIFT 0x0 +#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE_MASK 0x1 +#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE__SHIFT 0x0 +#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL_MASK 0xe +#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL__SHIFT 0x1 +#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE_MASK 0x10 +#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE__SHIFT 0x4 +#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL_MASK 0x3e0 +#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL__SHIFT 0x5 +#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE_MASK 0x400 +#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE__SHIFT 0xa +#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL_MASK 0x800 +#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL__SHIFT 0xb +#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE_MASK 0x1000 +#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE__SHIFT 0xc +#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL_MASK 0xe000 +#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL__SHIFT 0xd +#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE_MASK 0x20000 +#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE__SHIFT 0x11 +#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL_MASK 0x1c0000 +#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL__SHIFT 0x12 +#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE_MASK 0x200000 +#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE__SHIFT 0x15 +#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL_MASK 0xc00000 +#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL__SHIFT 0x16 +#define CB_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x1ff +#define CB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define CB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x7fc00 +#define CB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define CB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define CB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define CB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define CB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define CB_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define CB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x1ff +#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x7fc00 +#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000 +#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18 +#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c +#define CB_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x1ff +#define CB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define CB_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define CB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define CB_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x1ff +#define CB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define CB_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define CB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define CB_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x1ff +#define CB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define CB_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define CB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK 0xf +#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define CB_DEBUG_BUS_13__TILE_INTFC_BUSY_MASK 0x1 +#define CB_DEBUG_BUS_13__TILE_INTFC_BUSY__SHIFT 0x0 +#define CB_DEBUG_BUS_13__MU_BUSY_MASK 0x2 +#define CB_DEBUG_BUS_13__MU_BUSY__SHIFT 0x1 +#define CB_DEBUG_BUS_13__TQ_BUSY_MASK 0x4 +#define CB_DEBUG_BUS_13__TQ_BUSY__SHIFT 0x2 +#define CB_DEBUG_BUS_13__AC_BUSY_MASK 0x8 +#define CB_DEBUG_BUS_13__AC_BUSY__SHIFT 0x3 +#define CB_DEBUG_BUS_13__CRW_BUSY_MASK 0x10 +#define CB_DEBUG_BUS_13__CRW_BUSY__SHIFT 0x4 +#define CB_DEBUG_BUS_13__CACHE_CTRL_BUSY_MASK 0x20 +#define CB_DEBUG_BUS_13__CACHE_CTRL_BUSY__SHIFT 0x5 +#define CB_DEBUG_BUS_13__MC_WR_PENDING_MASK 0x40 +#define CB_DEBUG_BUS_13__MC_WR_PENDING__SHIFT 0x6 +#define CB_DEBUG_BUS_13__FC_WR_PENDING_MASK 0x80 +#define CB_DEBUG_BUS_13__FC_WR_PENDING__SHIFT 0x7 +#define CB_DEBUG_BUS_13__FC_RD_PENDING_MASK 0x100 +#define CB_DEBUG_BUS_13__FC_RD_PENDING__SHIFT 0x8 +#define CB_DEBUG_BUS_13__EVICT_PENDING_MASK 0x200 +#define CB_DEBUG_BUS_13__EVICT_PENDING__SHIFT 0x9 +#define CB_DEBUG_BUS_13__LAST_RD_ARB_WINNER_MASK 0x400 +#define CB_DEBUG_BUS_13__LAST_RD_ARB_WINNER__SHIFT 0xa +#define CB_DEBUG_BUS_13__MU_STATE_MASK 0x7f800 +#define CB_DEBUG_BUS_13__MU_STATE__SHIFT 0xb +#define CB_DEBUG_BUS_14__TILE_RETIREMENT_BUSY_MASK 0x1 +#define CB_DEBUG_BUS_14__TILE_RETIREMENT_BUSY__SHIFT 0x0 +#define CB_DEBUG_BUS_14__FOP_BUSY_MASK 0x2 +#define CB_DEBUG_BUS_14__FOP_BUSY__SHIFT 0x1 +#define CB_DEBUG_BUS_14__LAT_BUSY_MASK 0x4 +#define CB_DEBUG_BUS_14__LAT_BUSY__SHIFT 0x2 +#define CB_DEBUG_BUS_14__CACHE_CTL_BUSY_MASK 0x8 +#define CB_DEBUG_BUS_14__CACHE_CTL_BUSY__SHIFT 0x3 +#define CB_DEBUG_BUS_14__ADDR_BUSY_MASK 0x10 +#define CB_DEBUG_BUS_14__ADDR_BUSY__SHIFT 0x4 +#define CB_DEBUG_BUS_14__MERGE_BUSY_MASK 0x20 +#define CB_DEBUG_BUS_14__MERGE_BUSY__SHIFT 0x5 +#define CB_DEBUG_BUS_14__QUAD_BUSY_MASK 0x40 +#define CB_DEBUG_BUS_14__QUAD_BUSY__SHIFT 0x6 +#define CB_DEBUG_BUS_14__TILE_BUSY_MASK 0x80 +#define CB_DEBUG_BUS_14__TILE_BUSY__SHIFT 0x7 +#define CB_DEBUG_BUS_14__CLEAR_BUSY_MASK 0x100 +#define CB_DEBUG_BUS_14__CLEAR_BUSY__SHIFT 0x8 +#define CB_DEBUG_BUS_15__SURF_SYNC_STATE_MASK 0x3 +#define CB_DEBUG_BUS_15__SURF_SYNC_STATE__SHIFT 0x0 +#define CB_DEBUG_BUS_15__SURF_SYNC_START_MASK 0x4 +#define CB_DEBUG_BUS_15__SURF_SYNC_START__SHIFT 0x2 +#define CB_DEBUG_BUS_15__SF_BUSY_MASK 0x8 +#define CB_DEBUG_BUS_15__SF_BUSY__SHIFT 0x3 +#define CB_DEBUG_BUS_15__CS_BUSY_MASK 0x10 +#define CB_DEBUG_BUS_15__CS_BUSY__SHIFT 0x4 +#define CB_DEBUG_BUS_15__RB_BUSY_MASK 0x20 +#define CB_DEBUG_BUS_15__RB_BUSY__SHIFT 0x5 +#define CB_DEBUG_BUS_15__DS_BUSY_MASK 0x40 +#define CB_DEBUG_BUS_15__DS_BUSY__SHIFT 0x6 +#define CB_DEBUG_BUS_15__TB_BUSY_MASK 0x80 +#define CB_DEBUG_BUS_15__TB_BUSY__SHIFT 0x7 +#define CB_DEBUG_BUS_15__IB_BUSY_MASK 0x100 +#define CB_DEBUG_BUS_15__IB_BUSY__SHIFT 0x8 +#define CB_DEBUG_BUS_16__MC_RDREQ_CREDITS_MASK 0x3f +#define CB_DEBUG_BUS_16__MC_RDREQ_CREDITS__SHIFT 0x0 +#define CB_DEBUG_BUS_16__LAST_RD_GRANT_VEC_MASK 0x3c0 +#define CB_DEBUG_BUS_16__LAST_RD_GRANT_VEC__SHIFT 0x6 +#define CB_DEBUG_BUS_16__MC_WRREQ_CREDITS_MASK 0xfc00 +#define CB_DEBUG_BUS_16__MC_WRREQ_CREDITS__SHIFT 0xa +#define CB_DEBUG_BUS_16__LAST_WR_GRANT_VEC_MASK 0xf0000 +#define CB_DEBUG_BUS_16__LAST_WR_GRANT_VEC__SHIFT 0x10 +#define CB_DEBUG_BUS_16__CC_WRREQ_FIFO_EMPTY_MASK 0x100000 +#define CB_DEBUG_BUS_16__CC_WRREQ_FIFO_EMPTY__SHIFT 0x14 +#define CB_DEBUG_BUS_16__FC_WRREQ_FIFO_EMPTY_MASK 0x200000 +#define CB_DEBUG_BUS_16__FC_WRREQ_FIFO_EMPTY__SHIFT 0x15 +#define CB_DEBUG_BUS_16__CM_WRREQ_FIFO_EMPTY_MASK 0x400000 +#define CB_DEBUG_BUS_16__CM_WRREQ_FIFO_EMPTY__SHIFT 0x16 +#define CB_DEBUG_BUS_17__CM_BUSY_MASK 0x1 +#define CB_DEBUG_BUS_17__CM_BUSY__SHIFT 0x0 +#define CB_DEBUG_BUS_17__FC_BUSY_MASK 0x2 +#define CB_DEBUG_BUS_17__FC_BUSY__SHIFT 0x1 +#define CB_DEBUG_BUS_17__CC_BUSY_MASK 0x4 +#define CB_DEBUG_BUS_17__CC_BUSY__SHIFT 0x2 +#define CB_DEBUG_BUS_17__BB_BUSY_MASK 0x8 +#define CB_DEBUG_BUS_17__BB_BUSY__SHIFT 0x3 +#define CB_DEBUG_BUS_17__MA_BUSY_MASK 0x10 +#define CB_DEBUG_BUS_17__MA_BUSY__SHIFT 0x4 +#define CB_DEBUG_BUS_17__CORE_SCLK_VLD_MASK 0x20 +#define CB_DEBUG_BUS_17__CORE_SCLK_VLD__SHIFT 0x5 +#define CB_DEBUG_BUS_17__REG_SCLK1_VLD_MASK 0x40 +#define CB_DEBUG_BUS_17__REG_SCLK1_VLD__SHIFT 0x6 +#define CB_DEBUG_BUS_17__REG_SCLK0_VLD_MASK 0x80 +#define CB_DEBUG_BUS_17__REG_SCLK0_VLD__SHIFT 0x7 +#define CB_DEBUG_BUS_18__NOT_USED_MASK 0xffffff +#define CB_DEBUG_BUS_18__NOT_USED__SHIFT 0x0 +#define CP_DFY_CNTL__POLICY_MASK 0x300 +#define CP_DFY_CNTL__POLICY__SHIFT 0x8 +#define CP_DFY_CNTL__VOL_MASK 0x400 +#define CP_DFY_CNTL__VOL__SHIFT 0xa +#define CP_DFY_CNTL__ATC_MASK 0x800 +#define CP_DFY_CNTL__ATC__SHIFT 0xb +#define CP_DFY_STAT__BURST_COUNT_MASK 0xffff +#define CP_DFY_STAT__BURST_COUNT__SHIFT 0x0 +#define CP_DFY_STAT__TAGS_PENDING_MASK 0xff0000 +#define CP_DFY_STAT__TAGS_PENDING__SHIFT 0x10 +#define CP_DFY_STAT__BUSY_MASK 0x80000000 +#define CP_DFY_STAT__BUSY__SHIFT 0x1f +#define CP_DFY_ADDR_HI__ADDR_HI_MASK 0xffffffff +#define CP_DFY_ADDR_HI__ADDR_HI__SHIFT 0x0 +#define CP_DFY_ADDR_LO__ADDR_LO_MASK 0xffffffe0 +#define CP_DFY_ADDR_LO__ADDR_LO__SHIFT 0x5 +#define CP_DFY_DATA_0__DATA_MASK 0xffffffff +#define CP_DFY_DATA_0__DATA__SHIFT 0x0 +#define CP_DFY_DATA_1__DATA_MASK 0xffffffff +#define CP_DFY_DATA_1__DATA__SHIFT 0x0 +#define CP_DFY_DATA_2__DATA_MASK 0xffffffff +#define CP_DFY_DATA_2__DATA__SHIFT 0x0 +#define CP_DFY_DATA_3__DATA_MASK 0xffffffff +#define CP_DFY_DATA_3__DATA__SHIFT 0x0 +#define CP_DFY_DATA_4__DATA_MASK 0xffffffff +#define CP_DFY_DATA_4__DATA__SHIFT 0x0 +#define CP_DFY_DATA_5__DATA_MASK 0xffffffff +#define CP_DFY_DATA_5__DATA__SHIFT 0x0 +#define CP_DFY_DATA_6__DATA_MASK 0xffffffff +#define CP_DFY_DATA_6__DATA__SHIFT 0x0 +#define CP_DFY_DATA_7__DATA_MASK 0xffffffff +#define CP_DFY_DATA_7__DATA__SHIFT 0x0 +#define CP_DFY_DATA_8__DATA_MASK 0xffffffff +#define CP_DFY_DATA_8__DATA__SHIFT 0x0 +#define CP_DFY_DATA_9__DATA_MASK 0xffffffff +#define CP_DFY_DATA_9__DATA__SHIFT 0x0 +#define CP_DFY_DATA_10__DATA_MASK 0xffffffff +#define CP_DFY_DATA_10__DATA__SHIFT 0x0 +#define CP_DFY_DATA_11__DATA_MASK 0xffffffff +#define CP_DFY_DATA_11__DATA__SHIFT 0x0 +#define CP_DFY_DATA_12__DATA_MASK 0xffffffff +#define CP_DFY_DATA_12__DATA__SHIFT 0x0 +#define CP_DFY_DATA_13__DATA_MASK 0xffffffff +#define CP_DFY_DATA_13__DATA__SHIFT 0x0 +#define CP_DFY_DATA_14__DATA_MASK 0xffffffff +#define CP_DFY_DATA_14__DATA__SHIFT 0x0 +#define CP_DFY_DATA_15__DATA_MASK 0xffffffff +#define CP_DFY_DATA_15__DATA__SHIFT 0x0 +#define CP_RB0_BASE__RB_BASE_MASK 0xffffffff +#define CP_RB0_BASE__RB_BASE__SHIFT 0x0 +#define CP_RB0_BASE_HI__RB_BASE_HI_MASK 0xff +#define CP_RB0_BASE_HI__RB_BASE_HI__SHIFT 0x0 +#define CP_RB_BASE__RB_BASE_MASK 0xffffffff +#define CP_RB_BASE__RB_BASE__SHIFT 0x0 +#define CP_RB1_BASE__RB_BASE_MASK 0xffffffff +#define CP_RB1_BASE__RB_BASE__SHIFT 0x0 +#define CP_RB1_BASE_HI__RB_BASE_HI_MASK 0xff +#define CP_RB1_BASE_HI__RB_BASE_HI__SHIFT 0x0 +#define CP_RB2_BASE__RB_BASE_MASK 0xffffffff +#define CP_RB2_BASE__RB_BASE__SHIFT 0x0 +#define CP_RB0_CNTL__RB_BUFSZ_MASK 0x3f +#define CP_RB0_CNTL__RB_BUFSZ__SHIFT 0x0 +#define CP_RB0_CNTL__RB_BLKSZ_MASK 0x3f00 +#define CP_RB0_CNTL__RB_BLKSZ__SHIFT 0x8 +#define CP_RB0_CNTL__BUF_SWAP_MASK 0x30000 +#define CP_RB0_CNTL__BUF_SWAP__SHIFT 0x10 +#define CP_RB0_CNTL__MIN_AVAILSZ_MASK 0x300000 +#define CP_RB0_CNTL__MIN_AVAILSZ__SHIFT 0x14 +#define CP_RB0_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000 +#define CP_RB0_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16 +#define CP_RB0_CNTL__CACHE_POLICY_MASK 0x3000000 +#define CP_RB0_CNTL__CACHE_POLICY__SHIFT 0x18 +#define CP_RB0_CNTL__RB_VOLATILE_MASK 0x4000000 +#define CP_RB0_CNTL__RB_VOLATILE__SHIFT 0x1a +#define CP_RB0_CNTL__RB_NO_UPDATE_MASK 0x8000000 +#define CP_RB0_CNTL__RB_NO_UPDATE__SHIFT 0x1b +#define CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000 +#define CP_RB0_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f +#define CP_RB_CNTL__RB_BUFSZ_MASK 0x3f +#define CP_RB_CNTL__RB_BUFSZ__SHIFT 0x0 +#define CP_RB_CNTL__RB_BLKSZ_MASK 0x3f00 +#define CP_RB_CNTL__RB_BLKSZ__SHIFT 0x8 +#define CP_RB_CNTL__BUF_SWAP_MASK 0x30000 +#define CP_RB_CNTL__BUF_SWAP__SHIFT 0x10 +#define CP_RB_CNTL__MIN_AVAILSZ_MASK 0x300000 +#define CP_RB_CNTL__MIN_AVAILSZ__SHIFT 0x14 +#define CP_RB_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000 +#define CP_RB_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16 +#define CP_RB_CNTL__CACHE_POLICY_MASK 0x3000000 +#define CP_RB_CNTL__CACHE_POLICY__SHIFT 0x18 +#define CP_RB_CNTL__RB_VOLATILE_MASK 0x4000000 +#define CP_RB_CNTL__RB_VOLATILE__SHIFT 0x1a +#define CP_RB_CNTL__RB_NO_UPDATE_MASK 0x8000000 +#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT 0x1b +#define CP_RB_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000 +#define CP_RB_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f +#define CP_RB1_CNTL__RB_BUFSZ_MASK 0x3f +#define CP_RB1_CNTL__RB_BUFSZ__SHIFT 0x0 +#define CP_RB1_CNTL__RB_BLKSZ_MASK 0x3f00 +#define CP_RB1_CNTL__RB_BLKSZ__SHIFT 0x8 +#define CP_RB1_CNTL__MIN_AVAILSZ_MASK 0x300000 +#define CP_RB1_CNTL__MIN_AVAILSZ__SHIFT 0x14 +#define CP_RB1_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000 +#define CP_RB1_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16 +#define CP_RB1_CNTL__CACHE_POLICY_MASK 0x3000000 +#define CP_RB1_CNTL__CACHE_POLICY__SHIFT 0x18 +#define CP_RB1_CNTL__RB_VOLATILE_MASK 0x4000000 +#define CP_RB1_CNTL__RB_VOLATILE__SHIFT 0x1a +#define CP_RB1_CNTL__RB_NO_UPDATE_MASK 0x8000000 +#define CP_RB1_CNTL__RB_NO_UPDATE__SHIFT 0x1b +#define CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000 +#define CP_RB1_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f +#define CP_RB2_CNTL__RB_BUFSZ_MASK 0x3f +#define CP_RB2_CNTL__RB_BUFSZ__SHIFT 0x0 +#define CP_RB2_CNTL__RB_BLKSZ_MASK 0x3f00 +#define CP_RB2_CNTL__RB_BLKSZ__SHIFT 0x8 +#define CP_RB2_CNTL__MIN_AVAILSZ_MASK 0x300000 +#define CP_RB2_CNTL__MIN_AVAILSZ__SHIFT 0x14 +#define CP_RB2_CNTL__MIN_IB_AVAILSZ_MASK 0xc00000 +#define CP_RB2_CNTL__MIN_IB_AVAILSZ__SHIFT 0x16 +#define CP_RB2_CNTL__CACHE_POLICY_MASK 0x3000000 +#define CP_RB2_CNTL__CACHE_POLICY__SHIFT 0x18 +#define CP_RB2_CNTL__RB_VOLATILE_MASK 0x4000000 +#define CP_RB2_CNTL__RB_VOLATILE__SHIFT 0x1a +#define CP_RB2_CNTL__RB_NO_UPDATE_MASK 0x8000000 +#define CP_RB2_CNTL__RB_NO_UPDATE__SHIFT 0x1b +#define CP_RB2_CNTL__RB_RPTR_WR_ENA_MASK 0x80000000 +#define CP_RB2_CNTL__RB_RPTR_WR_ENA__SHIFT 0x1f +#define CP_RB_RPTR_WR__RB_RPTR_WR_MASK 0xfffff +#define CP_RB_RPTR_WR__RB_RPTR_WR__SHIFT 0x0 +#define CP_RB0_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3 +#define CP_RB0_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0 +#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc +#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2 +#define CP_RB_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3 +#define CP_RB_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0 +#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc +#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2 +#define CP_RB1_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3 +#define CP_RB1_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0 +#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc +#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2 +#define CP_RB2_RPTR_ADDR__RB_RPTR_SWAP_MASK 0x3 +#define CP_RB2_RPTR_ADDR__RB_RPTR_SWAP__SHIFT 0x0 +#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR_MASK 0xfffffffc +#define CP_RB2_RPTR_ADDR__RB_RPTR_ADDR__SHIFT 0x2 +#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff +#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0 +#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff +#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0 +#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff +#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0 +#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK 0xffff +#define CP_RB2_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT 0x0 +#define CP_RB0_WPTR__RB_WPTR_MASK 0xfffff +#define CP_RB0_WPTR__RB_WPTR__SHIFT 0x0 +#define CP_RB_WPTR__RB_WPTR_MASK 0xfffff +#define CP_RB_WPTR__RB_WPTR__SHIFT 0x0 +#define CP_RB1_WPTR__RB_WPTR_MASK 0xfffff +#define CP_RB1_WPTR__RB_WPTR__SHIFT 0x0 +#define CP_RB2_WPTR__RB_WPTR_MASK 0xfffff +#define CP_RB2_WPTR__RB_WPTR__SHIFT 0x0 +#define CP_RB_WPTR_POLL_ADDR_LO__OBSOLETE_MASK 0xfffffffc +#define CP_RB_WPTR_POLL_ADDR_LO__OBSOLETE__SHIFT 0x2 +#define CP_RB_WPTR_POLL_ADDR_HI__OBSOLETE_MASK 0xff +#define CP_RB_WPTR_POLL_ADDR_HI__OBSOLETE__SHIFT 0x0 +#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK 0x80000 +#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE__SHIFT 0x13 +#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK 0x100000 +#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14 +#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE_MASK 0x400000 +#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE__SHIFT 0x16 +#define CP_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK 0x80000 +#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE__SHIFT 0x13 +#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK 0x100000 +#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14 +#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK 0x400000 +#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE__SHIFT 0x16 +#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE_MASK 0x80000 +#define CP_INT_CNTL_RING1__CNTX_BUSY_INT_ENABLE__SHIFT 0x13 +#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE_MASK 0x100000 +#define CP_INT_CNTL_RING1__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14 +#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE_MASK 0x400000 +#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE__SHIFT 0x16 +#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_INT_CNTL_RING2__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_INT_CNTL_RING2__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE_MASK 0x80000 +#define CP_INT_CNTL_RING2__CNTX_BUSY_INT_ENABLE__SHIFT 0x13 +#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE_MASK 0x100000 +#define CP_INT_CNTL_RING2__CNTX_EMPTY_INT_ENABLE__SHIFT 0x14 +#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE_MASK 0x400000 +#define CP_INT_CNTL_RING2__PRIV_INSTR_INT_ENABLE__SHIFT 0x16 +#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_INT_CNTL_RING2__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_INT_CNTL_RING2__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_INT_CNTL_RING2__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_INT_CNTL_RING2__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_INT_CNTL_RING2__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_INT_CNTL_RING2__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_INT_CNTL_RING2__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT_MASK 0x4000 +#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT__SHIFT 0xe +#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000 +#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11 +#define CP_INT_STATUS__CNTX_BUSY_INT_STAT_MASK 0x80000 +#define CP_INT_STATUS__CNTX_BUSY_INT_STAT__SHIFT 0x13 +#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT_MASK 0x100000 +#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT__SHIFT 0x14 +#define CP_INT_STATUS__PRIV_INSTR_INT_STAT_MASK 0x400000 +#define CP_INT_STATUS__PRIV_INSTR_INT_STAT__SHIFT 0x16 +#define CP_INT_STATUS__PRIV_REG_INT_STAT_MASK 0x800000 +#define CP_INT_STATUS__PRIV_REG_INT_STAT__SHIFT 0x17 +#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT_MASK 0x1000000 +#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT__SHIFT 0x18 +#define CP_INT_STATUS__TIME_STAMP_INT_STAT_MASK 0x4000000 +#define CP_INT_STATUS__TIME_STAMP_INT_STAT__SHIFT 0x1a +#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000 +#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b +#define CP_INT_STATUS__GENERIC2_INT_STAT_MASK 0x20000000 +#define CP_INT_STATUS__GENERIC2_INT_STAT__SHIFT 0x1d +#define CP_INT_STATUS__GENERIC1_INT_STAT_MASK 0x40000000 +#define CP_INT_STATUS__GENERIC1_INT_STAT__SHIFT 0x1e +#define CP_INT_STATUS__GENERIC0_INT_STAT_MASK 0x80000000 +#define CP_INT_STATUS__GENERIC0_INT_STAT__SHIFT 0x1f +#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT_MASK 0x4000 +#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT__SHIFT 0xe +#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000 +#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11 +#define CP_INT_STATUS_RING0__CNTX_BUSY_INT_STAT_MASK 0x80000 +#define CP_INT_STATUS_RING0__CNTX_BUSY_INT_STAT__SHIFT 0x13 +#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT_MASK 0x100000 +#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT__SHIFT 0x14 +#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT_MASK 0x400000 +#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT__SHIFT 0x16 +#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT_MASK 0x800000 +#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT__SHIFT 0x17 +#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT_MASK 0x1000000 +#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT__SHIFT 0x18 +#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT_MASK 0x4000000 +#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT__SHIFT 0x1a +#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000 +#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b +#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT_MASK 0x20000000 +#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT__SHIFT 0x1d +#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT_MASK 0x40000000 +#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT__SHIFT 0x1e +#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT_MASK 0x80000000 +#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT__SHIFT 0x1f +#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT_MASK 0x4000 +#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT__SHIFT 0xe +#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000 +#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11 +#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT_MASK 0x80000 +#define CP_INT_STATUS_RING1__CNTX_BUSY_INT_STAT__SHIFT 0x13 +#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT_MASK 0x100000 +#define CP_INT_STATUS_RING1__CNTX_EMPTY_INT_STAT__SHIFT 0x14 +#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT_MASK 0x400000 +#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT__SHIFT 0x16 +#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT_MASK 0x800000 +#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT__SHIFT 0x17 +#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT_MASK 0x1000000 +#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT__SHIFT 0x18 +#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT_MASK 0x4000000 +#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT__SHIFT 0x1a +#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000 +#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b +#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT_MASK 0x20000000 +#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT__SHIFT 0x1d +#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT_MASK 0x40000000 +#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT__SHIFT 0x1e +#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT_MASK 0x80000000 +#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT__SHIFT 0x1f +#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT_MASK 0x4000 +#define CP_INT_STATUS_RING2__CP_ECC_ERROR_INT_STAT__SHIFT 0xe +#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT_MASK 0x20000 +#define CP_INT_STATUS_RING2__WRM_POLL_TIMEOUT_INT_STAT__SHIFT 0x11 +#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT_MASK 0x80000 +#define CP_INT_STATUS_RING2__CNTX_BUSY_INT_STAT__SHIFT 0x13 +#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT_MASK 0x100000 +#define CP_INT_STATUS_RING2__CNTX_EMPTY_INT_STAT__SHIFT 0x14 +#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT_MASK 0x400000 +#define CP_INT_STATUS_RING2__PRIV_INSTR_INT_STAT__SHIFT 0x16 +#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT_MASK 0x800000 +#define CP_INT_STATUS_RING2__PRIV_REG_INT_STAT__SHIFT 0x17 +#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT_MASK 0x1000000 +#define CP_INT_STATUS_RING2__OPCODE_ERROR_INT_STAT__SHIFT 0x18 +#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT_MASK 0x4000000 +#define CP_INT_STATUS_RING2__TIME_STAMP_INT_STAT__SHIFT 0x1a +#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT_MASK 0x8000000 +#define CP_INT_STATUS_RING2__RESERVED_BIT_ERROR_INT_STAT__SHIFT 0x1b +#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT_MASK 0x20000000 +#define CP_INT_STATUS_RING2__GENERIC2_INT_STAT__SHIFT 0x1d +#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT_MASK 0x40000000 +#define CP_INT_STATUS_RING2__GENERIC1_INT_STAT__SHIFT 0x1e +#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT_MASK 0x80000000 +#define CP_INT_STATUS_RING2__GENERIC0_INT_STAT__SHIFT 0x1f +#define CP_DEVICE_ID__DEVICE_ID_MASK 0xff +#define CP_DEVICE_ID__DEVICE_ID__SHIFT 0x0 +#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff +#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0 +#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00 +#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8 +#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000 +#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10 +#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000 +#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18 +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0 +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00 +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8 +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000 +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10 +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000 +#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18 +#define CP_RING0_PRIORITY__PRIORITY_MASK 0x3 +#define CP_RING0_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME0_PIPE0_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME0_PIPE0_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_RING1_PRIORITY__PRIORITY_MASK 0x3 +#define CP_RING1_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME0_PIPE1_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME0_PIPE1_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_RING2_PRIORITY__PRIORITY_MASK 0x3 +#define CP_RING2_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME0_PIPE2_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME0_PIPE2_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ENDIAN_SWAP__ENDIAN_SWAP_MASK 0x3 +#define CP_ENDIAN_SWAP__ENDIAN_SWAP__SHIFT 0x0 +#define CP_RB_VMID__RB0_VMID_MASK 0xf +#define CP_RB_VMID__RB0_VMID__SHIFT 0x0 +#define CP_RB_VMID__RB1_VMID_MASK 0xf00 +#define CP_RB_VMID__RB1_VMID__SHIFT 0x8 +#define CP_RB_VMID__RB2_VMID_MASK 0xf0000 +#define CP_RB_VMID__RB2_VMID__SHIFT 0x10 +#define CP_ME0_PIPE0_VMID__VMID_MASK 0xf +#define CP_ME0_PIPE0_VMID__VMID__SHIFT 0x0 +#define CP_ME0_PIPE1_VMID__VMID_MASK 0xf +#define CP_ME0_PIPE1_VMID__VMID__SHIFT 0x0 +#define CP_PFP_UCODE_ADDR__UCODE_ADDR_MASK 0xfff +#define CP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0 +#define CP_PFP_UCODE_DATA__UCODE_DATA_MASK 0xffffffff +#define CP_PFP_UCODE_DATA__UCODE_DATA__SHIFT 0x0 +#define CP_ME_RAM_RADDR__ME_RAM_RADDR_MASK 0xfff +#define CP_ME_RAM_RADDR__ME_RAM_RADDR__SHIFT 0x0 +#define CP_ME_RAM_WADDR__ME_RAM_WADDR_MASK 0xfff +#define CP_ME_RAM_WADDR__ME_RAM_WADDR__SHIFT 0x0 +#define CP_ME_RAM_DATA__ME_RAM_DATA_MASK 0xffffffff +#define CP_ME_RAM_DATA__ME_RAM_DATA__SHIFT 0x0 +#define CGTT_CPC_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_CPC_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000 +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000 +#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define CGTT_CPF_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_CPF_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000 +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000 +#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define CGTT_CP_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_CP_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000 +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000 +#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define CP_CE_UCODE_ADDR__UCODE_ADDR_MASK 0xfff +#define CP_CE_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0 +#define CP_CE_UCODE_DATA__UCODE_DATA_MASK 0xffffffff +#define CP_CE_UCODE_DATA__UCODE_DATA__SHIFT 0x0 +#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR_MASK 0x1fff +#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0 +#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA_MASK 0xffffffff +#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA__SHIFT 0x0 +#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR_MASK 0x1fff +#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0 +#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA_MASK 0xffffffff +#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA__SHIFT 0x0 +#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT_MASK 0x2 +#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT__SHIFT 0x1 +#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT_MASK 0x2 +#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT__SHIFT 0x1 +#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT_MASK 0x2 +#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT__SHIFT 0x1 +#define CP_PWR_CNTL__GFX_CLK_HALT_MASK 0x1 +#define CP_PWR_CNTL__GFX_CLK_HALT__SHIFT 0x0 +#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK 0x1 +#define CP_MEM_SLP_CNTL__CP_MEM_LS_EN__SHIFT 0x0 +#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN_MASK 0x2 +#define CP_MEM_SLP_CNTL__CP_MEM_DS_EN__SHIFT 0x1 +#define CP_MEM_SLP_CNTL__RESERVED_MASK 0xfc +#define CP_MEM_SLP_CNTL__RESERVED__SHIFT 0x2 +#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY_MASK 0xff00 +#define CP_MEM_SLP_CNTL__CP_MEM_LS_ON_DELAY__SHIFT 0x8 +#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY_MASK 0xff0000 +#define CP_MEM_SLP_CNTL__CP_MEM_LS_OFF_DELAY__SHIFT 0x10 +#define CP_MEM_SLP_CNTL__RESERVED1_MASK 0xff000000 +#define CP_MEM_SLP_CNTL__RESERVED1__SHIFT 0x18 +#define CP_ECC_FIRSTOCCURRENCE__INTERFACE_MASK 0x3 +#define CP_ECC_FIRSTOCCURRENCE__INTERFACE__SHIFT 0x0 +#define CP_ECC_FIRSTOCCURRENCE__REQUEST_CLIENT_MASK 0xf0 +#define CP_ECC_FIRSTOCCURRENCE__REQUEST_CLIENT__SHIFT 0x4 +#define CP_ECC_FIRSTOCCURRENCE__RING_ID_MASK 0x3c00 +#define CP_ECC_FIRSTOCCURRENCE__RING_ID__SHIFT 0xa +#define CP_ECC_FIRSTOCCURRENCE__VMID_MASK 0xf0000 +#define CP_ECC_FIRSTOCCURRENCE__VMID__SHIFT 0x10 +#define CP_ECC_FIRSTOCCURRENCE_RING0__INTERFACE_MASK 0x3 +#define CP_ECC_FIRSTOCCURRENCE_RING0__INTERFACE__SHIFT 0x0 +#define CP_ECC_FIRSTOCCURRENCE_RING0__REQUEST_CLIENT_MASK 0xf0 +#define CP_ECC_FIRSTOCCURRENCE_RING0__REQUEST_CLIENT__SHIFT 0x4 +#define CP_ECC_FIRSTOCCURRENCE_RING0__RING_ID_MASK 0x3c00 +#define CP_ECC_FIRSTOCCURRENCE_RING0__RING_ID__SHIFT 0xa +#define CP_ECC_FIRSTOCCURRENCE_RING0__VMID_MASK 0xf0000 +#define CP_ECC_FIRSTOCCURRENCE_RING0__VMID__SHIFT 0x10 +#define CP_ECC_FIRSTOCCURRENCE_RING1__INTERFACE_MASK 0x3 +#define CP_ECC_FIRSTOCCURRENCE_RING1__INTERFACE__SHIFT 0x0 +#define CP_ECC_FIRSTOCCURRENCE_RING1__REQUEST_CLIENT_MASK 0xf0 +#define CP_ECC_FIRSTOCCURRENCE_RING1__REQUEST_CLIENT__SHIFT 0x4 +#define CP_ECC_FIRSTOCCURRENCE_RING1__RING_ID_MASK 0x3c00 +#define CP_ECC_FIRSTOCCURRENCE_RING1__RING_ID__SHIFT 0xa +#define CP_ECC_FIRSTOCCURRENCE_RING1__VMID_MASK 0xf0000 +#define CP_ECC_FIRSTOCCURRENCE_RING1__VMID__SHIFT 0x10 +#define CP_ECC_FIRSTOCCURRENCE_RING2__INTERFACE_MASK 0x3 +#define CP_ECC_FIRSTOCCURRENCE_RING2__INTERFACE__SHIFT 0x0 +#define CP_ECC_FIRSTOCCURRENCE_RING2__REQUEST_CLIENT_MASK 0xf0 +#define CP_ECC_FIRSTOCCURRENCE_RING2__REQUEST_CLIENT__SHIFT 0x4 +#define CP_ECC_FIRSTOCCURRENCE_RING2__RING_ID_MASK 0x3c00 +#define CP_ECC_FIRSTOCCURRENCE_RING2__RING_ID__SHIFT 0xa +#define CP_ECC_FIRSTOCCURRENCE_RING2__VMID_MASK 0xf0000 +#define CP_ECC_FIRSTOCCURRENCE_RING2__VMID__SHIFT 0x10 +#define CP_FETCHER_SOURCE__ME_SRC_MASK 0x1 +#define CP_FETCHER_SOURCE__ME_SRC__SHIFT 0x0 +#define CP_PQ_WPTR_POLL_CNTL__PERIOD_MASK 0xff +#define CP_PQ_WPTR_POLL_CNTL__PERIOD__SHIFT 0x0 +#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE_MASK 0x40000000 +#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE__SHIFT 0x1e +#define CP_PQ_WPTR_POLL_CNTL__EN_MASK 0x80000000 +#define CP_PQ_WPTR_POLL_CNTL__EN__SHIFT 0x1f +#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK_MASK 0xffffffff +#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK__SHIFT 0x0 +#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CPC_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CPC_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CPC_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CPC_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CPC_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CPC_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK 0x2000 +#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT 0xd +#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK 0x4000 +#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT 0xe +#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK 0x20000 +#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT 0x11 +#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK 0x800000 +#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT 0x17 +#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK 0x1000000 +#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT 0x18 +#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK 0x4000000 +#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT 0x1a +#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK 0x8000000 +#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT 0x1b +#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK 0x20000000 +#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT 0x1d +#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK 0x40000000 +#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT 0x1e +#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK 0x80000000 +#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT 0x1f +#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CPC_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CPC_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CPC_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CPC_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CPC_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CPC_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CPC_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CPC_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK 0x2000 +#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT 0xd +#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK 0x4000 +#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT 0xe +#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK 0x20000 +#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT 0x11 +#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK 0x800000 +#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT 0x17 +#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK 0x1000000 +#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT 0x18 +#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK 0x4000000 +#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT 0x1a +#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK 0x8000000 +#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT 0x1b +#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK 0x20000000 +#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT 0x1d +#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK 0x40000000 +#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT 0x1e +#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK 0x80000000 +#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT 0x1f +#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK 0x2000 +#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT 0xd +#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x4000 +#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe +#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x20000 +#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11 +#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x800000 +#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17 +#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x1000000 +#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18 +#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x4000000 +#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a +#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x8000000 +#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b +#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000 +#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d +#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000 +#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e +#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000 +#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f +#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK 0x2000 +#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT 0xd +#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x4000 +#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe +#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x20000 +#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11 +#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x800000 +#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17 +#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x1000000 +#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18 +#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x4000000 +#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a +#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x8000000 +#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b +#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000 +#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d +#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000 +#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e +#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000 +#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0 +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00 +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8 +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000 +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10 +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000 +#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18 +#define CP_ME1_PIPE0_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME1_PIPE0_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME1_PIPE1_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME1_PIPE1_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME1_PIPE2_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME1_PIPE2_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME1_PIPE3_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME1_PIPE3_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK 0xff +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT 0x0 +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK 0xff00 +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT 0x8 +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK 0xff0000 +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT 0x10 +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK 0xff000000 +#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT 0x18 +#define CP_ME2_PIPE0_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME2_PIPE0_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME2_PIPE1_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME2_PIPE1_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME2_PIPE2_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME2_PIPE2_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_ME2_PIPE3_PRIORITY__PRIORITY_MASK 0x3 +#define CP_ME2_PIPE3_PRIORITY__PRIORITY__SHIFT 0x0 +#define CP_CE_PRGRM_CNTR_START__IP_START_MASK 0x7ff +#define CP_CE_PRGRM_CNTR_START__IP_START__SHIFT 0x0 +#define CP_PFP_PRGRM_CNTR_START__IP_START_MASK 0x7ff +#define CP_PFP_PRGRM_CNTR_START__IP_START__SHIFT 0x0 +#define CP_ME_PRGRM_CNTR_START__IP_START_MASK 0x7ff +#define CP_ME_PRGRM_CNTR_START__IP_START__SHIFT 0x0 +#define CP_MEC1_PRGRM_CNTR_START__IP_START_MASK 0xfff +#define CP_MEC1_PRGRM_CNTR_START__IP_START__SHIFT 0x0 +#define CP_MEC2_PRGRM_CNTR_START__IP_START_MASK 0xfff +#define CP_MEC2_PRGRM_CNTR_START__IP_START__SHIFT 0x0 +#define CP_CE_INTR_ROUTINE_START__IR_START_MASK 0x7ff +#define CP_CE_INTR_ROUTINE_START__IR_START__SHIFT 0x0 +#define CP_PFP_INTR_ROUTINE_START__IR_START_MASK 0x7ff +#define CP_PFP_INTR_ROUTINE_START__IR_START__SHIFT 0x0 +#define CP_ME_INTR_ROUTINE_START__IR_START_MASK 0x7ff +#define CP_ME_INTR_ROUTINE_START__IR_START__SHIFT 0x0 +#define CP_MEC1_INTR_ROUTINE_START__IR_START_MASK 0xfff +#define CP_MEC1_INTR_ROUTINE_START__IR_START__SHIFT 0x0 +#define CP_MEC2_INTR_ROUTINE_START__IR_START_MASK 0xfff +#define CP_MEC2_INTR_ROUTINE_START__IR_START__SHIFT 0x0 +#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX_MASK 0x7 +#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_WD_CNTX__SHIFT 0x0 +#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX_MASK 0x70 +#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX__SHIFT 0x4 +#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX_MASK 0x70000 +#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_WD_CNTX__SHIFT 0x10 +#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX_MASK 0x700000 +#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX__SHIFT 0x14 +#define CP_MAX_CONTEXT__MAX_CONTEXT_MASK 0x7 +#define CP_MAX_CONTEXT__MAX_CONTEXT__SHIFT 0x0 +#define CP_IQ_WAIT_TIME1__IB_OFFLOAD_MASK 0xff +#define CP_IQ_WAIT_TIME1__IB_OFFLOAD__SHIFT 0x0 +#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD_MASK 0xff00 +#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD__SHIFT 0x8 +#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD_MASK 0xff0000 +#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD__SHIFT 0x10 +#define CP_IQ_WAIT_TIME1__GWS_MASK 0xff000000 +#define CP_IQ_WAIT_TIME1__GWS__SHIFT 0x18 +#define CP_IQ_WAIT_TIME2__QUE_SLEEP_MASK 0xff +#define CP_IQ_WAIT_TIME2__QUE_SLEEP__SHIFT 0x0 +#define CP_IQ_WAIT_TIME2__SCH_WAVE_MASK 0xff00 +#define CP_IQ_WAIT_TIME2__SCH_WAVE__SHIFT 0x8 +#define CP_IQ_WAIT_TIME2__SEM_REARM_MASK 0xff0000 +#define CP_IQ_WAIT_TIME2__SEM_REARM__SHIFT 0x10 +#define CP_IQ_WAIT_TIME2__DEQ_RETRY_MASK 0xff000000 +#define CP_IQ_WAIT_TIME2__DEQ_RETRY__SHIFT 0x18 +#define CP_VMID_RESET__RESET_REQUEST_MASK 0xffff +#define CP_VMID_RESET__RESET_REQUEST__SHIFT 0x0 +#define CP_VMID_RESET__RESET_STATUS_MASK 0xffff0000 +#define CP_VMID_RESET__RESET_STATUS__SHIFT 0x10 +#define CP_VMID_PREEMPT__PREEMPT_REQUEST_MASK 0xffff +#define CP_VMID_PREEMPT__PREEMPT_REQUEST__SHIFT 0x0 +#define CP_VMID_PREEMPT__PREEMPT_STATUS_MASK 0xffff0000 +#define CP_VMID_PREEMPT__PREEMPT_STATUS__SHIFT 0x10 +#define CPC_INT_CNTX_ID__CNTX_ID_MASK 0xffff +#define CPC_INT_CNTX_ID__CNTX_ID__SHIFT 0x0 +#define CP_PQ_STATUS__DOORBELL_UPDATED_MASK 0x1 +#define CP_PQ_STATUS__DOORBELL_UPDATED__SHIFT 0x0 +#define CP_PQ_STATUS__DOORBELL_ENABLE_MASK 0x2 +#define CP_PQ_STATUS__DOORBELL_ENABLE__SHIFT 0x1 +#define CP_CPC_STATUS__MEC1_BUSY_MASK 0x1 +#define CP_CPC_STATUS__MEC1_BUSY__SHIFT 0x0 +#define CP_CPC_STATUS__MEC2_BUSY_MASK 0x2 +#define CP_CPC_STATUS__MEC2_BUSY__SHIFT 0x1 +#define CP_CPC_STATUS__DC0_BUSY_MASK 0x4 +#define CP_CPC_STATUS__DC0_BUSY__SHIFT 0x2 +#define CP_CPC_STATUS__DC1_BUSY_MASK 0x8 +#define CP_CPC_STATUS__DC1_BUSY__SHIFT 0x3 +#define CP_CPC_STATUS__RCIU1_BUSY_MASK 0x10 +#define CP_CPC_STATUS__RCIU1_BUSY__SHIFT 0x4 +#define CP_CPC_STATUS__RCIU2_BUSY_MASK 0x20 +#define CP_CPC_STATUS__RCIU2_BUSY__SHIFT 0x5 +#define CP_CPC_STATUS__ROQ1_BUSY_MASK 0x40 +#define CP_CPC_STATUS__ROQ1_BUSY__SHIFT 0x6 +#define CP_CPC_STATUS__ROQ2_BUSY_MASK 0x80 +#define CP_CPC_STATUS__ROQ2_BUSY__SHIFT 0x7 +#define CP_CPC_STATUS__MIU_RDREQ_BUSY_MASK 0x100 +#define CP_CPC_STATUS__MIU_RDREQ_BUSY__SHIFT 0x8 +#define CP_CPC_STATUS__MIU_WRREQ_BUSY_MASK 0x200 +#define CP_CPC_STATUS__MIU_WRREQ_BUSY__SHIFT 0x9 +#define CP_CPC_STATUS__TCIU_BUSY_MASK 0x400 +#define CP_CPC_STATUS__TCIU_BUSY__SHIFT 0xa +#define CP_CPC_STATUS__SCRATCH_RAM_BUSY_MASK 0x800 +#define CP_CPC_STATUS__SCRATCH_RAM_BUSY__SHIFT 0xb +#define CP_CPC_STATUS__QU_BUSY_MASK 0x1000 +#define CP_CPC_STATUS__QU_BUSY__SHIFT 0xc +#define CP_CPC_STATUS__CPG_CPC_BUSY_MASK 0x20000000 +#define CP_CPC_STATUS__CPG_CPC_BUSY__SHIFT 0x1d +#define CP_CPC_STATUS__CPF_CPC_BUSY_MASK 0x40000000 +#define CP_CPC_STATUS__CPF_CPC_BUSY__SHIFT 0x1e +#define CP_CPC_STATUS__CPC_BUSY_MASK 0x80000000 +#define CP_CPC_STATUS__CPC_BUSY__SHIFT 0x1f +#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY_MASK 0x1 +#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY__SHIFT 0x0 +#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY_MASK 0x2 +#define CP_CPC_BUSY_STAT__MEC1_SEMAPOHRE_BUSY__SHIFT 0x1 +#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY_MASK 0x4 +#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY__SHIFT 0x2 +#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY_MASK 0x8 +#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY__SHIFT 0x3 +#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY_MASK 0x10 +#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY__SHIFT 0x4 +#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY_MASK 0x20 +#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY__SHIFT 0x5 +#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY_MASK 0x40 +#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY__SHIFT 0x6 +#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY_MASK 0x80 +#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY__SHIFT 0x7 +#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY_MASK 0x100 +#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY__SHIFT 0x8 +#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY_MASK 0x200 +#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY__SHIFT 0x9 +#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY_MASK 0x400 +#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY__SHIFT 0xa +#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY_MASK 0x800 +#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY__SHIFT 0xb +#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY_MASK 0x1000 +#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY__SHIFT 0xc +#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY_MASK 0x2000 +#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY__SHIFT 0xd +#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY_MASK 0x10000 +#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY__SHIFT 0x10 +#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY_MASK 0x20000 +#define CP_CPC_BUSY_STAT__MEC2_SEMAPOHRE_BUSY__SHIFT 0x11 +#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY_MASK 0x40000 +#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY__SHIFT 0x12 +#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY_MASK 0x80000 +#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY__SHIFT 0x13 +#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY_MASK 0x100000 +#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY__SHIFT 0x14 +#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY_MASK 0x200000 +#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY__SHIFT 0x15 +#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY_MASK 0x400000 +#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY__SHIFT 0x16 +#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY_MASK 0x800000 +#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY__SHIFT 0x17 +#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY_MASK 0x1000000 +#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY__SHIFT 0x18 +#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY_MASK 0x2000000 +#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY__SHIFT 0x19 +#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY_MASK 0x4000000 +#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY__SHIFT 0x1a +#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY_MASK 0x8000000 +#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY__SHIFT 0x1b +#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY_MASK 0x10000000 +#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY__SHIFT 0x1c +#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY_MASK 0x20000000 +#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY__SHIFT 0x1d +#define CP_CPC_STALLED_STAT1__MIU_RDREQ_FREE_STALL_MASK 0x1 +#define CP_CPC_STALLED_STAT1__MIU_RDREQ_FREE_STALL__SHIFT 0x0 +#define CP_CPC_STALLED_STAT1__MIU_WRREQ_FREE_STALL_MASK 0x2 +#define CP_CPC_STALLED_STAT1__MIU_WRREQ_FREE_STALL__SHIFT 0x1 +#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL_MASK 0x8 +#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL__SHIFT 0x3 +#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION_MASK 0x10 +#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION__SHIFT 0x4 +#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL_MASK 0x40 +#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL__SHIFT 0x6 +#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET_MASK 0x100 +#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET__SHIFT 0x8 +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_MASK 0x200 +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU__SHIFT 0x9 +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ_MASK 0x400 +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ__SHIFT 0xa +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_MC_READ_MASK 0x800 +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_MC_READ__SHIFT 0xb +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_MC_WR_ACK_MASK 0x1000 +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_MC_WR_ACK__SHIFT 0xc +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA_MASK 0x2000 +#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA__SHIFT 0xd +#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET_MASK 0x10000 +#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET__SHIFT 0x10 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_MASK 0x20000 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU__SHIFT 0x11 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ_MASK 0x40000 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ__SHIFT 0x12 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_MC_READ_MASK 0x80000 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_MC_READ__SHIFT 0x13 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_MC_WR_ACK_MASK 0x100000 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_MC_WR_ACK__SHIFT 0x14 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA_MASK 0x200000 +#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA__SHIFT 0x15 +#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY_MASK 0x1 +#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY__SHIFT 0x0 +#define CP_CPF_STATUS__CSF_BUSY_MASK 0x2 +#define CP_CPF_STATUS__CSF_BUSY__SHIFT 0x1 +#define CP_CPF_STATUS__MIU_RDREQ_BUSY_MASK 0x4 +#define CP_CPF_STATUS__MIU_RDREQ_BUSY__SHIFT 0x2 +#define CP_CPF_STATUS__MIU_WRREQ_BUSY_MASK 0x8 +#define CP_CPF_STATUS__MIU_WRREQ_BUSY__SHIFT 0x3 +#define CP_CPF_STATUS__ROQ_ALIGN_BUSY_MASK 0x10 +#define CP_CPF_STATUS__ROQ_ALIGN_BUSY__SHIFT 0x4 +#define CP_CPF_STATUS__ROQ_RING_BUSY_MASK 0x20 +#define CP_CPF_STATUS__ROQ_RING_BUSY__SHIFT 0x5 +#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY_MASK 0x40 +#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY__SHIFT 0x6 +#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY_MASK 0x80 +#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY__SHIFT 0x7 +#define CP_CPF_STATUS__ROQ_STATE_BUSY_MASK 0x100 +#define CP_CPF_STATUS__ROQ_STATE_BUSY__SHIFT 0x8 +#define CP_CPF_STATUS__ROQ_CE_RING_BUSY_MASK 0x200 +#define CP_CPF_STATUS__ROQ_CE_RING_BUSY__SHIFT 0x9 +#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY_MASK 0x400 +#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY__SHIFT 0xa +#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY_MASK 0x800 +#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY__SHIFT 0xb +#define CP_CPF_STATUS__SEMAPHORE_BUSY_MASK 0x1000 +#define CP_CPF_STATUS__SEMAPHORE_BUSY__SHIFT 0xc +#define CP_CPF_STATUS__INTERRUPT_BUSY_MASK 0x2000 +#define CP_CPF_STATUS__INTERRUPT_BUSY__SHIFT 0xd +#define CP_CPF_STATUS__TCIU_BUSY_MASK 0x4000 +#define CP_CPF_STATUS__TCIU_BUSY__SHIFT 0xe +#define CP_CPF_STATUS__HQD_BUSY_MASK 0x8000 +#define CP_CPF_STATUS__HQD_BUSY__SHIFT 0xf +#define CP_CPF_STATUS__CPC_CPF_BUSY_MASK 0x40000000 +#define CP_CPF_STATUS__CPC_CPF_BUSY__SHIFT 0x1e +#define CP_CPF_STATUS__CPF_BUSY_MASK 0x80000000 +#define CP_CPF_STATUS__CPF_BUSY__SHIFT 0x1f +#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK 0x1 +#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT 0x0 +#define CP_CPF_BUSY_STAT__CSF_RING_BUSY_MASK 0x2 +#define CP_CPF_BUSY_STAT__CSF_RING_BUSY__SHIFT 0x1 +#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY_MASK 0x4 +#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY__SHIFT 0x2 +#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY_MASK 0x8 +#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY__SHIFT 0x3 +#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY_MASK 0x10 +#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY__SHIFT 0x4 +#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY_MASK 0x20 +#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY__SHIFT 0x5 +#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY_MASK 0x40 +#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY__SHIFT 0x6 +#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY_MASK 0x80 +#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY__SHIFT 0x7 +#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY_MASK 0x100 +#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY__SHIFT 0x8 +#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS_MASK 0x200 +#define CP_CPF_BUSY_STAT__OUTSTANDING_READ_TAGS__SHIFT 0x9 +#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY_MASK 0x800 +#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY__SHIFT 0xb +#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY_MASK 0x1000 +#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY__SHIFT 0xc +#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY_MASK 0x2000 +#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY__SHIFT 0xd +#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY_MASK 0x4000 +#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY__SHIFT 0xe +#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY_MASK 0x8000 +#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY__SHIFT 0xf +#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY_MASK 0x10000 +#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY__SHIFT 0x10 +#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY_MASK 0x20000 +#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY__SHIFT 0x11 +#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY_MASK 0x40000 +#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY__SHIFT 0x12 +#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY_MASK 0x80000 +#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY__SHIFT 0x13 +#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY_MASK 0x100000 +#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY__SHIFT 0x14 +#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY_MASK 0x200000 +#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY__SHIFT 0x15 +#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY_MASK 0x400000 +#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY__SHIFT 0x16 +#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY_MASK 0x800000 +#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY__SHIFT 0x17 +#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY_MASK 0x1000000 +#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY__SHIFT 0x18 +#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY_MASK 0x2000000 +#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY__SHIFT 0x19 +#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY_MASK 0x4000000 +#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY__SHIFT 0x1a +#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY_MASK 0x8000000 +#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY__SHIFT 0x1b +#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY_MASK 0x10000000 +#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY__SHIFT 0x1c +#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY_MASK 0x20000000 +#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY__SHIFT 0x1d +#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY_MASK 0x40000000 +#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY__SHIFT 0x1e +#define CP_CPF_BUSY_STAT__HQD_IB_BUSY_MASK 0x80000000 +#define CP_CPF_BUSY_STAT__HQD_IB_BUSY__SHIFT 0x1f +#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA_MASK 0x1 +#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA__SHIFT 0x0 +#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA_MASK 0x2 +#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA__SHIFT 0x1 +#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA_MASK 0x4 +#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA__SHIFT 0x2 +#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA_MASK 0x8 +#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA__SHIFT 0x3 +#define CP_CPF_STALLED_STAT1__MIU_WAITING_ON_RDREQ_FREE_MASK 0x10 +#define CP_CPF_STALLED_STAT1__MIU_WAITING_ON_RDREQ_FREE__SHIFT 0x4 +#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE_MASK 0x20 +#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE__SHIFT 0x5 +#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK 0x40 +#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT 0x6 +#define CP_CPC_MC_CNTL__PACK_DELAY_CNT_MASK 0x1f +#define CP_CPC_MC_CNTL__PACK_DELAY_CNT__SHIFT 0x0 +#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT_MASK 0x3f +#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT__SHIFT 0x0 +#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK 0x10 +#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE__SHIFT 0x4 +#define CP_MEC_CNTL__MEC_ME2_HALT_MASK 0x10000000 +#define CP_MEC_CNTL__MEC_ME2_HALT__SHIFT 0x1c +#define CP_MEC_CNTL__MEC_ME2_STEP_MASK 0x20000000 +#define CP_MEC_CNTL__MEC_ME2_STEP__SHIFT 0x1d +#define CP_MEC_CNTL__MEC_ME1_HALT_MASK 0x40000000 +#define CP_MEC_CNTL__MEC_ME1_HALT__SHIFT 0x1e +#define CP_MEC_CNTL__MEC_ME1_STEP_MASK 0x80000000 +#define CP_MEC_CNTL__MEC_ME1_STEP__SHIFT 0x1f +#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP_MASK 0xffffffff +#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP__SHIFT 0x0 +#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP_MASK 0xffffffff +#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP__SHIFT 0x0 +#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_MASK 0xff +#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT 0x0 +#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA_MASK 0xffffffff +#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA__SHIFT 0x0 +#define CPG_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f +#define CPG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3f +#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xfc00 +#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define CPG_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f +#define CPG_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xfc00 +#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CPC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f +#define CPC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3f +#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xfc00 +#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define CPC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f +#define CPC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xfc00 +#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CPF_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f +#define CPF_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3f +#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xfc00 +#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define CPF_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f +#define CPF_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xfc00 +#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CP_CPC_HALT_HYST_COUNT__COUNT_MASK 0xf +#define CP_CPC_HALT_HYST_COUNT__COUNT__SHIFT 0x0 +#define CP_DRAW_OBJECT__OBJECT_MASK 0xffffffff +#define CP_DRAW_OBJECT__OBJECT__SHIFT 0x0 +#define CP_DRAW_OBJECT_COUNTER__COUNT_MASK 0xffff +#define CP_DRAW_OBJECT_COUNTER__COUNT__SHIFT 0x0 +#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI_MASK 0xffffffff +#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI__SHIFT 0x0 +#define CP_DRAW_WINDOW_HI__WINDOW_HI_MASK 0xffffffff +#define CP_DRAW_WINDOW_HI__WINDOW_HI__SHIFT 0x0 +#define CP_DRAW_WINDOW_LO__MIN_MASK 0xffff +#define CP_DRAW_WINDOW_LO__MIN__SHIFT 0x0 +#define CP_DRAW_WINDOW_LO__MAX_MASK 0xffff0000 +#define CP_DRAW_WINDOW_LO__MAX__SHIFT 0x10 +#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX_MASK 0x1 +#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX__SHIFT 0x0 +#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN_MASK 0x2 +#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN__SHIFT 0x1 +#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI_MASK 0x4 +#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI__SHIFT 0x2 +#define CP_DRAW_WINDOW_CNTL__MODE_MASK 0x100 +#define CP_DRAW_WINDOW_CNTL__MODE__SHIFT 0x8 +#define CP_PRT_LOD_STATS_CNTL0__BU_SIZE_MASK 0xffffffff +#define CP_PRT_LOD_STATS_CNTL0__BU_SIZE__SHIFT 0x0 +#define CP_PRT_LOD_STATS_CNTL1__BASE_LO_MASK 0xffffffff +#define CP_PRT_LOD_STATS_CNTL1__BASE_LO__SHIFT 0x0 +#define CP_PRT_LOD_STATS_CNTL2__BASE_HI_MASK 0x3 +#define CP_PRT_LOD_STATS_CNTL2__BASE_HI__SHIFT 0x0 +#define CP_PRT_LOD_STATS_CNTL2__INTERVAL_MASK 0x3fc +#define CP_PRT_LOD_STATS_CNTL2__INTERVAL__SHIFT 0x2 +#define CP_PRT_LOD_STATS_CNTL2__RESET_CNT_MASK 0x3fc00 +#define CP_PRT_LOD_STATS_CNTL2__RESET_CNT__SHIFT 0xa +#define CP_PRT_LOD_STATS_CNTL2__RESET_FORCE_MASK 0x40000 +#define CP_PRT_LOD_STATS_CNTL2__RESET_FORCE__SHIFT 0x12 +#define CP_PRT_LOD_STATS_CNTL2__REPORT_AND_RESET_MASK 0x80000 +#define CP_PRT_LOD_STATS_CNTL2__REPORT_AND_RESET__SHIFT 0x13 +#define CP_PRT_LOD_STATS_CNTL2__MC_ENDIAN_SWAP_MASK 0x300000 +#define CP_PRT_LOD_STATS_CNTL2__MC_ENDIAN_SWAP__SHIFT 0x14 +#define CP_PRT_LOD_STATS_CNTL2__MC_VMID_MASK 0x7800000 +#define CP_PRT_LOD_STATS_CNTL2__MC_VMID__SHIFT 0x17 +#define CP_CE_COMPARE_COUNT__COMPARE_COUNT_MASK 0xffffffff +#define CP_CE_COMPARE_COUNT__COMPARE_COUNT__SHIFT 0x0 +#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT_MASK 0xffffffff +#define CP_CE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT 0x0 +#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT_MASK 0xffffffff +#define CP_DE_CE_COUNT__CONST_ENGINE_COUNT__SHIFT 0x0 +#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT_MASK 0xffffffff +#define CP_DE_LAST_INVAL_COUNT__LAST_INVAL_COUNT__SHIFT 0x0 +#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT_MASK 0xffffffff +#define CP_DE_DE_COUNT__DRAW_ENGINE_COUNT__SHIFT 0x0 +#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP_MASK 0x7f +#define CP_EOP_DONE_EVENT_CNTL__WBINV_TC_OP__SHIFT 0x0 +#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA_MASK 0x3f000 +#define CP_EOP_DONE_EVENT_CNTL__WBINV_ACTION_ENA__SHIFT 0xc +#define CP_EOP_DONE_EVENT_CNTL__CACHE_CONTROL_MASK 0x6000000 +#define CP_EOP_DONE_EVENT_CNTL__CACHE_CONTROL__SHIFT 0x19 +#define CP_EOP_DONE_EVENT_CNTL__EOP_VOLATILE_MASK 0x8000000 +#define CP_EOP_DONE_EVENT_CNTL__EOP_VOLATILE__SHIFT 0x1b +#define CP_EOP_DONE_DATA_CNTL__CNTX_ID_MASK 0xffff +#define CP_EOP_DONE_DATA_CNTL__CNTX_ID__SHIFT 0x0 +#define CP_EOP_DONE_DATA_CNTL__DST_SEL_MASK 0x30000 +#define CP_EOP_DONE_DATA_CNTL__DST_SEL__SHIFT 0x10 +#define CP_EOP_DONE_DATA_CNTL__INT_SEL_MASK 0x7000000 +#define CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT 0x18 +#define CP_EOP_DONE_DATA_CNTL__DATA_SEL_MASK 0xe0000000 +#define CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT 0x1d +#define CP_EOP_DONE_ADDR_LO__ADDR_SWAP_MASK 0x3 +#define CP_EOP_DONE_ADDR_LO__ADDR_SWAP__SHIFT 0x0 +#define CP_EOP_DONE_ADDR_LO__ADDR_LO_MASK 0xfffffffc +#define CP_EOP_DONE_ADDR_LO__ADDR_LO__SHIFT 0x2 +#define CP_EOP_DONE_ADDR_HI__ADDR_HI_MASK 0xffff +#define CP_EOP_DONE_ADDR_HI__ADDR_HI__SHIFT 0x0 +#define CP_EOP_DONE_DATA_LO__DATA_LO_MASK 0xffffffff +#define CP_EOP_DONE_DATA_LO__DATA_LO__SHIFT 0x0 +#define CP_EOP_DONE_DATA_HI__DATA_HI_MASK 0xffffffff +#define CP_EOP_DONE_DATA_HI__DATA_HI__SHIFT 0x0 +#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO_MASK 0xffffffff +#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO__SHIFT 0x0 +#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI_MASK 0xffffffff +#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI__SHIFT 0x0 +#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_SWAP_MASK 0x3 +#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_SWAP__SHIFT 0x0 +#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO_MASK 0xfffffffc +#define CP_STREAM_OUT_ADDR_LO__STREAM_OUT_ADDR_LO__SHIFT 0x2 +#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI_MASK 0xffff +#define CP_STREAM_OUT_ADDR_HI__STREAM_OUT_ADDR_HI__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT0_LO__NUM_PRIM_WRITTEN_CNT0_LO__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT0_HI__NUM_PRIM_WRITTEN_CNT0_HI__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT0_LO__NUM_PRIM_NEEDED_CNT0_LO__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT0_HI__NUM_PRIM_NEEDED_CNT0_HI__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT1_LO__NUM_PRIM_WRITTEN_CNT1_LO__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT1_HI__NUM_PRIM_WRITTEN_CNT1_HI__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT1_LO__NUM_PRIM_NEEDED_CNT1_LO__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT1_HI__NUM_PRIM_NEEDED_CNT1_HI__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT2_LO__NUM_PRIM_WRITTEN_CNT2_LO__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT2_HI__NUM_PRIM_WRITTEN_CNT2_HI__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT2_LO__NUM_PRIM_NEEDED_CNT2_LO__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT2_HI__NUM_PRIM_NEEDED_CNT2_HI__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT3_LO__NUM_PRIM_WRITTEN_CNT3_LO__SHIFT 0x0 +#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI_MASK 0xffffffff +#define CP_NUM_PRIM_WRITTEN_COUNT3_HI__NUM_PRIM_WRITTEN_CNT3_HI__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT3_LO__NUM_PRIM_NEEDED_CNT3_LO__SHIFT 0x0 +#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI_MASK 0xffffffff +#define CP_NUM_PRIM_NEEDED_COUNT3_HI__NUM_PRIM_NEEDED_CNT3_HI__SHIFT 0x0 +#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_SWAP_MASK 0x3 +#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_SWAP__SHIFT 0x0 +#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO_MASK 0xfffffffc +#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO__SHIFT 0x2 +#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI_MASK 0xffff +#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI__SHIFT 0x0 +#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO_MASK 0xffffffff +#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO__SHIFT 0x0 +#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI_MASK 0xffffffff +#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI__SHIFT 0x0 +#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO_MASK 0xffffffff +#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO__SHIFT 0x0 +#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI_MASK 0xffffffff +#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI__SHIFT 0x0 +#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO_MASK 0xffffffff +#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO__SHIFT 0x0 +#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI_MASK 0xffffffff +#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI__SHIFT 0x0 +#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO_MASK 0xffffffff +#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO__SHIFT 0x0 +#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI_MASK 0xffffffff +#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI__SHIFT 0x0 +#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO_MASK 0xffffffff +#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO__SHIFT 0x0 +#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI_MASK 0xffffffff +#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI__SHIFT 0x0 +#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO_MASK 0xffffffff +#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO__SHIFT 0x0 +#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI_MASK 0xffffffff +#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI__SHIFT 0x0 +#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO_MASK 0xffffffff +#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO__SHIFT 0x0 +#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI_MASK 0xffffffff +#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI__SHIFT 0x0 +#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO_MASK 0xffffffff +#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO__SHIFT 0x0 +#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI_MASK 0xffffffff +#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI__SHIFT 0x0 +#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO_MASK 0xffffffff +#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO__SHIFT 0x0 +#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI_MASK 0xffffffff +#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI__SHIFT 0x0 +#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO_MASK 0xffffffff +#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO__SHIFT 0x0 +#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI_MASK 0xffffffff +#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI__SHIFT 0x0 +#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE_MASK 0xffffffff +#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE__SHIFT 0x0 +#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE_MASK 0xffffffff +#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE__SHIFT 0x0 +#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO_MASK 0xffffffff +#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO__SHIFT 0x0 +#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI_MASK 0xffffffff +#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI__SHIFT 0x0 +#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE_MASK 0x1 +#define CP_STRMOUT_CNTL__OFFSET_UPDATE_DONE__SHIFT 0x0 +#define SCRATCH_REG0__SCRATCH_REG0_MASK 0xffffffff +#define SCRATCH_REG0__SCRATCH_REG0__SHIFT 0x0 +#define SCRATCH_REG1__SCRATCH_REG1_MASK 0xffffffff +#define SCRATCH_REG1__SCRATCH_REG1__SHIFT 0x0 +#define SCRATCH_REG2__SCRATCH_REG2_MASK 0xffffffff +#define SCRATCH_REG2__SCRATCH_REG2__SHIFT 0x0 +#define SCRATCH_REG3__SCRATCH_REG3_MASK 0xffffffff +#define SCRATCH_REG3__SCRATCH_REG3__SHIFT 0x0 +#define SCRATCH_REG4__SCRATCH_REG4_MASK 0xffffffff +#define SCRATCH_REG4__SCRATCH_REG4__SHIFT 0x0 +#define SCRATCH_REG5__SCRATCH_REG5_MASK 0xffffffff +#define SCRATCH_REG5__SCRATCH_REG5__SHIFT 0x0 +#define SCRATCH_REG6__SCRATCH_REG6_MASK 0xffffffff +#define SCRATCH_REG6__SCRATCH_REG6__SHIFT 0x0 +#define SCRATCH_REG7__SCRATCH_REG7_MASK 0xffffffff +#define SCRATCH_REG7__SCRATCH_REG7__SHIFT 0x0 +#define SCRATCH_UMSK__OBSOLETE_UMSK_MASK 0xff +#define SCRATCH_UMSK__OBSOLETE_UMSK__SHIFT 0x0 +#define SCRATCH_UMSK__OBSOLETE_SWAP_MASK 0x30000 +#define SCRATCH_UMSK__OBSOLETE_SWAP__SHIFT 0x10 +#define SCRATCH_ADDR__OBSOLETE_ADDR_MASK 0xffffffff +#define SCRATCH_ADDR__OBSOLETE_ADDR__SHIFT 0x0 +#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK 0xffffffff +#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT 0x0 +#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK 0xffffffff +#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT 0x0 +#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK 0xffffffff +#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT 0x0 +#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK 0xffffffff +#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT 0x0 +#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK 0xffffffff +#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT 0x0 +#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK 0xffffffff +#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT 0x0 +#define CP_APPEND_ADDR_LO__MEM_ADDR_LO_MASK 0xfffffffc +#define CP_APPEND_ADDR_LO__MEM_ADDR_LO__SHIFT 0x2 +#define CP_APPEND_ADDR_HI__MEM_ADDR_HI_MASK 0xffff +#define CP_APPEND_ADDR_HI__MEM_ADDR_HI__SHIFT 0x0 +#define CP_APPEND_ADDR_HI__CS_PS_SEL_MASK 0x10000 +#define CP_APPEND_ADDR_HI__CS_PS_SEL__SHIFT 0x10 +#define CP_APPEND_ADDR_HI__COMMAND_MASK 0xe0000000 +#define CP_APPEND_ADDR_HI__COMMAND__SHIFT 0x1d +#define CP_APPEND_DATA__DATA_MASK 0xffffffff +#define CP_APPEND_DATA__DATA__SHIFT 0x0 +#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE_MASK 0xffffffff +#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE__SHIFT 0x0 +#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE_MASK 0xffffffff +#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE__SHIFT 0x0 +#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK 0xffffffff +#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT 0x0 +#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK 0xffffffff +#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT 0x0 +#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK 0xffffffff +#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT 0x0 +#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK 0xffffffff +#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT 0x0 +#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK 0xffffffff +#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT 0x0 +#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK 0xffffffff +#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT 0x0 +#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK 0xffffffff +#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT 0x0 +#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK 0xffffffff +#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT 0x0 +#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK 0xffffffff +#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT 0x0 +#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK 0xffffffff +#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT 0x0 +#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK 0xffffffff +#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT 0x0 +#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK 0xffffffff +#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT 0x0 +#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_SWAP_MASK 0x3 +#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_SWAP__SHIFT 0x0 +#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO_MASK 0xfffffffc +#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO__SHIFT 0x2 +#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI_MASK 0xffff +#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI__SHIFT 0x0 +#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO_MASK 0xffffffff +#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO__SHIFT 0x0 +#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI_MASK 0xffffffff +#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI__SHIFT 0x0 +#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_SWAP_MASK 0x3 +#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_SWAP__SHIFT 0x0 +#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO_MASK 0xfffffffc +#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO__SHIFT 0x2 +#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI_MASK 0xffff +#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI__SHIFT 0x0 +#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER_MASK 0xffffffff +#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER__SHIFT 0x0 +#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK 0x3 +#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT 0x0 +#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO_MASK 0xfffffff8 +#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT 0x3 +#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI_MASK 0xffff +#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT 0x0 +#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK 0x10000 +#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT 0x10 +#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK 0x100000 +#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT 0x14 +#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK 0x3000000 +#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT 0x18 +#define CP_SIG_SEM_ADDR_HI__SEM_SELECT_MASK 0xe0000000 +#define CP_SIG_SEM_ADDR_HI__SEM_SELECT__SHIFT 0x1d +#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP_MASK 0x3 +#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_SWAP__SHIFT 0x0 +#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO_MASK 0xfffffff8 +#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT 0x3 +#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI_MASK 0xffff +#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT 0x0 +#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK 0x10000 +#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT 0x10 +#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK 0x100000 +#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT 0x14 +#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK 0x3000000 +#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT 0x18 +#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT_MASK 0xe0000000 +#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT__SHIFT 0x1d +#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT_MASK 0xffffffff +#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT__SHIFT 0x0 +#define CP_COHER_START_DELAY__START_DELAY_COUNT_MASK 0x3f +#define CP_COHER_START_DELAY__START_DELAY_COUNT__SHIFT 0x0 +#define CP_COHER_CNTL__DEST_BASE_0_ENA_MASK 0x1 +#define CP_COHER_CNTL__DEST_BASE_0_ENA__SHIFT 0x0 +#define CP_COHER_CNTL__DEST_BASE_1_ENA_MASK 0x2 +#define CP_COHER_CNTL__DEST_BASE_1_ENA__SHIFT 0x1 +#define CP_COHER_CNTL__CB0_DEST_BASE_ENA_MASK 0x40 +#define CP_COHER_CNTL__CB0_DEST_BASE_ENA__SHIFT 0x6 +#define CP_COHER_CNTL__CB1_DEST_BASE_ENA_MASK 0x80 +#define CP_COHER_CNTL__CB1_DEST_BASE_ENA__SHIFT 0x7 +#define CP_COHER_CNTL__CB2_DEST_BASE_ENA_MASK 0x100 +#define CP_COHER_CNTL__CB2_DEST_BASE_ENA__SHIFT 0x8 +#define CP_COHER_CNTL__CB3_DEST_BASE_ENA_MASK 0x200 +#define CP_COHER_CNTL__CB3_DEST_BASE_ENA__SHIFT 0x9 +#define CP_COHER_CNTL__CB4_DEST_BASE_ENA_MASK 0x400 +#define CP_COHER_CNTL__CB4_DEST_BASE_ENA__SHIFT 0xa +#define CP_COHER_CNTL__CB5_DEST_BASE_ENA_MASK 0x800 +#define CP_COHER_CNTL__CB5_DEST_BASE_ENA__SHIFT 0xb +#define CP_COHER_CNTL__CB6_DEST_BASE_ENA_MASK 0x1000 +#define CP_COHER_CNTL__CB6_DEST_BASE_ENA__SHIFT 0xc +#define CP_COHER_CNTL__CB7_DEST_BASE_ENA_MASK 0x2000 +#define CP_COHER_CNTL__CB7_DEST_BASE_ENA__SHIFT 0xd +#define CP_COHER_CNTL__DB_DEST_BASE_ENA_MASK 0x4000 +#define CP_COHER_CNTL__DB_DEST_BASE_ENA__SHIFT 0xe +#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA_MASK 0x8000 +#define CP_COHER_CNTL__TCL1_VOL_ACTION_ENA__SHIFT 0xf +#define CP_COHER_CNTL__TC_VOL_ACTION_ENA_MASK 0x10000 +#define CP_COHER_CNTL__TC_VOL_ACTION_ENA__SHIFT 0x10 +#define CP_COHER_CNTL__TC_WB_ACTION_ENA_MASK 0x40000 +#define CP_COHER_CNTL__TC_WB_ACTION_ENA__SHIFT 0x12 +#define CP_COHER_CNTL__DEST_BASE_2_ENA_MASK 0x80000 +#define CP_COHER_CNTL__DEST_BASE_2_ENA__SHIFT 0x13 +#define CP_COHER_CNTL__DEST_BASE_3_ENA_MASK 0x200000 +#define CP_COHER_CNTL__DEST_BASE_3_ENA__SHIFT 0x15 +#define CP_COHER_CNTL__TCL1_ACTION_ENA_MASK 0x400000 +#define CP_COHER_CNTL__TCL1_ACTION_ENA__SHIFT 0x16 +#define CP_COHER_CNTL__TC_ACTION_ENA_MASK 0x800000 +#define CP_COHER_CNTL__TC_ACTION_ENA__SHIFT 0x17 +#define CP_COHER_CNTL__CB_ACTION_ENA_MASK 0x2000000 +#define CP_COHER_CNTL__CB_ACTION_ENA__SHIFT 0x19 +#define CP_COHER_CNTL__DB_ACTION_ENA_MASK 0x4000000 +#define CP_COHER_CNTL__DB_ACTION_ENA__SHIFT 0x1a +#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA_MASK 0x8000000 +#define CP_COHER_CNTL__SH_KCACHE_ACTION_ENA__SHIFT 0x1b +#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA_MASK 0x10000000 +#define CP_COHER_CNTL__SH_KCACHE_VOL_ACTION_ENA__SHIFT 0x1c +#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA_MASK 0x20000000 +#define CP_COHER_CNTL__SH_ICACHE_ACTION_ENA__SHIFT 0x1d +#define CP_COHER_SIZE__COHER_SIZE_256B_MASK 0xffffffff +#define CP_COHER_SIZE__COHER_SIZE_256B__SHIFT 0x0 +#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK 0xff +#define CP_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT 0x0 +#define CP_COHER_BASE__COHER_BASE_256B_MASK 0xffffffff +#define CP_COHER_BASE__COHER_BASE_256B__SHIFT 0x0 +#define CP_COHER_BASE_HI__COHER_BASE_HI_256B_MASK 0xff +#define CP_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT 0x0 +#define CP_COHER_STATUS__MATCHING_GFX_CNTX_MASK 0xff +#define CP_COHER_STATUS__MATCHING_GFX_CNTX__SHIFT 0x0 +#define CP_COHER_STATUS__MEID_MASK 0x3000000 +#define CP_COHER_STATUS__MEID__SHIFT 0x18 +#define CP_COHER_STATUS__PHASE1_STATUS_MASK 0x40000000 +#define CP_COHER_STATUS__PHASE1_STATUS__SHIFT 0x1e +#define CP_COHER_STATUS__STATUS_MASK 0x80000000 +#define CP_COHER_STATUS__STATUS__SHIFT 0x1f +#define COHER_DEST_BASE_0__DEST_BASE_256B_MASK 0xffffffff +#define COHER_DEST_BASE_0__DEST_BASE_256B__SHIFT 0x0 +#define COHER_DEST_BASE_1__DEST_BASE_256B_MASK 0xffffffff +#define COHER_DEST_BASE_1__DEST_BASE_256B__SHIFT 0x0 +#define COHER_DEST_BASE_2__DEST_BASE_256B_MASK 0xffffffff +#define COHER_DEST_BASE_2__DEST_BASE_256B__SHIFT 0x0 +#define COHER_DEST_BASE_3__DEST_BASE_256B_MASK 0xffffffff +#define COHER_DEST_BASE_3__DEST_BASE_256B__SHIFT 0x0 +#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B_MASK 0xffffffff +#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B__SHIFT 0x0 +#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B_MASK 0xffffffff +#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B__SHIFT 0x0 +#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B_MASK 0xffffffff +#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B__SHIFT 0x0 +#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B_MASK 0xffffffff +#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B__SHIFT 0x0 +#define CP_DMA_ME_SRC_ADDR__SRC_ADDR_MASK 0xffffffff +#define CP_DMA_ME_SRC_ADDR__SRC_ADDR__SHIFT 0x0 +#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI_MASK 0xffff +#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT 0x0 +#define CP_DMA_ME_DST_ADDR__DST_ADDR_MASK 0xffffffff +#define CP_DMA_ME_DST_ADDR__DST_ADDR__SHIFT 0x0 +#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI_MASK 0xffff +#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI__SHIFT 0x0 +#define CP_DMA_ME_CONTROL__SRC_ATC_MASK 0x1000 +#define CP_DMA_ME_CONTROL__SRC_ATC__SHIFT 0xc +#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY_MASK 0x6000 +#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY__SHIFT 0xd +#define CP_DMA_ME_CONTROL__SRC_VOLATILE_MASK 0x8000 +#define CP_DMA_ME_CONTROL__SRC_VOLATILE__SHIFT 0xf +#define CP_DMA_ME_CONTROL__DST_SELECT_MASK 0x300000 +#define CP_DMA_ME_CONTROL__DST_SELECT__SHIFT 0x14 +#define CP_DMA_ME_CONTROL__DST_ATC_MASK 0x1000000 +#define CP_DMA_ME_CONTROL__DST_ATC__SHIFT 0x18 +#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY_MASK 0x6000000 +#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY__SHIFT 0x19 +#define CP_DMA_ME_CONTROL__DST_VOLATILE_MASK 0x8000000 +#define CP_DMA_ME_CONTROL__DST_VOLATILE__SHIFT 0x1b +#define CP_DMA_ME_CONTROL__SRC_SELECT_MASK 0x60000000 +#define CP_DMA_ME_CONTROL__SRC_SELECT__SHIFT 0x1d +#define CP_DMA_ME_COMMAND__BYTE_COUNT_MASK 0x1fffff +#define CP_DMA_ME_COMMAND__BYTE_COUNT__SHIFT 0x0 +#define CP_DMA_ME_COMMAND__DIS_WC_MASK 0x200000 +#define CP_DMA_ME_COMMAND__DIS_WC__SHIFT 0x15 +#define CP_DMA_ME_COMMAND__SRC_SWAP_MASK 0xc00000 +#define CP_DMA_ME_COMMAND__SRC_SWAP__SHIFT 0x16 +#define CP_DMA_ME_COMMAND__DST_SWAP_MASK 0x3000000 +#define CP_DMA_ME_COMMAND__DST_SWAP__SHIFT 0x18 +#define CP_DMA_ME_COMMAND__SAS_MASK 0x4000000 +#define CP_DMA_ME_COMMAND__SAS__SHIFT 0x1a +#define CP_DMA_ME_COMMAND__DAS_MASK 0x8000000 +#define CP_DMA_ME_COMMAND__DAS__SHIFT 0x1b +#define CP_DMA_ME_COMMAND__SAIC_MASK 0x10000000 +#define CP_DMA_ME_COMMAND__SAIC__SHIFT 0x1c +#define CP_DMA_ME_COMMAND__DAIC_MASK 0x20000000 +#define CP_DMA_ME_COMMAND__DAIC__SHIFT 0x1d +#define CP_DMA_ME_COMMAND__RAW_WAIT_MASK 0x40000000 +#define CP_DMA_ME_COMMAND__RAW_WAIT__SHIFT 0x1e +#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR_MASK 0xffffffff +#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR__SHIFT 0x0 +#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI_MASK 0xffff +#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT 0x0 +#define CP_DMA_PFP_DST_ADDR__DST_ADDR_MASK 0xffffffff +#define CP_DMA_PFP_DST_ADDR__DST_ADDR__SHIFT 0x0 +#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI_MASK 0xffff +#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI__SHIFT 0x0 +#define CP_DMA_PFP_CONTROL__SRC_ATC_MASK 0x1000 +#define CP_DMA_PFP_CONTROL__SRC_ATC__SHIFT 0xc +#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY_MASK 0x6000 +#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY__SHIFT 0xd +#define CP_DMA_PFP_CONTROL__SRC_VOLATILE_MASK 0x8000 +#define CP_DMA_PFP_CONTROL__SRC_VOLATILE__SHIFT 0xf +#define CP_DMA_PFP_CONTROL__DST_SELECT_MASK 0x300000 +#define CP_DMA_PFP_CONTROL__DST_SELECT__SHIFT 0x14 +#define CP_DMA_PFP_CONTROL__DST_ATC_MASK 0x1000000 +#define CP_DMA_PFP_CONTROL__DST_ATC__SHIFT 0x18 +#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY_MASK 0x6000000 +#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY__SHIFT 0x19 +#define CP_DMA_PFP_CONTROL__DST_VOLATILE_MASK 0x8000000 +#define CP_DMA_PFP_CONTROL__DST_VOLATILE__SHIFT 0x1b +#define CP_DMA_PFP_CONTROL__SRC_SELECT_MASK 0x60000000 +#define CP_DMA_PFP_CONTROL__SRC_SELECT__SHIFT 0x1d +#define CP_DMA_PFP_COMMAND__BYTE_COUNT_MASK 0x1fffff +#define CP_DMA_PFP_COMMAND__BYTE_COUNT__SHIFT 0x0 +#define CP_DMA_PFP_COMMAND__DIS_WC_MASK 0x200000 +#define CP_DMA_PFP_COMMAND__DIS_WC__SHIFT 0x15 +#define CP_DMA_PFP_COMMAND__SRC_SWAP_MASK 0xc00000 +#define CP_DMA_PFP_COMMAND__SRC_SWAP__SHIFT 0x16 +#define CP_DMA_PFP_COMMAND__DST_SWAP_MASK 0x3000000 +#define CP_DMA_PFP_COMMAND__DST_SWAP__SHIFT 0x18 +#define CP_DMA_PFP_COMMAND__SAS_MASK 0x4000000 +#define CP_DMA_PFP_COMMAND__SAS__SHIFT 0x1a +#define CP_DMA_PFP_COMMAND__DAS_MASK 0x8000000 +#define CP_DMA_PFP_COMMAND__DAS__SHIFT 0x1b +#define CP_DMA_PFP_COMMAND__SAIC_MASK 0x10000000 +#define CP_DMA_PFP_COMMAND__SAIC__SHIFT 0x1c +#define CP_DMA_PFP_COMMAND__DAIC_MASK 0x20000000 +#define CP_DMA_PFP_COMMAND__DAIC__SHIFT 0x1d +#define CP_DMA_PFP_COMMAND__RAW_WAIT_MASK 0x40000000 +#define CP_DMA_PFP_COMMAND__RAW_WAIT__SHIFT 0x1e +#define CP_DMA_CNTL__MIN_AVAILSZ_MASK 0x30 +#define CP_DMA_CNTL__MIN_AVAILSZ__SHIFT 0x4 +#define CP_DMA_CNTL__BUFFER_DEPTH_MASK 0xf0000 +#define CP_DMA_CNTL__BUFFER_DEPTH__SHIFT 0x10 +#define CP_DMA_CNTL__PIO_FIFO_EMPTY_MASK 0x10000000 +#define CP_DMA_CNTL__PIO_FIFO_EMPTY__SHIFT 0x1c +#define CP_DMA_CNTL__PIO_FIFO_FULL_MASK 0x20000000 +#define CP_DMA_CNTL__PIO_FIFO_FULL__SHIFT 0x1d +#define CP_DMA_CNTL__PIO_COUNT_MASK 0xc0000000 +#define CP_DMA_CNTL__PIO_COUNT__SHIFT 0x1e +#define CP_DMA_READ_TAGS__DMA_READ_TAG_MASK 0x3ffffff +#define CP_DMA_READ_TAGS__DMA_READ_TAG__SHIFT 0x0 +#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID_MASK 0x10000000 +#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID__SHIFT 0x1c +#define CP_PFP_IB_CONTROL__IB_EN_MASK 0xff +#define CP_PFP_IB_CONTROL__IB_EN__SHIFT 0x0 +#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN_MASK 0x1 +#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN__SHIFT 0x0 +#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN_MASK 0x2 +#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN__SHIFT 0x1 +#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN_MASK 0x8000 +#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN__SHIFT 0xf +#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN_MASK 0x10000 +#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN__SHIFT 0x10 +#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN_MASK 0x1000000 +#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN__SHIFT 0x18 +#define CP_SCRATCH_INDEX__SCRATCH_INDEX_MASK 0xff +#define CP_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT 0x0 +#define CP_SCRATCH_DATA__SCRATCH_DATA_MASK 0xffffffff +#define CP_SCRATCH_DATA__SCRATCH_DATA__SHIFT 0x0 +#define CP_RB_OFFSET__RB_OFFSET_MASK 0xfffff +#define CP_RB_OFFSET__RB_OFFSET__SHIFT 0x0 +#define CP_IB1_OFFSET__IB1_OFFSET_MASK 0xfffff +#define CP_IB1_OFFSET__IB1_OFFSET__SHIFT 0x0 +#define CP_IB2_OFFSET__IB2_OFFSET_MASK 0xfffff +#define CP_IB2_OFFSET__IB2_OFFSET__SHIFT 0x0 +#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN_MASK 0xfffff +#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN__SHIFT 0x0 +#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END_MASK 0xfffff +#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END__SHIFT 0x0 +#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN_MASK 0xfffff +#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN__SHIFT 0x0 +#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END_MASK 0xfffff +#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END__SHIFT 0x0 +#define CP_CE_IB1_OFFSET__IB1_OFFSET_MASK 0xfffff +#define CP_CE_IB1_OFFSET__IB1_OFFSET__SHIFT 0x0 +#define CP_CE_IB2_OFFSET__IB2_OFFSET_MASK 0xfffff +#define CP_CE_IB2_OFFSET__IB2_OFFSET__SHIFT 0x0 +#define CP_CE_COUNTER__CONST_ENGINE_COUNT_MASK 0xffffffff +#define CP_CE_COUNTER__CONST_ENGINE_COUNT__SHIFT 0x0 +#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV_MASK 0x1 +#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV__SHIFT 0x0 +#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_MASK 0x4 +#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV__SHIFT 0x2 +#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_MASK 0x10 +#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV__SHIFT 0x4 +#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG_MASK 0x400 +#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG__SHIFT 0xa +#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG_MASK 0x800 +#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG__SHIFT 0xb +#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM_MASK 0x1000 +#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM__SHIFT 0xc +#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA_MASK 0x2000 +#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA__SHIFT 0xd +#define CP_STALLED_STAT1__ME_WAITING_ON_MC_READ_DATA_MASK 0x4000 +#define CP_STALLED_STAT1__ME_WAITING_ON_MC_READ_DATA__SHIFT 0xe +#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA_MASK 0x8000 +#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA__SHIFT 0xf +#define CP_STALLED_STAT1__MIU_WAITING_ON_RDREQ_FREE_MASK 0x10000 +#define CP_STALLED_STAT1__MIU_WAITING_ON_RDREQ_FREE__SHIFT 0x10 +#define CP_STALLED_STAT1__MIU_WAITING_ON_WRREQ_FREE_MASK 0x20000 +#define CP_STALLED_STAT1__MIU_WAITING_ON_WRREQ_FREE__SHIFT 0x11 +#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE_MASK 0x800000 +#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE__SHIFT 0x17 +#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE_MASK 0x1000000 +#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE__SHIFT 0x18 +#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE_MASK 0x2000000 +#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE__SHIFT 0x19 +#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ_MASK 0x4000000 +#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ__SHIFT 0x1a +#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ_MASK 0x8000000 +#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ__SHIFT 0x1b +#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ_MASK 0x10000000 +#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ__SHIFT 0x1c +#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION_MASK 0x20000000 +#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION__SHIFT 0x1d +#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV_MASK 0x1 +#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV__SHIFT 0x0 +#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV_MASK 0x2 +#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV__SHIFT 0x1 +#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV_MASK 0x4 +#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV__SHIFT 0x2 +#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING_MASK 0x10 +#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING__SHIFT 0x4 +#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING_MASK 0x20 +#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING__SHIFT 0x5 +#define CP_STALLED_STAT2__PFP_MIU_READ_PENDING_MASK 0x40 +#define CP_STALLED_STAT2__PFP_MIU_READ_PENDING__SHIFT 0x6 +#define CP_STALLED_STAT2__PFP_TO_MIU_WRITE_NOT_RDY_TO_RCV_MASK 0x80 +#define CP_STALLED_STAT2__PFP_TO_MIU_WRITE_NOT_RDY_TO_RCV__SHIFT 0x7 +#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA_MASK 0x100 +#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA__SHIFT 0x8 +#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER_MASK 0x200 +#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER__SHIFT 0x9 +#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER_MASK 0x400 +#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER__SHIFT 0xa +#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME_MASK 0x800 +#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME__SHIFT 0xb +#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV_MASK 0x1000 +#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV__SHIFT 0xc +#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV_MASK 0x2000 +#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV__SHIFT 0xd +#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP_MASK 0x4000 +#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP__SHIFT 0xe +#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH_MASK 0x8000 +#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH__SHIFT 0xf +#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV_MASK 0x10000 +#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV__SHIFT 0x10 +#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV_MASK 0x20000 +#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV__SHIFT 0x11 +#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ_MASK 0x40000 +#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ__SHIFT 0x12 +#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM_MASK 0x80000 +#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM__SHIFT 0x13 +#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA_MASK 0x100000 +#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA__SHIFT 0x14 +#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE_MASK 0x200000 +#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_SC_EOP_DONE__SHIFT 0x15 +#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM_MASK 0x400000 +#define CP_STALLED_STAT2__EOPD_FIFO_NEEDS_WR_CONFIRM__SHIFT 0x16 +#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING_MASK 0x800000 +#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING__SHIFT 0x17 +#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING_MASK 0x1000000 +#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING__SHIFT 0x18 +#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE_MASK 0x2000000 +#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE__SHIFT 0x19 +#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE_MASK 0x4000000 +#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE__SHIFT 0x1a +#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM_MASK 0x8000000 +#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM__SHIFT 0x1b +#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION_MASK 0x10000000 +#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION__SHIFT 0x1c +#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE_MASK 0x20000000 +#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE__SHIFT 0x1d +#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS_MASK 0x40000000 +#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS__SHIFT 0x1e +#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN_MASK 0x80000000 +#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN__SHIFT 0x1f +#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV_MASK 0x1 +#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV__SHIFT 0x0 +#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV_MASK 0x2 +#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV__SHIFT 0x1 +#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER_MASK 0x4 +#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER__SHIFT 0x2 +#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY_MASK 0x8 +#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY__SHIFT 0x3 +#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY_MASK 0x10 +#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY__SHIFT 0x4 +#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY_MASK 0x20 +#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY__SHIFT 0x5 +#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV_MASK 0x40 +#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV__SHIFT 0x6 +#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV_MASK 0x80 +#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV__SHIFT 0x7 +#define CP_STALLED_STAT3__CE_TO_MIU_WRITE_NOT_RDY_TO_RCV_MASK 0x100 +#define CP_STALLED_STAT3__CE_TO_MIU_WRITE_NOT_RDY_TO_RCV__SHIFT 0x8 +#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA_MASK 0x400 +#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA__SHIFT 0xa +#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG_MASK 0x800 +#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG__SHIFT 0xb +#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_MASK 0x1000 +#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER__SHIFT 0xc +#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW_MASK 0x2000 +#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW__SHIFT 0xd +#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE_MASK 0x4000 +#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE__SHIFT 0xe +#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS_MASK 0x8000 +#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS__SHIFT 0xf +#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK 0x1 +#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT 0x0 +#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO_MASK 0x40 +#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO__SHIFT 0x6 +#define CP_BUSY_STAT__PFP_PARSING_PACKETS_MASK 0x80 +#define CP_BUSY_STAT__PFP_PARSING_PACKETS__SHIFT 0x7 +#define CP_BUSY_STAT__ME_PARSING_PACKETS_MASK 0x100 +#define CP_BUSY_STAT__ME_PARSING_PACKETS__SHIFT 0x8 +#define CP_BUSY_STAT__RCIU_PFP_BUSY_MASK 0x200 +#define CP_BUSY_STAT__RCIU_PFP_BUSY__SHIFT 0x9 +#define CP_BUSY_STAT__RCIU_ME_BUSY_MASK 0x400 +#define CP_BUSY_STAT__RCIU_ME_BUSY__SHIFT 0xa +#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY_MASK 0x1000 +#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY__SHIFT 0xc +#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING_MASK 0x2000 +#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING__SHIFT 0xd +#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS_MASK 0x4000 +#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS__SHIFT 0xe +#define CP_BUSY_STAT__GFX_CONTEXT_BUSY_MASK 0x8000 +#define CP_BUSY_STAT__GFX_CONTEXT_BUSY__SHIFT 0xf +#define CP_BUSY_STAT__ME_PARSER_BUSY_MASK 0x20000 +#define CP_BUSY_STAT__ME_PARSER_BUSY__SHIFT 0x11 +#define CP_BUSY_STAT__EOP_DONE_BUSY_MASK 0x40000 +#define CP_BUSY_STAT__EOP_DONE_BUSY__SHIFT 0x12 +#define CP_BUSY_STAT__STRM_OUT_BUSY_MASK 0x80000 +#define CP_BUSY_STAT__STRM_OUT_BUSY__SHIFT 0x13 +#define CP_BUSY_STAT__PIPE_STATS_BUSY_MASK 0x100000 +#define CP_BUSY_STAT__PIPE_STATS_BUSY__SHIFT 0x14 +#define CP_BUSY_STAT__RCIU_CE_BUSY_MASK 0x200000 +#define CP_BUSY_STAT__RCIU_CE_BUSY__SHIFT 0x15 +#define CP_BUSY_STAT__CE_PARSING_PACKETS_MASK 0x400000 +#define CP_BUSY_STAT__CE_PARSING_PACKETS__SHIFT 0x16 +#define CP_STAT__MIU_RDREQ_BUSY_MASK 0x80 +#define CP_STAT__MIU_RDREQ_BUSY__SHIFT 0x7 +#define CP_STAT__MIU_WRREQ_BUSY_MASK 0x100 +#define CP_STAT__MIU_WRREQ_BUSY__SHIFT 0x8 +#define CP_STAT__ROQ_RING_BUSY_MASK 0x200 +#define CP_STAT__ROQ_RING_BUSY__SHIFT 0x9 +#define CP_STAT__ROQ_INDIRECT1_BUSY_MASK 0x400 +#define CP_STAT__ROQ_INDIRECT1_BUSY__SHIFT 0xa +#define CP_STAT__ROQ_INDIRECT2_BUSY_MASK 0x800 +#define CP_STAT__ROQ_INDIRECT2_BUSY__SHIFT 0xb +#define CP_STAT__ROQ_STATE_BUSY_MASK 0x1000 +#define CP_STAT__ROQ_STATE_BUSY__SHIFT 0xc +#define CP_STAT__DC_BUSY_MASK 0x2000 +#define CP_STAT__DC_BUSY__SHIFT 0xd +#define CP_STAT__PFP_BUSY_MASK 0x8000 +#define CP_STAT__PFP_BUSY__SHIFT 0xf +#define CP_STAT__MEQ_BUSY_MASK 0x10000 +#define CP_STAT__MEQ_BUSY__SHIFT 0x10 +#define CP_STAT__ME_BUSY_MASK 0x20000 +#define CP_STAT__ME_BUSY__SHIFT 0x11 +#define CP_STAT__QUERY_BUSY_MASK 0x40000 +#define CP_STAT__QUERY_BUSY__SHIFT 0x12 +#define CP_STAT__SEMAPHORE_BUSY_MASK 0x80000 +#define CP_STAT__SEMAPHORE_BUSY__SHIFT 0x13 +#define CP_STAT__INTERRUPT_BUSY_MASK 0x100000 +#define CP_STAT__INTERRUPT_BUSY__SHIFT 0x14 +#define CP_STAT__SURFACE_SYNC_BUSY_MASK 0x200000 +#define CP_STAT__SURFACE_SYNC_BUSY__SHIFT 0x15 +#define CP_STAT__DMA_BUSY_MASK 0x400000 +#define CP_STAT__DMA_BUSY__SHIFT 0x16 +#define CP_STAT__RCIU_BUSY_MASK 0x800000 +#define CP_STAT__RCIU_BUSY__SHIFT 0x17 +#define CP_STAT__SCRATCH_RAM_BUSY_MASK 0x1000000 +#define CP_STAT__SCRATCH_RAM_BUSY__SHIFT 0x18 +#define CP_STAT__CPC_CPG_BUSY_MASK 0x2000000 +#define CP_STAT__CPC_CPG_BUSY__SHIFT 0x19 +#define CP_STAT__CE_BUSY_MASK 0x4000000 +#define CP_STAT__CE_BUSY__SHIFT 0x1a +#define CP_STAT__TCIU_BUSY_MASK 0x8000000 +#define CP_STAT__TCIU_BUSY__SHIFT 0x1b +#define CP_STAT__ROQ_CE_RING_BUSY_MASK 0x10000000 +#define CP_STAT__ROQ_CE_RING_BUSY__SHIFT 0x1c +#define CP_STAT__ROQ_CE_INDIRECT1_BUSY_MASK 0x20000000 +#define CP_STAT__ROQ_CE_INDIRECT1_BUSY__SHIFT 0x1d +#define CP_STAT__ROQ_CE_INDIRECT2_BUSY_MASK 0x40000000 +#define CP_STAT__ROQ_CE_INDIRECT2_BUSY__SHIFT 0x1e +#define CP_STAT__CP_BUSY_MASK 0x80000000 +#define CP_STAT__CP_BUSY__SHIFT 0x1f +#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP_MASK 0xffffffff +#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP__SHIFT 0x0 +#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP_MASK 0xffffffff +#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP__SHIFT 0x0 +#define CP_GRBM_FREE_COUNT__FREE_COUNT_MASK 0x3f +#define CP_GRBM_FREE_COUNT__FREE_COUNT__SHIFT 0x0 +#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS_MASK 0x3f00 +#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS__SHIFT 0x8 +#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP_MASK 0x3f0000 +#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP__SHIFT 0x10 +#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP_MASK 0xffffffff +#define CP_CE_HEADER_DUMP__CE_HEADER_DUMP__SHIFT 0x0 +#define CP_MC_PACK_DELAY_CNT__PACK_DELAY_CNT_MASK 0x1f +#define CP_MC_PACK_DELAY_CNT__PACK_DELAY_CNT__SHIFT 0x0 +#define CP_MC_TAG_CNTL__TAG_RAM_INDEX_MASK 0x3f +#define CP_MC_TAG_CNTL__TAG_RAM_INDEX__SHIFT 0x0 +#define CP_MC_TAG_CNTL__TAG_RAM_SEL_MASK 0x30000 +#define CP_MC_TAG_CNTL__TAG_RAM_SEL__SHIFT 0x10 +#define CP_MC_TAG_DATA__TAG_RAM_DATA_MASK 0xffffffff +#define CP_MC_TAG_DATA__TAG_RAM_DATA__SHIFT 0x0 +#define CP_CSF_STAT__BUFFER_SLOTS_ALLOCATED_MASK 0xf +#define CP_CSF_STAT__BUFFER_SLOTS_ALLOCATED__SHIFT 0x0 +#define CP_CSF_STAT__BUFFER_REQUEST_COUNT_MASK 0x3f00 +#define CP_CSF_STAT__BUFFER_REQUEST_COUNT__SHIFT 0x8 +#define CP_CSF_CNTL__FETCH_BUFFER_DEPTH_MASK 0xf +#define CP_CSF_CNTL__FETCH_BUFFER_DEPTH__SHIFT 0x0 +#define CP_ME_CNTL__CE_INVALIDATE_ICACHE_MASK 0x10 +#define CP_ME_CNTL__CE_INVALIDATE_ICACHE__SHIFT 0x4 +#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE_MASK 0x40 +#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE__SHIFT 0x6 +#define CP_ME_CNTL__ME_INVALIDATE_ICACHE_MASK 0x100 +#define CP_ME_CNTL__ME_INVALIDATE_ICACHE__SHIFT 0x8 +#define CP_ME_CNTL__CE_HALT_MASK 0x1000000 +#define CP_ME_CNTL__CE_HALT__SHIFT 0x18 +#define CP_ME_CNTL__CE_STEP_MASK 0x2000000 +#define CP_ME_CNTL__CE_STEP__SHIFT 0x19 +#define CP_ME_CNTL__PFP_HALT_MASK 0x4000000 +#define CP_ME_CNTL__PFP_HALT__SHIFT 0x1a +#define CP_ME_CNTL__PFP_STEP_MASK 0x8000000 +#define CP_ME_CNTL__PFP_STEP__SHIFT 0x1b +#define CP_ME_CNTL__ME_HALT_MASK 0x10000000 +#define CP_ME_CNTL__ME_HALT__SHIFT 0x1c +#define CP_ME_CNTL__ME_STEP_MASK 0x20000000 +#define CP_ME_CNTL__ME_STEP__SHIFT 0x1d +#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS_MASK 0xff +#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS__SHIFT 0x0 +#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT_MASK 0x700 +#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT__SHIFT 0x8 +#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS_MASK 0xff00000 +#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS__SHIFT 0x14 +#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT_MASK 0x70000000 +#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT__SHIFT 0x1c +#define CP_ME_PREEMPTION__ME_CNTXSW_PREEMPTION_MASK 0x1 +#define CP_ME_PREEMPTION__ME_CNTXSW_PREEMPTION__SHIFT 0x0 +#define CP_RB0_RPTR__RB_RPTR_MASK 0xfffff +#define CP_RB0_RPTR__RB_RPTR__SHIFT 0x0 +#define CP_RB_RPTR__RB_RPTR_MASK 0xfffff +#define CP_RB_RPTR__RB_RPTR__SHIFT 0x0 +#define CP_RB1_RPTR__RB_RPTR_MASK 0xfffff +#define CP_RB1_RPTR__RB_RPTR__SHIFT 0x0 +#define CP_RB2_RPTR__RB_RPTR_MASK 0xfffff +#define CP_RB2_RPTR__RB_RPTR__SHIFT 0x0 +#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER_MASK 0xfffffff +#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER__SHIFT 0x0 +#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT_MASK 0xf0000000 +#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT__SHIFT 0x1c +#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK 0xffff +#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT 0x0 +#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK 0xffff0000 +#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT 0x10 +#define CP_CE_INIT_BASE_LO__INIT_BASE_LO_MASK 0xffffffe0 +#define CP_CE_INIT_BASE_LO__INIT_BASE_LO__SHIFT 0x5 +#define CP_CE_INIT_BASE_HI__INIT_BASE_HI_MASK 0xffff +#define CP_CE_INIT_BASE_HI__INIT_BASE_HI__SHIFT 0x0 +#define CP_CE_INIT_BUFSZ__INIT_BUFSZ_MASK 0xfff +#define CP_CE_INIT_BUFSZ__INIT_BUFSZ__SHIFT 0x0 +#define CP_CE_IB1_BASE_LO__IB1_BASE_LO_MASK 0xfffffffc +#define CP_CE_IB1_BASE_LO__IB1_BASE_LO__SHIFT 0x2 +#define CP_CE_IB1_BASE_HI__IB1_BASE_HI_MASK 0xffff +#define CP_CE_IB1_BASE_HI__IB1_BASE_HI__SHIFT 0x0 +#define CP_CE_IB1_BUFSZ__IB1_BUFSZ_MASK 0xfffff +#define CP_CE_IB1_BUFSZ__IB1_BUFSZ__SHIFT 0x0 +#define CP_CE_IB2_BASE_LO__IB2_BASE_LO_MASK 0xfffffffc +#define CP_CE_IB2_BASE_LO__IB2_BASE_LO__SHIFT 0x2 +#define CP_CE_IB2_BASE_HI__IB2_BASE_HI_MASK 0xffff +#define CP_CE_IB2_BASE_HI__IB2_BASE_HI__SHIFT 0x0 +#define CP_CE_IB2_BUFSZ__IB2_BUFSZ_MASK 0xfffff +#define CP_CE_IB2_BUFSZ__IB2_BUFSZ__SHIFT 0x0 +#define CP_IB1_BASE_LO__IB1_BASE_LO_MASK 0xfffffffc +#define CP_IB1_BASE_LO__IB1_BASE_LO__SHIFT 0x2 +#define CP_IB1_BASE_HI__IB1_BASE_HI_MASK 0xffff +#define CP_IB1_BASE_HI__IB1_BASE_HI__SHIFT 0x0 +#define CP_IB1_BUFSZ__IB1_BUFSZ_MASK 0xfffff +#define CP_IB1_BUFSZ__IB1_BUFSZ__SHIFT 0x0 +#define CP_IB2_BASE_LO__IB2_BASE_LO_MASK 0xfffffffc +#define CP_IB2_BASE_LO__IB2_BASE_LO__SHIFT 0x2 +#define CP_IB2_BASE_HI__IB2_BASE_HI_MASK 0xffff +#define CP_IB2_BASE_HI__IB2_BASE_HI__SHIFT 0x0 +#define CP_IB2_BUFSZ__IB2_BUFSZ_MASK 0xfffff +#define CP_IB2_BUFSZ__IB2_BUFSZ__SHIFT 0x0 +#define CP_ST_BASE_LO__ST_BASE_LO_MASK 0xfffffffc +#define CP_ST_BASE_LO__ST_BASE_LO__SHIFT 0x2 +#define CP_ST_BASE_HI__ST_BASE_HI_MASK 0xffff +#define CP_ST_BASE_HI__ST_BASE_HI__SHIFT 0x0 +#define CP_ST_BUFSZ__ST_BUFSZ_MASK 0xfffff +#define CP_ST_BUFSZ__ST_BUFSZ__SHIFT 0x0 +#define CP_ROQ_THRESHOLDS__IB1_START_MASK 0xff +#define CP_ROQ_THRESHOLDS__IB1_START__SHIFT 0x0 +#define CP_ROQ_THRESHOLDS__IB2_START_MASK 0xff00 +#define CP_ROQ_THRESHOLDS__IB2_START__SHIFT 0x8 +#define CP_MEQ_STQ_THRESHOLD__STQ_START_MASK 0xff +#define CP_MEQ_STQ_THRESHOLD__STQ_START__SHIFT 0x0 +#define CP_ROQ1_THRESHOLDS__RB1_START_MASK 0xff +#define CP_ROQ1_THRESHOLDS__RB1_START__SHIFT 0x0 +#define CP_ROQ1_THRESHOLDS__RB2_START_MASK 0xff00 +#define CP_ROQ1_THRESHOLDS__RB2_START__SHIFT 0x8 +#define CP_ROQ1_THRESHOLDS__R0_IB1_START_MASK 0xff0000 +#define CP_ROQ1_THRESHOLDS__R0_IB1_START__SHIFT 0x10 +#define CP_ROQ1_THRESHOLDS__R1_IB1_START_MASK 0xff000000 +#define CP_ROQ1_THRESHOLDS__R1_IB1_START__SHIFT 0x18 +#define CP_ROQ2_THRESHOLDS__R2_IB1_START_MASK 0xff +#define CP_ROQ2_THRESHOLDS__R2_IB1_START__SHIFT 0x0 +#define CP_ROQ2_THRESHOLDS__R0_IB2_START_MASK 0xff00 +#define CP_ROQ2_THRESHOLDS__R0_IB2_START__SHIFT 0x8 +#define CP_ROQ2_THRESHOLDS__R1_IB2_START_MASK 0xff0000 +#define CP_ROQ2_THRESHOLDS__R1_IB2_START__SHIFT 0x10 +#define CP_ROQ2_THRESHOLDS__R2_IB2_START_MASK 0xff000000 +#define CP_ROQ2_THRESHOLDS__R2_IB2_START__SHIFT 0x18 +#define CP_STQ_THRESHOLDS__STQ0_START_MASK 0xff +#define CP_STQ_THRESHOLDS__STQ0_START__SHIFT 0x0 +#define CP_STQ_THRESHOLDS__STQ1_START_MASK 0xff00 +#define CP_STQ_THRESHOLDS__STQ1_START__SHIFT 0x8 +#define CP_STQ_THRESHOLDS__STQ2_START_MASK 0xff0000 +#define CP_STQ_THRESHOLDS__STQ2_START__SHIFT 0x10 +#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START_MASK 0x3f +#define CP_QUEUE_THRESHOLDS__ROQ_IB1_START__SHIFT 0x0 +#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START_MASK 0x3f00 +#define CP_QUEUE_THRESHOLDS__ROQ_IB2_START__SHIFT 0x8 +#define CP_MEQ_THRESHOLDS__MEQ1_START_MASK 0xff +#define CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT 0x0 +#define CP_MEQ_THRESHOLDS__MEQ2_START_MASK 0xff00 +#define CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT 0x8 +#define CP_ROQ_AVAIL__ROQ_CNT_RING_MASK 0x7ff +#define CP_ROQ_AVAIL__ROQ_CNT_RING__SHIFT 0x0 +#define CP_ROQ_AVAIL__ROQ_CNT_IB1_MASK 0x7ff0000 +#define CP_ROQ_AVAIL__ROQ_CNT_IB1__SHIFT 0x10 +#define CP_STQ_AVAIL__STQ_CNT_MASK 0x1ff +#define CP_STQ_AVAIL__STQ_CNT__SHIFT 0x0 +#define CP_ROQ2_AVAIL__ROQ_CNT_IB2_MASK 0x7ff +#define CP_ROQ2_AVAIL__ROQ_CNT_IB2__SHIFT 0x0 +#define CP_MEQ_AVAIL__MEQ_CNT_MASK 0x3ff +#define CP_MEQ_AVAIL__MEQ_CNT__SHIFT 0x0 +#define CP_CMD_INDEX__CMD_INDEX_MASK 0x7ff +#define CP_CMD_INDEX__CMD_INDEX__SHIFT 0x0 +#define CP_CMD_INDEX__CMD_ME_SEL_MASK 0x3000 +#define CP_CMD_INDEX__CMD_ME_SEL__SHIFT 0xc +#define CP_CMD_INDEX__CMD_QUEUE_SEL_MASK 0x30000 +#define CP_CMD_INDEX__CMD_QUEUE_SEL__SHIFT 0x10 +#define CP_CMD_DATA__CMD_DATA_MASK 0xffffffff +#define CP_CMD_DATA__CMD_DATA__SHIFT 0x0 +#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY_MASK 0x3ff +#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY__SHIFT 0x0 +#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY_MASK 0x3ff0000 +#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY__SHIFT 0x10 +#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1_MASK 0x3ff +#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1__SHIFT 0x0 +#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1_MASK 0x3ff0000 +#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1__SHIFT 0x10 +#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2_MASK 0x3ff +#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2__SHIFT 0x0 +#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2_MASK 0x3ff0000 +#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2__SHIFT 0x10 +#define CP_STQ_STAT__STQ_RPTR_MASK 0x3ff +#define CP_STQ_STAT__STQ_RPTR__SHIFT 0x0 +#define CP_STQ_WR_STAT__STQ_WPTR_MASK 0x3ff +#define CP_STQ_WR_STAT__STQ_WPTR__SHIFT 0x0 +#define CP_MEQ_STAT__MEQ_RPTR_MASK 0x3ff +#define CP_MEQ_STAT__MEQ_RPTR__SHIFT 0x0 +#define CP_MEQ_STAT__MEQ_WPTR_MASK 0x3ff0000 +#define CP_MEQ_STAT__MEQ_WPTR__SHIFT 0x10 +#define CP_CEQ1_AVAIL__CEQ_CNT_RING_MASK 0x7ff +#define CP_CEQ1_AVAIL__CEQ_CNT_RING__SHIFT 0x0 +#define CP_CEQ1_AVAIL__CEQ_CNT_IB1_MASK 0x7ff0000 +#define CP_CEQ1_AVAIL__CEQ_CNT_IB1__SHIFT 0x10 +#define CP_CEQ2_AVAIL__CEQ_CNT_IB2_MASK 0x7ff +#define CP_CEQ2_AVAIL__CEQ_CNT_IB2__SHIFT 0x0 +#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY_MASK 0x3ff +#define CP_CE_ROQ_RB_STAT__CEQ_RPTR_PRIMARY__SHIFT 0x0 +#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY_MASK 0x3ff0000 +#define CP_CE_ROQ_RB_STAT__CEQ_WPTR_PRIMARY__SHIFT 0x10 +#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1_MASK 0x3ff +#define CP_CE_ROQ_IB1_STAT__CEQ_RPTR_INDIRECT1__SHIFT 0x0 +#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1_MASK 0x3ff0000 +#define CP_CE_ROQ_IB1_STAT__CEQ_WPTR_INDIRECT1__SHIFT 0x10 +#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2_MASK 0x3ff +#define CP_CE_ROQ_IB2_STAT__CEQ_RPTR_INDIRECT2__SHIFT 0x0 +#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2_MASK 0x3ff0000 +#define CP_CE_ROQ_IB2_STAT__CEQ_WPTR_INDIRECT2__SHIFT 0x10 +#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK 0x4000 +#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT 0xe +#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK 0x20000 +#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT 0x11 +#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED_MASK 0x80000 +#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED__SHIFT 0x13 +#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED_MASK 0x100000 +#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED__SHIFT 0x14 +#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED_MASK 0x400000 +#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED__SHIFT 0x16 +#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK 0x800000 +#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT 0x17 +#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK 0x1000000 +#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT 0x18 +#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK 0x4000000 +#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT 0x1a +#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK 0x8000000 +#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT 0x1b +#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK 0x20000000 +#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT 0x1d +#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK 0x40000000 +#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT 0x1e +#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK 0x80000000 +#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT 0x1f +#define CP_PERFMON_CNTL__PERFMON_STATE_MASK 0xf +#define CP_PERFMON_CNTL__PERFMON_STATE__SHIFT 0x0 +#define CP_PERFMON_CNTL__SPM_PERFMON_STATE_MASK 0xf0 +#define CP_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT 0x4 +#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK 0x300 +#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT 0x8 +#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK 0x400 +#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT 0xa +#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK 0x80000000 +#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT 0x1f +#define CP_RINGID__RINGID_MASK 0x3 +#define CP_RINGID__RINGID__SHIFT 0x0 +#define CP_PIPEID__PIPE_ID_MASK 0x3 +#define CP_PIPEID__PIPE_ID__SHIFT 0x0 +#define CP_VMID__VMID_MASK 0xf +#define CP_VMID__VMID__SHIFT 0x0 +#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET_MASK 0x7 +#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET__SHIFT 0x0 +#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET_MASK 0x3f00 +#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET__SHIFT 0x8 +#define CP_HPD_ROQ_OFFSETS__IB_OFFSET_MASK 0x3f0000 +#define CP_HPD_ROQ_OFFSETS__IB_OFFSET__SHIFT 0x10 +#define CP_HPD_EOP_BASE_ADDR__BASE_ADDR_MASK 0xffffffff +#define CP_HPD_EOP_BASE_ADDR__BASE_ADDR__SHIFT 0x0 +#define CP_HPD_EOP_BASE_ADDR_HI__BASE_ADDR_HI_MASK 0xff +#define CP_HPD_EOP_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT 0x0 +#define CP_HPD_EOP_VMID__VMID_MASK 0xf +#define CP_HPD_EOP_VMID__VMID__SHIFT 0x0 +#define CP_HPD_EOP_CONTROL__EOP_SIZE_MASK 0x3f +#define CP_HPD_EOP_CONTROL__EOP_SIZE__SHIFT 0x0 +#define CP_HPD_EOP_CONTROL__PROCESSING_EOP_MASK 0x100 +#define CP_HPD_EOP_CONTROL__PROCESSING_EOP__SHIFT 0x8 +#define CP_HPD_EOP_CONTROL__PROCESSING_QID_MASK 0xe00 +#define CP_HPD_EOP_CONTROL__PROCESSING_QID__SHIFT 0x9 +#define CP_HPD_EOP_CONTROL__PROCESS_EOP_EN_MASK 0x1000 +#define CP_HPD_EOP_CONTROL__PROCESS_EOP_EN__SHIFT 0xc +#define CP_HPD_EOP_CONTROL__PROCESSING_EOPIB_MASK 0x2000 +#define CP_HPD_EOP_CONTROL__PROCESSING_EOPIB__SHIFT 0xd +#define CP_HPD_EOP_CONTROL__PROCESS_EOPIB_EN_MASK 0x4000 +#define CP_HPD_EOP_CONTROL__PROCESS_EOPIB_EN__SHIFT 0xe +#define CP_HPD_EOP_CONTROL__EOP_ATC_MASK 0x800000 +#define CP_HPD_EOP_CONTROL__EOP_ATC__SHIFT 0x17 +#define CP_HPD_EOP_CONTROL__CACHE_POLICY_MASK 0x3000000 +#define CP_HPD_EOP_CONTROL__CACHE_POLICY__SHIFT 0x18 +#define CP_HPD_EOP_CONTROL__EOP_VOLATILE_MASK 0x4000000 +#define CP_HPD_EOP_CONTROL__EOP_VOLATILE__SHIFT 0x1a +#define CP_HPD_EOP_CONTROL__PEND_Q_SEM_MASK 0x70000000 +#define CP_HPD_EOP_CONTROL__PEND_Q_SEM__SHIFT 0x1c +#define CP_HPD_EOP_CONTROL__PEND_SIG_SEM_MASK 0x80000000 +#define CP_HPD_EOP_CONTROL__PEND_SIG_SEM__SHIFT 0x1f +#define CP_MQD_BASE_ADDR__BASE_ADDR_MASK 0xfffffffc +#define CP_MQD_BASE_ADDR__BASE_ADDR__SHIFT 0x2 +#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK 0xffff +#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT 0x0 +#define CP_HQD_ACTIVE__ACTIVE_MASK 0x1 +#define CP_HQD_ACTIVE__ACTIVE__SHIFT 0x0 +#define CP_HQD_VMID__VMID_MASK 0xf +#define CP_HQD_VMID__VMID__SHIFT 0x0 +#define CP_HQD_VMID__IB_VMID_MASK 0xf00 +#define CP_HQD_VMID__IB_VMID__SHIFT 0x8 +#define CP_HQD_VMID__VQID_MASK 0x3ff0000 +#define CP_HQD_VMID__VQID__SHIFT 0x10 +#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK 0x1 +#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ__SHIFT 0x0 +#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE_MASK 0x3ff00 +#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT 0x8 +#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE_MASK 0x80000000 +#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE__SHIFT 0x1f +#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY_MASK 0x3 +#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY__SHIFT 0x0 +#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK 0xf +#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT 0x0 +#define CP_HQD_QUANTUM__QUANTUM_EN_MASK 0x1 +#define CP_HQD_QUANTUM__QUANTUM_EN__SHIFT 0x0 +#define CP_HQD_QUANTUM__QUANTUM_SCALE_MASK 0x10 +#define CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT 0x4 +#define CP_HQD_QUANTUM__QUANTUM_DURATION_MASK 0x3f00 +#define CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT 0x8 +#define CP_HQD_PQ_BASE__ADDR_MASK 0xffffffff +#define CP_HQD_PQ_BASE__ADDR__SHIFT 0x0 +#define CP_HQD_PQ_BASE_HI__ADDR_HI_MASK 0xff +#define CP_HQD_PQ_BASE_HI__ADDR_HI__SHIFT 0x0 +#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET_MASK 0xffffffff +#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET__SHIFT 0x0 +#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR_MASK 0xfffffffc +#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR__SHIFT 0x2 +#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI_MASK 0xffff +#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI__SHIFT 0x0 +#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR_MASK 0xfffffffc +#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR__SHIFT 0x2 +#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI_MASK 0xffff +#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI__SHIFT 0x0 +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK 0x7ffffc +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT 0x2 +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK 0x10000000 +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE__SHIFT 0x1c +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT_MASK 0x20000000 +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT__SHIFT 0x1d +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK 0x40000000 +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT 0x1e +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK 0x80000000 +#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT 0x1f +#define CP_HQD_PQ_WPTR__OFFSET_MASK 0xffffffff +#define CP_HQD_PQ_WPTR__OFFSET__SHIFT 0x0 +#define CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK 0x3f +#define CP_HQD_PQ_CONTROL__QUEUE_SIZE__SHIFT 0x0 +#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK 0x3f00 +#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT 0x8 +#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP_MASK 0x30000 +#define CP_HQD_PQ_CONTROL__ENDIAN_SWAP__SHIFT 0x10 +#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE_MASK 0x300000 +#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE__SHIFT 0x14 +#define CP_HQD_PQ_CONTROL__PQ_ATC_MASK 0x800000 +#define CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT 0x17 +#define CP_HQD_PQ_CONTROL__CACHE_POLICY_MASK 0x3000000 +#define CP_HQD_PQ_CONTROL__CACHE_POLICY__SHIFT 0x18 +#define CP_HQD_PQ_CONTROL__PQ_VOLATILE_MASK 0x4000000 +#define CP_HQD_PQ_CONTROL__PQ_VOLATILE__SHIFT 0x1a +#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK 0x8000000 +#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR__SHIFT 0x1b +#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK 0x10000000 +#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH__SHIFT 0x1c +#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP_MASK 0x20000000 +#define CP_HQD_PQ_CONTROL__ROQ_PQ_IB_FLIP__SHIFT 0x1d +#define CP_HQD_PQ_CONTROL__PRIV_STATE_MASK 0x40000000 +#define CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT 0x1e +#define CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK 0x80000000 +#define CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT 0x1f +#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR_MASK 0xfffffffc +#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR__SHIFT 0x2 +#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI_MASK 0xffff +#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI__SHIFT 0x0 +#define CP_HQD_IB_RPTR__CONSUMED_OFFSET_MASK 0xfffff +#define CP_HQD_IB_RPTR__CONSUMED_OFFSET__SHIFT 0x0 +#define CP_HQD_IB_CONTROL__IB_SIZE_MASK 0xfffff +#define CP_HQD_IB_CONTROL__IB_SIZE__SHIFT 0x0 +#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE_MASK 0x300000 +#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT 0x14 +#define CP_HQD_IB_CONTROL__IB_ATC_MASK 0x800000 +#define CP_HQD_IB_CONTROL__IB_ATC__SHIFT 0x17 +#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY_MASK 0x3000000 +#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY__SHIFT 0x18 +#define CP_HQD_IB_CONTROL__IB_VOLATILE_MASK 0x4000000 +#define CP_HQD_IB_CONTROL__IB_VOLATILE__SHIFT 0x1a +#define CP_HQD_IB_CONTROL__PROCESSING_IB_MASK 0x80000000 +#define CP_HQD_IB_CONTROL__PROCESSING_IB__SHIFT 0x1f +#define CP_HQD_IQ_TIMER__WAIT_TIME_MASK 0xff +#define CP_HQD_IQ_TIMER__WAIT_TIME__SHIFT 0x0 +#define CP_HQD_IQ_TIMER__RETRY_TYPE_MASK 0x700 +#define CP_HQD_IQ_TIMER__RETRY_TYPE__SHIFT 0x8 +#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK 0x3000 +#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT 0xc +#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE_MASK 0x3f0000 +#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE__SHIFT 0x10 +#define CP_HQD_IQ_TIMER__IQ_ATC_MASK 0x800000 +#define CP_HQD_IQ_TIMER__IQ_ATC__SHIFT 0x17 +#define CP_HQD_IQ_TIMER__CACHE_POLICY_MASK 0x3000000 +#define CP_HQD_IQ_TIMER__CACHE_POLICY__SHIFT 0x18 +#define CP_HQD_IQ_TIMER__IQ_VOLATILE_MASK 0x4000000 +#define CP_HQD_IQ_TIMER__IQ_VOLATILE__SHIFT 0x1a +#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN_MASK 0x20000000 +#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN__SHIFT 0x1d +#define CP_HQD_IQ_TIMER__PROCESSING_IQ_MASK 0x40000000 +#define CP_HQD_IQ_TIMER__PROCESSING_IQ__SHIFT 0x1e +#define CP_HQD_IQ_TIMER__ACTIVE_MASK 0x80000000 +#define CP_HQD_IQ_TIMER__ACTIVE__SHIFT 0x1f +#define CP_HQD_IQ_RPTR__OFFSET_MASK 0x3f +#define CP_HQD_IQ_RPTR__OFFSET__SHIFT 0x0 +#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK 0x3 +#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT 0x0 +#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK 0x10 +#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT 0x4 +#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT_MASK 0x100 +#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT__SHIFT 0x8 +#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_MASK 0x1 +#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD__SHIFT 0x0 +#define CP_HQD_SEMA_CMD__RETRY_MASK 0x1 +#define CP_HQD_SEMA_CMD__RETRY__SHIFT 0x0 +#define CP_HQD_SEMA_CMD__RESULT_MASK 0x6 +#define CP_HQD_SEMA_CMD__RESULT__SHIFT 0x1 +#define CP_HQD_MSG_TYPE__ACTION_MASK 0x3 +#define CP_HQD_MSG_TYPE__ACTION__SHIFT 0x0 +#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO_MASK 0xffffffff +#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO__SHIFT 0x0 +#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI_MASK 0xffffffff +#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI__SHIFT 0x0 +#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO_MASK 0xffffffff +#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO__SHIFT 0x0 +#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI_MASK 0xffffffff +#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI__SHIFT 0x0 +#define CP_HQD_HQ_SCHEDULER0__DEQUEUE_STATUS_MASK 0x3 +#define CP_HQD_HQ_SCHEDULER0__DEQUEUE_STATUS__SHIFT 0x0 +#define CP_HQD_HQ_SCHEDULER0__DEQUEUE_RETRY_CNT_MASK 0xc +#define CP_HQD_HQ_SCHEDULER0__DEQUEUE_RETRY_CNT__SHIFT 0x2 +#define CP_HQD_HQ_SCHEDULER0__RSV_5_4_MASK 0x30 +#define CP_HQD_HQ_SCHEDULER0__RSV_5_4__SHIFT 0x4 +#define CP_HQD_HQ_SCHEDULER0__QUEUE_RUN_ONCE_MASK 0x40 +#define CP_HQD_HQ_SCHEDULER0__QUEUE_RUN_ONCE__SHIFT 0x6 +#define CP_HQD_HQ_SCHEDULER0__SCRATCH_RAM_INIT_MASK 0x80 +#define CP_HQD_HQ_SCHEDULER0__SCRATCH_RAM_INIT__SHIFT 0x7 +#define CP_HQD_HQ_SCHEDULER0__TCL2_DIRTY_MASK 0x100 +#define CP_HQD_HQ_SCHEDULER0__TCL2_DIRTY__SHIFT 0x8 +#define CP_HQD_HQ_SCHEDULER0__PG_ACTIVATED_MASK 0x200 +#define CP_HQD_HQ_SCHEDULER0__PG_ACTIVATED__SHIFT 0x9 +#define CP_HQD_HQ_SCHEDULER0__CG_ACTIVATED_MASK 0x400 +#define CP_HQD_HQ_SCHEDULER0__CG_ACTIVATED__SHIFT 0xa +#define CP_HQD_HQ_SCHEDULER0__RSVR_31_11_MASK 0xfffff800 +#define CP_HQD_HQ_SCHEDULER0__RSVR_31_11__SHIFT 0xb +#define CP_HQD_HQ_SCHEDULER1__SCHEDULER_MASK 0xffffffff +#define CP_HQD_HQ_SCHEDULER1__SCHEDULER__SHIFT 0x0 +#define CP_MQD_CONTROL__VMID_MASK 0xf +#define CP_MQD_CONTROL__VMID__SHIFT 0x0 +#define CP_MQD_CONTROL__MQD_ATC_MASK 0x800000 +#define CP_MQD_CONTROL__MQD_ATC__SHIFT 0x17 +#define CP_MQD_CONTROL__CACHE_POLICY_MASK 0x3000000 +#define CP_MQD_CONTROL__CACHE_POLICY__SHIFT 0x18 +#define CP_MQD_CONTROL__MQD_VOLATILE_MASK 0x4000000 +#define CP_MQD_CONTROL__MQD_VOLATILE__SHIFT 0x1a +#define DB_Z_READ_BASE__BASE_256B_MASK 0xffffffff +#define DB_Z_READ_BASE__BASE_256B__SHIFT 0x0 +#define DB_STENCIL_READ_BASE__BASE_256B_MASK 0xffffffff +#define DB_STENCIL_READ_BASE__BASE_256B__SHIFT 0x0 +#define DB_Z_WRITE_BASE__BASE_256B_MASK 0xffffffff +#define DB_Z_WRITE_BASE__BASE_256B__SHIFT 0x0 +#define DB_STENCIL_WRITE_BASE__BASE_256B_MASK 0xffffffff +#define DB_STENCIL_WRITE_BASE__BASE_256B__SHIFT 0x0 +#define DB_DEPTH_INFO__ADDR5_SWIZZLE_MASK_MASK 0xf +#define DB_DEPTH_INFO__ADDR5_SWIZZLE_MASK__SHIFT 0x0 +#define DB_DEPTH_INFO__ARRAY_MODE_MASK 0xf0 +#define DB_DEPTH_INFO__ARRAY_MODE__SHIFT 0x4 +#define DB_DEPTH_INFO__PIPE_CONFIG_MASK 0x1f00 +#define DB_DEPTH_INFO__PIPE_CONFIG__SHIFT 0x8 +#define DB_DEPTH_INFO__BANK_WIDTH_MASK 0x6000 +#define DB_DEPTH_INFO__BANK_WIDTH__SHIFT 0xd +#define DB_DEPTH_INFO__BANK_HEIGHT_MASK 0x18000 +#define DB_DEPTH_INFO__BANK_HEIGHT__SHIFT 0xf +#define DB_DEPTH_INFO__MACRO_TILE_ASPECT_MASK 0x60000 +#define DB_DEPTH_INFO__MACRO_TILE_ASPECT__SHIFT 0x11 +#define DB_DEPTH_INFO__NUM_BANKS_MASK 0x180000 +#define DB_DEPTH_INFO__NUM_BANKS__SHIFT 0x13 +#define DB_Z_INFO__FORMAT_MASK 0x3 +#define DB_Z_INFO__FORMAT__SHIFT 0x0 +#define DB_Z_INFO__NUM_SAMPLES_MASK 0xc +#define DB_Z_INFO__NUM_SAMPLES__SHIFT 0x2 +#define DB_Z_INFO__TILE_SPLIT_MASK 0xe000 +#define DB_Z_INFO__TILE_SPLIT__SHIFT 0xd +#define DB_Z_INFO__TILE_MODE_INDEX_MASK 0x700000 +#define DB_Z_INFO__TILE_MODE_INDEX__SHIFT 0x14 +#define DB_Z_INFO__ALLOW_EXPCLEAR_MASK 0x8000000 +#define DB_Z_INFO__ALLOW_EXPCLEAR__SHIFT 0x1b +#define DB_Z_INFO__READ_SIZE_MASK 0x10000000 +#define DB_Z_INFO__READ_SIZE__SHIFT 0x1c +#define DB_Z_INFO__TILE_SURFACE_ENABLE_MASK 0x20000000 +#define DB_Z_INFO__TILE_SURFACE_ENABLE__SHIFT 0x1d +#define DB_Z_INFO__ZRANGE_PRECISION_MASK 0x80000000 +#define DB_Z_INFO__ZRANGE_PRECISION__SHIFT 0x1f +#define DB_STENCIL_INFO__FORMAT_MASK 0x1 +#define DB_STENCIL_INFO__FORMAT__SHIFT 0x0 +#define DB_STENCIL_INFO__TILE_SPLIT_MASK 0xe000 +#define DB_STENCIL_INFO__TILE_SPLIT__SHIFT 0xd +#define DB_STENCIL_INFO__TILE_MODE_INDEX_MASK 0x700000 +#define DB_STENCIL_INFO__TILE_MODE_INDEX__SHIFT 0x14 +#define DB_STENCIL_INFO__ALLOW_EXPCLEAR_MASK 0x8000000 +#define DB_STENCIL_INFO__ALLOW_EXPCLEAR__SHIFT 0x1b +#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE_MASK 0x20000000 +#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE__SHIFT 0x1d +#define DB_DEPTH_SIZE__PITCH_TILE_MAX_MASK 0x7ff +#define DB_DEPTH_SIZE__PITCH_TILE_MAX__SHIFT 0x0 +#define DB_DEPTH_SIZE__HEIGHT_TILE_MAX_MASK 0x3ff800 +#define DB_DEPTH_SIZE__HEIGHT_TILE_MAX__SHIFT 0xb +#define DB_DEPTH_SLICE__SLICE_TILE_MAX_MASK 0x3fffff +#define DB_DEPTH_SLICE__SLICE_TILE_MAX__SHIFT 0x0 +#define DB_DEPTH_VIEW__SLICE_START_MASK 0x7ff +#define DB_DEPTH_VIEW__SLICE_START__SHIFT 0x0 +#define DB_DEPTH_VIEW__SLICE_MAX_MASK 0xffe000 +#define DB_DEPTH_VIEW__SLICE_MAX__SHIFT 0xd +#define DB_DEPTH_VIEW__Z_READ_ONLY_MASK 0x1000000 +#define DB_DEPTH_VIEW__Z_READ_ONLY__SHIFT 0x18 +#define DB_DEPTH_VIEW__STENCIL_READ_ONLY_MASK 0x2000000 +#define DB_DEPTH_VIEW__STENCIL_READ_ONLY__SHIFT 0x19 +#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE_MASK 0x1 +#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE__SHIFT 0x0 +#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE_MASK 0x2 +#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE__SHIFT 0x1 +#define DB_RENDER_CONTROL__DEPTH_COPY_MASK 0x4 +#define DB_RENDER_CONTROL__DEPTH_COPY__SHIFT 0x2 +#define DB_RENDER_CONTROL__STENCIL_COPY_MASK 0x8 +#define DB_RENDER_CONTROL__STENCIL_COPY__SHIFT 0x3 +#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE_MASK 0x10 +#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE__SHIFT 0x4 +#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE_MASK 0x20 +#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE__SHIFT 0x5 +#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE_MASK 0x40 +#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE__SHIFT 0x6 +#define DB_RENDER_CONTROL__COPY_CENTROID_MASK 0x80 +#define DB_RENDER_CONTROL__COPY_CENTROID__SHIFT 0x7 +#define DB_RENDER_CONTROL__COPY_SAMPLE_MASK 0xf00 +#define DB_RENDER_CONTROL__COPY_SAMPLE__SHIFT 0x8 +#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE_MASK 0x1 +#define DB_COUNT_CONTROL__ZPASS_INCREMENT_DISABLE__SHIFT 0x0 +#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS_MASK 0x2 +#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS__SHIFT 0x1 +#define DB_COUNT_CONTROL__SAMPLE_RATE_MASK 0x70 +#define DB_COUNT_CONTROL__SAMPLE_RATE__SHIFT 0x4 +#define DB_COUNT_CONTROL__ZPASS_ENABLE_MASK 0xf00 +#define DB_COUNT_CONTROL__ZPASS_ENABLE__SHIFT 0x8 +#define DB_COUNT_CONTROL__ZFAIL_ENABLE_MASK 0xf000 +#define DB_COUNT_CONTROL__ZFAIL_ENABLE__SHIFT 0xc +#define DB_COUNT_CONTROL__SFAIL_ENABLE_MASK 0xf0000 +#define DB_COUNT_CONTROL__SFAIL_ENABLE__SHIFT 0x10 +#define DB_COUNT_CONTROL__DBFAIL_ENABLE_MASK 0xf00000 +#define DB_COUNT_CONTROL__DBFAIL_ENABLE__SHIFT 0x14 +#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE_MASK 0xf000000 +#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE__SHIFT 0x18 +#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE_MASK 0xf0000000 +#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE__SHIFT 0x1c +#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK 0x3 +#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT 0x0 +#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0_MASK 0xc +#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0__SHIFT 0x2 +#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1_MASK 0x30 +#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1__SHIFT 0x4 +#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER_MASK 0x40 +#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER__SHIFT 0x6 +#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK 0x80 +#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT 0x7 +#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK 0x100 +#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT 0x8 +#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE_MASK 0x200 +#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE__SHIFT 0x9 +#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL_MASK 0x400 +#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL__SHIFT 0xa +#define DB_RENDER_OVERRIDE__FORCE_Z_READ_MASK 0x800 +#define DB_RENDER_OVERRIDE__FORCE_Z_READ__SHIFT 0xb +#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ_MASK 0x1000 +#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ__SHIFT 0xc +#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK 0x6000 +#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT 0xd +#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT_MASK 0x8000 +#define DB_RENDER_OVERRIDE__FORCE_QC_SMASK_CONFLICT__SHIFT 0xf +#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP_MASK 0x10000 +#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP__SHIFT 0x10 +#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE_MASK 0x20000 +#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE__SHIFT 0x11 +#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED_MASK 0x40000 +#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED__SHIFT 0x12 +#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM_MASK 0x180000 +#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM__SHIFT 0x13 +#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT_MASK 0x3e00000 +#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT__SHIFT 0x15 +#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK 0x4000000 +#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT 0x1a +#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY_MASK 0x8000000 +#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY__SHIFT 0x1b +#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY_MASK 0x10000000 +#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY__SHIFT 0x1c +#define DB_RENDER_OVERRIDE__FORCE_Z_VALID_MASK 0x20000000 +#define DB_RENDER_OVERRIDE__FORCE_Z_VALID__SHIFT 0x1d +#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID_MASK 0x40000000 +#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID__SHIFT 0x1e +#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION_MASK 0x80000000 +#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION__SHIFT 0x1f +#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK 0x3 +#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL__SHIFT 0x0 +#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK 0x1c +#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN__SHIFT 0x2 +#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION_MASK 0x20 +#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION__SHIFT 0x5 +#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION_MASK 0x40 +#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION__SHIFT 0x6 +#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION_MASK 0x80 +#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION__SHIFT 0x7 +#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH_MASK 0x100 +#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH__SHIFT 0x8 +#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP_MASK 0x200 +#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP__SHIFT 0x9 +#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK 0x400 +#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT 0xa +#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE_MASK 0x800 +#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE__SHIFT 0xb +#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC_MASK 0x7000 +#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC__SHIFT 0xc +#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF_MASK 0x38000 +#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF__SHIFT 0xf +#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF_MASK 0x1c0000 +#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF__SHIFT 0x12 +#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE_MASK 0x200000 +#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE__SHIFT 0x15 +#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS_MASK 0x400000 +#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS__SHIFT 0x16 +#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK 0x800000 +#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT 0x17 +#define DB_EQAA__MAX_ANCHOR_SAMPLES_MASK 0x7 +#define DB_EQAA__MAX_ANCHOR_SAMPLES__SHIFT 0x0 +#define DB_EQAA__PS_ITER_SAMPLES_MASK 0x70 +#define DB_EQAA__PS_ITER_SAMPLES__SHIFT 0x4 +#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES_MASK 0x700 +#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES__SHIFT 0x8 +#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES_MASK 0x7000 +#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES__SHIFT 0xc +#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS_MASK 0x10000 +#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS__SHIFT 0x10 +#define DB_EQAA__INCOHERENT_EQAA_READS_MASK 0x20000 +#define DB_EQAA__INCOHERENT_EQAA_READS__SHIFT 0x11 +#define DB_EQAA__INTERPOLATE_COMP_Z_MASK 0x40000 +#define DB_EQAA__INTERPOLATE_COMP_Z__SHIFT 0x12 +#define DB_EQAA__INTERPOLATE_SRC_Z_MASK 0x80000 +#define DB_EQAA__INTERPOLATE_SRC_Z__SHIFT 0x13 +#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS_MASK 0x100000 +#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS__SHIFT 0x14 +#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE_MASK 0x200000 +#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE__SHIFT 0x15 +#define DB_EQAA__OVERRASTERIZATION_AMOUNT_MASK 0x7000000 +#define DB_EQAA__OVERRASTERIZATION_AMOUNT__SHIFT 0x18 +#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION_MASK 0x8000000 +#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION__SHIFT 0x1b +#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE_MASK 0x1 +#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE__SHIFT 0x0 +#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE_MASK 0x2 +#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE__SHIFT 0x1 +#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE_MASK 0x4 +#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE__SHIFT 0x2 +#define DB_SHADER_CONTROL__Z_ORDER_MASK 0x30 +#define DB_SHADER_CONTROL__Z_ORDER__SHIFT 0x4 +#define DB_SHADER_CONTROL__KILL_ENABLE_MASK 0x40 +#define DB_SHADER_CONTROL__KILL_ENABLE__SHIFT 0x6 +#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE_MASK 0x80 +#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE__SHIFT 0x7 +#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE_MASK 0x100 +#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE__SHIFT 0x8 +#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL_MASK 0x200 +#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL__SHIFT 0x9 +#define DB_SHADER_CONTROL__EXEC_ON_NOOP_MASK 0x400 +#define DB_SHADER_CONTROL__EXEC_ON_NOOP__SHIFT 0xa +#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE_MASK 0x800 +#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE__SHIFT 0xb +#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER_MASK 0x1000 +#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER__SHIFT 0xc +#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT_MASK 0x6000 +#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT 0xd +#define DB_DEPTH_BOUNDS_MIN__MIN_MASK 0xffffffff +#define DB_DEPTH_BOUNDS_MIN__MIN__SHIFT 0x0 +#define DB_DEPTH_BOUNDS_MAX__MAX_MASK 0xffffffff +#define DB_DEPTH_BOUNDS_MAX__MAX__SHIFT 0x0 +#define DB_STENCIL_CLEAR__CLEAR_MASK 0xff +#define DB_STENCIL_CLEAR__CLEAR__SHIFT 0x0 +#define DB_DEPTH_CLEAR__DEPTH_CLEAR_MASK 0xffffffff +#define DB_DEPTH_CLEAR__DEPTH_CLEAR__SHIFT 0x0 +#define DB_HTILE_DATA_BASE__BASE_256B_MASK 0xffffffff +#define DB_HTILE_DATA_BASE__BASE_256B__SHIFT 0x0 +#define DB_HTILE_SURFACE__LINEAR_MASK 0x1 +#define DB_HTILE_SURFACE__LINEAR__SHIFT 0x0 +#define DB_HTILE_SURFACE__FULL_CACHE_MASK 0x2 +#define DB_HTILE_SURFACE__FULL_CACHE__SHIFT 0x1 +#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN_MASK 0x4 +#define DB_HTILE_SURFACE__HTILE_USES_PRELOAD_WIN__SHIFT 0x2 +#define DB_HTILE_SURFACE__PRELOAD_MASK 0x8 +#define DB_HTILE_SURFACE__PRELOAD__SHIFT 0x3 +#define DB_HTILE_SURFACE__PREFETCH_WIDTH_MASK 0x3f0 +#define DB_HTILE_SURFACE__PREFETCH_WIDTH__SHIFT 0x4 +#define DB_HTILE_SURFACE__PREFETCH_HEIGHT_MASK 0xfc00 +#define DB_HTILE_SURFACE__PREFETCH_HEIGHT__SHIFT 0xa +#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK 0x10000 +#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT 0x10 +#define DB_PRELOAD_CONTROL__START_X_MASK 0xff +#define DB_PRELOAD_CONTROL__START_X__SHIFT 0x0 +#define DB_PRELOAD_CONTROL__START_Y_MASK 0xff00 +#define DB_PRELOAD_CONTROL__START_Y__SHIFT 0x8 +#define DB_PRELOAD_CONTROL__MAX_X_MASK 0xff0000 +#define DB_PRELOAD_CONTROL__MAX_X__SHIFT 0x10 +#define DB_PRELOAD_CONTROL__MAX_Y_MASK 0xff000000 +#define DB_PRELOAD_CONTROL__MAX_Y__SHIFT 0x18 +#define DB_STENCILREFMASK__STENCILTESTVAL_MASK 0xff +#define DB_STENCILREFMASK__STENCILTESTVAL__SHIFT 0x0 +#define DB_STENCILREFMASK__STENCILMASK_MASK 0xff00 +#define DB_STENCILREFMASK__STENCILMASK__SHIFT 0x8 +#define DB_STENCILREFMASK__STENCILWRITEMASK_MASK 0xff0000 +#define DB_STENCILREFMASK__STENCILWRITEMASK__SHIFT 0x10 +#define DB_STENCILREFMASK__STENCILOPVAL_MASK 0xff000000 +#define DB_STENCILREFMASK__STENCILOPVAL__SHIFT 0x18 +#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF_MASK 0xff +#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF__SHIFT 0x0 +#define DB_STENCILREFMASK_BF__STENCILMASK_BF_MASK 0xff00 +#define DB_STENCILREFMASK_BF__STENCILMASK_BF__SHIFT 0x8 +#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF_MASK 0xff0000 +#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF__SHIFT 0x10 +#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF_MASK 0xff000000 +#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF__SHIFT 0x18 +#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0_MASK 0x7 +#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT 0x0 +#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0_MASK 0xff0 +#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT 0x4 +#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0_MASK 0xff000 +#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0__SHIFT 0xc +#define DB_SRESULTS_COMPARE_STATE0__ENABLE0_MASK 0x1000000 +#define DB_SRESULTS_COMPARE_STATE0__ENABLE0__SHIFT 0x18 +#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1_MASK 0x7 +#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1__SHIFT 0x0 +#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1_MASK 0xff0 +#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1__SHIFT 0x4 +#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1_MASK 0xff000 +#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1__SHIFT 0xc +#define DB_SRESULTS_COMPARE_STATE1__ENABLE1_MASK 0x1000000 +#define DB_SRESULTS_COMPARE_STATE1__ENABLE1__SHIFT 0x18 +#define DB_DEPTH_CONTROL__STENCIL_ENABLE_MASK 0x1 +#define DB_DEPTH_CONTROL__STENCIL_ENABLE__SHIFT 0x0 +#define DB_DEPTH_CONTROL__Z_ENABLE_MASK 0x2 +#define DB_DEPTH_CONTROL__Z_ENABLE__SHIFT 0x1 +#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE_MASK 0x4 +#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE__SHIFT 0x2 +#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE_MASK 0x8 +#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE__SHIFT 0x3 +#define DB_DEPTH_CONTROL__ZFUNC_MASK 0x70 +#define DB_DEPTH_CONTROL__ZFUNC__SHIFT 0x4 +#define DB_DEPTH_CONTROL__BACKFACE_ENABLE_MASK 0x80 +#define DB_DEPTH_CONTROL__BACKFACE_ENABLE__SHIFT 0x7 +#define DB_DEPTH_CONTROL__STENCILFUNC_MASK 0x700 +#define DB_DEPTH_CONTROL__STENCILFUNC__SHIFT 0x8 +#define DB_DEPTH_CONTROL__STENCILFUNC_BF_MASK 0x700000 +#define DB_DEPTH_CONTROL__STENCILFUNC_BF__SHIFT 0x14 +#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_MASK 0x40000000 +#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL__SHIFT 0x1e +#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS_MASK 0x80000000 +#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS__SHIFT 0x1f +#define DB_STENCIL_CONTROL__STENCILFAIL_MASK 0xf +#define DB_STENCIL_CONTROL__STENCILFAIL__SHIFT 0x0 +#define DB_STENCIL_CONTROL__STENCILZPASS_MASK 0xf0 +#define DB_STENCIL_CONTROL__STENCILZPASS__SHIFT 0x4 +#define DB_STENCIL_CONTROL__STENCILZFAIL_MASK 0xf00 +#define DB_STENCIL_CONTROL__STENCILZFAIL__SHIFT 0x8 +#define DB_STENCIL_CONTROL__STENCILFAIL_BF_MASK 0xf000 +#define DB_STENCIL_CONTROL__STENCILFAIL_BF__SHIFT 0xc +#define DB_STENCIL_CONTROL__STENCILZPASS_BF_MASK 0xf0000 +#define DB_STENCIL_CONTROL__STENCILZPASS_BF__SHIFT 0x10 +#define DB_STENCIL_CONTROL__STENCILZFAIL_BF_MASK 0xf00000 +#define DB_STENCIL_CONTROL__STENCILZFAIL_BF__SHIFT 0x14 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE_MASK 0x1 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE__SHIFT 0x0 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0_MASK 0x300 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0__SHIFT 0x8 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1_MASK 0xc00 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1__SHIFT 0xa +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2_MASK 0x3000 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2__SHIFT 0xc +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3_MASK 0xc000 +#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3__SHIFT 0xe +#define DB_ALPHA_TO_MASK__OFFSET_ROUND_MASK 0x10000 +#define DB_ALPHA_TO_MASK__OFFSET_ROUND__SHIFT 0x10 +#define DB_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define DB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define DB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define DB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define DB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define DB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define DB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define DB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define DB_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define DB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define DB_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define DB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define DB_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00 +#define DB_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define DB_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define DB_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define DB_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000 +#define DB_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18 +#define DB_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define DB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define DB_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define DB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define DB_PERFCOUNTER2_SELECT__PERF_SEL1_MASK 0xffc00 +#define DB_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT 0xa +#define DB_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define DB_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define DB_PERFCOUNTER2_SELECT__PERF_MODE1_MASK 0xf000000 +#define DB_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT 0x18 +#define DB_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define DB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define DB_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define DB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define DB_PERFCOUNTER3_SELECT__PERF_SEL1_MASK 0xffc00 +#define DB_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT 0xa +#define DB_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define DB_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define DB_PERFCOUNTER3_SELECT__PERF_MODE1_MASK 0xf000000 +#define DB_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT 0x18 +#define DB_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define DB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000 +#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18 +#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c +#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff +#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0 +#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00 +#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa +#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf000000 +#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x18 +#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x1c +#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE_MASK 0x1 +#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE__SHIFT 0x0 +#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE_MASK 0x2 +#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE__SHIFT 0x1 +#define DB_DEBUG__FETCH_FULL_Z_TILE_MASK 0x4 +#define DB_DEBUG__FETCH_FULL_Z_TILE__SHIFT 0x2 +#define DB_DEBUG__FETCH_FULL_STENCIL_TILE_MASK 0x8 +#define DB_DEBUG__FETCH_FULL_STENCIL_TILE__SHIFT 0x3 +#define DB_DEBUG__FORCE_Z_MODE_MASK 0x30 +#define DB_DEBUG__FORCE_Z_MODE__SHIFT 0x4 +#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ_MASK 0x40 +#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ__SHIFT 0x6 +#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ_MASK 0x80 +#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ__SHIFT 0x7 +#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE_MASK 0x300 +#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE__SHIFT 0x8 +#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0_MASK 0xc00 +#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0__SHIFT 0xa +#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1_MASK 0x3000 +#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1__SHIFT 0xc +#define DB_DEBUG__DEBUG_FAST_Z_DISABLE_MASK 0x4000 +#define DB_DEBUG__DEBUG_FAST_Z_DISABLE__SHIFT 0xe +#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE_MASK 0x8000 +#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE__SHIFT 0xf +#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE_MASK 0x10000 +#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE__SHIFT 0x10 +#define DB_DEBUG__DISABLE_SUMM_SQUADS_MASK 0x20000 +#define DB_DEBUG__DISABLE_SUMM_SQUADS__SHIFT 0x11 +#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS_MASK 0x40000 +#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS__SHIFT 0x12 +#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE_MASK 0x180000 +#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE__SHIFT 0x13 +#define DB_DEBUG__NEVER_FREE_Z_ONLY_MASK 0x200000 +#define DB_DEBUG__NEVER_FREE_Z_ONLY__SHIFT 0x15 +#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS_MASK 0x400000 +#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS__SHIFT 0x16 +#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK 0x800000 +#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT 0x17 +#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES_MASK 0xf000000 +#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES__SHIFT 0x18 +#define DB_DEBUG__ONE_FREE_IN_FLIGHT_MASK 0x10000000 +#define DB_DEBUG__ONE_FREE_IN_FLIGHT__SHIFT 0x1c +#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT_MASK 0x20000000 +#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT__SHIFT 0x1d +#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC_MASK 0x40000000 +#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC__SHIFT 0x1e +#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC_MASK 0x80000000 +#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC__SHIFT 0x1f +#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING_MASK 0x1 +#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING__SHIFT 0x0 +#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE_MASK 0x2 +#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE__SHIFT 0x1 +#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE_MASK 0x4 +#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE__SHIFT 0x2 +#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB_MASK 0x8 +#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB__SHIFT 0x3 +#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM_MASK 0x10 +#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM__SHIFT 0x4 +#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_MASK 0x20 +#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL__SHIFT 0x5 +#define DB_DEBUG2__ENABLE_PREZL_CB_STALL_MASK 0x40 +#define DB_DEBUG2__ENABLE_PREZL_CB_STALL__SHIFT 0x6 +#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_REZ_MASK 0x80 +#define DB_DEBUG2__DISABLE_PREZL_LPF_STALL_REZ__SHIFT 0x7 +#define DB_DEBUG2__DISABLE_PREZL_CB_STALL_REZ_MASK 0x100 +#define DB_DEBUG2__DISABLE_PREZL_CB_STALL_REZ__SHIFT 0x8 +#define DB_DEBUG2__CLK_OFF_DELAY_MASK 0x3e00 +#define DB_DEBUG2__CLK_OFF_DELAY__SHIFT 0x9 +#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER_MASK 0x4000 +#define DB_DEBUG2__DISABLE_TILE_COVERED_FOR_PS_ITER__SHIFT 0xe +#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING_MASK 0x8000 +#define DB_DEBUG2__ENABLE_SUBTILE_GROUPING__SHIFT 0xf +#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES_MASK 0x10000 +#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES__SHIFT 0x10 +#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING_MASK 0x20000 +#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING__SHIFT 0x11 +#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING_MASK 0x40000 +#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING__SHIFT 0x12 +#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL_MASK 0x80000 +#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL__SHIFT 0x13 +#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM_MASK 0x10000000 +#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM__SHIFT 0x1c +#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL_MASK 0x20000000 +#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL__SHIFT 0x1d +#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM_MASK 0x40000000 +#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM__SHIFT 0x1e +#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT_MASK 0x80000000 +#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT__SHIFT 0x1f +#define DB_DEBUG3__FORCE_DB_IS_GOOD_MASK 0x4 +#define DB_DEBUG3__FORCE_DB_IS_GOOD__SHIFT 0x2 +#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION_MASK 0x8 +#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION__SHIFT 0x3 +#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP_MASK 0x10 +#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT 0x4 +#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z_MASK 0x20 +#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z__SHIFT 0x5 +#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z_MASK 0x40 +#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z__SHIFT 0x6 +#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS_MASK 0x80 +#define DB_DEBUG3__DISABLE_TCP_CAM_BYPASS__SHIFT 0x7 +#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION_MASK 0x100 +#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION__SHIFT 0x8 +#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT_MASK 0x200 +#define DB_DEBUG3__DISABLE_REDUNDANT_PLANE_FLUSHES_OPT__SHIFT 0x9 +#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP_MASK 0x400 +#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP__SHIFT 0xa +#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS_MASK 0x800 +#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS__SHIFT 0xb +#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING_MASK 0x1000 +#define DB_DEBUG3__DISABLE_OP_Z_DATA_FORWARDING__SHIFT 0xc +#define DB_DEBUG3__DISABLE_OP_DF_BYPASS_MASK 0x2000 +#define DB_DEBUG3__DISABLE_OP_DF_BYPASS__SHIFT 0xd +#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE_MASK 0x4000 +#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE__SHIFT 0xe +#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK_MASK 0x8000 +#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK__SHIFT 0xf +#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION_MASK 0x10000 +#define DB_DEBUG3__ALLOW_RF2P_RW_COLLISION__SHIFT 0x10 +#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE_MASK 0x20000 +#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE__SHIFT 0x11 +#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING_MASK 0x40000 +#define DB_DEBUG3__DISABLE_OP_S_DATA_FORWARDING__SHIFT 0x12 +#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE_MASK 0x80000 +#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE__SHIFT 0x13 +#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE_MASK 0x100000 +#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE__SHIFT 0x14 +#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT_MASK 0x200000 +#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT__SHIFT 0x15 +#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB_MASK 0x400000 +#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB__SHIFT 0x16 +#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD_MASK 0x800000 +#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD__SHIFT 0x17 +#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT_MASK 0x1000000 +#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT__SHIFT 0x18 +#define DB_DEBUG3__DISABLE_DI_DT_STALL_MASK 0x2000000 +#define DB_DEBUG3__DISABLE_DI_DT_STALL__SHIFT 0x19 +#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET_MASK 0x4000000 +#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET__SHIFT 0x1a +#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX_MASK 0x8000000 +#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX__SHIFT 0x1b +#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND_MASK 0x10000000 +#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND__SHIFT 0x1c +#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND_MASK 0x20000000 +#define DB_DEBUG3__DONT_DELETE_CONTEXT_SUSPEND__SHIFT 0x1d +#define DB_DEBUG3__DB_EXTRA_DEBUG3_MASK 0xc0000000 +#define DB_DEBUG3__DB_EXTRA_DEBUG3__SHIFT 0x1e +#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION_MASK 0x1 +#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION__SHIFT 0x0 +#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION_MASK 0x2 +#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION__SHIFT 0x1 +#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL_MASK 0x4 +#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL__SHIFT 0x2 +#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL_MASK 0x8 +#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL__SHIFT 0x3 +#define DB_DEBUG4__DB_EXTRA_DEBUG4_MASK 0xfffffff0 +#define DB_DEBUG4__DB_EXTRA_DEBUG4__SHIFT 0x4 +#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS_MASK 0x1f +#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS__SHIFT 0x0 +#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS_MASK 0x3e0 +#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS__SHIFT 0x5 +#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS_MASK 0x1c00 +#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS__SHIFT 0xa +#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS_MASK 0x7f000000 +#define DB_CREDIT_LIMIT__DB_CB_TILE_CREDITS__SHIFT 0x18 +#define DB_WATERMARKS__DEPTH_FREE_MASK 0x1f +#define DB_WATERMARKS__DEPTH_FREE__SHIFT 0x0 +#define DB_WATERMARKS__DEPTH_FLUSH_MASK 0x7e0 +#define DB_WATERMARKS__DEPTH_FLUSH__SHIFT 0x5 +#define DB_WATERMARKS__FORCE_SUMMARIZE_MASK 0x7800 +#define DB_WATERMARKS__FORCE_SUMMARIZE__SHIFT 0xb +#define DB_WATERMARKS__DEPTH_PENDING_FREE_MASK 0xf8000 +#define DB_WATERMARKS__DEPTH_PENDING_FREE__SHIFT 0xf +#define DB_WATERMARKS__DEPTH_CACHELINE_FREE_MASK 0x7f00000 +#define DB_WATERMARKS__DEPTH_CACHELINE_FREE__SHIFT 0x14 +#define DB_WATERMARKS__EARLY_Z_PANIC_DISABLE_MASK 0x8000000 +#define DB_WATERMARKS__EARLY_Z_PANIC_DISABLE__SHIFT 0x1b +#define DB_WATERMARKS__LATE_Z_PANIC_DISABLE_MASK 0x10000000 +#define DB_WATERMARKS__LATE_Z_PANIC_DISABLE__SHIFT 0x1c +#define DB_WATERMARKS__RE_Z_PANIC_DISABLE_MASK 0x20000000 +#define DB_WATERMARKS__RE_Z_PANIC_DISABLE__SHIFT 0x1d +#define DB_WATERMARKS__AUTO_FLUSH_HTILE_MASK 0x40000000 +#define DB_WATERMARKS__AUTO_FLUSH_HTILE__SHIFT 0x1e +#define DB_WATERMARKS__AUTO_FLUSH_QUAD_MASK 0x80000000 +#define DB_WATERMARKS__AUTO_FLUSH_QUAD__SHIFT 0x1f +#define DB_SUBTILE_CONTROL__MSAA1_X_MASK 0x3 +#define DB_SUBTILE_CONTROL__MSAA1_X__SHIFT 0x0 +#define DB_SUBTILE_CONTROL__MSAA1_Y_MASK 0xc +#define DB_SUBTILE_CONTROL__MSAA1_Y__SHIFT 0x2 +#define DB_SUBTILE_CONTROL__MSAA2_X_MASK 0x30 +#define DB_SUBTILE_CONTROL__MSAA2_X__SHIFT 0x4 +#define DB_SUBTILE_CONTROL__MSAA2_Y_MASK 0xc0 +#define DB_SUBTILE_CONTROL__MSAA2_Y__SHIFT 0x6 +#define DB_SUBTILE_CONTROL__MSAA4_X_MASK 0x300 +#define DB_SUBTILE_CONTROL__MSAA4_X__SHIFT 0x8 +#define DB_SUBTILE_CONTROL__MSAA4_Y_MASK 0xc00 +#define DB_SUBTILE_CONTROL__MSAA4_Y__SHIFT 0xa +#define DB_SUBTILE_CONTROL__MSAA8_X_MASK 0x3000 +#define DB_SUBTILE_CONTROL__MSAA8_X__SHIFT 0xc +#define DB_SUBTILE_CONTROL__MSAA8_Y_MASK 0xc000 +#define DB_SUBTILE_CONTROL__MSAA8_Y__SHIFT 0xe +#define DB_SUBTILE_CONTROL__MSAA16_X_MASK 0x30000 +#define DB_SUBTILE_CONTROL__MSAA16_X__SHIFT 0x10 +#define DB_SUBTILE_CONTROL__MSAA16_Y_MASK 0xc0000 +#define DB_SUBTILE_CONTROL__MSAA16_Y__SHIFT 0x12 +#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH_MASK 0x7f +#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH__SHIFT 0x0 +#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH_MASK 0x3f80 +#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH__SHIFT 0x7 +#define DB_FREE_CACHELINES__FREE_Z_DEPTH_MASK 0x1fc000 +#define DB_FREE_CACHELINES__FREE_Z_DEPTH__SHIFT 0xe +#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH_MASK 0x1e00000 +#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH__SHIFT 0x15 +#define DB_FREE_CACHELINES__QUAD_READ_REQS_MASK 0xfe000000 +#define DB_FREE_CACHELINES__QUAD_READ_REQS__SHIFT 0x19 +#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH_MASK 0x1f +#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH__SHIFT 0x0 +#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH_MASK 0x3e0 +#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH__SHIFT 0x5 +#define DB_FIFO_DEPTH1__MCC_DEPTH_MASK 0xfc00 +#define DB_FIFO_DEPTH1__MCC_DEPTH__SHIFT 0xa +#define DB_FIFO_DEPTH1__QC_DEPTH_MASK 0x1f0000 +#define DB_FIFO_DEPTH1__QC_DEPTH__SHIFT 0x10 +#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH_MASK 0x1fe00000 +#define DB_FIFO_DEPTH1__LTILE_PROBE_FIFO_DEPTH__SHIFT 0x15 +#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH_MASK 0xff +#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH__SHIFT 0x0 +#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH_MASK 0x7f00 +#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH__SHIFT 0x8 +#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH_MASK 0x1ff8000 +#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH__SHIFT 0xf +#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH_MASK 0xfe000000 +#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH__SHIFT 0x19 +#define DB_CGTT_CLK_CTRL_0__ON_DELAY_MASK 0xf +#define DB_CGTT_CLK_CTRL_0__ON_DELAY__SHIFT 0x0 +#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS_MASK 0xff0 +#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS__SHIFT 0x4 +#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK 0xfff000 +#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT 0xc +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK 0x1000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT 0x18 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK 0x2000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT 0x19 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK 0x4000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT 0x1a +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK 0x8000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT 0x1b +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK 0x10000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT 0x1c +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK 0x20000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT 0x1d +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK 0x40000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT 0x1e +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK 0x80000000 +#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT 0x1f +#define DB_ZPASS_COUNT_LOW__COUNT_LOW_MASK 0xffffffff +#define DB_ZPASS_COUNT_LOW__COUNT_LOW__SHIFT 0x0 +#define DB_ZPASS_COUNT_HI__COUNT_HI_MASK 0x7fffffff +#define DB_ZPASS_COUNT_HI__COUNT_HI__SHIFT 0x0 +#define DB_RING_CONTROL__COUNTER_CONTROL_MASK 0x3 +#define DB_RING_CONTROL__COUNTER_CONTROL__SHIFT 0x0 +#define DB_READ_DEBUG_0__BUSY_DATA0_MASK 0xffffffff +#define DB_READ_DEBUG_0__BUSY_DATA0__SHIFT 0x0 +#define DB_READ_DEBUG_1__BUSY_DATA1_MASK 0xffffffff +#define DB_READ_DEBUG_1__BUSY_DATA1__SHIFT 0x0 +#define DB_READ_DEBUG_2__BUSY_DATA2_MASK 0xffffffff +#define DB_READ_DEBUG_2__BUSY_DATA2__SHIFT 0x0 +#define DB_READ_DEBUG_3__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_3__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_4__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_4__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_5__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_5__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_6__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_6__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_7__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_7__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_8__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_8__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_9__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_9__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_A__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_A__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_B__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_B__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_C__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_C__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_D__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_D__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_E__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_E__DEBUG_DATA__SHIFT 0x0 +#define DB_READ_DEBUG_F__DEBUG_DATA_MASK 0xffffffff +#define DB_READ_DEBUG_F__DEBUG_DATA__SHIFT 0x0 +#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW_MASK 0xffffffff +#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW__SHIFT 0x0 +#define DB_OCCLUSION_COUNT0_HI__COUNT_HI_MASK 0x7fffffff +#define DB_OCCLUSION_COUNT0_HI__COUNT_HI__SHIFT 0x0 +#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW_MASK 0xffffffff +#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW__SHIFT 0x0 +#define DB_OCCLUSION_COUNT1_HI__COUNT_HI_MASK 0x7fffffff +#define DB_OCCLUSION_COUNT1_HI__COUNT_HI__SHIFT 0x0 +#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW_MASK 0xffffffff +#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW__SHIFT 0x0 +#define DB_OCCLUSION_COUNT2_HI__COUNT_HI_MASK 0x7fffffff +#define DB_OCCLUSION_COUNT2_HI__COUNT_HI__SHIFT 0x0 +#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW_MASK 0xffffffff +#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW__SHIFT 0x0 +#define DB_OCCLUSION_COUNT3_HI__COUNT_HI_MASK 0x7fffffff +#define DB_OCCLUSION_COUNT3_HI__COUNT_HI__SHIFT 0x0 +#define CC_RB_REDUNDANCY__FAILED_RB0_MASK 0xf00 +#define CC_RB_REDUNDANCY__FAILED_RB0__SHIFT 0x8 +#define CC_RB_REDUNDANCY__EN_REDUNDANCY0_MASK 0x1000 +#define CC_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT 0xc +#define CC_RB_REDUNDANCY__FAILED_RB1_MASK 0xf0000 +#define CC_RB_REDUNDANCY__FAILED_RB1__SHIFT 0x10 +#define CC_RB_REDUNDANCY__EN_REDUNDANCY1_MASK 0x100000 +#define CC_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT 0x14 +#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0xff0000 +#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x10 +#define GC_USER_RB_REDUNDANCY__FAILED_RB0_MASK 0xf00 +#define GC_USER_RB_REDUNDANCY__FAILED_RB0__SHIFT 0x8 +#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0_MASK 0x1000 +#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT 0xc +#define GC_USER_RB_REDUNDANCY__FAILED_RB1_MASK 0xf0000 +#define GC_USER_RB_REDUNDANCY__FAILED_RB1__SHIFT 0x10 +#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1_MASK 0x100000 +#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT 0x14 +#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK 0xff0000 +#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT 0x10 +#define GB_ADDR_CONFIG__NUM_PIPES_MASK 0x7 +#define GB_ADDR_CONFIG__NUM_PIPES__SHIFT 0x0 +#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK 0x70 +#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT 0x4 +#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE_MASK 0x700 +#define GB_ADDR_CONFIG__BANK_INTERLEAVE_SIZE__SHIFT 0x8 +#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK 0x3000 +#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT 0xc +#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE_MASK 0x70000 +#define GB_ADDR_CONFIG__SHADER_ENGINE_TILE_SIZE__SHIFT 0x10 +#define GB_ADDR_CONFIG__NUM_GPUS_MASK 0x700000 +#define GB_ADDR_CONFIG__NUM_GPUS__SHIFT 0x14 +#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE_MASK 0x3000000 +#define GB_ADDR_CONFIG__MULTI_GPU_TILE_SIZE__SHIFT 0x18 +#define GB_ADDR_CONFIG__ROW_SIZE_MASK 0x30000000 +#define GB_ADDR_CONFIG__ROW_SIZE__SHIFT 0x1c +#define GB_ADDR_CONFIG__NUM_LOWER_PIPES_MASK 0x40000000 +#define GB_ADDR_CONFIG__NUM_LOWER_PIPES__SHIFT 0x1e +#define GB_BACKEND_MAP__BACKEND_MAP_MASK 0xffffffff +#define GB_BACKEND_MAP__BACKEND_MAP__SHIFT 0x0 +#define GB_GPU_ID__GPU_ID_MASK 0xf +#define GB_GPU_ID__GPU_ID__SHIFT 0x0 +#define CC_RB_DAISY_CHAIN__RB_0_MASK 0xf +#define CC_RB_DAISY_CHAIN__RB_0__SHIFT 0x0 +#define CC_RB_DAISY_CHAIN__RB_1_MASK 0xf0 +#define CC_RB_DAISY_CHAIN__RB_1__SHIFT 0x4 +#define CC_RB_DAISY_CHAIN__RB_2_MASK 0xf00 +#define CC_RB_DAISY_CHAIN__RB_2__SHIFT 0x8 +#define CC_RB_DAISY_CHAIN__RB_3_MASK 0xf000 +#define CC_RB_DAISY_CHAIN__RB_3__SHIFT 0xc +#define CC_RB_DAISY_CHAIN__RB_4_MASK 0xf0000 +#define CC_RB_DAISY_CHAIN__RB_4__SHIFT 0x10 +#define CC_RB_DAISY_CHAIN__RB_5_MASK 0xf00000 +#define CC_RB_DAISY_CHAIN__RB_5__SHIFT 0x14 +#define CC_RB_DAISY_CHAIN__RB_6_MASK 0xf000000 +#define CC_RB_DAISY_CHAIN__RB_6__SHIFT 0x18 +#define CC_RB_DAISY_CHAIN__RB_7_MASK 0xf0000000 +#define CC_RB_DAISY_CHAIN__RB_7__SHIFT 0x1c +#define GB_TILE_MODE0__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE0__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE0__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE0__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE0__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE0__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE0__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE1__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE1__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE1__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE1__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE1__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE1__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE1__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE1__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE1__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE2__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE2__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE2__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE2__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE2__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE2__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE2__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE2__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE2__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE3__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE3__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE3__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE3__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE3__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE3__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE3__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE3__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE3__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE4__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE4__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE4__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE4__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE4__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE4__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE4__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE4__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE4__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE5__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE5__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE5__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE5__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE5__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE5__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE5__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE5__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE5__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE6__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE6__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE6__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE6__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE6__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE6__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE6__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE6__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE6__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE7__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE7__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE7__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE7__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE7__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE7__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE7__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE7__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE7__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE8__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE8__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE8__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE8__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE8__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE8__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE8__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE8__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE8__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE9__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE9__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE9__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE9__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE9__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE9__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE9__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE9__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE9__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE10__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE10__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE10__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE10__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE10__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE10__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE10__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE10__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE10__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE11__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE11__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE11__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE11__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE11__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE11__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE11__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE11__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE11__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE12__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE12__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE12__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE12__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE12__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE12__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE12__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE12__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE12__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE13__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE13__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE13__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE13__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE13__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE13__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE13__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE13__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE13__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE14__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE14__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE14__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE14__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE14__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE14__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE14__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE14__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE14__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE15__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE15__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE15__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE15__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE15__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE15__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE15__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE15__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE15__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE16__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE16__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE16__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE16__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE16__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE16__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE16__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE16__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE16__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE17__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE17__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE17__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE17__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE17__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE17__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE17__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE17__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE17__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE18__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE18__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE18__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE18__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE18__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE18__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE18__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE18__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE18__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE19__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE19__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE19__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE19__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE19__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE19__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE19__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE19__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE19__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE20__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE20__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE20__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE20__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE20__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE20__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE20__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE20__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE20__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE21__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE21__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE21__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE21__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE21__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE21__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE21__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE21__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE21__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE22__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE22__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE22__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE22__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE22__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE22__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE22__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE22__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE22__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE23__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE23__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE23__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE23__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE23__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE23__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE23__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE23__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE23__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE24__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE24__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE24__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE24__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE24__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE24__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE24__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE24__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE24__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE25__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE25__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE25__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE25__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE25__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE25__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE25__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE25__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE25__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE26__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE26__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE26__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE26__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE26__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE26__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE26__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE26__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE26__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE27__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE27__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE27__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE27__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE27__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE27__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE27__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE27__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE27__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE28__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE28__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE28__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE28__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE28__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE28__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE28__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE28__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE28__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE29__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE29__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE29__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE29__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE29__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE29__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE29__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE29__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE29__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE30__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE30__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE30__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE30__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE30__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE30__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE30__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE30__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE30__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_TILE_MODE31__ARRAY_MODE_MASK 0x3c +#define GB_TILE_MODE31__ARRAY_MODE__SHIFT 0x2 +#define GB_TILE_MODE31__PIPE_CONFIG_MASK 0x7c0 +#define GB_TILE_MODE31__PIPE_CONFIG__SHIFT 0x6 +#define GB_TILE_MODE31__TILE_SPLIT_MASK 0x3800 +#define GB_TILE_MODE31__TILE_SPLIT__SHIFT 0xb +#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW_MASK 0x1c00000 +#define GB_TILE_MODE31__MICRO_TILE_MODE_NEW__SHIFT 0x16 +#define GB_TILE_MODE31__SAMPLE_SPLIT_MASK 0x6000000 +#define GB_TILE_MODE31__SAMPLE_SPLIT__SHIFT 0x19 +#define GB_MACROTILE_MODE0__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE0__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE0__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE0__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE1__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE1__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE1__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE1__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE1__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE1__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE1__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE2__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE2__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE2__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE2__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE2__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE2__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE2__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE3__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE3__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE3__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE3__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE3__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE3__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE3__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE4__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE4__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE4__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE4__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE4__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE4__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE4__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE5__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE5__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE5__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE5__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE5__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE5__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE5__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE6__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE6__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE6__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE6__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE6__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE6__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE6__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE7__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE7__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE7__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE7__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE7__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE7__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE7__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE8__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE8__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE8__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE8__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE8__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE8__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE8__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE9__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE9__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE9__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE9__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE9__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE9__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE9__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE10__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE10__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE10__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE10__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE10__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE10__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE10__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE11__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE11__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE11__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE11__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE11__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE11__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE11__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE12__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE12__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE12__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE12__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE12__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE12__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE12__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE13__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE13__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE13__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE13__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE13__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE13__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE13__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE14__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE14__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE14__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE14__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE14__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE14__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE14__NUM_BANKS__SHIFT 0x6 +#define GB_MACROTILE_MODE15__BANK_WIDTH_MASK 0x3 +#define GB_MACROTILE_MODE15__BANK_WIDTH__SHIFT 0x0 +#define GB_MACROTILE_MODE15__BANK_HEIGHT_MASK 0xc +#define GB_MACROTILE_MODE15__BANK_HEIGHT__SHIFT 0x2 +#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT_MASK 0x30 +#define GB_MACROTILE_MODE15__MACRO_TILE_ASPECT__SHIFT 0x4 +#define GB_MACROTILE_MODE15__NUM_BANKS_MASK 0xc0 +#define GB_MACROTILE_MODE15__NUM_BANKS__SHIFT 0x6 +#define GB_EDC_MODE__FORCE_SEC_ON_DED_MASK 0x10000 +#define GB_EDC_MODE__FORCE_SEC_ON_DED__SHIFT 0x10 +#define GB_EDC_MODE__DED_MODE_MASK 0x300000 +#define GB_EDC_MODE__DED_MODE__SHIFT 0x14 +#define GB_EDC_MODE__PROP_FED_MASK 0x20000000 +#define GB_EDC_MODE__PROP_FED__SHIFT 0x1d +#define GB_EDC_MODE__BYPASS_MASK 0x80000000 +#define GB_EDC_MODE__BYPASS__SHIFT 0x1f +#define CC_GC_EDC_CONFIG__DIS_EDC_MASK 0x2 +#define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT 0x1 +#define RAS_SIGNATURE_CONTROL__ENABLE_MASK 0x1 +#define RAS_SIGNATURE_CONTROL__ENABLE__SHIFT 0x0 +#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK_MASK 0xffffffff +#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK__SHIFT 0x0 +#define RAS_SX_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_SX_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_SX_SIGNATURE1__SIGNATURE_MASK 0xffffffff +#define RAS_SX_SIGNATURE1__SIGNATURE__SHIFT 0x0 +#define RAS_SX_SIGNATURE2__SIGNATURE_MASK 0xffffffff +#define RAS_SX_SIGNATURE2__SIGNATURE__SHIFT 0x0 +#define RAS_SX_SIGNATURE3__SIGNATURE_MASK 0xffffffff +#define RAS_SX_SIGNATURE3__SIGNATURE__SHIFT 0x0 +#define RAS_DB_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_DB_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_PA_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_PA_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_VGT_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_VGT_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_SQ_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_SQ_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE1__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE1__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE2__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE2__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE3__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE3__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE4__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE4__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE5__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE5__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE6__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE6__SIGNATURE__SHIFT 0x0 +#define RAS_SC_SIGNATURE7__SIGNATURE_MASK 0xffffffff +#define RAS_SC_SIGNATURE7__SIGNATURE__SHIFT 0x0 +#define RAS_IA_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_IA_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_IA_SIGNATURE1__SIGNATURE_MASK 0xffffffff +#define RAS_IA_SIGNATURE1__SIGNATURE__SHIFT 0x0 +#define RAS_SPI_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_SPI_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_SPI_SIGNATURE1__SIGNATURE_MASK 0xffffffff +#define RAS_SPI_SIGNATURE1__SIGNATURE__SHIFT 0x0 +#define RAS_TA_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_TA_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_TD_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_TD_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_CB_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_CB_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_BCI_SIGNATURE0__SIGNATURE_MASK 0xffffffff +#define RAS_BCI_SIGNATURE0__SIGNATURE__SHIFT 0x0 +#define RAS_BCI_SIGNATURE1__SIGNATURE_MASK 0xffffffff +#define RAS_BCI_SIGNATURE1__SIGNATURE__SHIFT 0x0 +#define GRBM_CAM_INDEX__CAM_INDEX_MASK 0x7 +#define GRBM_CAM_INDEX__CAM_INDEX__SHIFT 0x0 +#define GRBM_CAM_DATA__CAM_ADDR_MASK 0xffff +#define GRBM_CAM_DATA__CAM_ADDR__SHIFT 0x0 +#define GRBM_CAM_DATA__CAM_REMAPADDR_MASK 0xffff0000 +#define GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT 0x10 +#define GRBM_CNTL__READ_TIMEOUT_MASK 0xff +#define GRBM_CNTL__READ_TIMEOUT__SHIFT 0x0 +#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD_MASK 0x3f +#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD__SHIFT 0x0 +#define GRBM_SKEW_CNTL__SKEW_COUNT_MASK 0xfc0 +#define GRBM_SKEW_CNTL__SKEW_COUNT__SHIFT 0x6 +#define GRBM_PWR_CNTL__REQ_TYPE_MASK 0xf +#define GRBM_PWR_CNTL__REQ_TYPE__SHIFT 0x0 +#define GRBM_PWR_CNTL__RSP_TYPE_MASK 0xf0 +#define GRBM_PWR_CNTL__RSP_TYPE__SHIFT 0x4 +#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL_MASK 0xf +#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL__SHIFT 0x0 +#define GRBM_STATUS__SRBM_RQ_PENDING_MASK 0x20 +#define GRBM_STATUS__SRBM_RQ_PENDING__SHIFT 0x5 +#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING_MASK 0x80 +#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING__SHIFT 0x7 +#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING_MASK 0x100 +#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING__SHIFT 0x8 +#define GRBM_STATUS__GDS_DMA_RQ_PENDING_MASK 0x200 +#define GRBM_STATUS__GDS_DMA_RQ_PENDING__SHIFT 0x9 +#define GRBM_STATUS__DB_CLEAN_MASK 0x1000 +#define GRBM_STATUS__DB_CLEAN__SHIFT 0xc +#define GRBM_STATUS__CB_CLEAN_MASK 0x2000 +#define GRBM_STATUS__CB_CLEAN__SHIFT 0xd +#define GRBM_STATUS__TA_BUSY_MASK 0x4000 +#define GRBM_STATUS__TA_BUSY__SHIFT 0xe +#define GRBM_STATUS__GDS_BUSY_MASK 0x8000 +#define GRBM_STATUS__GDS_BUSY__SHIFT 0xf +#define GRBM_STATUS__WD_BUSY_NO_DMA_MASK 0x10000 +#define GRBM_STATUS__WD_BUSY_NO_DMA__SHIFT 0x10 +#define GRBM_STATUS__VGT_BUSY_MASK 0x20000 +#define GRBM_STATUS__VGT_BUSY__SHIFT 0x11 +#define GRBM_STATUS__IA_BUSY_NO_DMA_MASK 0x40000 +#define GRBM_STATUS__IA_BUSY_NO_DMA__SHIFT 0x12 +#define GRBM_STATUS__IA_BUSY_MASK 0x80000 +#define GRBM_STATUS__IA_BUSY__SHIFT 0x13 +#define GRBM_STATUS__SX_BUSY_MASK 0x100000 +#define GRBM_STATUS__SX_BUSY__SHIFT 0x14 +#define GRBM_STATUS__WD_BUSY_MASK 0x200000 +#define GRBM_STATUS__WD_BUSY__SHIFT 0x15 +#define GRBM_STATUS__SPI_BUSY_MASK 0x400000 +#define GRBM_STATUS__SPI_BUSY__SHIFT 0x16 +#define GRBM_STATUS__BCI_BUSY_MASK 0x800000 +#define GRBM_STATUS__BCI_BUSY__SHIFT 0x17 +#define GRBM_STATUS__SC_BUSY_MASK 0x1000000 +#define GRBM_STATUS__SC_BUSY__SHIFT 0x18 +#define GRBM_STATUS__PA_BUSY_MASK 0x2000000 +#define GRBM_STATUS__PA_BUSY__SHIFT 0x19 +#define GRBM_STATUS__DB_BUSY_MASK 0x4000000 +#define GRBM_STATUS__DB_BUSY__SHIFT 0x1a +#define GRBM_STATUS__CP_COHERENCY_BUSY_MASK 0x10000000 +#define GRBM_STATUS__CP_COHERENCY_BUSY__SHIFT 0x1c +#define GRBM_STATUS__CP_BUSY_MASK 0x20000000 +#define GRBM_STATUS__CP_BUSY__SHIFT 0x1d +#define GRBM_STATUS__CB_BUSY_MASK 0x40000000 +#define GRBM_STATUS__CB_BUSY__SHIFT 0x1e +#define GRBM_STATUS__GUI_ACTIVE_MASK 0x80000000 +#define GRBM_STATUS__GUI_ACTIVE__SHIFT 0x1f +#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL_MASK 0xf +#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL__SHIFT 0x0 +#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING_MASK 0x10 +#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING__SHIFT 0x4 +#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING_MASK 0x20 +#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING__SHIFT 0x5 +#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING_MASK 0x40 +#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING__SHIFT 0x6 +#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING_MASK 0x80 +#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING__SHIFT 0x7 +#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING_MASK 0x100 +#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING__SHIFT 0x8 +#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING_MASK 0x200 +#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING__SHIFT 0x9 +#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING_MASK 0x400 +#define GRBM_STATUS2__ME2PIPE0_RQ_PENDING__SHIFT 0xa +#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING_MASK 0x800 +#define GRBM_STATUS2__ME2PIPE1_RQ_PENDING__SHIFT 0xb +#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING_MASK 0x1000 +#define GRBM_STATUS2__ME2PIPE2_RQ_PENDING__SHIFT 0xc +#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING_MASK 0x2000 +#define GRBM_STATUS2__ME2PIPE3_RQ_PENDING__SHIFT 0xd +#define GRBM_STATUS2__RLC_RQ_PENDING_MASK 0x4000 +#define GRBM_STATUS2__RLC_RQ_PENDING__SHIFT 0xe +#define GRBM_STATUS2__RLC_BUSY_MASK 0x1000000 +#define GRBM_STATUS2__RLC_BUSY__SHIFT 0x18 +#define GRBM_STATUS2__TC_BUSY_MASK 0x2000000 +#define GRBM_STATUS2__TC_BUSY__SHIFT 0x19 +#define GRBM_STATUS2__CPF_BUSY_MASK 0x10000000 +#define GRBM_STATUS2__CPF_BUSY__SHIFT 0x1c +#define GRBM_STATUS2__CPC_BUSY_MASK 0x20000000 +#define GRBM_STATUS2__CPC_BUSY__SHIFT 0x1d +#define GRBM_STATUS2__CPG_BUSY_MASK 0x40000000 +#define GRBM_STATUS2__CPG_BUSY__SHIFT 0x1e +#define GRBM_STATUS_SE0__DB_CLEAN_MASK 0x2 +#define GRBM_STATUS_SE0__DB_CLEAN__SHIFT 0x1 +#define GRBM_STATUS_SE0__CB_CLEAN_MASK 0x4 +#define GRBM_STATUS_SE0__CB_CLEAN__SHIFT 0x2 +#define GRBM_STATUS_SE0__BCI_BUSY_MASK 0x400000 +#define GRBM_STATUS_SE0__BCI_BUSY__SHIFT 0x16 +#define GRBM_STATUS_SE0__VGT_BUSY_MASK 0x800000 +#define GRBM_STATUS_SE0__VGT_BUSY__SHIFT 0x17 +#define GRBM_STATUS_SE0__PA_BUSY_MASK 0x1000000 +#define GRBM_STATUS_SE0__PA_BUSY__SHIFT 0x18 +#define GRBM_STATUS_SE0__TA_BUSY_MASK 0x2000000 +#define GRBM_STATUS_SE0__TA_BUSY__SHIFT 0x19 +#define GRBM_STATUS_SE0__SX_BUSY_MASK 0x4000000 +#define GRBM_STATUS_SE0__SX_BUSY__SHIFT 0x1a +#define GRBM_STATUS_SE0__SPI_BUSY_MASK 0x8000000 +#define GRBM_STATUS_SE0__SPI_BUSY__SHIFT 0x1b +#define GRBM_STATUS_SE0__SC_BUSY_MASK 0x20000000 +#define GRBM_STATUS_SE0__SC_BUSY__SHIFT 0x1d +#define GRBM_STATUS_SE0__DB_BUSY_MASK 0x40000000 +#define GRBM_STATUS_SE0__DB_BUSY__SHIFT 0x1e +#define GRBM_STATUS_SE0__CB_BUSY_MASK 0x80000000 +#define GRBM_STATUS_SE0__CB_BUSY__SHIFT 0x1f +#define GRBM_STATUS_SE1__DB_CLEAN_MASK 0x2 +#define GRBM_STATUS_SE1__DB_CLEAN__SHIFT 0x1 +#define GRBM_STATUS_SE1__CB_CLEAN_MASK 0x4 +#define GRBM_STATUS_SE1__CB_CLEAN__SHIFT 0x2 +#define GRBM_STATUS_SE1__BCI_BUSY_MASK 0x400000 +#define GRBM_STATUS_SE1__BCI_BUSY__SHIFT 0x16 +#define GRBM_STATUS_SE1__VGT_BUSY_MASK 0x800000 +#define GRBM_STATUS_SE1__VGT_BUSY__SHIFT 0x17 +#define GRBM_STATUS_SE1__PA_BUSY_MASK 0x1000000 +#define GRBM_STATUS_SE1__PA_BUSY__SHIFT 0x18 +#define GRBM_STATUS_SE1__TA_BUSY_MASK 0x2000000 +#define GRBM_STATUS_SE1__TA_BUSY__SHIFT 0x19 +#define GRBM_STATUS_SE1__SX_BUSY_MASK 0x4000000 +#define GRBM_STATUS_SE1__SX_BUSY__SHIFT 0x1a +#define GRBM_STATUS_SE1__SPI_BUSY_MASK 0x8000000 +#define GRBM_STATUS_SE1__SPI_BUSY__SHIFT 0x1b +#define GRBM_STATUS_SE1__SC_BUSY_MASK 0x20000000 +#define GRBM_STATUS_SE1__SC_BUSY__SHIFT 0x1d +#define GRBM_STATUS_SE1__DB_BUSY_MASK 0x40000000 +#define GRBM_STATUS_SE1__DB_BUSY__SHIFT 0x1e +#define GRBM_STATUS_SE1__CB_BUSY_MASK 0x80000000 +#define GRBM_STATUS_SE1__CB_BUSY__SHIFT 0x1f +#define GRBM_STATUS_SE2__DB_CLEAN_MASK 0x2 +#define GRBM_STATUS_SE2__DB_CLEAN__SHIFT 0x1 +#define GRBM_STATUS_SE2__CB_CLEAN_MASK 0x4 +#define GRBM_STATUS_SE2__CB_CLEAN__SHIFT 0x2 +#define GRBM_STATUS_SE2__BCI_BUSY_MASK 0x400000 +#define GRBM_STATUS_SE2__BCI_BUSY__SHIFT 0x16 +#define GRBM_STATUS_SE2__VGT_BUSY_MASK 0x800000 +#define GRBM_STATUS_SE2__VGT_BUSY__SHIFT 0x17 +#define GRBM_STATUS_SE2__PA_BUSY_MASK 0x1000000 +#define GRBM_STATUS_SE2__PA_BUSY__SHIFT 0x18 +#define GRBM_STATUS_SE2__TA_BUSY_MASK 0x2000000 +#define GRBM_STATUS_SE2__TA_BUSY__SHIFT 0x19 +#define GRBM_STATUS_SE2__SX_BUSY_MASK 0x4000000 +#define GRBM_STATUS_SE2__SX_BUSY__SHIFT 0x1a +#define GRBM_STATUS_SE2__SPI_BUSY_MASK 0x8000000 +#define GRBM_STATUS_SE2__SPI_BUSY__SHIFT 0x1b +#define GRBM_STATUS_SE2__SC_BUSY_MASK 0x20000000 +#define GRBM_STATUS_SE2__SC_BUSY__SHIFT 0x1d +#define GRBM_STATUS_SE2__DB_BUSY_MASK 0x40000000 +#define GRBM_STATUS_SE2__DB_BUSY__SHIFT 0x1e +#define GRBM_STATUS_SE2__CB_BUSY_MASK 0x80000000 +#define GRBM_STATUS_SE2__CB_BUSY__SHIFT 0x1f +#define GRBM_STATUS_SE3__DB_CLEAN_MASK 0x2 +#define GRBM_STATUS_SE3__DB_CLEAN__SHIFT 0x1 +#define GRBM_STATUS_SE3__CB_CLEAN_MASK 0x4 +#define GRBM_STATUS_SE3__CB_CLEAN__SHIFT 0x2 +#define GRBM_STATUS_SE3__BCI_BUSY_MASK 0x400000 +#define GRBM_STATUS_SE3__BCI_BUSY__SHIFT 0x16 +#define GRBM_STATUS_SE3__VGT_BUSY_MASK 0x800000 +#define GRBM_STATUS_SE3__VGT_BUSY__SHIFT 0x17 +#define GRBM_STATUS_SE3__PA_BUSY_MASK 0x1000000 +#define GRBM_STATUS_SE3__PA_BUSY__SHIFT 0x18 +#define GRBM_STATUS_SE3__TA_BUSY_MASK 0x2000000 +#define GRBM_STATUS_SE3__TA_BUSY__SHIFT 0x19 +#define GRBM_STATUS_SE3__SX_BUSY_MASK 0x4000000 +#define GRBM_STATUS_SE3__SX_BUSY__SHIFT 0x1a +#define GRBM_STATUS_SE3__SPI_BUSY_MASK 0x8000000 +#define GRBM_STATUS_SE3__SPI_BUSY__SHIFT 0x1b +#define GRBM_STATUS_SE3__SC_BUSY_MASK 0x20000000 +#define GRBM_STATUS_SE3__SC_BUSY__SHIFT 0x1d +#define GRBM_STATUS_SE3__DB_BUSY_MASK 0x40000000 +#define GRBM_STATUS_SE3__DB_BUSY__SHIFT 0x1e +#define GRBM_STATUS_SE3__CB_BUSY_MASK 0x80000000 +#define GRBM_STATUS_SE3__CB_BUSY__SHIFT 0x1f +#define GRBM_SOFT_RESET__SOFT_RESET_CP_MASK 0x1 +#define GRBM_SOFT_RESET__SOFT_RESET_CP__SHIFT 0x0 +#define GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK 0x4 +#define GRBM_SOFT_RESET__SOFT_RESET_RLC__SHIFT 0x2 +#define GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK 0x10000 +#define GRBM_SOFT_RESET__SOFT_RESET_GFX__SHIFT 0x10 +#define GRBM_SOFT_RESET__SOFT_RESET_CPF_MASK 0x20000 +#define GRBM_SOFT_RESET__SOFT_RESET_CPF__SHIFT 0x11 +#define GRBM_SOFT_RESET__SOFT_RESET_CPC_MASK 0x40000 +#define GRBM_SOFT_RESET__SOFT_RESET_CPC__SHIFT 0x12 +#define GRBM_SOFT_RESET__SOFT_RESET_CPG_MASK 0x80000 +#define GRBM_SOFT_RESET__SOFT_RESET_CPG__SHIFT 0x13 +#define GRBM_DEBUG_CNTL__GRBM_DEBUG_INDEX_MASK 0x3f +#define GRBM_DEBUG_CNTL__GRBM_DEBUG_INDEX__SHIFT 0x0 +#define GRBM_DEBUG_DATA__DATA_MASK 0xffffffff +#define GRBM_DEBUG_DATA__DATA__SHIFT 0x0 +#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK 0xff +#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT 0x0 +#define GRBM_GFX_INDEX__SH_INDEX_MASK 0xff00 +#define GRBM_GFX_INDEX__SH_INDEX__SHIFT 0x8 +#define GRBM_GFX_INDEX__SE_INDEX_MASK 0xff0000 +#define GRBM_GFX_INDEX__SE_INDEX__SHIFT 0x10 +#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK 0x20000000 +#define GRBM_GFX_INDEX__SH_BROADCAST_WRITES__SHIFT 0x1d +#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK 0x40000000 +#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT 0x1e +#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK 0x80000000 +#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT 0x1f +#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK 0xf +#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT 0x0 +#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK 0x1f00 +#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT 0x8 +#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS_MASK 0xff +#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS__SHIFT 0x0 +#define GRBM_DEBUG__IGNORE_RDY_MASK 0x2 +#define GRBM_DEBUG__IGNORE_RDY__SHIFT 0x1 +#define GRBM_DEBUG__IGNORE_FAO_MASK 0x20 +#define GRBM_DEBUG__IGNORE_FAO__SHIFT 0x5 +#define GRBM_DEBUG__DISABLE_READ_TIMEOUT_MASK 0x40 +#define GRBM_DEBUG__DISABLE_READ_TIMEOUT__SHIFT 0x6 +#define GRBM_DEBUG__SNAPSHOT_FREE_CNTRS_MASK 0x80 +#define GRBM_DEBUG__SNAPSHOT_FREE_CNTRS__SHIFT 0x7 +#define GRBM_DEBUG__HYSTERESIS_GUI_ACTIVE_MASK 0xf00 +#define GRBM_DEBUG__HYSTERESIS_GUI_ACTIVE__SHIFT 0x8 +#define GRBM_DEBUG__GFX_CLOCK_DOMAIN_OVERRIDE_MASK 0x1000 +#define GRBM_DEBUG__GFX_CLOCK_DOMAIN_OVERRIDE__SHIFT 0xc +#define GRBM_DEBUG_SNAPSHOT__CPF_RDY_MASK 0x1 +#define GRBM_DEBUG_SNAPSHOT__CPF_RDY__SHIFT 0x0 +#define GRBM_DEBUG_SNAPSHOT__CPG_RDY_MASK 0x2 +#define GRBM_DEBUG_SNAPSHOT__CPG_RDY__SHIFT 0x1 +#define GRBM_DEBUG_SNAPSHOT__SRBM_RDY_MASK 0x4 +#define GRBM_DEBUG_SNAPSHOT__SRBM_RDY__SHIFT 0x2 +#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE0_RDY_MASK 0x8 +#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE0_RDY__SHIFT 0x3 +#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE1_RDY_MASK 0x10 +#define GRBM_DEBUG_SNAPSHOT__WD_ME0PIPE1_RDY__SHIFT 0x4 +#define GRBM_DEBUG_SNAPSHOT__GDS_RDY_MASK 0x20 +#define GRBM_DEBUG_SNAPSHOT__GDS_RDY__SHIFT 0x5 +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY0_MASK 0x40 +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY0__SHIFT 0x6 +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY0_MASK 0x80 +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY0__SHIFT 0x7 +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY0_MASK 0x100 +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY0__SHIFT 0x8 +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY0_MASK 0x200 +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY0__SHIFT 0x9 +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY0_MASK 0x400 +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY0__SHIFT 0xa +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY0_MASK 0x800 +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY0__SHIFT 0xb +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY0_MASK 0x1000 +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY0__SHIFT 0xc +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY0_MASK 0x2000 +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY0__SHIFT 0xd +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY1_MASK 0x4000 +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE0_RDY1__SHIFT 0xe +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY1_MASK 0x8000 +#define GRBM_DEBUG_SNAPSHOT__SE0SPI_ME0PIPE1_RDY1__SHIFT 0xf +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY1_MASK 0x10000 +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE0_RDY1__SHIFT 0x10 +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY1_MASK 0x20000 +#define GRBM_DEBUG_SNAPSHOT__SE1SPI_ME0PIPE1_RDY1__SHIFT 0x11 +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY1_MASK 0x40000 +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE0_RDY1__SHIFT 0x12 +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY1_MASK 0x80000 +#define GRBM_DEBUG_SNAPSHOT__SE2SPI_ME0PIPE1_RDY1__SHIFT 0x13 +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY1_MASK 0x100000 +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE0_RDY1__SHIFT 0x14 +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY1_MASK 0x200000 +#define GRBM_DEBUG_SNAPSHOT__SE3SPI_ME0PIPE1_RDY1__SHIFT 0x15 +#define GRBM_READ_ERROR__READ_ADDRESS_MASK 0x3fffc +#define GRBM_READ_ERROR__READ_ADDRESS__SHIFT 0x2 +#define GRBM_READ_ERROR__READ_PIPEID_MASK 0x300000 +#define GRBM_READ_ERROR__READ_PIPEID__SHIFT 0x14 +#define GRBM_READ_ERROR__READ_MEID_MASK 0xc00000 +#define GRBM_READ_ERROR__READ_MEID__SHIFT 0x16 +#define GRBM_READ_ERROR__READ_ERROR_MASK 0x80000000 +#define GRBM_READ_ERROR__READ_ERROR__SHIFT 0x1f +#define GRBM_READ_ERROR2__READ_REQUESTER_SRBM_MASK 0x20000 +#define GRBM_READ_ERROR2__READ_REQUESTER_SRBM__SHIFT 0x11 +#define GRBM_READ_ERROR2__READ_REQUESTER_RLC_MASK 0x40000 +#define GRBM_READ_ERROR2__READ_REQUESTER_RLC__SHIFT 0x12 +#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA_MASK 0x80000 +#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA__SHIFT 0x13 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF_MASK 0x100000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF__SHIFT 0x14 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF_MASK 0x200000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF__SHIFT 0x15 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF_MASK 0x400000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF__SHIFT 0x16 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF_MASK 0x800000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF__SHIFT 0x17 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0_MASK 0x1000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0__SHIFT 0x18 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1_MASK 0x2000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1__SHIFT 0x19 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2_MASK 0x4000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2__SHIFT 0x1a +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3_MASK 0x8000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3__SHIFT 0x1b +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0_MASK 0x10000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0__SHIFT 0x1c +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1_MASK 0x20000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1__SHIFT 0x1d +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2_MASK 0x40000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2__SHIFT 0x1e +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3_MASK 0x80000000 +#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3__SHIFT 0x1f +#define GRBM_INT_CNTL__RDERR_INT_ENABLE_MASK 0x1 +#define GRBM_INT_CNTL__RDERR_INT_ENABLE__SHIFT 0x0 +#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE_MASK 0x80000 +#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE__SHIFT 0x13 +#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3f +#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400 +#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa +#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800 +#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb +#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x1000 +#define GRBM_PERFCOUNTER0_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0xc +#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x2000 +#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xd +#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x4000 +#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xe +#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x10000 +#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x10 +#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x20000 +#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x11 +#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x40000 +#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x12 +#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK 0x80000 +#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT 0x13 +#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x100000 +#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x14 +#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x200000 +#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x15 +#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK 0x400000 +#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT 0x16 +#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK 0x800000 +#define GRBM_PERFCOUNTER0_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT 0x17 +#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK 0x1000000 +#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT 0x18 +#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x2000000 +#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x19 +#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK 0x4000000 +#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT 0x1a +#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK 0x8000000 +#define GRBM_PERFCOUNTER0_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT 0x1b +#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK 0x10000000 +#define GRBM_PERFCOUNTER0_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT 0x1c +#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3f +#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400 +#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa +#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800 +#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb +#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x1000 +#define GRBM_PERFCOUNTER1_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0xc +#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x2000 +#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xd +#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x4000 +#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xe +#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x10000 +#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0x10 +#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x20000 +#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x11 +#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x40000 +#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x12 +#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK 0x80000 +#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT 0x13 +#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x100000 +#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x14 +#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x200000 +#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x15 +#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK 0x400000 +#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT 0x16 +#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK_MASK 0x800000 +#define GRBM_PERFCOUNTER1_SELECT__IA_BUSY_USER_DEFINED_MASK__SHIFT 0x17 +#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK 0x1000000 +#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT 0x18 +#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x2000000 +#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x19 +#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK 0x4000000 +#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT 0x1a +#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK_MASK 0x8000000 +#define GRBM_PERFCOUNTER1_SELECT__TC_BUSY_USER_DEFINED_MASK__SHIFT 0x1b +#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK_MASK 0x10000000 +#define GRBM_PERFCOUNTER1_SELECT__WD_BUSY_USER_DEFINED_MASK__SHIFT 0x1c +#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f +#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0 +#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400 +#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa +#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800 +#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb +#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000 +#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc +#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000 +#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd +#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000 +#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf +#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000 +#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10 +#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000 +#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11 +#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000 +#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12 +#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000 +#define GRBM_SE0_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13 +#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000 +#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14 +#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000 +#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15 +#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f +#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0 +#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400 +#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa +#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800 +#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb +#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000 +#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc +#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000 +#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd +#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000 +#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf +#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000 +#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10 +#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000 +#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11 +#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000 +#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12 +#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000 +#define GRBM_SE1_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13 +#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000 +#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14 +#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000 +#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15 +#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f +#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0 +#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400 +#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa +#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800 +#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb +#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000 +#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc +#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000 +#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd +#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000 +#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf +#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000 +#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10 +#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000 +#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11 +#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000 +#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12 +#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000 +#define GRBM_SE2_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13 +#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000 +#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14 +#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000 +#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15 +#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL_MASK 0x3f +#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL__SHIFT 0x0 +#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK 0x400 +#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT 0xa +#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK 0x800 +#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT 0xb +#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK 0x1000 +#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT 0xc +#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK 0x2000 +#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT 0xd +#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK 0x8000 +#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT 0xf +#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK 0x10000 +#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT 0x10 +#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK 0x20000 +#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT 0x11 +#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK 0x40000 +#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT 0x12 +#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK_MASK 0x80000 +#define GRBM_SE3_PERFCOUNTER_SELECT__VGT_BUSY_USER_DEFINED_MASK__SHIFT 0x13 +#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK 0x100000 +#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT 0x14 +#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK 0x200000 +#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT 0x15 +#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GRBM_SCRATCH_REG0__SCRATCH_REG0_MASK 0xffffffff +#define GRBM_SCRATCH_REG0__SCRATCH_REG0__SHIFT 0x0 +#define GRBM_SCRATCH_REG1__SCRATCH_REG1_MASK 0xffffffff +#define GRBM_SCRATCH_REG1__SCRATCH_REG1__SHIFT 0x0 +#define GRBM_SCRATCH_REG2__SCRATCH_REG2_MASK 0xffffffff +#define GRBM_SCRATCH_REG2__SCRATCH_REG2__SHIFT 0x0 +#define GRBM_SCRATCH_REG3__SCRATCH_REG3_MASK 0xffffffff +#define GRBM_SCRATCH_REG3__SCRATCH_REG3__SHIFT 0x0 +#define GRBM_SCRATCH_REG4__SCRATCH_REG4_MASK 0xffffffff +#define GRBM_SCRATCH_REG4__SCRATCH_REG4__SHIFT 0x0 +#define GRBM_SCRATCH_REG5__SCRATCH_REG5_MASK 0xffffffff +#define GRBM_SCRATCH_REG5__SCRATCH_REG5__SHIFT 0x0 +#define GRBM_SCRATCH_REG6__SCRATCH_REG6_MASK 0xffffffff +#define GRBM_SCRATCH_REG6__SCRATCH_REG6__SHIFT 0x0 +#define GRBM_SCRATCH_REG7__SCRATCH_REG7_MASK 0xffffffff +#define GRBM_SCRATCH_REG7__SCRATCH_REG7__SHIFT 0x0 +#define DEBUG_INDEX__DEBUG_INDEX_MASK 0x3ffff +#define DEBUG_INDEX__DEBUG_INDEX__SHIFT 0x0 +#define DEBUG_DATA__DEBUG_DATA_MASK 0xffffffff +#define DEBUG_DATA__DEBUG_DATA__SHIFT 0x0 +#define GRBM_NOWHERE__DATA_MASK 0xffffffff +#define GRBM_NOWHERE__DATA__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE_MASK 0xffffffff +#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE_MASK 0xffffffff +#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE_MASK 0xffffffff +#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET_MASK 0xffffffff +#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET__SHIFT 0x0 +#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA_MASK 0x1 +#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA__SHIFT 0x0 +#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA_MASK 0x2 +#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA__SHIFT 0x1 +#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA_MASK 0x4 +#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA__SHIFT 0x2 +#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA_MASK 0x8 +#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA__SHIFT 0x3 +#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA_MASK 0x10 +#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA__SHIFT 0x4 +#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA_MASK 0x20 +#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA__SHIFT 0x5 +#define PA_CL_VTE_CNTL__VTX_XY_FMT_MASK 0x100 +#define PA_CL_VTE_CNTL__VTX_XY_FMT__SHIFT 0x8 +#define PA_CL_VTE_CNTL__VTX_Z_FMT_MASK 0x200 +#define PA_CL_VTE_CNTL__VTX_Z_FMT__SHIFT 0x9 +#define PA_CL_VTE_CNTL__VTX_W0_FMT_MASK 0x400 +#define PA_CL_VTE_CNTL__VTX_W0_FMT__SHIFT 0xa +#define PA_CL_VTE_CNTL__PERFCOUNTER_REF_MASK 0x800 +#define PA_CL_VTE_CNTL__PERFCOUNTER_REF__SHIFT 0xb +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK 0x1 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0__SHIFT 0x0 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1_MASK 0x2 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1__SHIFT 0x1 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2_MASK 0x4 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2__SHIFT 0x2 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3_MASK 0x8 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3__SHIFT 0x3 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4_MASK 0x10 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4__SHIFT 0x4 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5_MASK 0x20 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5__SHIFT 0x5 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6_MASK 0x40 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6__SHIFT 0x6 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7_MASK 0x80 +#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7__SHIFT 0x7 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0_MASK 0x100 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0__SHIFT 0x8 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1_MASK 0x200 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1__SHIFT 0x9 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2_MASK 0x400 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2__SHIFT 0xa +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3_MASK 0x800 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3__SHIFT 0xb +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4_MASK 0x1000 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4__SHIFT 0xc +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5_MASK 0x2000 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5__SHIFT 0xd +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6_MASK 0x4000 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6__SHIFT 0xe +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7_MASK 0x8000 +#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7__SHIFT 0xf +#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE_MASK 0x10000 +#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE__SHIFT 0x10 +#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG_MASK 0x20000 +#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG__SHIFT 0x11 +#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX_MASK 0x40000 +#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX__SHIFT 0x12 +#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX_MASK 0x80000 +#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX__SHIFT 0x13 +#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG_MASK 0x100000 +#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG__SHIFT 0x14 +#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA_MASK 0x200000 +#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA__SHIFT 0x15 +#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA_MASK 0x400000 +#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA__SHIFT 0x16 +#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA_MASK 0x800000 +#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA__SHIFT 0x17 +#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK 0x1000000 +#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT 0x18 +#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG_MASK 0x2000000 +#define PA_CL_VS_OUT_CNTL__USE_VTX_GS_CUT_FLAG__SHIFT 0x19 +#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD_MASK 0x1 +#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD__SHIFT 0x0 +#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD_MASK 0x2 +#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD__SHIFT 0x1 +#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD_MASK 0x4 +#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD__SHIFT 0x2 +#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0_MASK 0x8 +#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0__SHIFT 0x3 +#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN_MASK 0x10 +#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN__SHIFT 0x4 +#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN_MASK 0x20 +#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN__SHIFT 0x5 +#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN_MASK 0x40 +#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN__SHIFT 0x6 +#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0_MASK 0x80 +#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0__SHIFT 0x7 +#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF_MASK 0x100 +#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF__SHIFT 0x8 +#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN_MASK 0x200 +#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN__SHIFT 0x9 +#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF_MASK 0x400 +#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF__SHIFT 0xa +#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN_MASK 0x800 +#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN__SHIFT 0xb +#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF_MASK 0x1000 +#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF__SHIFT 0xc +#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN_MASK 0x2000 +#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN__SHIFT 0xd +#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD_MASK 0x4000 +#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD__SHIFT 0xe +#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0_MASK 0x100000 +#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0__SHIFT 0x14 +#define PA_CL_CLIP_CNTL__UCP_ENA_0_MASK 0x1 +#define PA_CL_CLIP_CNTL__UCP_ENA_0__SHIFT 0x0 +#define PA_CL_CLIP_CNTL__UCP_ENA_1_MASK 0x2 +#define PA_CL_CLIP_CNTL__UCP_ENA_1__SHIFT 0x1 +#define PA_CL_CLIP_CNTL__UCP_ENA_2_MASK 0x4 +#define PA_CL_CLIP_CNTL__UCP_ENA_2__SHIFT 0x2 +#define PA_CL_CLIP_CNTL__UCP_ENA_3_MASK 0x8 +#define PA_CL_CLIP_CNTL__UCP_ENA_3__SHIFT 0x3 +#define PA_CL_CLIP_CNTL__UCP_ENA_4_MASK 0x10 +#define PA_CL_CLIP_CNTL__UCP_ENA_4__SHIFT 0x4 +#define PA_CL_CLIP_CNTL__UCP_ENA_5_MASK 0x20 +#define PA_CL_CLIP_CNTL__UCP_ENA_5__SHIFT 0x5 +#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG_MASK 0x2000 +#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG__SHIFT 0xd +#define PA_CL_CLIP_CNTL__PS_UCP_MODE_MASK 0xc000 +#define PA_CL_CLIP_CNTL__PS_UCP_MODE__SHIFT 0xe +#define PA_CL_CLIP_CNTL__CLIP_DISABLE_MASK 0x10000 +#define PA_CL_CLIP_CNTL__CLIP_DISABLE__SHIFT 0x10 +#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA_MASK 0x20000 +#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA__SHIFT 0x11 +#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA_MASK 0x40000 +#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA__SHIFT 0x12 +#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF_MASK 0x80000 +#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF__SHIFT 0x13 +#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT_MASK 0x100000 +#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT__SHIFT 0x14 +#define PA_CL_CLIP_CNTL__VTX_KILL_OR_MASK 0x200000 +#define PA_CL_CLIP_CNTL__VTX_KILL_OR__SHIFT 0x15 +#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL_MASK 0x400000 +#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL__SHIFT 0x16 +#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA_MASK 0x1000000 +#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA__SHIFT 0x18 +#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE_MASK 0x2000000 +#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE__SHIFT 0x19 +#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE_MASK 0x4000000 +#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE__SHIFT 0x1a +#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE_MASK 0x8000000 +#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE__SHIFT 0x1b +#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_0_X__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_0_X__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_0_Y__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_0_Y__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_0_Z__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_0_Z__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_0_W__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_0_W__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_1_X__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_1_X__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_1_Y__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_1_Y__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_1_Z__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_1_Z__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_1_W__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_1_W__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_2_X__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_2_X__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_2_Y__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_2_Y__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_2_Z__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_2_Z__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_2_W__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_2_W__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_3_X__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_3_X__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_3_Y__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_3_Y__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_3_Z__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_3_Z__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_3_W__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_3_W__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_4_X__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_4_X__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_4_Y__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_4_Y__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_4_Z__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_4_Z__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_4_W__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_4_W__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_5_X__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_5_X__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_5_Y__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_5_Y__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_5_Z__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_5_Z__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_UCP_5_W__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_UCP_5_W__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_POINT_X_RAD__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_POINT_X_RAD__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_POINT_Y_RAD__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_POINT_Y_RAD__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_POINT_SIZE__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_POINT_SIZE__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_POINT_CULL_RAD__DATA_REGISTER_MASK 0xffffffff +#define PA_CL_POINT_CULL_RAD__DATA_REGISTER__SHIFT 0x0 +#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK 0x1 +#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA__SHIFT 0x0 +#define PA_CL_ENHANCE__NUM_CLIP_SEQ_MASK 0x6 +#define PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT 0x1 +#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL_MASK 0x8 +#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL__SHIFT 0x3 +#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE_MASK 0x10 +#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE__SHIFT 0x4 +#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL_MASK 0x20 +#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL__SHIFT 0x5 +#define PA_CL_ENHANCE__ECO_SPARE3_MASK 0x10000000 +#define PA_CL_ENHANCE__ECO_SPARE3__SHIFT 0x1c +#define PA_CL_ENHANCE__ECO_SPARE2_MASK 0x20000000 +#define PA_CL_ENHANCE__ECO_SPARE2__SHIFT 0x1d +#define PA_CL_ENHANCE__ECO_SPARE1_MASK 0x40000000 +#define PA_CL_ENHANCE__ECO_SPARE1__SHIFT 0x1e +#define PA_CL_ENHANCE__ECO_SPARE0_MASK 0x80000000 +#define PA_CL_ENHANCE__ECO_SPARE0__SHIFT 0x1f +#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE_MASK 0x1 +#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE__SHIFT 0x0 +#define PA_SU_VTX_CNTL__PIX_CENTER_MASK 0x1 +#define PA_SU_VTX_CNTL__PIX_CENTER__SHIFT 0x0 +#define PA_SU_VTX_CNTL__ROUND_MODE_MASK 0x6 +#define PA_SU_VTX_CNTL__ROUND_MODE__SHIFT 0x1 +#define PA_SU_VTX_CNTL__QUANT_MODE_MASK 0x38 +#define PA_SU_VTX_CNTL__QUANT_MODE__SHIFT 0x3 +#define PA_SU_POINT_SIZE__HEIGHT_MASK 0xffff +#define PA_SU_POINT_SIZE__HEIGHT__SHIFT 0x0 +#define PA_SU_POINT_SIZE__WIDTH_MASK 0xffff0000 +#define PA_SU_POINT_SIZE__WIDTH__SHIFT 0x10 +#define PA_SU_POINT_MINMAX__MIN_SIZE_MASK 0xffff +#define PA_SU_POINT_MINMAX__MIN_SIZE__SHIFT 0x0 +#define PA_SU_POINT_MINMAX__MAX_SIZE_MASK 0xffff0000 +#define PA_SU_POINT_MINMAX__MAX_SIZE__SHIFT 0x10 +#define PA_SU_LINE_CNTL__WIDTH_MASK 0xffff +#define PA_SU_LINE_CNTL__WIDTH__SHIFT 0x0 +#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET_MASK 0x3 +#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET__SHIFT 0x0 +#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH_MASK 0x4 +#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH__SHIFT 0x2 +#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM_MASK 0x8 +#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM__SHIFT 0x3 +#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST_MASK 0x10 +#define PA_SU_LINE_STIPPLE_CNTL__DIAMOND_ADJUST__SHIFT 0x4 +#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE_MASK 0xffffffff +#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE__SHIFT 0x0 +#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK 0x1 +#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT 0x0 +#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK 0x2 +#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT 0x1 +#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK 0x4 +#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT 0x2 +#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK 0x8 +#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT 0x3 +#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA_MASK 0x10 +#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA__SHIFT 0x4 +#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA_MASK 0x20 +#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA__SHIFT 0x5 +#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA_MASK 0x40 +#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA__SHIFT 0x6 +#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA_MASK 0x80 +#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA__SHIFT 0x7 +#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT_MASK 0xff00 +#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT__SHIFT 0x8 +#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION_MASK 0x40000000 +#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION__SHIFT 0x1e +#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION_MASK 0x80000000 +#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION__SHIFT 0x1f +#define PA_SU_SC_MODE_CNTL__CULL_FRONT_MASK 0x1 +#define PA_SU_SC_MODE_CNTL__CULL_FRONT__SHIFT 0x0 +#define PA_SU_SC_MODE_CNTL__CULL_BACK_MASK 0x2 +#define PA_SU_SC_MODE_CNTL__CULL_BACK__SHIFT 0x1 +#define PA_SU_SC_MODE_CNTL__FACE_MASK 0x4 +#define PA_SU_SC_MODE_CNTL__FACE__SHIFT 0x2 +#define PA_SU_SC_MODE_CNTL__POLY_MODE_MASK 0x18 +#define PA_SU_SC_MODE_CNTL__POLY_MODE__SHIFT 0x3 +#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE_MASK 0xe0 +#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE__SHIFT 0x5 +#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE_MASK 0x700 +#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE__SHIFT 0x8 +#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE_MASK 0x800 +#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE__SHIFT 0xb +#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE_MASK 0x1000 +#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE__SHIFT 0xc +#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE_MASK 0x2000 +#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE__SHIFT 0xd +#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE_MASK 0x10000 +#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE__SHIFT 0x10 +#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST_MASK 0x80000 +#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST__SHIFT 0x13 +#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS_MASK 0x100000 +#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS__SHIFT 0x14 +#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA_MASK 0x200000 +#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA__SHIFT 0x15 +#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS_MASK 0xff +#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS__SHIFT 0x0 +#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT_MASK 0x100 +#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT__SHIFT 0x8 +#define PA_SU_POLY_OFFSET_CLAMP__CLAMP_MASK 0xffffffff +#define PA_SU_POLY_OFFSET_CLAMP__CLAMP__SHIFT 0x0 +#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE_MASK 0xffffffff +#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE__SHIFT 0x0 +#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET_MASK 0xffffffff +#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET__SHIFT 0x0 +#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE_MASK 0xffffffff +#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE__SHIFT 0x0 +#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET_MASK 0xffffffff +#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET__SHIFT 0x0 +#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X_MASK 0x1ff +#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X__SHIFT 0x0 +#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y_MASK 0x1ff0000 +#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y__SHIFT 0x10 +#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE_MASK 0xffffff +#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE__SHIFT 0x0 +#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00 +#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff +#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0 +#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00 +#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa +#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffff +#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffff +#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffff +#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffff +#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES_MASK 0x7 +#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES__SHIFT 0x0 +#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN_MASK 0x10 +#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN__SHIFT 0x4 +#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST_MASK 0x1e000 +#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST__SHIFT 0xd +#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES_MASK 0x700000 +#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES__SHIFT 0x14 +#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE_MASK 0x3000000 +#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE__SHIFT 0x18 +#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0_MASK 0xffff +#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0__SHIFT 0x0 +#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0_MASK 0xffff0000 +#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0__SHIFT 0x10 +#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1_MASK 0xffff +#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1__SHIFT 0x0 +#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1_MASK 0xffff0000 +#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y__SHIFT 0x1c +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X_MASK 0xf +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X__SHIFT 0x0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y_MASK 0xf0 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y__SHIFT 0x4 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X_MASK 0xf00 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X__SHIFT 0x8 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y_MASK 0xf000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y__SHIFT 0xc +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X_MASK 0xf0000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X__SHIFT 0x10 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y_MASK 0xf00000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y__SHIFT 0x14 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X_MASK 0xf000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X__SHIFT 0x18 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y_MASK 0xf0000000 +#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y__SHIFT 0x1c +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0_MASK 0xf +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0__SHIFT 0x0 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1_MASK 0xf0 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1__SHIFT 0x4 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2_MASK 0xf00 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2__SHIFT 0x8 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3_MASK 0xf000 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3__SHIFT 0xc +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4_MASK 0xf0000 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4__SHIFT 0x10 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5_MASK 0xf00000 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5__SHIFT 0x14 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6_MASK 0xf000000 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6__SHIFT 0x18 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7_MASK 0xf0000000 +#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7__SHIFT 0x1c +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8_MASK 0xf +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8__SHIFT 0x0 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9_MASK 0xf0 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9__SHIFT 0x4 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10_MASK 0xf00 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10__SHIFT 0x8 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11_MASK 0xf000 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11__SHIFT 0xc +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12_MASK 0xf0000 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12__SHIFT 0x10 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13_MASK 0xf00000 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13__SHIFT 0x14 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14_MASK 0xf000000 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14__SHIFT 0x18 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15_MASK 0xf0000000 +#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15__SHIFT 0x1c +#define PA_SC_CLIPRECT_0_TL__TL_X_MASK 0x7fff +#define PA_SC_CLIPRECT_0_TL__TL_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_0_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_0_TL__TL_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_0_BR__BR_X_MASK 0x7fff +#define PA_SC_CLIPRECT_0_BR__BR_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_0_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_0_BR__BR_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_1_TL__TL_X_MASK 0x7fff +#define PA_SC_CLIPRECT_1_TL__TL_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_1_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_1_TL__TL_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_1_BR__BR_X_MASK 0x7fff +#define PA_SC_CLIPRECT_1_BR__BR_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_1_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_1_BR__BR_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_2_TL__TL_X_MASK 0x7fff +#define PA_SC_CLIPRECT_2_TL__TL_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_2_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_2_TL__TL_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_2_BR__BR_X_MASK 0x7fff +#define PA_SC_CLIPRECT_2_BR__BR_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_2_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_2_BR__BR_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_3_TL__TL_X_MASK 0x7fff +#define PA_SC_CLIPRECT_3_TL__TL_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_3_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_3_TL__TL_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_3_BR__BR_X_MASK 0x7fff +#define PA_SC_CLIPRECT_3_BR__BR_X__SHIFT 0x0 +#define PA_SC_CLIPRECT_3_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_CLIPRECT_3_BR__BR_Y__SHIFT 0x10 +#define PA_SC_CLIPRECT_RULE__CLIP_RULE_MASK 0xffff +#define PA_SC_CLIPRECT_RULE__CLIP_RULE__SHIFT 0x0 +#define PA_SC_EDGERULE__ER_TRI_MASK 0xf +#define PA_SC_EDGERULE__ER_TRI__SHIFT 0x0 +#define PA_SC_EDGERULE__ER_POINT_MASK 0xf0 +#define PA_SC_EDGERULE__ER_POINT__SHIFT 0x4 +#define PA_SC_EDGERULE__ER_RECT_MASK 0xf00 +#define PA_SC_EDGERULE__ER_RECT__SHIFT 0x8 +#define PA_SC_EDGERULE__ER_LINE_LR_MASK 0x3f000 +#define PA_SC_EDGERULE__ER_LINE_LR__SHIFT 0xc +#define PA_SC_EDGERULE__ER_LINE_RL_MASK 0xfc0000 +#define PA_SC_EDGERULE__ER_LINE_RL__SHIFT 0x12 +#define PA_SC_EDGERULE__ER_LINE_TB_MASK 0xf000000 +#define PA_SC_EDGERULE__ER_LINE_TB__SHIFT 0x18 +#define PA_SC_EDGERULE__ER_LINE_BT_MASK 0xf0000000 +#define PA_SC_EDGERULE__ER_LINE_BT__SHIFT 0x1c +#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH_MASK 0x200 +#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH__SHIFT 0x9 +#define PA_SC_LINE_CNTL__LAST_PIXEL_MASK 0x400 +#define PA_SC_LINE_CNTL__LAST_PIXEL__SHIFT 0xa +#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA_MASK 0x800 +#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA__SHIFT 0xb +#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA_MASK 0x1000 +#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA__SHIFT 0xc +#define PA_SC_LINE_STIPPLE__LINE_PATTERN_MASK 0xffff +#define PA_SC_LINE_STIPPLE__LINE_PATTERN__SHIFT 0x0 +#define PA_SC_LINE_STIPPLE__REPEAT_COUNT_MASK 0xff0000 +#define PA_SC_LINE_STIPPLE__REPEAT_COUNT__SHIFT 0x10 +#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER_MASK 0x10000000 +#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER__SHIFT 0x1c +#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL_MASK 0x60000000 +#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL__SHIFT 0x1d +#define PA_SC_MODE_CNTL_0__MSAA_ENABLE_MASK 0x1 +#define PA_SC_MODE_CNTL_0__MSAA_ENABLE__SHIFT 0x0 +#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE_MASK 0x2 +#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE__SHIFT 0x1 +#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE_MASK 0x4 +#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE__SHIFT 0x2 +#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR_MASK 0x8 +#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR__SHIFT 0x3 +#define PA_SC_MODE_CNTL_1__WALK_SIZE_MASK 0x1 +#define PA_SC_MODE_CNTL_1__WALK_SIZE__SHIFT 0x0 +#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT_MASK 0x2 +#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT__SHIFT 0x1 +#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST_MASK 0x4 +#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST__SHIFT 0x2 +#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE_MASK 0x8 +#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE__SHIFT 0x3 +#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE_MASK 0x70 +#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE__SHIFT 0x4 +#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE_MASK 0x80 +#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE__SHIFT 0x7 +#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE_MASK 0x100 +#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE__SHIFT 0x8 +#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE_MASK 0x200 +#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE__SHIFT 0x9 +#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR_MASK 0x400 +#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR__SHIFT 0xa +#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT_MASK 0x800 +#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT__SHIFT 0xb +#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET_MASK 0x1000 +#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET__SHIFT 0xc +#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT_MASK 0x2000 +#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT__SHIFT 0xd +#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z_MASK 0x4000 +#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z__SHIFT 0xe +#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK_MASK 0x8000 +#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK__SHIFT 0xf +#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE_MASK 0x10000 +#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE__SHIFT 0x10 +#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE_MASK 0x20000 +#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE__SHIFT 0x11 +#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE_MASK 0x40000 +#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE__SHIFT 0x12 +#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE_MASK 0x80000 +#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE__SHIFT 0x13 +#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_MASK 0xf00000 +#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE__SHIFT 0x14 +#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE_MASK 0x1000000 +#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE__SHIFT 0x18 +#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE_MASK 0x2000000 +#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE__SHIFT 0x19 +#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE_MASK 0x4000000 +#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE__SHIFT 0x1a +#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE_MASK 0x8000000 +#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE__SHIFT 0x1b +#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK_MASK 0x70000000 +#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK__SHIFT 0x1c +#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK 0x3 +#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT 0x0 +#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK 0xc +#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT 0x2 +#define PA_SC_RASTER_CONFIG__RB_XSEL2_MASK 0x30 +#define PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT 0x4 +#define PA_SC_RASTER_CONFIG__RB_XSEL_MASK 0x40 +#define PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT 0x6 +#define PA_SC_RASTER_CONFIG__RB_YSEL_MASK 0x80 +#define PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT 0x7 +#define PA_SC_RASTER_CONFIG__PKR_MAP_MASK 0x300 +#define PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT 0x8 +#define PA_SC_RASTER_CONFIG__PKR_XSEL_MASK 0xc00 +#define PA_SC_RASTER_CONFIG__PKR_XSEL__SHIFT 0xa +#define PA_SC_RASTER_CONFIG__PKR_YSEL_MASK 0x3000 +#define PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT 0xc +#define PA_SC_RASTER_CONFIG__PKR_XSEL2_MASK 0xc000 +#define PA_SC_RASTER_CONFIG__PKR_XSEL2__SHIFT 0xe +#define PA_SC_RASTER_CONFIG__SC_MAP_MASK 0x30000 +#define PA_SC_RASTER_CONFIG__SC_MAP__SHIFT 0x10 +#define PA_SC_RASTER_CONFIG__SC_XSEL_MASK 0xc0000 +#define PA_SC_RASTER_CONFIG__SC_XSEL__SHIFT 0x12 +#define PA_SC_RASTER_CONFIG__SC_YSEL_MASK 0x300000 +#define PA_SC_RASTER_CONFIG__SC_YSEL__SHIFT 0x14 +#define PA_SC_RASTER_CONFIG__SE_MAP_MASK 0x3000000 +#define PA_SC_RASTER_CONFIG__SE_MAP__SHIFT 0x18 +#define PA_SC_RASTER_CONFIG__SE_XSEL_MASK 0xc000000 +#define PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT 0x1a +#define PA_SC_RASTER_CONFIG__SE_YSEL_MASK 0x30000000 +#define PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT 0x1c +#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP_MASK 0x3 +#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP__SHIFT 0x0 +#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL_MASK 0xc +#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL__SHIFT 0x2 +#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL_MASK 0x30 +#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL__SHIFT 0x4 +#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE_MASK 0x3 +#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE__SHIFT 0x0 +#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE_MASK 0xc +#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE__SHIFT 0x2 +#define PA_SC_GENERIC_SCISSOR_TL__TL_X_MASK 0x7fff +#define PA_SC_GENERIC_SCISSOR_TL__TL_X__SHIFT 0x0 +#define PA_SC_GENERIC_SCISSOR_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_GENERIC_SCISSOR_TL__TL_Y__SHIFT 0x10 +#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_GENERIC_SCISSOR_BR__BR_X_MASK 0x7fff +#define PA_SC_GENERIC_SCISSOR_BR__BR_X__SHIFT 0x0 +#define PA_SC_GENERIC_SCISSOR_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_GENERIC_SCISSOR_BR__BR_Y__SHIFT 0x10 +#define PA_SC_SCREEN_SCISSOR_TL__TL_X_MASK 0xffff +#define PA_SC_SCREEN_SCISSOR_TL__TL_X__SHIFT 0x0 +#define PA_SC_SCREEN_SCISSOR_TL__TL_Y_MASK 0xffff0000 +#define PA_SC_SCREEN_SCISSOR_TL__TL_Y__SHIFT 0x10 +#define PA_SC_SCREEN_SCISSOR_BR__BR_X_MASK 0xffff +#define PA_SC_SCREEN_SCISSOR_BR__BR_X__SHIFT 0x0 +#define PA_SC_SCREEN_SCISSOR_BR__BR_Y_MASK 0xffff0000 +#define PA_SC_SCREEN_SCISSOR_BR__BR_Y__SHIFT 0x10 +#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET_MASK 0xffff +#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET__SHIFT 0x0 +#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET_MASK 0xffff0000 +#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET__SHIFT 0x10 +#define PA_SC_WINDOW_SCISSOR_TL__TL_X_MASK 0x7fff +#define PA_SC_WINDOW_SCISSOR_TL__TL_X__SHIFT 0x0 +#define PA_SC_WINDOW_SCISSOR_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_WINDOW_SCISSOR_TL__TL_Y__SHIFT 0x10 +#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_WINDOW_SCISSOR_BR__BR_X_MASK 0x7fff +#define PA_SC_WINDOW_SCISSOR_BR__BR_X__SHIFT 0x0 +#define PA_SC_WINDOW_SCISSOR_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_WINDOW_SCISSOR_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_0_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_0_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_1_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_1_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_2_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_2_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_3_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_3_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_4_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_4_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_5_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_5_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_6_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_6_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_7_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_7_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_8_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_8_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_9_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_9_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_10_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_10_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_11_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_11_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_12_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_12_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_13_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_13_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_14_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_14_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_15_TL__TL_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_15_TL__TL_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE_MASK 0x80000000 +#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE__SHIFT 0x1f +#define PA_SC_VPORT_SCISSOR_0_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_0_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_1_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_1_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_2_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_2_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_3_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_3_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_4_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_4_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_5_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_5_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_6_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_6_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_7_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_7_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_8_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_8_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_9_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_9_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_10_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_10_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_11_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_11_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_12_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_12_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_13_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_13_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_14_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_14_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_SCISSOR_15_BR__BR_X_MASK 0x7fff +#define PA_SC_VPORT_SCISSOR_15_BR__BR_X__SHIFT 0x0 +#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y_MASK 0x7fff0000 +#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y__SHIFT 0x10 +#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN_MASK 0xffffffff +#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX_MASK 0xffffffff +#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX__SHIFT 0x0 +#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK 0x1 +#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER__SHIFT 0x0 +#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX_MASK 0x2 +#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX__SHIFT 0x1 +#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX_MASK 0x4 +#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX__SHIFT 0x2 +#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS_MASK 0x8 +#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS__SHIFT 0x3 +#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID_MASK 0x10 +#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID__SHIFT 0x4 +#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX_MASK 0x20 +#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX__SHIFT 0x5 +#define PA_SC_ENHANCE__DISABLE_PW_BUBBLE_COLLAPSE_MASK 0xc0 +#define PA_SC_ENHANCE__DISABLE_PW_BUBBLE_COLLAPSE__SHIFT 0x6 +#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER_MASK 0x100 +#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER__SHIFT 0x8 +#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION_MASK 0x200 +#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION__SHIFT 0x9 +#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM_MASK 0x400 +#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM__SHIFT 0xa +#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE_MASK 0x800 +#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE__SHIFT 0xb +#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE_MASK 0x1000 +#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE__SHIFT 0xc +#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE_MASK 0x2000 +#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE__SHIFT 0xd +#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS_MASK 0x4000 +#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS__SHIFT 0xe +#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE_MASK 0x8000 +#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE__SHIFT 0xf +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE_MASK 0x10000 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE__SHIFT 0x10 +#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE_MASK 0x20000 +#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE__SHIFT 0x11 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST_MASK 0x40000 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST__SHIFT 0x12 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING_MASK 0x80000 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING__SHIFT 0x13 +#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY_MASK 0x100000 +#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY__SHIFT 0x14 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING_MASK 0x200000 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING__SHIFT 0x15 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING_MASK 0x400000 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING__SHIFT 0x16 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS_MASK 0x800000 +#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS__SHIFT 0x17 +#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID_MASK 0x1000000 +#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID__SHIFT 0x18 +#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO_MASK 0x2000000 +#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO__SHIFT 0x19 +#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK 0x4000000 +#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT 0x1a +#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING_MASK 0x8000000 +#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING__SHIFT 0x1b +#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET_MASK 0x10000000 +#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET__SHIFT 0x1c +#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET_MASK 0x20000000 +#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET__SHIFT 0x1d +#define PA_SC_ENHANCE__ECO_SPARE1_MASK 0x40000000 +#define PA_SC_ENHANCE__ECO_SPARE1__SHIFT 0x1e +#define PA_SC_ENHANCE__ECO_SPARE0_MASK 0x80000000 +#define PA_SC_ENHANCE__ECO_SPARE0__SHIFT 0x1f +#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE_MASK 0x3f +#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT 0x0 +#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE_MASK 0x7fc0 +#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT 0x6 +#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE_MASK 0x1f8000 +#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT 0xf +#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE_MASK 0xff800000 +#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT 0x17 +#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE_MASK 0x3f +#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE__SHIFT 0x0 +#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE_MASK 0xfc0 +#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE__SHIFT 0x6 +#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE_MASK 0x3f000 +#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE__SHIFT 0xc +#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE_MASK 0xfc0000 +#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE__SHIFT 0x12 +#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT_MASK 0xffff +#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT 0x0 +#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT_MASK 0xffff0000 +#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT 0x10 +#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR_MASK 0xf +#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR__SHIFT 0x0 +#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT_MASK 0xff00 +#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT__SHIFT 0x8 +#define PA_SC_SCREEN_EXTENT_MIN_0__X_MASK 0xffff +#define PA_SC_SCREEN_EXTENT_MIN_0__X__SHIFT 0x0 +#define PA_SC_SCREEN_EXTENT_MIN_0__Y_MASK 0xffff0000 +#define PA_SC_SCREEN_EXTENT_MIN_0__Y__SHIFT 0x10 +#define PA_SC_SCREEN_EXTENT_MAX_0__X_MASK 0xffff +#define PA_SC_SCREEN_EXTENT_MAX_0__X__SHIFT 0x0 +#define PA_SC_SCREEN_EXTENT_MAX_0__Y_MASK 0xffff0000 +#define PA_SC_SCREEN_EXTENT_MAX_0__Y__SHIFT 0x10 +#define PA_SC_SCREEN_EXTENT_MIN_1__X_MASK 0xffff +#define PA_SC_SCREEN_EXTENT_MIN_1__X__SHIFT 0x0 +#define PA_SC_SCREEN_EXTENT_MIN_1__Y_MASK 0xffff0000 +#define PA_SC_SCREEN_EXTENT_MIN_1__Y__SHIFT 0x10 +#define PA_SC_SCREEN_EXTENT_MAX_1__X_MASK 0xffff +#define PA_SC_SCREEN_EXTENT_MAX_1__X__SHIFT 0x0 +#define PA_SC_SCREEN_EXTENT_MAX_1__Y_MASK 0xffff0000 +#define PA_SC_SCREEN_EXTENT_MAX_1__Y__SHIFT 0x10 +#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL_MASK 0x3ff +#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT 0x0 +#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK 0x1 +#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT 0x0 +#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK 0x2 +#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT 0x1 +#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD_MASK 0x3fff +#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD__SHIFT 0x0 +#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD_MASK 0x3fff +#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD__SHIFT 0x0 +#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK 0xffff +#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT 0x0 +#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT_MASK 0xffff +#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT__SHIFT 0x0 +#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK 0x1 +#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT 0x0 +#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK 0x2 +#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT 0x1 +#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD_MASK 0x3fff +#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD__SHIFT 0x0 +#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD_MASK 0x3fff +#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD__SHIFT 0x0 +#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK 0xffff +#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT 0x0 +#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT_MASK 0xffff +#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT__SHIFT 0x0 +#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK 0x1 +#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT 0x0 +#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK 0x2 +#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT 0x1 +#define PA_SC_TRAP_SCREEN_H__X_COORD_MASK 0x3fff +#define PA_SC_TRAP_SCREEN_H__X_COORD__SHIFT 0x0 +#define PA_SC_TRAP_SCREEN_V__Y_COORD_MASK 0x3fff +#define PA_SC_TRAP_SCREEN_V__Y_COORD__SHIFT 0x0 +#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT_MASK 0xffff +#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT 0x0 +#define PA_SC_TRAP_SCREEN_COUNT__COUNT_MASK 0xffff +#define PA_SC_TRAP_SCREEN_COUNT__COUNT__SHIFT 0x0 +#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK 0x1 +#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT 0x0 +#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK 0x1 +#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT 0x0 +#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK 0x1 +#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT 0x0 +#define PA_CL_CNTL_STATUS__CL_BUSY_MASK 0x80000000 +#define PA_CL_CNTL_STATUS__CL_BUSY__SHIFT 0x1f +#define PA_SU_CNTL_STATUS__SU_BUSY_MASK 0x80000000 +#define PA_SU_CNTL_STATUS__SU_BUSY__SHIFT 0x1f +#define PA_SC_FIFO_DEPTH_CNTL__DEPTH_MASK 0x3ff +#define PA_SC_FIFO_DEPTH_CNTL__DEPTH__SHIFT 0x0 +#define CGTT_PA_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_PA_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK 0x20000000 +#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT 0x1d +#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK 0x40000000 +#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT 0x1e +#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE_MASK 0x80000000 +#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE__SHIFT 0x1f +#define CGTT_SC_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_SC_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_SC_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX_MASK 0x1f +#define PA_SU_DEBUG_CNTL__SU_DEBUG_INDX__SHIFT 0x0 +#define PA_SU_DEBUG_DATA__DATA_MASK 0xffffffff +#define PA_SU_DEBUG_DATA__DATA__SHIFT 0x0 +#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX_MASK 0x3f +#define PA_SC_DEBUG_CNTL__SC_DEBUG_INDX__SHIFT 0x0 +#define PA_SC_DEBUG_DATA__DATA_MASK 0xffffffff +#define PA_SC_DEBUG_DATA__DATA__SHIFT 0x0 +#define CLIPPER_DEBUG_REG00__ALWAYS_ZERO_MASK 0xff +#define CLIPPER_DEBUG_REG00__ALWAYS_ZERO__SHIFT 0x0 +#define CLIPPER_DEBUG_REG00__clip_ga_bc_fifo_write_MASK 0x100 +#define CLIPPER_DEBUG_REG00__clip_ga_bc_fifo_write__SHIFT 0x8 +#define CLIPPER_DEBUG_REG00__su_clip_baryc_free_MASK 0x600 +#define CLIPPER_DEBUG_REG00__su_clip_baryc_free__SHIFT 0x9 +#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_write_MASK 0x800 +#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_write__SHIFT 0xb +#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_full_MASK 0x1000 +#define CLIPPER_DEBUG_REG00__clip_to_ga_fifo_full__SHIFT 0xc +#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_empty_MASK 0x2000 +#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_empty__SHIFT 0xd +#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_full_MASK 0x4000 +#define CLIPPER_DEBUG_REG00__primic_to_clprim_fifo_full__SHIFT 0xe +#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_empty_MASK 0x8000 +#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_empty__SHIFT 0xf +#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_full_MASK 0x10000 +#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_full__SHIFT 0x10 +#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_empty_MASK 0x20000 +#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_empty__SHIFT 0x11 +#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_full_MASK 0x40000 +#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_full__SHIFT 0x12 +#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_empty_MASK 0x80000 +#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_empty__SHIFT 0x13 +#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_full_MASK 0x100000 +#define CLIPPER_DEBUG_REG00__vgt_to_clips_fifo_full__SHIFT 0x14 +#define CLIPPER_DEBUG_REG00__clipcode_fifo_fifo_empty_MASK 0x200000 +#define CLIPPER_DEBUG_REG00__clipcode_fifo_fifo_empty__SHIFT 0x15 +#define CLIPPER_DEBUG_REG00__clipcode_fifo_full_MASK 0x400000 +#define CLIPPER_DEBUG_REG00__clipcode_fifo_full__SHIFT 0x16 +#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_empty_MASK 0x800000 +#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_empty__SHIFT 0x17 +#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_full_MASK 0x1000000 +#define CLIPPER_DEBUG_REG00__vte_out_clip_fifo_fifo_full__SHIFT 0x18 +#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_empty_MASK 0x2000000 +#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_empty__SHIFT 0x19 +#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_full_MASK 0x4000000 +#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_full__SHIFT 0x1a +#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_empty_MASK 0x8000000 +#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_empty__SHIFT 0x1b +#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_full_MASK 0x10000000 +#define CLIPPER_DEBUG_REG00__ccgen_to_clipcc_fifo_full__SHIFT 0x1c +#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_write_MASK 0x20000000 +#define CLIPPER_DEBUG_REG00__clip_to_outsm_fifo_write__SHIFT 0x1d +#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_write_MASK 0x40000000 +#define CLIPPER_DEBUG_REG00__vte_out_orig_fifo_fifo_write__SHIFT 0x1e +#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_write_MASK 0x80000000 +#define CLIPPER_DEBUG_REG00__vgt_to_clipp_fifo_write__SHIFT 0x1f +#define CLIPPER_DEBUG_REG01__ALWAYS_ZERO_MASK 0xff +#define CLIPPER_DEBUG_REG01__ALWAYS_ZERO__SHIFT 0x0 +#define CLIPPER_DEBUG_REG01__clip_extra_bc_valid_MASK 0x700 +#define CLIPPER_DEBUG_REG01__clip_extra_bc_valid__SHIFT 0x8 +#define CLIPPER_DEBUG_REG01__clip_vert_vte_valid_MASK 0x3800 +#define CLIPPER_DEBUG_REG01__clip_vert_vte_valid__SHIFT 0xb +#define CLIPPER_DEBUG_REG01__clip_to_outsm_vertex_deallocate_MASK 0x1c000 +#define CLIPPER_DEBUG_REG01__clip_to_outsm_vertex_deallocate__SHIFT 0xe +#define CLIPPER_DEBUG_REG01__clip_to_outsm_deallocate_slot_MASK 0xe0000 +#define CLIPPER_DEBUG_REG01__clip_to_outsm_deallocate_slot__SHIFT 0x11 +#define CLIPPER_DEBUG_REG01__clip_to_outsm_null_primitive_MASK 0x100000 +#define CLIPPER_DEBUG_REG01__clip_to_outsm_null_primitive__SHIFT 0x14 +#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_2_MASK 0x200000 +#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_2__SHIFT 0x15 +#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_1_MASK 0x400000 +#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_1__SHIFT 0x16 +#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_0_MASK 0x800000 +#define CLIPPER_DEBUG_REG01__vte_positions_vte_clip_vte_naninf_kill_0__SHIFT 0x17 +#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_extra_bc_valid_MASK 0x1000000 +#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_extra_bc_valid__SHIFT 0x18 +#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vte_naninf_kill_MASK 0x2000000 +#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vte_naninf_kill__SHIFT 0x19 +#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vertex_store_indx_MASK 0xc000000 +#define CLIPPER_DEBUG_REG01__vte_out_clip_rd_vertex_store_indx__SHIFT 0x1a +#define CLIPPER_DEBUG_REG01__clip_ga_bc_fifo_write_MASK 0x10000000 +#define CLIPPER_DEBUG_REG01__clip_ga_bc_fifo_write__SHIFT 0x1c +#define CLIPPER_DEBUG_REG01__clip_to_ga_fifo_write_MASK 0x20000000 +#define CLIPPER_DEBUG_REG01__clip_to_ga_fifo_write__SHIFT 0x1d +#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_advanceread_MASK 0x40000000 +#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_advanceread__SHIFT 0x1e +#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_empty_MASK 0x80000000 +#define CLIPPER_DEBUG_REG01__vte_out_clip_fifo_fifo_empty__SHIFT 0x1f +#define CLIPPER_DEBUG_REG02__clip_extra_bc_valid_MASK 0x7 +#define CLIPPER_DEBUG_REG02__clip_extra_bc_valid__SHIFT 0x0 +#define CLIPPER_DEBUG_REG02__clip_vert_vte_valid_MASK 0x38 +#define CLIPPER_DEBUG_REG02__clip_vert_vte_valid__SHIFT 0x3 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_clip_seq_indx_MASK 0xc0 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_clip_seq_indx__SHIFT 0x6 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_2_MASK 0xf00 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_2__SHIFT 0x8 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_1_MASK 0xf000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_1__SHIFT 0xc +#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_0_MASK 0xf0000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_vertex_store_indx_0__SHIFT 0x10 +#define CLIPPER_DEBUG_REG02__clip_to_clipga_extra_bc_coords_MASK 0x100000 +#define CLIPPER_DEBUG_REG02__clip_to_clipga_extra_bc_coords__SHIFT 0x14 +#define CLIPPER_DEBUG_REG02__clip_to_clipga_vte_naninf_kill_MASK 0x200000 +#define CLIPPER_DEBUG_REG02__clip_to_clipga_vte_naninf_kill__SHIFT 0x15 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_end_of_packet_MASK 0x400000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_end_of_packet__SHIFT 0x16 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_first_prim_of_slot_MASK 0x800000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_first_prim_of_slot__SHIFT 0x17 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_clipped_prim_MASK 0x1000000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_clipped_prim__SHIFT 0x18 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_null_primitive_MASK 0x2000000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_null_primitive__SHIFT 0x19 +#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_full_MASK 0x4000000 +#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_full__SHIFT 0x1a +#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_full_MASK 0x8000000 +#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_full__SHIFT 0x1b +#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_write_MASK 0x10000000 +#define CLIPPER_DEBUG_REG02__clip_ga_bc_fifo_write__SHIFT 0x1c +#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_write_MASK 0x20000000 +#define CLIPPER_DEBUG_REG02__clip_to_ga_fifo_write__SHIFT 0x1d +#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_advanceread_MASK 0x40000000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_advanceread__SHIFT 0x1e +#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_empty_MASK 0x80000000 +#define CLIPPER_DEBUG_REG02__clip_to_outsm_fifo_empty__SHIFT 0x1f +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_code_or_MASK 0x3fff +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_code_or__SHIFT 0x0 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_id_MASK 0xfc000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_id__SHIFT 0xe +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_state_var_indx_MASK 0x700000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_state_var_indx__SHIFT 0x14 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_primitive_MASK 0x800000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_clip_primitive__SHIFT 0x17 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_deallocate_slot_MASK 0x7000000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_deallocate_slot__SHIFT 0x18 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_first_prim_of_slot_MASK 0x8000000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_end_of_packet_MASK 0x10000000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_end_of_packet__SHIFT 0x1c +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG03__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_param_cache_indx_0_MASK 0x7fe +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_param_cache_indx_0__SHIFT 0x1 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG04__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_code_or_MASK 0x3fff +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_code_or__SHIFT 0x0 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_id_MASK 0xfc000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_id__SHIFT 0xe +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_state_var_indx_MASK 0x700000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_state_var_indx__SHIFT 0x14 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_primitive_MASK 0x800000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_clip_primitive__SHIFT 0x17 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_deallocate_slot_MASK 0x7000000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_deallocate_slot__SHIFT 0x18 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_first_prim_of_slot_MASK 0x8000000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_end_of_packet_MASK 0x10000000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_end_of_packet__SHIFT 0x1c +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG05__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_param_cache_indx_0_MASK 0x7fe +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_param_cache_indx_0__SHIFT 0x1 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG06__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_code_or_MASK 0x3fff +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_code_or__SHIFT 0x0 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_id_MASK 0xfc000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_id__SHIFT 0xe +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_state_var_indx_MASK 0x700000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_state_var_indx__SHIFT 0x14 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_primitive_MASK 0x800000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_clip_primitive__SHIFT 0x17 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_deallocate_slot_MASK 0x7000000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_deallocate_slot__SHIFT 0x18 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_first_prim_of_slot_MASK 0x8000000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_end_of_packet_MASK 0x10000000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_end_of_packet__SHIFT 0x1c +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG07__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_param_cache_indx_0_MASK 0x7fe +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_param_cache_indx_0__SHIFT 0x1 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG08__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_code_or_MASK 0x3fff +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_code_or__SHIFT 0x0 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_id_MASK 0xfc000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_id__SHIFT 0xe +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_state_var_indx_MASK 0x700000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_state_var_indx__SHIFT 0x14 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_primitive_MASK 0x800000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_clip_primitive__SHIFT 0x17 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_deallocate_slot_MASK 0x7000000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_deallocate_slot__SHIFT 0x18 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_first_prim_of_slot_MASK 0x8000000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_first_prim_of_slot__SHIFT 0x1b +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_end_of_packet_MASK 0x10000000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_end_of_packet__SHIFT 0x1c +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG09__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_param_cache_indx_0_MASK 0x7fe +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_param_cache_indx_0__SHIFT 0x1 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_2_MASK 0x1f800 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_2__SHIFT 0xb +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_1_MASK 0x7e0000 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_1__SHIFT 0x11 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_0_MASK 0x1f800000 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_vertex_store_indx_0__SHIFT 0x17 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_event_MASK 0x20000000 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_event__SHIFT 0x1d +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_null_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_null_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG10__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_event_MASK 0x1 +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_event__SHIFT 0x0 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_event_MASK 0x2 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_event__SHIFT 0x1 +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_event_MASK 0x4 +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_event__SHIFT 0x2 +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_event_MASK 0x8 +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_event__SHIFT 0x3 +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_primitive_MASK 0x10 +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_primitive__SHIFT 0x4 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_primitive_MASK 0x20 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_primitive__SHIFT 0x5 +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_primitive_MASK 0x40 +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_primitive__SHIFT 0x6 +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_primitive_MASK 0x80 +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_primitive__SHIFT 0x7 +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf00 +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x8 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf000 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0xc +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf0000 +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x10 +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_to_outsm_cnt_MASK 0xf00000 +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x14 +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_prim_valid_MASK 0x1000000 +#define CLIPPER_DEBUG_REG11__clipsm3_clip_to_clipga_prim_valid__SHIFT 0x18 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_prim_valid_MASK 0x2000000 +#define CLIPPER_DEBUG_REG11__clipsm2_clip_to_clipga_prim_valid__SHIFT 0x19 +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_prim_valid_MASK 0x4000000 +#define CLIPPER_DEBUG_REG11__clipsm1_clip_to_clipga_prim_valid__SHIFT 0x1a +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_prim_valid_MASK 0x8000000 +#define CLIPPER_DEBUG_REG11__clipsm0_clip_to_clipga_prim_valid__SHIFT 0x1b +#define CLIPPER_DEBUG_REG11__clipsm3_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x10000000 +#define CLIPPER_DEBUG_REG11__clipsm3_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1c +#define CLIPPER_DEBUG_REG11__clipsm2_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x20000000 +#define CLIPPER_DEBUG_REG11__clipsm2_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1d +#define CLIPPER_DEBUG_REG11__clipsm1_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x40000000 +#define CLIPPER_DEBUG_REG11__clipsm1_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1e +#define CLIPPER_DEBUG_REG11__clipsm0_inc_clip_to_clipga_clip_to_outsm_cnt_MASK 0x80000000 +#define CLIPPER_DEBUG_REG11__clipsm0_inc_clip_to_clipga_clip_to_outsm_cnt__SHIFT 0x1f +#define CLIPPER_DEBUG_REG12__ALWAYS_ZERO_MASK 0xff +#define CLIPPER_DEBUG_REG12__ALWAYS_ZERO__SHIFT 0x0 +#define CLIPPER_DEBUG_REG12__clip_priority_available_vte_out_clip_MASK 0x1f00 +#define CLIPPER_DEBUG_REG12__clip_priority_available_vte_out_clip__SHIFT 0x8 +#define CLIPPER_DEBUG_REG12__clip_priority_available_clip_verts_MASK 0x3e000 +#define CLIPPER_DEBUG_REG12__clip_priority_available_clip_verts__SHIFT 0xd +#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_out_MASK 0xc0000 +#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_out__SHIFT 0x12 +#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_vert_MASK 0x300000 +#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_vert__SHIFT 0x14 +#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_load_MASK 0xc00000 +#define CLIPPER_DEBUG_REG12__clip_priority_seq_indx_load__SHIFT 0x16 +#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_clip_primitive_MASK 0x1000000 +#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_clip_primitive__SHIFT 0x18 +#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_prim_valid_MASK 0x2000000 +#define CLIPPER_DEBUG_REG12__clipsm3_clprim_to_clip_prim_valid__SHIFT 0x19 +#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_clip_primitive_MASK 0x4000000 +#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_clip_primitive__SHIFT 0x1a +#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_prim_valid_MASK 0x8000000 +#define CLIPPER_DEBUG_REG12__clipsm2_clprim_to_clip_prim_valid__SHIFT 0x1b +#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_clip_primitive_MASK 0x10000000 +#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_clip_primitive__SHIFT 0x1c +#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_prim_valid_MASK 0x20000000 +#define CLIPPER_DEBUG_REG12__clipsm1_clprim_to_clip_prim_valid__SHIFT 0x1d +#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_clip_primitive_MASK 0x40000000 +#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_clip_primitive__SHIFT 0x1e +#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG12__clipsm0_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG13__clprim_in_back_state_var_indx_MASK 0x7 +#define CLIPPER_DEBUG_REG13__clprim_in_back_state_var_indx__SHIFT 0x0 +#define CLIPPER_DEBUG_REG13__point_clip_candidate_MASK 0x8 +#define CLIPPER_DEBUG_REG13__point_clip_candidate__SHIFT 0x3 +#define CLIPPER_DEBUG_REG13__prim_nan_kill_MASK 0x10 +#define CLIPPER_DEBUG_REG13__prim_nan_kill__SHIFT 0x4 +#define CLIPPER_DEBUG_REG13__clprim_clip_primitive_MASK 0x20 +#define CLIPPER_DEBUG_REG13__clprim_clip_primitive__SHIFT 0x5 +#define CLIPPER_DEBUG_REG13__clprim_cull_primitive_MASK 0x40 +#define CLIPPER_DEBUG_REG13__clprim_cull_primitive__SHIFT 0x6 +#define CLIPPER_DEBUG_REG13__prim_back_valid_MASK 0x80 +#define CLIPPER_DEBUG_REG13__prim_back_valid__SHIFT 0x7 +#define CLIPPER_DEBUG_REG13__vertval_bits_vertex_cc_next_valid_MASK 0xf00 +#define CLIPPER_DEBUG_REG13__vertval_bits_vertex_cc_next_valid__SHIFT 0x8 +#define CLIPPER_DEBUG_REG13__clipcc_vertex_store_indx_MASK 0x3000 +#define CLIPPER_DEBUG_REG13__clipcc_vertex_store_indx__SHIFT 0xc +#define CLIPPER_DEBUG_REG13__vte_out_orig_fifo_fifo_empty_MASK 0x4000 +#define CLIPPER_DEBUG_REG13__vte_out_orig_fifo_fifo_empty__SHIFT 0xe +#define CLIPPER_DEBUG_REG13__clipcode_fifo_fifo_empty_MASK 0x8000 +#define CLIPPER_DEBUG_REG13__clipcode_fifo_fifo_empty__SHIFT 0xf +#define CLIPPER_DEBUG_REG13__ccgen_to_clipcc_fifo_empty_MASK 0x10000 +#define CLIPPER_DEBUG_REG13__ccgen_to_clipcc_fifo_empty__SHIFT 0x10 +#define CLIPPER_DEBUG_REG13__clip_priority_seq_indx_out_cnt_MASK 0x1e0000 +#define CLIPPER_DEBUG_REG13__clip_priority_seq_indx_out_cnt__SHIFT 0x11 +#define CLIPPER_DEBUG_REG13__outsm_clr_rd_orig_vertices_MASK 0x600000 +#define CLIPPER_DEBUG_REG13__outsm_clr_rd_orig_vertices__SHIFT 0x15 +#define CLIPPER_DEBUG_REG13__outsm_clr_rd_clipsm_wait_MASK 0x800000 +#define CLIPPER_DEBUG_REG13__outsm_clr_rd_clipsm_wait__SHIFT 0x17 +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_contents_MASK 0x1f000000 +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_contents__SHIFT 0x18 +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_full_MASK 0x20000000 +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_full__SHIFT 0x1d +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_advanceread_MASK 0x40000000 +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_advanceread__SHIFT 0x1e +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_write_MASK 0x80000000 +#define CLIPPER_DEBUG_REG13__outsm_clr_fifo_write__SHIFT 0x1f +#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_2_MASK 0x3f +#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_2__SHIFT 0x0 +#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_1_MASK 0xfc0 +#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_1__SHIFT 0x6 +#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_0_MASK 0x3f000 +#define CLIPPER_DEBUG_REG14__clprim_in_back_vertex_store_indx_0__SHIFT 0xc +#define CLIPPER_DEBUG_REG14__outputclprimtoclip_null_primitive_MASK 0x40000 +#define CLIPPER_DEBUG_REG14__outputclprimtoclip_null_primitive__SHIFT 0x12 +#define CLIPPER_DEBUG_REG14__clprim_in_back_end_of_packet_MASK 0x80000 +#define CLIPPER_DEBUG_REG14__clprim_in_back_end_of_packet__SHIFT 0x13 +#define CLIPPER_DEBUG_REG14__clprim_in_back_first_prim_of_slot_MASK 0x100000 +#define CLIPPER_DEBUG_REG14__clprim_in_back_first_prim_of_slot__SHIFT 0x14 +#define CLIPPER_DEBUG_REG14__clprim_in_back_deallocate_slot_MASK 0xe00000 +#define CLIPPER_DEBUG_REG14__clprim_in_back_deallocate_slot__SHIFT 0x15 +#define CLIPPER_DEBUG_REG14__clprim_in_back_event_id_MASK 0x3f000000 +#define CLIPPER_DEBUG_REG14__clprim_in_back_event_id__SHIFT 0x18 +#define CLIPPER_DEBUG_REG14__clprim_in_back_event_MASK 0x40000000 +#define CLIPPER_DEBUG_REG14__clprim_in_back_event__SHIFT 0x1e +#define CLIPPER_DEBUG_REG14__prim_back_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG14__prim_back_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG15__vertval_bits_vertex_vertex_store_msb_MASK 0xffff +#define CLIPPER_DEBUG_REG15__vertval_bits_vertex_vertex_store_msb__SHIFT 0x0 +#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_2_MASK 0x1f0000 +#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_2__SHIFT 0x10 +#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_1_MASK 0x3e00000 +#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_1__SHIFT 0x15 +#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_0_MASK 0x7c000000 +#define CLIPPER_DEBUG_REG15__primic_to_clprim_fifo_vertex_store_indx_0__SHIFT 0x1a +#define CLIPPER_DEBUG_REG15__primic_to_clprim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG15__primic_to_clprim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG16__sm0_prim_end_state_MASK 0x7f +#define CLIPPER_DEBUG_REG16__sm0_prim_end_state__SHIFT 0x0 +#define CLIPPER_DEBUG_REG16__sm0_ps_expand_MASK 0x80 +#define CLIPPER_DEBUG_REG16__sm0_ps_expand__SHIFT 0x7 +#define CLIPPER_DEBUG_REG16__sm0_clip_vert_cnt_MASK 0x1f00 +#define CLIPPER_DEBUG_REG16__sm0_clip_vert_cnt__SHIFT 0x8 +#define CLIPPER_DEBUG_REG16__sm0_vertex_clip_cnt_MASK 0x3e000 +#define CLIPPER_DEBUG_REG16__sm0_vertex_clip_cnt__SHIFT 0xd +#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_1_MASK 0x40000 +#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_1__SHIFT 0x12 +#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_0_MASK 0x80000 +#define CLIPPER_DEBUG_REG16__sm0_inv_to_clip_data_valid_0__SHIFT 0x13 +#define CLIPPER_DEBUG_REG16__sm0_current_state_MASK 0x7f00000 +#define CLIPPER_DEBUG_REG16__sm0_current_state__SHIFT 0x14 +#define CLIPPER_DEBUG_REG16__sm0_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000 +#define CLIPPER_DEBUG_REG16__sm0_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b +#define CLIPPER_DEBUG_REG16__sm0_clip_to_outsm_fifo_full_MASK 0x10000000 +#define CLIPPER_DEBUG_REG16__sm0_clip_to_outsm_fifo_full__SHIFT 0x1c +#define CLIPPER_DEBUG_REG16__sm0_highest_priority_seq_MASK 0x20000000 +#define CLIPPER_DEBUG_REG16__sm0_highest_priority_seq__SHIFT 0x1d +#define CLIPPER_DEBUG_REG16__sm0_outputcliptoclipga_0_MASK 0x40000000 +#define CLIPPER_DEBUG_REG16__sm0_outputcliptoclipga_0__SHIFT 0x1e +#define CLIPPER_DEBUG_REG16__sm0_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG16__sm0_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG17__sm1_prim_end_state_MASK 0x7f +#define CLIPPER_DEBUG_REG17__sm1_prim_end_state__SHIFT 0x0 +#define CLIPPER_DEBUG_REG17__sm1_ps_expand_MASK 0x80 +#define CLIPPER_DEBUG_REG17__sm1_ps_expand__SHIFT 0x7 +#define CLIPPER_DEBUG_REG17__sm1_clip_vert_cnt_MASK 0x1f00 +#define CLIPPER_DEBUG_REG17__sm1_clip_vert_cnt__SHIFT 0x8 +#define CLIPPER_DEBUG_REG17__sm1_vertex_clip_cnt_MASK 0x3e000 +#define CLIPPER_DEBUG_REG17__sm1_vertex_clip_cnt__SHIFT 0xd +#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_1_MASK 0x40000 +#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_1__SHIFT 0x12 +#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_0_MASK 0x80000 +#define CLIPPER_DEBUG_REG17__sm1_inv_to_clip_data_valid_0__SHIFT 0x13 +#define CLIPPER_DEBUG_REG17__sm1_current_state_MASK 0x7f00000 +#define CLIPPER_DEBUG_REG17__sm1_current_state__SHIFT 0x14 +#define CLIPPER_DEBUG_REG17__sm1_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000 +#define CLIPPER_DEBUG_REG17__sm1_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b +#define CLIPPER_DEBUG_REG17__sm1_clip_to_outsm_fifo_full_MASK 0x10000000 +#define CLIPPER_DEBUG_REG17__sm1_clip_to_outsm_fifo_full__SHIFT 0x1c +#define CLIPPER_DEBUG_REG17__sm1_highest_priority_seq_MASK 0x20000000 +#define CLIPPER_DEBUG_REG17__sm1_highest_priority_seq__SHIFT 0x1d +#define CLIPPER_DEBUG_REG17__sm1_outputcliptoclipga_0_MASK 0x40000000 +#define CLIPPER_DEBUG_REG17__sm1_outputcliptoclipga_0__SHIFT 0x1e +#define CLIPPER_DEBUG_REG17__sm1_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG17__sm1_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG18__sm2_prim_end_state_MASK 0x7f +#define CLIPPER_DEBUG_REG18__sm2_prim_end_state__SHIFT 0x0 +#define CLIPPER_DEBUG_REG18__sm2_ps_expand_MASK 0x80 +#define CLIPPER_DEBUG_REG18__sm2_ps_expand__SHIFT 0x7 +#define CLIPPER_DEBUG_REG18__sm2_clip_vert_cnt_MASK 0x1f00 +#define CLIPPER_DEBUG_REG18__sm2_clip_vert_cnt__SHIFT 0x8 +#define CLIPPER_DEBUG_REG18__sm2_vertex_clip_cnt_MASK 0x3e000 +#define CLIPPER_DEBUG_REG18__sm2_vertex_clip_cnt__SHIFT 0xd +#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_1_MASK 0x40000 +#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_1__SHIFT 0x12 +#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_0_MASK 0x80000 +#define CLIPPER_DEBUG_REG18__sm2_inv_to_clip_data_valid_0__SHIFT 0x13 +#define CLIPPER_DEBUG_REG18__sm2_current_state_MASK 0x7f00000 +#define CLIPPER_DEBUG_REG18__sm2_current_state__SHIFT 0x14 +#define CLIPPER_DEBUG_REG18__sm2_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000 +#define CLIPPER_DEBUG_REG18__sm2_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b +#define CLIPPER_DEBUG_REG18__sm2_clip_to_outsm_fifo_full_MASK 0x10000000 +#define CLIPPER_DEBUG_REG18__sm2_clip_to_outsm_fifo_full__SHIFT 0x1c +#define CLIPPER_DEBUG_REG18__sm2_highest_priority_seq_MASK 0x20000000 +#define CLIPPER_DEBUG_REG18__sm2_highest_priority_seq__SHIFT 0x1d +#define CLIPPER_DEBUG_REG18__sm2_outputcliptoclipga_0_MASK 0x40000000 +#define CLIPPER_DEBUG_REG18__sm2_outputcliptoclipga_0__SHIFT 0x1e +#define CLIPPER_DEBUG_REG18__sm2_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG18__sm2_clprim_to_clip_prim_valid__SHIFT 0x1f +#define CLIPPER_DEBUG_REG19__sm3_prim_end_state_MASK 0x7f +#define CLIPPER_DEBUG_REG19__sm3_prim_end_state__SHIFT 0x0 +#define CLIPPER_DEBUG_REG19__sm3_ps_expand_MASK 0x80 +#define CLIPPER_DEBUG_REG19__sm3_ps_expand__SHIFT 0x7 +#define CLIPPER_DEBUG_REG19__sm3_clip_vert_cnt_MASK 0x1f00 +#define CLIPPER_DEBUG_REG19__sm3_clip_vert_cnt__SHIFT 0x8 +#define CLIPPER_DEBUG_REG19__sm3_vertex_clip_cnt_MASK 0x3e000 +#define CLIPPER_DEBUG_REG19__sm3_vertex_clip_cnt__SHIFT 0xd +#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_1_MASK 0x40000 +#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_1__SHIFT 0x12 +#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_0_MASK 0x80000 +#define CLIPPER_DEBUG_REG19__sm3_inv_to_clip_data_valid_0__SHIFT 0x13 +#define CLIPPER_DEBUG_REG19__sm3_current_state_MASK 0x7f00000 +#define CLIPPER_DEBUG_REG19__sm3_current_state__SHIFT 0x14 +#define CLIPPER_DEBUG_REG19__sm3_clip_to_clipga_clip_to_outsm_cnt_eq0_MASK 0x8000000 +#define CLIPPER_DEBUG_REG19__sm3_clip_to_clipga_clip_to_outsm_cnt_eq0__SHIFT 0x1b +#define CLIPPER_DEBUG_REG19__sm3_clip_to_outsm_fifo_full_MASK 0x10000000 +#define CLIPPER_DEBUG_REG19__sm3_clip_to_outsm_fifo_full__SHIFT 0x1c +#define CLIPPER_DEBUG_REG19__sm3_highest_priority_seq_MASK 0x20000000 +#define CLIPPER_DEBUG_REG19__sm3_highest_priority_seq__SHIFT 0x1d +#define CLIPPER_DEBUG_REG19__sm3_outputcliptoclipga_0_MASK 0x40000000 +#define CLIPPER_DEBUG_REG19__sm3_outputcliptoclipga_0__SHIFT 0x1e +#define CLIPPER_DEBUG_REG19__sm3_clprim_to_clip_prim_valid_MASK 0x80000000 +#define CLIPPER_DEBUG_REG19__sm3_clprim_to_clip_prim_valid__SHIFT 0x1f +#define SXIFCCG_DEBUG_REG0__position_address_MASK 0x3f +#define SXIFCCG_DEBUG_REG0__position_address__SHIFT 0x0 +#define SXIFCCG_DEBUG_REG0__point_address_MASK 0x1c0 +#define SXIFCCG_DEBUG_REG0__point_address__SHIFT 0x6 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_state_var_indx_MASK 0xe00 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_state_var_indx__SHIFT 0x9 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_req_mask_MASK 0xf000 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_req_mask__SHIFT 0xc +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_pci_MASK 0x3ff0000 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_pci__SHIFT 0x10 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_sel_MASK 0xc000000 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_sel__SHIFT 0x1a +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_sp_id_MASK 0x30000000 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_sp_id__SHIFT 0x1c +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_inc_MASK 0x40000000 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_aux_inc__SHIFT 0x1e +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_advance_MASK 0x80000000 +#define SXIFCCG_DEBUG_REG0__sx_pending_rd_advance__SHIFT 0x1f +#define SXIFCCG_DEBUG_REG1__available_positions_MASK 0x7f +#define SXIFCCG_DEBUG_REG1__available_positions__SHIFT 0x0 +#define SXIFCCG_DEBUG_REG1__sx_receive_indx_MASK 0x380 +#define SXIFCCG_DEBUG_REG1__sx_receive_indx__SHIFT 0x7 +#define SXIFCCG_DEBUG_REG1__sx_pending_fifo_contents_MASK 0x7c00 +#define SXIFCCG_DEBUG_REG1__sx_pending_fifo_contents__SHIFT 0xa +#define SXIFCCG_DEBUG_REG1__statevar_bits_vs_out_misc_vec_ena_MASK 0x8000 +#define SXIFCCG_DEBUG_REG1__statevar_bits_vs_out_misc_vec_ena__SHIFT 0xf +#define SXIFCCG_DEBUG_REG1__statevar_bits_disable_sp_MASK 0xf0000 +#define SXIFCCG_DEBUG_REG1__statevar_bits_disable_sp__SHIFT 0x10 +#define SXIFCCG_DEBUG_REG1__aux_sel_MASK 0x300000 +#define SXIFCCG_DEBUG_REG1__aux_sel__SHIFT 0x14 +#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_1_MASK 0x400000 +#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_1__SHIFT 0x16 +#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_0_MASK 0x800000 +#define SXIFCCG_DEBUG_REG1__sx_to_pa_empty_0__SHIFT 0x17 +#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_1_MASK 0xf000000 +#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_1__SHIFT 0x18 +#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_0_MASK 0xf0000000 +#define SXIFCCG_DEBUG_REG1__pasx_req_cnt_0__SHIFT 0x1c +#define SXIFCCG_DEBUG_REG2__param_cache_base_MASK 0x7f +#define SXIFCCG_DEBUG_REG2__param_cache_base__SHIFT 0x0 +#define SXIFCCG_DEBUG_REG2__sx_aux_MASK 0x180 +#define SXIFCCG_DEBUG_REG2__sx_aux__SHIFT 0x7 +#define SXIFCCG_DEBUG_REG2__sx_request_indx_MASK 0x7e00 +#define SXIFCCG_DEBUG_REG2__sx_request_indx__SHIFT 0x9 +#define SXIFCCG_DEBUG_REG2__req_active_verts_loaded_MASK 0x8000 +#define SXIFCCG_DEBUG_REG2__req_active_verts_loaded__SHIFT 0xf +#define SXIFCCG_DEBUG_REG2__req_active_verts_MASK 0x7f0000 +#define SXIFCCG_DEBUG_REG2__req_active_verts__SHIFT 0x10 +#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_state_var_indx_MASK 0x3800000 +#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_state_var_indx__SHIFT 0x17 +#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_active_verts_MASK 0xfc000000 +#define SXIFCCG_DEBUG_REG2__vgt_to_ccgen_active_verts__SHIFT 0x1a +#define SXIFCCG_DEBUG_REG3__ALWAYS_ZERO_MASK 0xff +#define SXIFCCG_DEBUG_REG3__ALWAYS_ZERO__SHIFT 0x0 +#define SXIFCCG_DEBUG_REG3__vertex_fifo_entriesavailable_MASK 0xf00 +#define SXIFCCG_DEBUG_REG3__vertex_fifo_entriesavailable__SHIFT 0x8 +#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist1_vec_ena_MASK 0x1000 +#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist1_vec_ena__SHIFT 0xc +#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist0_vec_ena_MASK 0x2000 +#define SXIFCCG_DEBUG_REG3__statevar_bits_vs_out_ccdist0_vec_ena__SHIFT 0xd +#define SXIFCCG_DEBUG_REG3__available_positions_MASK 0x1fc000 +#define SXIFCCG_DEBUG_REG3__available_positions__SHIFT 0xe +#define SXIFCCG_DEBUG_REG3__current_state_MASK 0x600000 +#define SXIFCCG_DEBUG_REG3__current_state__SHIFT 0x15 +#define SXIFCCG_DEBUG_REG3__vertex_fifo_empty_MASK 0x800000 +#define SXIFCCG_DEBUG_REG3__vertex_fifo_empty__SHIFT 0x17 +#define SXIFCCG_DEBUG_REG3__vertex_fifo_full_MASK 0x1000000 +#define SXIFCCG_DEBUG_REG3__vertex_fifo_full__SHIFT 0x18 +#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_empty_MASK 0x2000000 +#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_empty__SHIFT 0x19 +#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_full_MASK 0x4000000 +#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_full__SHIFT 0x1a +#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_empty_MASK 0x8000000 +#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_empty__SHIFT 0x1b +#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_full_MASK 0x10000000 +#define SXIFCCG_DEBUG_REG3__vgt_to_ccgen_fifo_full__SHIFT 0x1c +#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_fifo_full_MASK 0x20000000 +#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_fifo_full__SHIFT 0x1d +#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_write_MASK 0x40000000 +#define SXIFCCG_DEBUG_REG3__sx0_receive_fifo_write__SHIFT 0x1e +#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_write_MASK 0x80000000 +#define SXIFCCG_DEBUG_REG3__ccgen_to_clipcc_write__SHIFT 0x1f +#define SETUP_DEBUG_REG0__su_baryc_cntl_state_MASK 0x3 +#define SETUP_DEBUG_REG0__su_baryc_cntl_state__SHIFT 0x0 +#define SETUP_DEBUG_REG0__su_cntl_state_MASK 0x3c +#define SETUP_DEBUG_REG0__su_cntl_state__SHIFT 0x2 +#define SETUP_DEBUG_REG0__pmode_state_MASK 0x3f00 +#define SETUP_DEBUG_REG0__pmode_state__SHIFT 0x8 +#define SETUP_DEBUG_REG0__ge_stallb_MASK 0x4000 +#define SETUP_DEBUG_REG0__ge_stallb__SHIFT 0xe +#define SETUP_DEBUG_REG0__geom_enable_MASK 0x8000 +#define SETUP_DEBUG_REG0__geom_enable__SHIFT 0xf +#define SETUP_DEBUG_REG0__su_clip_baryc_free_MASK 0x30000 +#define SETUP_DEBUG_REG0__su_clip_baryc_free__SHIFT 0x10 +#define SETUP_DEBUG_REG0__su_clip_rtr_MASK 0x40000 +#define SETUP_DEBUG_REG0__su_clip_rtr__SHIFT 0x12 +#define SETUP_DEBUG_REG0__pfifo_busy_MASK 0x80000 +#define SETUP_DEBUG_REG0__pfifo_busy__SHIFT 0x13 +#define SETUP_DEBUG_REG0__su_cntl_busy_MASK 0x100000 +#define SETUP_DEBUG_REG0__su_cntl_busy__SHIFT 0x14 +#define SETUP_DEBUG_REG0__geom_busy_MASK 0x200000 +#define SETUP_DEBUG_REG0__geom_busy__SHIFT 0x15 +#define SETUP_DEBUG_REG0__event_id_gated_MASK 0xfc00000 +#define SETUP_DEBUG_REG0__event_id_gated__SHIFT 0x16 +#define SETUP_DEBUG_REG0__event_gated_MASK 0x10000000 +#define SETUP_DEBUG_REG0__event_gated__SHIFT 0x1c +#define SETUP_DEBUG_REG0__pmode_prim_gated_MASK 0x20000000 +#define SETUP_DEBUG_REG0__pmode_prim_gated__SHIFT 0x1d +#define SETUP_DEBUG_REG0__su_dyn_sclk_vld_MASK 0x40000000 +#define SETUP_DEBUG_REG0__su_dyn_sclk_vld__SHIFT 0x1e +#define SETUP_DEBUG_REG0__cl_dyn_sclk_vld_MASK 0x80000000 +#define SETUP_DEBUG_REG0__cl_dyn_sclk_vld__SHIFT 0x1f +#define SETUP_DEBUG_REG1__y_sort0_gated_23_8_MASK 0xffff +#define SETUP_DEBUG_REG1__y_sort0_gated_23_8__SHIFT 0x0 +#define SETUP_DEBUG_REG1__x_sort0_gated_23_8_MASK 0xffff0000 +#define SETUP_DEBUG_REG1__x_sort0_gated_23_8__SHIFT 0x10 +#define SETUP_DEBUG_REG2__y_sort1_gated_23_8_MASK 0xffff +#define SETUP_DEBUG_REG2__y_sort1_gated_23_8__SHIFT 0x0 +#define SETUP_DEBUG_REG2__x_sort1_gated_23_8_MASK 0xffff0000 +#define SETUP_DEBUG_REG2__x_sort1_gated_23_8__SHIFT 0x10 +#define SETUP_DEBUG_REG3__y_sort2_gated_23_8_MASK 0xffff +#define SETUP_DEBUG_REG3__y_sort2_gated_23_8__SHIFT 0x0 +#define SETUP_DEBUG_REG3__x_sort2_gated_23_8_MASK 0xffff0000 +#define SETUP_DEBUG_REG3__x_sort2_gated_23_8__SHIFT 0x10 +#define SETUP_DEBUG_REG4__attr_indx_sort0_gated_MASK 0x3fff +#define SETUP_DEBUG_REG4__attr_indx_sort0_gated__SHIFT 0x0 +#define SETUP_DEBUG_REG4__null_prim_gated_MASK 0x4000 +#define SETUP_DEBUG_REG4__null_prim_gated__SHIFT 0xe +#define SETUP_DEBUG_REG4__backfacing_gated_MASK 0x8000 +#define SETUP_DEBUG_REG4__backfacing_gated__SHIFT 0xf +#define SETUP_DEBUG_REG4__st_indx_gated_MASK 0x70000 +#define SETUP_DEBUG_REG4__st_indx_gated__SHIFT 0x10 +#define SETUP_DEBUG_REG4__clipped_gated_MASK 0x80000 +#define SETUP_DEBUG_REG4__clipped_gated__SHIFT 0x13 +#define SETUP_DEBUG_REG4__dealloc_slot_gated_MASK 0x700000 +#define SETUP_DEBUG_REG4__dealloc_slot_gated__SHIFT 0x14 +#define SETUP_DEBUG_REG4__xmajor_gated_MASK 0x800000 +#define SETUP_DEBUG_REG4__xmajor_gated__SHIFT 0x17 +#define SETUP_DEBUG_REG4__diamond_rule_gated_MASK 0x3000000 +#define SETUP_DEBUG_REG4__diamond_rule_gated__SHIFT 0x18 +#define SETUP_DEBUG_REG4__type_gated_MASK 0x1c000000 +#define SETUP_DEBUG_REG4__type_gated__SHIFT 0x1a +#define SETUP_DEBUG_REG4__fpov_gated_MASK 0x60000000 +#define SETUP_DEBUG_REG4__fpov_gated__SHIFT 0x1d +#define SETUP_DEBUG_REG4__eop_gated_MASK 0x80000000 +#define SETUP_DEBUG_REG4__eop_gated__SHIFT 0x1f +#define SETUP_DEBUG_REG5__attr_indx_sort2_gated_MASK 0x3fff +#define SETUP_DEBUG_REG5__attr_indx_sort2_gated__SHIFT 0x0 +#define SETUP_DEBUG_REG5__attr_indx_sort1_gated_MASK 0xfffc000 +#define SETUP_DEBUG_REG5__attr_indx_sort1_gated__SHIFT 0xe +#define SETUP_DEBUG_REG5__provoking_vtx_gated_MASK 0x30000000 +#define SETUP_DEBUG_REG5__provoking_vtx_gated__SHIFT 0x1c +#define SETUP_DEBUG_REG5__valid_prim_gated_MASK 0x40000000 +#define SETUP_DEBUG_REG5__valid_prim_gated__SHIFT 0x1e +#define SETUP_DEBUG_REG5__pa_reg_sclk_vld_MASK 0x80000000 +#define SETUP_DEBUG_REG5__pa_reg_sclk_vld__SHIFT 0x1f +#define PA_SC_DEBUG_REG0__REG0_FIELD0_MASK 0x3 +#define PA_SC_DEBUG_REG0__REG0_FIELD0__SHIFT 0x0 +#define PA_SC_DEBUG_REG0__REG0_FIELD1_MASK 0xc +#define PA_SC_DEBUG_REG0__REG0_FIELD1__SHIFT 0x2 +#define PA_SC_DEBUG_REG1__REG1_FIELD0_MASK 0x3 +#define PA_SC_DEBUG_REG1__REG1_FIELD0__SHIFT 0x0 +#define PA_SC_DEBUG_REG1__REG1_FIELD1_MASK 0xc +#define PA_SC_DEBUG_REG1__REG1_FIELD1__SHIFT 0x2 +#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK 0x1 +#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT 0x0 +#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK 0x2 +#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT 0x1 +#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK 0x4 +#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT 0x2 +#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK 0x8 +#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT 0x3 +#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK 0x10 +#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT 0x4 +#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK 0x20 +#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT 0x5 +#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK 0x40 +#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT 0x6 +#define COMPUTE_DISPATCH_INITIATOR__DISPATCH_CACHE_CNTL_MASK 0x380 +#define COMPUTE_DISPATCH_INITIATOR__DISPATCH_CACHE_CNTL__SHIFT 0x7 +#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL_MASK 0x400 +#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL__SHIFT 0xa +#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL_MASK 0x800 +#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL__SHIFT 0xb +#define COMPUTE_DISPATCH_INITIATOR__DATA_ATC_MASK 0x1000 +#define COMPUTE_DISPATCH_INITIATOR__DATA_ATC__SHIFT 0xc +#define COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK 0x4000 +#define COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT 0xe +#define COMPUTE_DIM_X__SIZE_MASK 0xffffffff +#define COMPUTE_DIM_X__SIZE__SHIFT 0x0 +#define COMPUTE_DIM_Y__SIZE_MASK 0xffffffff +#define COMPUTE_DIM_Y__SIZE__SHIFT 0x0 +#define COMPUTE_DIM_Z__SIZE_MASK 0xffffffff +#define COMPUTE_DIM_Z__SIZE__SHIFT 0x0 +#define COMPUTE_START_X__START_MASK 0xffffffff +#define COMPUTE_START_X__START__SHIFT 0x0 +#define COMPUTE_START_Y__START_MASK 0xffffffff +#define COMPUTE_START_Y__START__SHIFT 0x0 +#define COMPUTE_START_Z__START_MASK 0xffffffff +#define COMPUTE_START_Z__START__SHIFT 0x0 +#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK 0xffff +#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT 0x0 +#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK 0xffff0000 +#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT 0x10 +#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK 0xffff +#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT 0x0 +#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK 0xffff0000 +#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT 0x10 +#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK 0xffff +#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT 0x0 +#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK 0xffff0000 +#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT 0x10 +#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE_MASK 0x1 +#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE__SHIFT 0x0 +#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE_MASK 0x1 +#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE__SHIFT 0x0 +#define COMPUTE_PGM_LO__DATA_MASK 0xffffffff +#define COMPUTE_PGM_LO__DATA__SHIFT 0x0 +#define COMPUTE_PGM_HI__DATA_MASK 0xff +#define COMPUTE_PGM_HI__DATA__SHIFT 0x0 +#define COMPUTE_PGM_HI__INST_ATC_MASK 0x100 +#define COMPUTE_PGM_HI__INST_ATC__SHIFT 0x8 +#define COMPUTE_TBA_LO__DATA_MASK 0xffffffff +#define COMPUTE_TBA_LO__DATA__SHIFT 0x0 +#define COMPUTE_TBA_HI__DATA_MASK 0xff +#define COMPUTE_TBA_HI__DATA__SHIFT 0x0 +#define COMPUTE_TMA_LO__DATA_MASK 0xffffffff +#define COMPUTE_TMA_LO__DATA__SHIFT 0x0 +#define COMPUTE_TMA_HI__DATA_MASK 0xff +#define COMPUTE_TMA_HI__DATA__SHIFT 0x0 +#define COMPUTE_PGM_RSRC1__VGPRS_MASK 0x3f +#define COMPUTE_PGM_RSRC1__VGPRS__SHIFT 0x0 +#define COMPUTE_PGM_RSRC1__SGPRS_MASK 0x3c0 +#define COMPUTE_PGM_RSRC1__SGPRS__SHIFT 0x6 +#define COMPUTE_PGM_RSRC1__PRIORITY_MASK 0xc00 +#define COMPUTE_PGM_RSRC1__PRIORITY__SHIFT 0xa +#define COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK 0xff000 +#define COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT 0xc +#define COMPUTE_PGM_RSRC1__PRIV_MASK 0x100000 +#define COMPUTE_PGM_RSRC1__PRIV__SHIFT 0x14 +#define COMPUTE_PGM_RSRC1__DX10_CLAMP_MASK 0x200000 +#define COMPUTE_PGM_RSRC1__DX10_CLAMP__SHIFT 0x15 +#define COMPUTE_PGM_RSRC1__DEBUG_MODE_MASK 0x400000 +#define COMPUTE_PGM_RSRC1__DEBUG_MODE__SHIFT 0x16 +#define COMPUTE_PGM_RSRC1__IEEE_MODE_MASK 0x800000 +#define COMPUTE_PGM_RSRC1__IEEE_MODE__SHIFT 0x17 +#define COMPUTE_PGM_RSRC1__BULKY_MASK 0x1000000 +#define COMPUTE_PGM_RSRC1__BULKY__SHIFT 0x18 +#define COMPUTE_PGM_RSRC1__CDBG_USER_MASK 0x2000000 +#define COMPUTE_PGM_RSRC1__CDBG_USER__SHIFT 0x19 +#define COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK 0x1 +#define COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT 0x0 +#define COMPUTE_PGM_RSRC2__USER_SGPR_MASK 0x3e +#define COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT 0x1 +#define COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK 0x40 +#define COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT 0x6 +#define COMPUTE_PGM_RSRC2__TGID_X_EN_MASK 0x80 +#define COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT 0x7 +#define COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK 0x100 +#define COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT 0x8 +#define COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK 0x200 +#define COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT 0x9 +#define COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK 0x400 +#define COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT 0xa +#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK 0x1800 +#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT 0xb +#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK 0x6000 +#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT 0xd +#define COMPUTE_PGM_RSRC2__LDS_SIZE_MASK 0xff8000 +#define COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT 0xf +#define COMPUTE_PGM_RSRC2__EXCP_EN_MASK 0x7f000000 +#define COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT 0x18 +#define COMPUTE_VMID__DATA_MASK 0xf +#define COMPUTE_VMID__DATA__SHIFT 0x0 +#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK 0x3ff +#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT 0x0 +#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK 0xf000 +#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT 0xc +#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK 0x3f0000 +#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT 0x10 +#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK 0x400000 +#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT 0x16 +#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK 0x800000 +#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT 0x17 +#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK 0x7000000 +#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT 0x18 +#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN_MASK 0xffff +#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH0_CU_EN__SHIFT 0x0 +#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN_MASK 0xffff0000 +#define COMPUTE_STATIC_THREAD_MGMT_SE0__SH1_CU_EN__SHIFT 0x10 +#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN_MASK 0xffff +#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH0_CU_EN__SHIFT 0x0 +#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN_MASK 0xffff0000 +#define COMPUTE_STATIC_THREAD_MGMT_SE1__SH1_CU_EN__SHIFT 0x10 +#define COMPUTE_TMPRING_SIZE__WAVES_MASK 0xfff +#define COMPUTE_TMPRING_SIZE__WAVES__SHIFT 0x0 +#define COMPUTE_TMPRING_SIZE__WAVESIZE_MASK 0x1fff000 +#define COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT 0xc +#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN_MASK 0xffff +#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH0_CU_EN__SHIFT 0x0 +#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN_MASK 0xffff0000 +#define COMPUTE_STATIC_THREAD_MGMT_SE2__SH1_CU_EN__SHIFT 0x10 +#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN_MASK 0xffff +#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH0_CU_EN__SHIFT 0x0 +#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN_MASK 0xffff0000 +#define COMPUTE_STATIC_THREAD_MGMT_SE3__SH1_CU_EN__SHIFT 0x10 +#define COMPUTE_RESTART_X__RESTART_MASK 0xffffffff +#define COMPUTE_RESTART_X__RESTART__SHIFT 0x0 +#define COMPUTE_RESTART_Y__RESTART_MASK 0xffffffff +#define COMPUTE_RESTART_Y__RESTART__SHIFT 0x0 +#define COMPUTE_RESTART_Z__RESTART_MASK 0xffffffff +#define COMPUTE_RESTART_Z__RESTART__SHIFT 0x0 +#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE_MASK 0x1 +#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE__SHIFT 0x0 +#define COMPUTE_MISC_RESERVED__SEND_SEID_MASK 0x3 +#define COMPUTE_MISC_RESERVED__SEND_SEID__SHIFT 0x0 +#define COMPUTE_MISC_RESERVED__RESERVED2_MASK 0x4 +#define COMPUTE_MISC_RESERVED__RESERVED2__SHIFT 0x2 +#define COMPUTE_MISC_RESERVED__RESERVED3_MASK 0x8 +#define COMPUTE_MISC_RESERVED__RESERVED3__SHIFT 0x3 +#define COMPUTE_MISC_RESERVED__RESERVED4_MASK 0x10 +#define COMPUTE_MISC_RESERVED__RESERVED4__SHIFT 0x4 +#define COMPUTE_USER_DATA_0__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_0__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_1__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_1__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_2__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_2__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_3__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_3__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_4__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_4__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_5__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_5__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_6__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_6__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_7__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_7__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_8__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_8__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_9__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_9__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_10__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_10__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_11__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_11__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_12__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_12__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_13__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_13__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_14__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_14__DATA__SHIFT 0x0 +#define COMPUTE_USER_DATA_15__DATA_MASK 0xffffffff +#define COMPUTE_USER_DATA_15__DATA__SHIFT 0x0 +#define CSPRIV_CONNECT__DOORBELL_OFFSET_MASK 0x1fffff +#define CSPRIV_CONNECT__DOORBELL_OFFSET__SHIFT 0x0 +#define CSPRIV_CONNECT__QUEUE_ID_MASK 0xe00000 +#define CSPRIV_CONNECT__QUEUE_ID__SHIFT 0x15 +#define CSPRIV_CONNECT__VMID_MASK 0x3c000000 +#define CSPRIV_CONNECT__VMID__SHIFT 0x1a +#define CSPRIV_CONNECT__UNORD_DISP_MASK 0x80000000 +#define CSPRIV_CONNECT__UNORD_DISP__SHIFT 0x1f +#define CSPRIV_THREAD_TRACE_TG0__TGID_X_MASK 0xffffffff +#define CSPRIV_THREAD_TRACE_TG0__TGID_X__SHIFT 0x0 +#define CSPRIV_THREAD_TRACE_TG1__TGID_Y_MASK 0xffffffff +#define CSPRIV_THREAD_TRACE_TG1__TGID_Y__SHIFT 0x0 +#define CSPRIV_THREAD_TRACE_TG2__TGID_Z_MASK 0xffffffff +#define CSPRIV_THREAD_TRACE_TG2__TGID_Z__SHIFT 0x0 +#define CSPRIV_THREAD_TRACE_TG3__WAVE_ID_BASE_MASK 0xfff +#define CSPRIV_THREAD_TRACE_TG3__WAVE_ID_BASE__SHIFT 0x0 +#define CSPRIV_THREAD_TRACE_TG3__THREADS_IN_GROUP_MASK 0xfff000 +#define CSPRIV_THREAD_TRACE_TG3__THREADS_IN_GROUP__SHIFT 0xc +#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_X_FLAG_MASK 0x1000000 +#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_X_FLAG__SHIFT 0x18 +#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Y_FLAG_MASK 0x2000000 +#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Y_FLAG__SHIFT 0x19 +#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Z_FLAG_MASK 0x4000000 +#define CSPRIV_THREAD_TRACE_TG3__PARTIAL_Z_FLAG__SHIFT 0x1a +#define CSPRIV_THREAD_TRACE_TG3__LAST_TG_MASK 0x8000000 +#define CSPRIV_THREAD_TRACE_TG3__LAST_TG__SHIFT 0x1b +#define CSPRIV_THREAD_TRACE_TG3__FIRST_TG_MASK 0x10000000 +#define CSPRIV_THREAD_TRACE_TG3__FIRST_TG__SHIFT 0x1c +#define CSPRIV_THREAD_TRACE_EVENT__EVENT_ID_MASK 0x1f +#define CSPRIV_THREAD_TRACE_EVENT__EVENT_ID__SHIFT 0x0 +#define RLC_CNTL__RLC_ENABLE_F32_MASK 0x1 +#define RLC_CNTL__RLC_ENABLE_F32__SHIFT 0x0 +#define RLC_CNTL__FORCE_RETRY_MASK 0x2 +#define RLC_CNTL__FORCE_RETRY__SHIFT 0x1 +#define RLC_CNTL__READ_CACHE_DISABLE_MASK 0x4 +#define RLC_CNTL__READ_CACHE_DISABLE__SHIFT 0x2 +#define RLC_CNTL__RLC_STEP_F32_MASK 0x8 +#define RLC_CNTL__RLC_STEP_F32__SHIFT 0x3 +#define RLC_CNTL__SOFT_RESET_DEBUG_MODE_MASK 0x10 +#define RLC_CNTL__SOFT_RESET_DEBUG_MODE__SHIFT 0x4 +#define RLC_CNTL__RESERVED_MASK 0xffffff00 +#define RLC_CNTL__RESERVED__SHIFT 0x8 +#define RLC_DEBUG_SELECT__SELECT_MASK 0xff +#define RLC_DEBUG_SELECT__SELECT__SHIFT 0x0 +#define RLC_DEBUG_SELECT__RESERVED_MASK 0xffffff00 +#define RLC_DEBUG_SELECT__RESERVED__SHIFT 0x8 +#define RLC_DEBUG__DATA_MASK 0xffffffff +#define RLC_DEBUG__DATA__SHIFT 0x0 +#define RLC_MC_CNTL__WRREQ_SWAP_MASK 0x3 +#define RLC_MC_CNTL__WRREQ_SWAP__SHIFT 0x0 +#define RLC_MC_CNTL__WRREQ_TRAN_MASK 0x4 +#define RLC_MC_CNTL__WRREQ_TRAN__SHIFT 0x2 +#define RLC_MC_CNTL__WRREQ_PRIV_MASK 0x8 +#define RLC_MC_CNTL__WRREQ_PRIV__SHIFT 0x3 +#define RLC_MC_CNTL__WRNFO_STALL_MASK 0x10 +#define RLC_MC_CNTL__WRNFO_STALL__SHIFT 0x4 +#define RLC_MC_CNTL__WRNFO_URG_MASK 0x1e0 +#define RLC_MC_CNTL__WRNFO_URG__SHIFT 0x5 +#define RLC_MC_CNTL__WRREQ_DW_IMASK_MASK 0x1e00 +#define RLC_MC_CNTL__WRREQ_DW_IMASK__SHIFT 0x9 +#define RLC_MC_CNTL__RESERVED_B_MASK 0xfe000 +#define RLC_MC_CNTL__RESERVED_B__SHIFT 0xd +#define RLC_MC_CNTL__RDNFO_URG_MASK 0xf00000 +#define RLC_MC_CNTL__RDNFO_URG__SHIFT 0x14 +#define RLC_MC_CNTL__RDREQ_SWAP_MASK 0x3000000 +#define RLC_MC_CNTL__RDREQ_SWAP__SHIFT 0x18 +#define RLC_MC_CNTL__RDREQ_TRAN_MASK 0x4000000 +#define RLC_MC_CNTL__RDREQ_TRAN__SHIFT 0x1a +#define RLC_MC_CNTL__RDREQ_PRIV_MASK 0x8000000 +#define RLC_MC_CNTL__RDREQ_PRIV__SHIFT 0x1b +#define RLC_MC_CNTL__RDNFO_STALL_MASK 0x10000000 +#define RLC_MC_CNTL__RDNFO_STALL__SHIFT 0x1c +#define RLC_MC_CNTL__RESERVED_MASK 0xe0000000 +#define RLC_MC_CNTL__RESERVED__SHIFT 0x1d +#define RLC_STAT__RLC_BUSY_MASK 0x1 +#define RLC_STAT__RLC_BUSY__SHIFT 0x0 +#define RLC_STAT__RLC_GPM_BUSY_MASK 0x2 +#define RLC_STAT__RLC_GPM_BUSY__SHIFT 0x1 +#define RLC_STAT__RLC_SPM_BUSY_MASK 0x4 +#define RLC_STAT__RLC_SPM_BUSY__SHIFT 0x2 +#define RLC_STAT__RESERVED_MASK 0xfffffff8 +#define RLC_STAT__RESERVED__SHIFT 0x3 +#define RLC_SAFE_MODE__REQ_MASK 0x1 +#define RLC_SAFE_MODE__REQ__SHIFT 0x0 +#define RLC_SAFE_MODE__MESSAGE_MASK 0x1e +#define RLC_SAFE_MODE__MESSAGE__SHIFT 0x1 +#define RLC_SAFE_MODE__RESERVED_MASK 0xffffffe0 +#define RLC_SAFE_MODE__RESERVED__SHIFT 0x5 +#define RLC_SOFT_RESET_GPU__SOFT_RESET_GPU_MASK 0x1 +#define RLC_SOFT_RESET_GPU__SOFT_RESET_GPU__SHIFT 0x0 +#define RLC_SOFT_RESET_GPU__RESERVED_MASK 0xfffffffe +#define RLC_SOFT_RESET_GPU__RESERVED__SHIFT 0x1 +#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK 0x1 +#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN__SHIFT 0x0 +#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN_MASK 0x2 +#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN__SHIFT 0x1 +#define RLC_MEM_SLP_CNTL__RESERVED_MASK 0xfc +#define RLC_MEM_SLP_CNTL__RESERVED__SHIFT 0x2 +#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY_MASK 0xff00 +#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY__SHIFT 0x8 +#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY_MASK 0xff0000 +#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY__SHIFT 0x10 +#define RLC_MEM_SLP_CNTL__RESERVED1_MASK 0xff000000 +#define RLC_MEM_SLP_CNTL__RESERVED1__SHIFT 0x18 +#define RLC_PERFMON_CNTL__PERFMON_STATE_MASK 0x7 +#define RLC_PERFMON_CNTL__PERFMON_STATE__SHIFT 0x0 +#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK 0x400 +#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT 0xa +#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0xff +#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0xff +#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define CGTT_RLC_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_RLC_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000 +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000 +#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE_MASK 0x1 +#define RLC_LB_CNTL__LOAD_BALANCE_ENABLE__SHIFT 0x0 +#define RLC_LB_CNTL__LB_CNT_CP_BUSY_MASK 0x2 +#define RLC_LB_CNTL__LB_CNT_CP_BUSY__SHIFT 0x1 +#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK 0x4 +#define RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE__SHIFT 0x2 +#define RLC_LB_CNTL__LB_CNT_REG_INC_MASK 0x8 +#define RLC_LB_CNTL__LB_CNT_REG_INC__SHIFT 0x3 +#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST_MASK 0xff0 +#define RLC_LB_CNTL__CU_MASK_USED_OFF_HYST__SHIFT 0x4 +#define RLC_LB_CNTL__RESERVED_MASK 0xfffff000 +#define RLC_LB_CNTL__RESERVED__SHIFT 0xc +#define RLC_LB_CNTR_MAX__LB_CNTR_MAX_MASK 0xffffffff +#define RLC_LB_CNTR_MAX__LB_CNTR_MAX__SHIFT 0x0 +#define RLC_LB_CNTR_INIT__LB_CNTR_INIT_MASK 0xffffffff +#define RLC_LB_CNTR_INIT__LB_CNTR_INIT__SHIFT 0x0 +#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR_MASK 0xffffffff +#define RLC_LOAD_BALANCE_CNTR__RLC_LOAD_BALANCE_CNTR__SHIFT 0x0 +#define RLC_SAVE_AND_RESTORE_BASE__BASE_MASK 0xffffffff +#define RLC_SAVE_AND_RESTORE_BASE__BASE__SHIFT 0x0 +#define RLC_JUMP_TABLE_RESTORE__ADDR_MASK 0xffffffff +#define RLC_JUMP_TABLE_RESTORE__ADDR__SHIFT 0x0 +#define RLC_DRIVER_CPDMA_STATUS__DRIVER_REQUEST_MASK 0x1 +#define RLC_DRIVER_CPDMA_STATUS__DRIVER_REQUEST__SHIFT 0x0 +#define RLC_DRIVER_CPDMA_STATUS__RESERVED1_MASK 0xe +#define RLC_DRIVER_CPDMA_STATUS__RESERVED1__SHIFT 0x1 +#define RLC_DRIVER_CPDMA_STATUS__DRIVER_ACK_MASK 0x10 +#define RLC_DRIVER_CPDMA_STATUS__DRIVER_ACK__SHIFT 0x4 +#define RLC_DRIVER_CPDMA_STATUS__RESERVED_MASK 0xffffffe0 +#define RLC_DRIVER_CPDMA_STATUS__RESERVED__SHIFT 0x5 +#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE_MASK 0xff +#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE__SHIFT 0x0 +#define RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK 0xff00 +#define RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT 0x8 +#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE_MASK 0xffff0000 +#define RLC_PG_DELAY_2__PERCU_TIMEOUT_VALUE__SHIFT 0x10 +#define RLC_GPM_DEBUG_SELECT__SELECT_MASK 0xff +#define RLC_GPM_DEBUG_SELECT__SELECT__SHIFT 0x0 +#define RLC_GPM_DEBUG_SELECT__RESERVED_MASK 0xffffff00 +#define RLC_GPM_DEBUG_SELECT__RESERVED__SHIFT 0x8 +#define RLC_GPM_DEBUG__DATA_MASK 0xffffffff +#define RLC_GPM_DEBUG__DATA__SHIFT 0x0 +#define RLC_GPM_UCODE_ADDR__UCODE_ADDR_MASK 0xfff +#define RLC_GPM_UCODE_ADDR__UCODE_ADDR__SHIFT 0x0 +#define RLC_GPM_UCODE_ADDR__RESERVED_MASK 0xfffff000 +#define RLC_GPM_UCODE_ADDR__RESERVED__SHIFT 0xc +#define RLC_GPM_UCODE_DATA__UCODE_DATA_MASK 0xffffffff +#define RLC_GPM_UCODE_DATA__UCODE_DATA__SHIFT 0x0 +#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB_MASK 0xffffffff +#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB__SHIFT 0x0 +#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB_MASK 0xffffffff +#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB__SHIFT 0x0 +#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE_MASK 0x1 +#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE__SHIFT 0x0 +#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED_MASK 0xfffffffe +#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED__SHIFT 0x1 +#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS_MASK 0xffffffff +#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS__SHIFT 0x0 +#define RLC_GPM_STAT__RLC_BUSY_MASK 0x1 +#define RLC_GPM_STAT__RLC_BUSY__SHIFT 0x0 +#define RLC_GPM_STAT__GFX_POWER_STATUS_MASK 0x2 +#define RLC_GPM_STAT__GFX_POWER_STATUS__SHIFT 0x1 +#define RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK 0x4 +#define RLC_GPM_STAT__GFX_CLOCK_STATUS__SHIFT 0x2 +#define RLC_GPM_STAT__GFX_LS_STATUS_MASK 0x8 +#define RLC_GPM_STAT__GFX_LS_STATUS__SHIFT 0x3 +#define RLC_GPM_STAT__RESERVED_MASK 0xfffffff0 +#define RLC_GPM_STAT__RESERVED__SHIFT 0x4 +#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK 0x3f +#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL__SHIFT 0x0 +#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED_MASK 0xffffffc0 +#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED__SHIFT 0x6 +#define RLC_GPU_CLOCK_32__GPU_CLOCK_32_MASK 0xffffffff +#define RLC_GPU_CLOCK_32__GPU_CLOCK_32__SHIFT 0x0 +#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK 0x1 +#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE__SHIFT 0x0 +#define RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK 0x2 +#define RLC_PG_CNTL__GFX_POWER_GATING_SRC__SHIFT 0x1 +#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK 0x4 +#define RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE__SHIFT 0x2 +#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK 0x8 +#define RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE__SHIFT 0x3 +#define RLC_PG_CNTL__RESERVED_MASK 0xfff0 +#define RLC_PG_CNTL__RESERVED__SHIFT 0x4 +#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE_MASK 0x10000 +#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE__SHIFT 0x10 +#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK 0x20000 +#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE__SHIFT 0x11 +#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK 0x40000 +#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT 0x12 +#define RLC_PG_CNTL__RESERVED1_MASK 0xf80000 +#define RLC_PG_CNTL__RESERVED1__SHIFT 0x13 +#define RLC_PG_CNTL__PG_ERROR_STATUS_MASK 0xff000000 +#define RLC_PG_CNTL__PG_ERROR_STATUS__SHIFT 0x18 +#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK 0xff +#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT 0x0 +#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK 0xff00 +#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY__SHIFT 0x8 +#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY_MASK 0xff0000 +#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY__SHIFT 0x10 +#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY_MASK 0xff000000 +#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY__SHIFT 0x18 +#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE_MASK 0x1 +#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE__SHIFT 0x0 +#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE_MASK 0x2 +#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE__SHIFT 0x1 +#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE_MASK 0x4 +#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE__SHIFT 0x2 +#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE_MASK 0x8 +#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE__SHIFT 0x3 +#define RLC_GPM_THREAD_ENABLE__RESERVED_MASK 0xfffffff0 +#define RLC_GPM_THREAD_ENABLE__RESERVED__SHIFT 0x4 +#define RLC_GPM_VMID_THREAD0__RLC_VMID_MASK 0xf +#define RLC_GPM_VMID_THREAD0__RLC_VMID__SHIFT 0x0 +#define RLC_GPM_VMID_THREAD0__RESERVED_MASK 0xfffffff0 +#define RLC_GPM_VMID_THREAD0__RESERVED__SHIFT 0x4 +#define RLC_GPM_VMID_THREAD1__RLC_VMID_MASK 0xf +#define RLC_GPM_VMID_THREAD1__RLC_VMID__SHIFT 0x0 +#define RLC_GPM_VMID_THREAD1__RESERVED_MASK 0xfffffff0 +#define RLC_GPM_VMID_THREAD1__RESERVED__SHIFT 0x4 +#define RLC_CGTT_MGCG_OVERRIDE__OVERRIDE_MASK 0xffffffff +#define RLC_CGTT_MGCG_OVERRIDE__OVERRIDE__SHIFT 0x0 +#define RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK 0x1 +#define RLC_CGCG_CGLS_CTRL__CGCG_EN__SHIFT 0x0 +#define RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK 0x2 +#define RLC_CGCG_CGLS_CTRL__CGLS_EN__SHIFT 0x1 +#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK 0xfc +#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT 0x2 +#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK 0x7ffff00 +#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT 0x8 +#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER_MASK 0x8000000 +#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER__SHIFT 0x1b +#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL_MASK 0x10000000 +#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL__SHIFT 0x1c +#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE_MASK 0x60000000 +#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE__SHIFT 0x1d +#define RLC_CGCG_CGLS_CTRL__SPARE_MASK 0x80000000 +#define RLC_CGCG_CGLS_CTRL__SPARE__SHIFT 0x1f +#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT_MASK 0xf +#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT__SHIFT 0x0 +#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT_MASK 0xf0 +#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT__SHIFT 0x4 +#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT_MASK 0xf00 +#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT__SHIFT 0x8 +#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT_MASK 0xf000 +#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT__SHIFT 0xc +#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT_MASK 0xfff0000 +#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT__SHIFT 0x10 +#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT_MASK 0xf0000000 +#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT__SHIFT 0x1c +#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK_MASK 0xffffffff +#define RLC_DYN_PG_STATUS__PG_STATUS_CU_MASK__SHIFT 0x0 +#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK_MASK 0xffffffff +#define RLC_DYN_PG_REQUEST__PG_REQUEST_CU_MASK__SHIFT 0x0 +#define RLC_PG_DELAY__POWER_UP_DELAY_MASK 0xff +#define RLC_PG_DELAY__POWER_UP_DELAY__SHIFT 0x0 +#define RLC_PG_DELAY__POWER_DOWN_DELAY_MASK 0xff00 +#define RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT 0x8 +#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY_MASK 0xff0000 +#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT 0x10 +#define RLC_PG_DELAY__MEM_SLEEP_DELAY_MASK 0xff000000 +#define RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT 0x18 +#define RLC_CU_STATUS__WORK_PENDING_MASK 0xffffffff +#define RLC_CU_STATUS__WORK_PENDING__SHIFT 0x0 +#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK_MASK 0xffffffff +#define RLC_LB_INIT_CU_MASK__INIT_CU_MASK__SHIFT 0x0 +#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK_MASK 0xffffffff +#define RLC_LB_ALWAYS_ACTIVE_CU_MASK__ALWAYS_ACTIVE_CU_MASK__SHIFT 0x0 +#define RLC_LB_PARAMS__SKIP_L2_CHECK_MASK 0x1 +#define RLC_LB_PARAMS__SKIP_L2_CHECK__SHIFT 0x0 +#define RLC_LB_PARAMS__FIFO_SAMPLES_MASK 0xfe +#define RLC_LB_PARAMS__FIFO_SAMPLES__SHIFT 0x1 +#define RLC_LB_PARAMS__PG_IDLE_SAMPLES_MASK 0xff00 +#define RLC_LB_PARAMS__PG_IDLE_SAMPLES__SHIFT 0x8 +#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL_MASK 0xffff0000 +#define RLC_LB_PARAMS__PG_IDLE_SAMPLE_INTERVAL__SHIFT 0x10 +#define RLC_THREAD1_DELAY__CU_IDEL_DELAY_MASK 0xff +#define RLC_THREAD1_DELAY__CU_IDEL_DELAY__SHIFT 0x0 +#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY_MASK 0xff00 +#define RLC_THREAD1_DELAY__LBPW_INNER_LOOP_DELAY__SHIFT 0x8 +#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY_MASK 0xff0000 +#define RLC_THREAD1_DELAY__LBPW_OUTER_LOOP_DELAY__SHIFT 0x10 +#define RLC_THREAD1_DELAY__SPARE_MASK 0xff000000 +#define RLC_THREAD1_DELAY__SPARE__SHIFT 0x18 +#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK_MASK 0xffffffff +#define RLC_PG_ALWAYS_ON_CU_MASK__AON_CU_MASK__SHIFT 0x0 +#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU_MASK 0xff +#define RLC_MAX_PG_CU__MAX_POWERED_UP_CU__SHIFT 0x0 +#define RLC_MAX_PG_CU__SPARE_MASK 0xffffff00 +#define RLC_MAX_PG_CU__SPARE__SHIFT 0x8 +#define RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK 0x1 +#define RLC_AUTO_PG_CTRL__AUTO_PG_EN__SHIFT 0x0 +#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN_MASK 0x2 +#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN__SHIFT 0x1 +#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN_MASK 0x4 +#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN__SHIFT 0x2 +#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK 0x7fff8 +#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT 0x3 +#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK 0xfff80000 +#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD__SHIFT 0x13 +#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE_MASK 0x1 +#define RLC_SMU_GRBM_REG_SAVE_CTRL__START_GRBM_REG_SAVE__SHIFT 0x0 +#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE_MASK 0xfffffffe +#define RLC_SMU_GRBM_REG_SAVE_CTRL__SPARE__SHIFT 0x1 +#define RLC_SMU_PG_CTRL__START_PG_MASK 0x1 +#define RLC_SMU_PG_CTRL__START_PG__SHIFT 0x0 +#define RLC_SMU_PG_CTRL__SPARE_MASK 0xfffffffe +#define RLC_SMU_PG_CTRL__SPARE__SHIFT 0x1 +#define RLC_SMU_PG_WAKE_UP_CTRL__START_PG_WAKE_UP_MASK 0x1 +#define RLC_SMU_PG_WAKE_UP_CTRL__START_PG_WAKE_UP__SHIFT 0x0 +#define RLC_SMU_PG_WAKE_UP_CTRL__SPARE_MASK 0xfffffffe +#define RLC_SMU_PG_WAKE_UP_CTRL__SPARE__SHIFT 0x1 +#define RLC_SERDES_RD_MASTER_INDEX__CU_ID_MASK 0xf +#define RLC_SERDES_RD_MASTER_INDEX__CU_ID__SHIFT 0x0 +#define RLC_SERDES_RD_MASTER_INDEX__SH_ID_MASK 0x30 +#define RLC_SERDES_RD_MASTER_INDEX__SH_ID__SHIFT 0x4 +#define RLC_SERDES_RD_MASTER_INDEX__SE_ID_MASK 0x1c0 +#define RLC_SERDES_RD_MASTER_INDEX__SE_ID__SHIFT 0x6 +#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID_MASK 0x200 +#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_ID__SHIFT 0x9 +#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU_MASK 0x400 +#define RLC_SERDES_RD_MASTER_INDEX__SE_NONCU__SHIFT 0xa +#define RLC_SERDES_RD_MASTER_INDEX__NON_SE_MASK 0x3800 +#define RLC_SERDES_RD_MASTER_INDEX__NON_SE__SHIFT 0xb +#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID_MASK 0xc000 +#define RLC_SERDES_RD_MASTER_INDEX__DATA_REG_ID__SHIFT 0xe +#define RLC_SERDES_RD_MASTER_INDEX__SPARE_MASK 0xffff0000 +#define RLC_SERDES_RD_MASTER_INDEX__SPARE__SHIFT 0x10 +#define RLC_SERDES_RD_DATA_0__DATA_MASK 0xffffffff +#define RLC_SERDES_RD_DATA_0__DATA__SHIFT 0x0 +#define RLC_SERDES_RD_DATA_1__DATA_MASK 0xffffffff +#define RLC_SERDES_RD_DATA_1__DATA__SHIFT 0x0 +#define RLC_SERDES_RD_DATA_2__DATA_MASK 0xffffffff +#define RLC_SERDES_RD_DATA_2__DATA__SHIFT 0x0 +#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK_MASK 0xffffffff +#define RLC_SERDES_WR_CU_MASTER_MASK__MASTER_MASK__SHIFT 0x0 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK_MASK 0xffff +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SE_MASTER_MASK__SHIFT 0x0 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK_MASK 0x10000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__GC_MASTER_MASK__SHIFT 0x10 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK_MASK 0x20000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC0_MASTER_MASK__SHIFT 0x11 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK_MASK 0x40000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__TC1_MASTER_MASK__SHIFT 0x12 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK_MASK 0x80000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE0_MASTER_MASK__SHIFT 0x13 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK_MASK 0x100000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE1_MASTER_MASK__SHIFT 0x14 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK_MASK 0x200000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE2_MASTER_MASK__SHIFT 0x15 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK_MASK 0x400000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__SPARE3_MASTER_MASK__SHIFT 0x16 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED_MASK 0xff800000 +#define RLC_SERDES_WR_NONCU_MASTER_MASK__RESERVED__SHIFT 0x17 +#define RLC_SERDES_WR_CTRL__BPM_ADDR_MASK 0xff +#define RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT 0x0 +#define RLC_SERDES_WR_CTRL__POWER_DOWN_MASK 0x100 +#define RLC_SERDES_WR_CTRL__POWER_DOWN__SHIFT 0x8 +#define RLC_SERDES_WR_CTRL__POWER_UP_MASK 0x200 +#define RLC_SERDES_WR_CTRL__POWER_UP__SHIFT 0x9 +#define RLC_SERDES_WR_CTRL__P1_SELECT_MASK 0x400 +#define RLC_SERDES_WR_CTRL__P1_SELECT__SHIFT 0xa +#define RLC_SERDES_WR_CTRL__P2_SELECT_MASK 0x800 +#define RLC_SERDES_WR_CTRL__P2_SELECT__SHIFT 0xb +#define RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK 0x1000 +#define RLC_SERDES_WR_CTRL__WRITE_COMMAND__SHIFT 0xc +#define RLC_SERDES_WR_CTRL__READ_COMMAND_MASK 0x2000 +#define RLC_SERDES_WR_CTRL__READ_COMMAND__SHIFT 0xd +#define RLC_SERDES_WR_CTRL__RESERVED_1_MASK 0xc000 +#define RLC_SERDES_WR_CTRL__RESERVED_1__SHIFT 0xe +#define RLC_SERDES_WR_CTRL__CGLS_ENABLE_MASK 0x10000 +#define RLC_SERDES_WR_CTRL__CGLS_ENABLE__SHIFT 0x10 +#define RLC_SERDES_WR_CTRL__CGLS_DISABLE_MASK 0x20000 +#define RLC_SERDES_WR_CTRL__CGLS_DISABLE__SHIFT 0x11 +#define RLC_SERDES_WR_CTRL__CGLS_ON_MASK 0x40000 +#define RLC_SERDES_WR_CTRL__CGLS_ON__SHIFT 0x12 +#define RLC_SERDES_WR_CTRL__CGLS_OFF_MASK 0x80000 +#define RLC_SERDES_WR_CTRL__CGLS_OFF__SHIFT 0x13 +#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_0_MASK 0x100000 +#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_0__SHIFT 0x14 +#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_1_MASK 0x200000 +#define RLC_SERDES_WR_CTRL__CGCG_OVERRIDE_1__SHIFT 0x15 +#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_0_MASK 0x400000 +#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_0__SHIFT 0x16 +#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1_MASK 0x800000 +#define RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1__SHIFT 0x17 +#define RLC_SERDES_WR_CTRL__RESERVED_2_MASK 0xf000000 +#define RLC_SERDES_WR_CTRL__RESERVED_2__SHIFT 0x18 +#define RLC_SERDES_WR_CTRL__REG_ADDR_MASK 0xf0000000 +#define RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT 0x1c +#define RLC_SERDES_WR_DATA__DATA_MASK 0xffffffff +#define RLC_SERDES_WR_DATA__DATA__SHIFT 0x0 +#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY_MASK 0xffffffff +#define RLC_SERDES_CU_MASTER_BUSY__BUSY_BUSY__SHIFT 0x0 +#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK 0xffff +#define RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY__SHIFT 0x0 +#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK 0x10000 +#define RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY__SHIFT 0x10 +#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK 0x20000 +#define RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY__SHIFT 0x11 +#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK 0x40000 +#define RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY__SHIFT 0x12 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY_MASK 0x80000 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE0_MASTER_BUSY__SHIFT 0x13 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY_MASK 0x100000 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE1_MASTER_BUSY__SHIFT 0x14 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY_MASK 0x200000 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE2_MASTER_BUSY__SHIFT 0x15 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY_MASK 0x400000 +#define RLC_SERDES_NONCU_MASTER_BUSY__SPARE3_MASTER_BUSY__SHIFT 0x16 +#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED_MASK 0xff800000 +#define RLC_SERDES_NONCU_MASTER_BUSY__RESERVED__SHIFT 0x17 +#define RLC_GPM_GENERAL_0__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_0__DATA__SHIFT 0x0 +#define RLC_GPM_GENERAL_1__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_1__DATA__SHIFT 0x0 +#define RLC_GPM_GENERAL_2__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_2__DATA__SHIFT 0x0 +#define RLC_GPM_GENERAL_3__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_3__DATA__SHIFT 0x0 +#define RLC_GPM_GENERAL_4__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_4__DATA__SHIFT 0x0 +#define RLC_GPM_GENERAL_5__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_5__DATA__SHIFT 0x0 +#define RLC_GPM_GENERAL_6__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_6__DATA__SHIFT 0x0 +#define RLC_GPM_GENERAL_7__DATA_MASK 0xffffffff +#define RLC_GPM_GENERAL_7__DATA__SHIFT 0x0 +#define RLC_GPM_CU_PD_TIMEOUT__TIMEOUT_MASK 0xffffffff +#define RLC_GPM_CU_PD_TIMEOUT__TIMEOUT__SHIFT 0x0 +#define RLC_GPM_SCRATCH_ADDR__ADDR_MASK 0x1ff +#define RLC_GPM_SCRATCH_ADDR__ADDR__SHIFT 0x0 +#define RLC_GPM_SCRATCH_ADDR__RESERVED_MASK 0xfffffe00 +#define RLC_GPM_SCRATCH_ADDR__RESERVED__SHIFT 0x9 +#define RLC_GPM_SCRATCH_DATA__DATA_MASK 0xffffffff +#define RLC_GPM_SCRATCH_DATA__DATA__SHIFT 0x0 +#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK_MASK 0xffffffff +#define RLC_STATIC_PG_STATUS__PG_STATUS_CU_MASK__SHIFT 0x0 +#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL_MASK 0xf +#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL__SHIFT 0x0 +#define RLC_GPM_PERF_COUNT_0__SE_INDEX_MASK 0xf0 +#define RLC_GPM_PERF_COUNT_0__SE_INDEX__SHIFT 0x4 +#define RLC_GPM_PERF_COUNT_0__SH_INDEX_MASK 0xf00 +#define RLC_GPM_PERF_COUNT_0__SH_INDEX__SHIFT 0x8 +#define RLC_GPM_PERF_COUNT_0__CU_INDEX_MASK 0xf000 +#define RLC_GPM_PERF_COUNT_0__CU_INDEX__SHIFT 0xc +#define RLC_GPM_PERF_COUNT_0__EVENT_SEL_MASK 0x30000 +#define RLC_GPM_PERF_COUNT_0__EVENT_SEL__SHIFT 0x10 +#define RLC_GPM_PERF_COUNT_0__UNUSED_MASK 0xc0000 +#define RLC_GPM_PERF_COUNT_0__UNUSED__SHIFT 0x12 +#define RLC_GPM_PERF_COUNT_0__ENABLE_MASK 0x100000 +#define RLC_GPM_PERF_COUNT_0__ENABLE__SHIFT 0x14 +#define RLC_GPM_PERF_COUNT_0__RESERVED_MASK 0xffe00000 +#define RLC_GPM_PERF_COUNT_0__RESERVED__SHIFT 0x15 +#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL_MASK 0xf +#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL__SHIFT 0x0 +#define RLC_GPM_PERF_COUNT_1__SE_INDEX_MASK 0xf0 +#define RLC_GPM_PERF_COUNT_1__SE_INDEX__SHIFT 0x4 +#define RLC_GPM_PERF_COUNT_1__SH_INDEX_MASK 0xf00 +#define RLC_GPM_PERF_COUNT_1__SH_INDEX__SHIFT 0x8 +#define RLC_GPM_PERF_COUNT_1__CU_INDEX_MASK 0xf000 +#define RLC_GPM_PERF_COUNT_1__CU_INDEX__SHIFT 0xc +#define RLC_GPM_PERF_COUNT_1__EVENT_SEL_MASK 0x30000 +#define RLC_GPM_PERF_COUNT_1__EVENT_SEL__SHIFT 0x10 +#define RLC_GPM_PERF_COUNT_1__UNUSED_MASK 0xc0000 +#define RLC_GPM_PERF_COUNT_1__UNUSED__SHIFT 0x12 +#define RLC_GPM_PERF_COUNT_1__ENABLE_MASK 0x100000 +#define RLC_GPM_PERF_COUNT_1__ENABLE__SHIFT 0x14 +#define RLC_GPM_PERF_COUNT_1__RESERVED_MASK 0xffe00000 +#define RLC_GPM_PERF_COUNT_1__RESERVED__SHIFT 0x15 +#define RLC_GPR_REG1__DATA_MASK 0xffffffff +#define RLC_GPR_REG1__DATA__SHIFT 0x0 +#define RLC_GPR_REG2__DATA_MASK 0xffffffff +#define RLC_GPR_REG2__DATA__SHIFT 0x0 +#define RLC_SPM_VMID__RLC_SPM_VMID_MASK 0xf +#define RLC_SPM_VMID__RLC_SPM_VMID__SHIFT 0x0 +#define RLC_SPM_VMID__RESERVED_MASK 0xfffffff0 +#define RLC_SPM_VMID__RESERVED__SHIFT 0x4 +#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL_MASK 0x1 +#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL__SHIFT 0x0 +#define RLC_SPM_INT_CNTL__RESERVED_MASK 0xfffffffe +#define RLC_SPM_INT_CNTL__RESERVED__SHIFT 0x1 +#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS_MASK 0x1 +#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS__SHIFT 0x0 +#define RLC_SPM_INT_STATUS__RESERVED_MASK 0xfffffffe +#define RLC_SPM_INT_STATUS__RESERVED__SHIFT 0x1 +#define RLC_SPM_DEBUG_SELECT__SELECT_MASK 0xff +#define RLC_SPM_DEBUG_SELECT__SELECT__SHIFT 0x0 +#define RLC_SPM_DEBUG_SELECT__RESERVED_MASK 0x7f00 +#define RLC_SPM_DEBUG_SELECT__RESERVED__SHIFT 0x8 +#define RLC_SPM_DEBUG_SELECT__RLC_SPM_DEBUG_MODE_MASK 0x8000 +#define RLC_SPM_DEBUG_SELECT__RLC_SPM_DEBUG_MODE__SHIFT 0xf +#define RLC_SPM_DEBUG_SELECT__RLC_SPM_NUM_SAMPLE_MASK 0xffff0000 +#define RLC_SPM_DEBUG_SELECT__RLC_SPM_NUM_SAMPLE__SHIFT 0x10 +#define RLC_SPM_DEBUG__DATA_MASK 0xffffffff +#define RLC_SPM_DEBUG__DATA__SHIFT 0x0 +#define RLC_GPM_LOG_ADDR__ADDR_MASK 0xffffffff +#define RLC_GPM_LOG_ADDR__ADDR__SHIFT 0x0 +#define RLC_GPM_LOG_SIZE__SIZE_MASK 0xffffffff +#define RLC_GPM_LOG_SIZE__SIZE__SHIFT 0x0 +#define RLC_GPM_LOG_CONT__CONT_MASK 0xffffffff +#define RLC_GPM_LOG_CONT__CONT__SHIFT 0x0 +#define RLC_SPM_PERFMON_CNTL__RESERVED1_MASK 0xfff +#define RLC_SPM_PERFMON_CNTL__RESERVED1__SHIFT 0x0 +#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE_MASK 0x3000 +#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE__SHIFT 0xc +#define RLC_SPM_PERFMON_CNTL__RESERVED_MASK 0xc000 +#define RLC_SPM_PERFMON_CNTL__RESERVED__SHIFT 0xe +#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_MASK 0xffff0000 +#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL__SHIFT 0x10 +#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO_MASK 0xffffffff +#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO__SHIFT 0x0 +#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI_MASK 0xffff +#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI__SHIFT 0x0 +#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED_MASK 0xffff0000 +#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED__SHIFT 0x10 +#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE_MASK 0xffffffff +#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE__SHIFT 0x0 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE_MASK 0xff +#define RLC_SPM_PERFMON_SEGMENT_SIZE__PERFMON_SEGMENT_SIZE__SHIFT 0x0 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1_MASK 0x700 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED1__SHIFT 0x8 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE_MASK 0xf800 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_LINE__SHIFT 0xb +#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE_MASK 0x1f0000 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE0_NUM_LINE__SHIFT 0x10 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE_MASK 0x3e00000 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE1_NUM_LINE__SHIFT 0x15 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE_MASK 0x7c000000 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE2_NUM_LINE__SHIFT 0x1a +#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED_MASK 0x80000000 +#define RLC_SPM_PERFMON_SEGMENT_SIZE__RESERVED__SHIFT 0x1f +#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK 0xffffffff +#define RLC_SPM_SE_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT 0x0 +#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA_MASK 0xffffffff +#define RLC_SPM_SE_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT 0x0 +#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_CPG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_CPC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_CPF_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_CB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_DB_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_PA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_GDS_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_IA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_SC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_TCC_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_TCA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_TCP_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_TA_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_TD_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_VGT_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_SPI_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_SQG_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_TCS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_TCS_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_TCS_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_TCS_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_SX_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR_MASK 0xffffffff +#define RLC_SPM_GLOBAL_MUXSEL_ADDR__PERFMON_SEL_ADDR__SHIFT 0x0 +#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA_MASK 0xffffffff +#define RLC_SPM_GLOBAL_MUXSEL_DATA__PERFMON_SEL_DATA__SHIFT 0x0 +#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR_MASK 0xffffffff +#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR__SHIFT 0x0 +#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD_MASK 0xffffffff +#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD__SHIFT 0x0 +#define RLC_SPM_DBR0_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_DBR0_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_DBR0_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_DBR0_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_DBR1_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_DBR1_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_DBR1_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_DBR1_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_CBR0_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_CBR0_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_CBR0_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_CBR0_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define RLC_SPM_CBR1_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY_MASK 0xff +#define RLC_SPM_CBR1_PERFMON_SAMPLE_DELAY__PERFMON_SAMPLE_DELAY__SHIFT 0x0 +#define RLC_SPM_CBR1_PERFMON_SAMPLE_DELAY__RESERVED_MASK 0xffffff00 +#define RLC_SPM_CBR1_PERFMON_SAMPLE_DELAY__RESERVED__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_0__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_0__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_0__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_0__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_0__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_0__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_1__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_1__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_1__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_1__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_1__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_1__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_2__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_2__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_2__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_2__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_2__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_2__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_3__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_3__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_3__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_3__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_3__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_3__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_4__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_4__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_4__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_4__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_4__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_4__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_5__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_5__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_5__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_5__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_5__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_5__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_6__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_6__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_6__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_6__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_6__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_6__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_7__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_7__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_7__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_7__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_7__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_7__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_8__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_8__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_8__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_8__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_8__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_8__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_9__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_9__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_9__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_9__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_9__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_9__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_10__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_10__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_10__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_10__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_10__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_10__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_11__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_11__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_11__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_11__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_11__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_11__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_12__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_12__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_12__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_12__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_12__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_12__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_13__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_13__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_13__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_13__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_13__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_13__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_14__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_14__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_14__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_14__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_14__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_14__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_15__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_15__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_15__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_15__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_15__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_15__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_16__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_16__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_16__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_16__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_16__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_16__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_17__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_17__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_17__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_17__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_17__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_17__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_18__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_18__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_18__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_18__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_18__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_18__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_19__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_19__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_19__CYL_WRAP_MASK 0x1e000 +#define SPI_PS_INPUT_CNTL_19__CYL_WRAP__SHIFT 0xd +#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_MASK 0x20000 +#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX__SHIFT 0x11 +#define SPI_PS_INPUT_CNTL_19__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_19__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_20__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_20__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_20__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_20__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_21__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_21__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_21__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_21__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_22__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_22__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_22__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_22__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_23__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_23__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_23__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_23__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_24__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_24__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_24__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_24__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_25__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_25__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_25__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_25__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_26__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_26__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_26__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_26__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_27__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_27__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_27__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_27__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_28__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_28__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_28__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_28__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_29__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_29__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_29__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_29__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_30__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_30__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_30__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_30__DUP__SHIFT 0x12 +#define SPI_PS_INPUT_CNTL_31__OFFSET_MASK 0x3f +#define SPI_PS_INPUT_CNTL_31__OFFSET__SHIFT 0x0 +#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_MASK 0x300 +#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL__SHIFT 0x8 +#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE_MASK 0x400 +#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE__SHIFT 0xa +#define SPI_PS_INPUT_CNTL_31__DUP_MASK 0x40000 +#define SPI_PS_INPUT_CNTL_31__DUP__SHIFT 0x12 +#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT_MASK 0x3e +#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT__SHIFT 0x1 +#define SPI_VS_OUT_CONFIG__VS_HALF_PACK_MASK 0x40 +#define SPI_VS_OUT_CONFIG__VS_HALF_PACK__SHIFT 0x6 +#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA_MASK 0x1 +#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA__SHIFT 0x0 +#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA_MASK 0x2 +#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA__SHIFT 0x1 +#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA_MASK 0x4 +#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA__SHIFT 0x2 +#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA_MASK 0x8 +#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA__SHIFT 0x3 +#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA_MASK 0x10 +#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA__SHIFT 0x4 +#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA_MASK 0x20 +#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA__SHIFT 0x5 +#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA_MASK 0x40 +#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA__SHIFT 0x6 +#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA_MASK 0x80 +#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA__SHIFT 0x7 +#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA_MASK 0x100 +#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA__SHIFT 0x8 +#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA_MASK 0x200 +#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA__SHIFT 0x9 +#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA_MASK 0x400 +#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA__SHIFT 0xa +#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA_MASK 0x800 +#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA__SHIFT 0xb +#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA_MASK 0x1000 +#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA__SHIFT 0xc +#define SPI_PS_INPUT_ENA__ANCILLARY_ENA_MASK 0x2000 +#define SPI_PS_INPUT_ENA__ANCILLARY_ENA__SHIFT 0xd +#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA_MASK 0x4000 +#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA__SHIFT 0xe +#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA_MASK 0x8000 +#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA__SHIFT 0xf +#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA_MASK 0x1 +#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA__SHIFT 0x0 +#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA_MASK 0x2 +#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA__SHIFT 0x1 +#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA_MASK 0x4 +#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA__SHIFT 0x2 +#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA_MASK 0x8 +#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA__SHIFT 0x3 +#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA_MASK 0x10 +#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA__SHIFT 0x4 +#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA_MASK 0x20 +#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA__SHIFT 0x5 +#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA_MASK 0x40 +#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA__SHIFT 0x6 +#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA_MASK 0x80 +#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA__SHIFT 0x7 +#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA_MASK 0x100 +#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA__SHIFT 0x8 +#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA_MASK 0x200 +#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA__SHIFT 0x9 +#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA_MASK 0x400 +#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA__SHIFT 0xa +#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA_MASK 0x800 +#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA__SHIFT 0xb +#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA_MASK 0x1000 +#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA__SHIFT 0xc +#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA_MASK 0x2000 +#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA__SHIFT 0xd +#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA_MASK 0x4000 +#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA__SHIFT 0xe +#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA_MASK 0x8000 +#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA__SHIFT 0xf +#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA_MASK 0x1 +#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA__SHIFT 0x0 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA_MASK 0x2 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA__SHIFT 0x1 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X_MASK 0x1c +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X__SHIFT 0x2 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y_MASK 0xe0 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y__SHIFT 0x5 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z_MASK 0x700 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z__SHIFT 0x8 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W_MASK 0x3800 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W__SHIFT 0xb +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1_MASK 0x4000 +#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1__SHIFT 0xe +#define SPI_PS_IN_CONTROL__NUM_INTERP_MASK 0x3f +#define SPI_PS_IN_CONTROL__NUM_INTERP__SHIFT 0x0 +#define SPI_PS_IN_CONTROL__PARAM_GEN_MASK 0x40 +#define SPI_PS_IN_CONTROL__PARAM_GEN__SHIFT 0x6 +#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE_MASK 0x4000 +#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE__SHIFT 0xe +#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL_MASK 0x1 +#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL__SHIFT 0x0 +#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL_MASK 0x10 +#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL__SHIFT 0x4 +#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL_MASK 0x100 +#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL__SHIFT 0x8 +#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL_MASK 0x1000 +#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL__SHIFT 0xc +#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION_MASK 0x30000 +#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION__SHIFT 0x10 +#define SPI_BARYC_CNTL__POS_FLOAT_ULC_MASK 0x100000 +#define SPI_BARYC_CNTL__POS_FLOAT_ULC__SHIFT 0x14 +#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS_MASK 0x1000000 +#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS__SHIFT 0x18 +#define SPI_TMPRING_SIZE__WAVES_MASK 0xfff +#define SPI_TMPRING_SIZE__WAVES__SHIFT 0x0 +#define SPI_TMPRING_SIZE__WAVESIZE_MASK 0x1fff000 +#define SPI_TMPRING_SIZE__WAVESIZE__SHIFT 0xc +#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT_MASK 0xf +#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT__SHIFT 0x0 +#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT_MASK 0xf0 +#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT__SHIFT 0x4 +#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT_MASK 0xf00 +#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT__SHIFT 0x8 +#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT_MASK 0xf000 +#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT__SHIFT 0xc +#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT_MASK 0xf +#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT__SHIFT 0x0 +#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT_MASK 0xf +#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT__SHIFT 0x0 +#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT_MASK 0xf0 +#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT__SHIFT 0x4 +#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT_MASK 0xf00 +#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT__SHIFT 0x8 +#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT_MASK 0xf000 +#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT__SHIFT 0xc +#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT_MASK 0xf0000 +#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT__SHIFT 0x10 +#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT_MASK 0xf00000 +#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT__SHIFT 0x14 +#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT_MASK 0xf000000 +#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT__SHIFT 0x18 +#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT_MASK 0xf0000000 +#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT__SHIFT 0x1c +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0_MASK 0x7 +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0__SHIFT 0x0 +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1_MASK 0x38 +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1__SHIFT 0x3 +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2_MASK 0x1c0 +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2__SHIFT 0x6 +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3_MASK 0xe00 +#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3__SHIFT 0x9 +#define SPI_ARB_PRIORITY__TS0_DUR_MULT_MASK 0x3000 +#define SPI_ARB_PRIORITY__TS0_DUR_MULT__SHIFT 0xc +#define SPI_ARB_PRIORITY__TS1_DUR_MULT_MASK 0xc000 +#define SPI_ARB_PRIORITY__TS1_DUR_MULT__SHIFT 0xe +#define SPI_ARB_PRIORITY__TS2_DUR_MULT_MASK 0x30000 +#define SPI_ARB_PRIORITY__TS2_DUR_MULT__SHIFT 0x10 +#define SPI_ARB_PRIORITY__TS3_DUR_MULT_MASK 0xc0000 +#define SPI_ARB_PRIORITY__TS3_DUR_MULT__SHIFT 0x12 +#define SPI_ARB_CYCLES_0__TS0_DURATION_MASK 0xffff +#define SPI_ARB_CYCLES_0__TS0_DURATION__SHIFT 0x0 +#define SPI_ARB_CYCLES_0__TS1_DURATION_MASK 0xffff0000 +#define SPI_ARB_CYCLES_0__TS1_DURATION__SHIFT 0x10 +#define SPI_ARB_CYCLES_1__TS2_DURATION_MASK 0xffff +#define SPI_ARB_CYCLES_1__TS2_DURATION__SHIFT 0x0 +#define SPI_ARB_CYCLES_1__TS3_DURATION_MASK 0xffff0000 +#define SPI_ARB_CYCLES_1__TS3_DURATION__SHIFT 0x10 +#define SPI_CDBG_SYS_GFX__PS_EN_MASK 0x1 +#define SPI_CDBG_SYS_GFX__PS_EN__SHIFT 0x0 +#define SPI_CDBG_SYS_GFX__VS_EN_MASK 0x2 +#define SPI_CDBG_SYS_GFX__VS_EN__SHIFT 0x1 +#define SPI_CDBG_SYS_GFX__GS_EN_MASK 0x4 +#define SPI_CDBG_SYS_GFX__GS_EN__SHIFT 0x2 +#define SPI_CDBG_SYS_GFX__ES_EN_MASK 0x8 +#define SPI_CDBG_SYS_GFX__ES_EN__SHIFT 0x3 +#define SPI_CDBG_SYS_GFX__HS_EN_MASK 0x10 +#define SPI_CDBG_SYS_GFX__HS_EN__SHIFT 0x4 +#define SPI_CDBG_SYS_GFX__LS_EN_MASK 0x20 +#define SPI_CDBG_SYS_GFX__LS_EN__SHIFT 0x5 +#define SPI_CDBG_SYS_GFX__CS_EN_MASK 0x40 +#define SPI_CDBG_SYS_GFX__CS_EN__SHIFT 0x6 +#define SPI_CDBG_SYS_HP3D__PS_EN_MASK 0x1 +#define SPI_CDBG_SYS_HP3D__PS_EN__SHIFT 0x0 +#define SPI_CDBG_SYS_HP3D__VS_EN_MASK 0x2 +#define SPI_CDBG_SYS_HP3D__VS_EN__SHIFT 0x1 +#define SPI_CDBG_SYS_HP3D__GS_EN_MASK 0x4 +#define SPI_CDBG_SYS_HP3D__GS_EN__SHIFT 0x2 +#define SPI_CDBG_SYS_HP3D__ES_EN_MASK 0x8 +#define SPI_CDBG_SYS_HP3D__ES_EN__SHIFT 0x3 +#define SPI_CDBG_SYS_HP3D__HS_EN_MASK 0x10 +#define SPI_CDBG_SYS_HP3D__HS_EN__SHIFT 0x4 +#define SPI_CDBG_SYS_HP3D__LS_EN_MASK 0x20 +#define SPI_CDBG_SYS_HP3D__LS_EN__SHIFT 0x5 +#define SPI_CDBG_SYS_CS0__PIPE0_MASK 0xff +#define SPI_CDBG_SYS_CS0__PIPE0__SHIFT 0x0 +#define SPI_CDBG_SYS_CS0__PIPE1_MASK 0xff00 +#define SPI_CDBG_SYS_CS0__PIPE1__SHIFT 0x8 +#define SPI_CDBG_SYS_CS0__PIPE2_MASK 0xff0000 +#define SPI_CDBG_SYS_CS0__PIPE2__SHIFT 0x10 +#define SPI_CDBG_SYS_CS0__PIPE3_MASK 0xff000000 +#define SPI_CDBG_SYS_CS0__PIPE3__SHIFT 0x18 +#define SPI_CDBG_SYS_CS1__PIPE0_MASK 0xff +#define SPI_CDBG_SYS_CS1__PIPE0__SHIFT 0x0 +#define SPI_CDBG_SYS_CS1__PIPE1_MASK 0xff00 +#define SPI_CDBG_SYS_CS1__PIPE1__SHIFT 0x8 +#define SPI_CDBG_SYS_CS1__PIPE2_MASK 0xff0000 +#define SPI_CDBG_SYS_CS1__PIPE2__SHIFT 0x10 +#define SPI_CDBG_SYS_CS1__PIPE3_MASK 0xff000000 +#define SPI_CDBG_SYS_CS1__PIPE3__SHIFT 0x18 +#define SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_GFX__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS0__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS0__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS1__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS1__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS2__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS2__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS3__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS3__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS4__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS4__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS5__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS5__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS6__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS6__VALUE__SHIFT 0x0 +#define SPI_WCL_PIPE_PERCENT_CS7__VALUE_MASK 0x1f +#define SPI_WCL_PIPE_PERCENT_CS7__VALUE__SHIFT 0x0 +#define SPI_GDBG_WAVE_CNTL__STALL_RA_MASK 0x1 +#define SPI_GDBG_WAVE_CNTL__STALL_RA__SHIFT 0x0 +#define SPI_GDBG_TRAP_CONFIG__ME_SEL_MASK 0x3 +#define SPI_GDBG_TRAP_CONFIG__ME_SEL__SHIFT 0x0 +#define SPI_GDBG_TRAP_CONFIG__PIPE_SEL_MASK 0xc +#define SPI_GDBG_TRAP_CONFIG__PIPE_SEL__SHIFT 0x2 +#define SPI_GDBG_TRAP_CONFIG__QUEUE_SEL_MASK 0x70 +#define SPI_GDBG_TRAP_CONFIG__QUEUE_SEL__SHIFT 0x4 +#define SPI_GDBG_TRAP_CONFIG__ME_MATCH_MASK 0x80 +#define SPI_GDBG_TRAP_CONFIG__ME_MATCH__SHIFT 0x7 +#define SPI_GDBG_TRAP_CONFIG__PIPE_MATCH_MASK 0x100 +#define SPI_GDBG_TRAP_CONFIG__PIPE_MATCH__SHIFT 0x8 +#define SPI_GDBG_TRAP_CONFIG__QUEUE_MATCH_MASK 0x200 +#define SPI_GDBG_TRAP_CONFIG__QUEUE_MATCH__SHIFT 0x9 +#define SPI_GDBG_TRAP_CONFIG__TRAP_EN_MASK 0x8000 +#define SPI_GDBG_TRAP_CONFIG__TRAP_EN__SHIFT 0xf +#define SPI_GDBG_TRAP_CONFIG__VMID_SEL_MASK 0xffff0000 +#define SPI_GDBG_TRAP_CONFIG__VMID_SEL__SHIFT 0x10 +#define SPI_GDBG_TRAP_MASK__EXCP_EN_MASK 0x1ff +#define SPI_GDBG_TRAP_MASK__EXCP_EN__SHIFT 0x0 +#define SPI_GDBG_TRAP_MASK__REPLACE_MASK 0x200 +#define SPI_GDBG_TRAP_MASK__REPLACE__SHIFT 0x9 +#define SPI_GDBG_TBA_LO__MEM_BASE_MASK 0xffffffff +#define SPI_GDBG_TBA_LO__MEM_BASE__SHIFT 0x0 +#define SPI_GDBG_TBA_HI__MEM_BASE_MASK 0xff +#define SPI_GDBG_TBA_HI__MEM_BASE__SHIFT 0x0 +#define SPI_GDBG_TMA_LO__MEM_BASE_MASK 0xffffffff +#define SPI_GDBG_TMA_LO__MEM_BASE__SHIFT 0x0 +#define SPI_GDBG_TMA_HI__MEM_BASE_MASK 0xff +#define SPI_GDBG_TMA_HI__MEM_BASE__SHIFT 0x0 +#define SPI_GDBG_TRAP_DATA0__DATA_MASK 0xffffffff +#define SPI_GDBG_TRAP_DATA0__DATA__SHIFT 0x0 +#define SPI_GDBG_TRAP_DATA1__DATA_MASK 0xffffffff +#define SPI_GDBG_TRAP_DATA1__DATA__SHIFT 0x0 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_MASK 0x1 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET__SHIFT 0x0 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID_MASK 0x2 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID__SHIFT 0x1 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID_MASK 0x4 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID__SHIFT 0x2 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE_MASK 0x8 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE__SHIFT 0x3 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY_MASK 0x10 +#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY__SHIFT 0x4 +#define SPI_COMPUTE_QUEUE_RESET__RESET_MASK 0x1 +#define SPI_COMPUTE_QUEUE_RESET__RESET__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_0__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_0__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_0__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_0__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_0__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_0__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_0__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_0__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_1__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_1__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_1__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_1__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_1__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_1__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_1__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_1__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_2__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_2__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_2__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_2__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_2__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_2__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_2__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_2__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_3__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_3__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_3__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_3__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_3__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_3__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_3__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_3__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_4__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_4__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_4__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_4__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_4__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_4__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_4__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_4__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_5__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_5__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_5__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_5__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_5__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_5__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_5__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_5__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_6__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_6__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_6__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_6__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_6__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_6__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_6__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_6__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_7__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_7__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_7__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_7__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_7__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_7__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_7__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_7__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_8__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_8__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_8__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_8__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_8__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_8__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_8__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_8__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_9__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_9__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_9__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_9__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_9__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_9__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_9__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_9__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_10__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_10__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_10__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_10__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_10__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_10__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_10__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_10__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_CU_11__VGPR_MASK 0xf +#define SPI_RESOURCE_RESERVE_CU_11__VGPR__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_CU_11__SGPR_MASK 0xf0 +#define SPI_RESOURCE_RESERVE_CU_11__SGPR__SHIFT 0x4 +#define SPI_RESOURCE_RESERVE_CU_11__LDS_MASK 0xf00 +#define SPI_RESOURCE_RESERVE_CU_11__LDS__SHIFT 0x8 +#define SPI_RESOURCE_RESERVE_CU_11__WAVES_MASK 0x7000 +#define SPI_RESOURCE_RESERVE_CU_11__WAVES__SHIFT 0xc +#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS_MASK 0x78000 +#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS__SHIFT 0xf +#define SPI_RESOURCE_RESERVE_EN_CU_0__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_0__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_0__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_1__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_1__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_1__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_2__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_2__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_2__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_3__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_3__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_3__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_4__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_4__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_4__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_5__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_5__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_5__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_6__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_6__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_6__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_7__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_7__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_7__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_8__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_8__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_8__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_9__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_9__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_9__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_10__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_10__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_10__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_RESOURCE_RESERVE_EN_CU_11__EN_MASK 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_11__EN__SHIFT 0x0 +#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK_MASK 0xfffe +#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK__SHIFT 0x1 +#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK_MASK 0xff0000 +#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK__SHIFT 0x10 +#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY_MASK 0x1000000 +#define SPI_RESOURCE_RESERVE_EN_CU_11__RESERVE_SPACE_ONLY__SHIFT 0x18 +#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID_MASK 0xfff +#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT 0x0 +#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY_MASK 0x1fffff +#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY__SHIFT 0x0 +#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER_MASK 0xe00000 +#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER__SHIFT 0x15 +#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS_MASK 0x1000000 +#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS__SHIFT 0x18 +#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS_MASK 0x2000000 +#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS__SHIFT 0x19 +#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET_MASK 0x4000000 +#define SPI_CONFIG_CNTL__RSRC_MGMT_RESET__SHIFT 0x1a +#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL_MASK 0x8000000 +#define SPI_CONFIG_CNTL__TTRACE_STALL_ALL__SHIFT 0x1b +#define SPI_DEBUG_CNTL__DEBUG_GRBM_OVERRIDE_MASK 0x1 +#define SPI_DEBUG_CNTL__DEBUG_GRBM_OVERRIDE__SHIFT 0x0 +#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL_MASK 0xe +#define SPI_DEBUG_CNTL__DEBUG_THREAD_TYPE_SEL__SHIFT 0x1 +#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL_MASK 0x3f0 +#define SPI_DEBUG_CNTL__DEBUG_GROUP_SEL__SHIFT 0x4 +#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL_MASK 0xfc00 +#define SPI_DEBUG_CNTL__DEBUG_SIMD_SEL__SHIFT 0xa +#define SPI_DEBUG_CNTL__DEBUG_SH_SEL_MASK 0x10000 +#define SPI_DEBUG_CNTL__DEBUG_SH_SEL__SHIFT 0x10 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0_MASK 0x20000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_0__SHIFT 0x11 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1_MASK 0x40000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_1__SHIFT 0x12 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_2_MASK 0x80000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_2__SHIFT 0x13 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3_MASK 0x100000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_3__SHIFT 0x14 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_4_MASK 0x200000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_4__SHIFT 0x15 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_5_MASK 0x400000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_5__SHIFT 0x16 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_6_MASK 0x800000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_6__SHIFT 0x17 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_7_MASK 0x1000000 +#define SPI_DEBUG_CNTL__SPI_ECO_SPARE_7__SHIFT 0x18 +#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL_MASK 0xe000000 +#define SPI_DEBUG_CNTL__DEBUG_PIPE_SEL__SHIFT 0x19 +#define SPI_DEBUG_CNTL__DEBUG_REG_EN_MASK 0x80000000 +#define SPI_DEBUG_CNTL__DEBUG_REG_EN__SHIFT 0x1f +#define SPI_DEBUG_READ__DATA_MASK 0xffffff +#define SPI_DEBUG_READ__DATA__SHIFT 0x0 +#define SPI_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define SPI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define SPI_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define SPI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00 +#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define SPI_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define SPI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK 0xffc00 +#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT 0xa +#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define SPI_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define SPI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1_MASK 0xffc00 +#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT 0xa +#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff +#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0 +#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00 +#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa +#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK 0x3ff +#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT 0x0 +#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK 0xffc00 +#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT 0xa +#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK 0x3ff +#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT 0x0 +#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK 0xffc00 +#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT 0xa +#define SPI_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0xff +#define SPI_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x0 +#define SPI_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0xff +#define SPI_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x0 +#define SPI_PERFCOUNTER_BINS__BIN0_MIN_MASK 0xf +#define SPI_PERFCOUNTER_BINS__BIN0_MIN__SHIFT 0x0 +#define SPI_PERFCOUNTER_BINS__BIN0_MAX_MASK 0xf0 +#define SPI_PERFCOUNTER_BINS__BIN0_MAX__SHIFT 0x4 +#define SPI_PERFCOUNTER_BINS__BIN1_MIN_MASK 0xf00 +#define SPI_PERFCOUNTER_BINS__BIN1_MIN__SHIFT 0x8 +#define SPI_PERFCOUNTER_BINS__BIN1_MAX_MASK 0xf000 +#define SPI_PERFCOUNTER_BINS__BIN1_MAX__SHIFT 0xc +#define SPI_PERFCOUNTER_BINS__BIN2_MIN_MASK 0xf0000 +#define SPI_PERFCOUNTER_BINS__BIN2_MIN__SHIFT 0x10 +#define SPI_PERFCOUNTER_BINS__BIN2_MAX_MASK 0xf00000 +#define SPI_PERFCOUNTER_BINS__BIN2_MAX__SHIFT 0x14 +#define SPI_PERFCOUNTER_BINS__BIN3_MIN_MASK 0xf000000 +#define SPI_PERFCOUNTER_BINS__BIN3_MIN__SHIFT 0x18 +#define SPI_PERFCOUNTER_BINS__BIN3_MAX_MASK 0xf0000000 +#define SPI_PERFCOUNTER_BINS__BIN3_MAX__SHIFT 0x1c +#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY_MASK 0xf +#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT 0x0 +#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW_MASK 0x10 +#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW__SHIFT 0x4 +#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE_MASK 0x40 +#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE__SHIFT 0x6 +#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT_MASK 0x80 +#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT__SHIFT 0x7 +#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE_MASK 0x100 +#define SPI_CONFIG_CNTL_1__CRC_SIMD_ID_WADDR_DISABLE__SHIFT 0x8 +#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE_MASK 0x200 +#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_MODE__SHIFT 0x9 +#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT_MASK 0x3c00 +#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT__SHIFT 0xa +#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE_MASK 0xffff0000 +#define SPI_CONFIG_CNTL_1__PC_LIMIT_SIZE__SHIFT 0x10 +#define SPI_DEBUG_BUSY__LS_BUSY_MASK 0x1 +#define SPI_DEBUG_BUSY__LS_BUSY__SHIFT 0x0 +#define SPI_DEBUG_BUSY__HS_BUSY_MASK 0x2 +#define SPI_DEBUG_BUSY__HS_BUSY__SHIFT 0x1 +#define SPI_DEBUG_BUSY__ES_BUSY_MASK 0x4 +#define SPI_DEBUG_BUSY__ES_BUSY__SHIFT 0x2 +#define SPI_DEBUG_BUSY__GS_BUSY_MASK 0x8 +#define SPI_DEBUG_BUSY__GS_BUSY__SHIFT 0x3 +#define SPI_DEBUG_BUSY__VS_BUSY_MASK 0x10 +#define SPI_DEBUG_BUSY__VS_BUSY__SHIFT 0x4 +#define SPI_DEBUG_BUSY__PS0_BUSY_MASK 0x20 +#define SPI_DEBUG_BUSY__PS0_BUSY__SHIFT 0x5 +#define SPI_DEBUG_BUSY__PS1_BUSY_MASK 0x40 +#define SPI_DEBUG_BUSY__PS1_BUSY__SHIFT 0x6 +#define SPI_DEBUG_BUSY__CSG_BUSY_MASK 0x80 +#define SPI_DEBUG_BUSY__CSG_BUSY__SHIFT 0x7 +#define SPI_DEBUG_BUSY__CS0_BUSY_MASK 0x100 +#define SPI_DEBUG_BUSY__CS0_BUSY__SHIFT 0x8 +#define SPI_DEBUG_BUSY__CS1_BUSY_MASK 0x200 +#define SPI_DEBUG_BUSY__CS1_BUSY__SHIFT 0x9 +#define SPI_DEBUG_BUSY__CS2_BUSY_MASK 0x400 +#define SPI_DEBUG_BUSY__CS2_BUSY__SHIFT 0xa +#define SPI_DEBUG_BUSY__CS3_BUSY_MASK 0x800 +#define SPI_DEBUG_BUSY__CS3_BUSY__SHIFT 0xb +#define SPI_DEBUG_BUSY__CS4_BUSY_MASK 0x1000 +#define SPI_DEBUG_BUSY__CS4_BUSY__SHIFT 0xc +#define SPI_DEBUG_BUSY__CS5_BUSY_MASK 0x2000 +#define SPI_DEBUG_BUSY__CS5_BUSY__SHIFT 0xd +#define SPI_DEBUG_BUSY__CS6_BUSY_MASK 0x4000 +#define SPI_DEBUG_BUSY__CS6_BUSY__SHIFT 0xe +#define SPI_DEBUG_BUSY__CS7_BUSY_MASK 0x8000 +#define SPI_DEBUG_BUSY__CS7_BUSY__SHIFT 0xf +#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY_MASK 0x10000 +#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY__SHIFT 0x10 +#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY_MASK 0x20000 +#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY__SHIFT 0x11 +#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY_MASK 0x40000 +#define SPI_DEBUG_BUSY__RSRC_ALLOC0_BUSY__SHIFT 0x12 +#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY_MASK 0x80000 +#define SPI_DEBUG_BUSY__RSRC_ALLOC1_BUSY__SHIFT 0x13 +#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY_MASK 0x100000 +#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY__SHIFT 0x14 +#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY_MASK 0x200000 +#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY__SHIFT 0x15 +#define SPI_DEBUG_BUSY__GRBM_BUSY_MASK 0x400000 +#define SPI_DEBUG_BUSY__GRBM_BUSY__SHIFT 0x16 +#define SPI_DEBUG_BUSY__SPIS_BUSY_MASK 0x800000 +#define SPI_DEBUG_BUSY__SPIS_BUSY__SHIFT 0x17 +#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY_MASK 0xf +#define CGTS_SM_CTRL_REG__ON_SEQ_DELAY__SHIFT 0x0 +#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY_MASK 0xff0 +#define CGTS_SM_CTRL_REG__OFF_SEQ_DELAY__SHIFT 0x4 +#define CGTS_SM_CTRL_REG__MGCG_ENABLED_MASK 0x1000 +#define CGTS_SM_CTRL_REG__MGCG_ENABLED__SHIFT 0xc +#define CGTS_SM_CTRL_REG__BASE_MODE_MASK 0x10000 +#define CGTS_SM_CTRL_REG__BASE_MODE__SHIFT 0x10 +#define CGTS_SM_CTRL_REG__SM_MODE_MASK 0xe0000 +#define CGTS_SM_CTRL_REG__SM_MODE__SHIFT 0x11 +#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK 0x100000 +#define CGTS_SM_CTRL_REG__SM_MODE_ENABLE__SHIFT 0x14 +#define CGTS_SM_CTRL_REG__OVERRIDE_MASK 0x200000 +#define CGTS_SM_CTRL_REG__OVERRIDE__SHIFT 0x15 +#define CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK 0x400000 +#define CGTS_SM_CTRL_REG__LS_OVERRIDE__SHIFT 0x16 +#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK 0x800000 +#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN__SHIFT 0x17 +#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD_MASK 0xff000000 +#define CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT 0x18 +#define CGTS_RD_CTRL_REG__ROW_MUX_SEL_MASK 0x1f +#define CGTS_RD_CTRL_REG__ROW_MUX_SEL__SHIFT 0x0 +#define CGTS_RD_CTRL_REG__REG_MUX_SEL_MASK 0x1f00 +#define CGTS_RD_CTRL_REG__REG_MUX_SEL__SHIFT 0x8 +#define CGTS_RD_REG__READ_DATA_MASK 0x3fff +#define CGTS_RD_REG__READ_DATA__SHIFT 0x0 +#define CGTS_TCC_DISABLE__TCC_DISABLE_MASK 0xffff0000 +#define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT 0x10 +#define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK 0xffff0000 +#define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT 0x10 +#define CGTS_CU0_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU0_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU0_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU0_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU0_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU0_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU0_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU0_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU0_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU0_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU0_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU0_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU0_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU0_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU0_TA_SQC_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU0_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC__SHIFT 0x10 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU0_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU0_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU0_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU0_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU0_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU0_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU0_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU0_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU0_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU0_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU0_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU0_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU0_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU0_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU0_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU0_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU1_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU1_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU1_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU1_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU1_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU1_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU1_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU1_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU1_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU1_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU1_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU1_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU1_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU1_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU1_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU1_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU1_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU1_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU1_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU1_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU1_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU1_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU1_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU1_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU1_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU1_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU1_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU1_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU1_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU1_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU1_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU1_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU1_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU1_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU1_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU1_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU1_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU1_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU1_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU2_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU2_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU2_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU2_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU2_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU2_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU2_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU2_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU2_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU2_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU2_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU2_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU2_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU2_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU2_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU2_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU2_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU2_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU2_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU2_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU2_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU2_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU2_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU2_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU2_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU2_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU2_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU2_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU2_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU2_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU2_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU2_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU2_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU2_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU2_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU2_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU2_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU2_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU2_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU3_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU3_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU3_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU3_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU3_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU3_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU3_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU3_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU3_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU3_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU3_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU3_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU3_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU3_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU3_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU3_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU3_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU3_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU3_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU3_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU3_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU3_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU3_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU3_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU3_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU3_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU3_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU3_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU3_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU3_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU3_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU3_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU3_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU3_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU3_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU3_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU3_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU3_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU3_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU4_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU4_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU4_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU4_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU4_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU4_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU4_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU4_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU4_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU4_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU4_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU4_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU4_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU4_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU4_TA_SQC_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU4_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC__SHIFT 0x10 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU4_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU4_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU4_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU4_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU4_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU4_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU4_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU4_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU4_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU4_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU4_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU4_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU4_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU4_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU4_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU4_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU5_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU5_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU5_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU5_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU5_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU5_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU5_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU5_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU5_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU5_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU5_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU5_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU5_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU5_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU5_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU5_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU5_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU5_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU5_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU5_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU5_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU5_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU5_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU5_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU5_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU5_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU5_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU5_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU5_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU5_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU5_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU5_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU5_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU5_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU5_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU5_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU5_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU5_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU5_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU6_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU6_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU6_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU6_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU6_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU6_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU6_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU6_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU6_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU6_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU6_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU6_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU6_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU6_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU6_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU6_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU6_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU6_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU6_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU6_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU6_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU6_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU6_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU6_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU6_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU6_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU6_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU6_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU6_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU6_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU6_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU6_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU6_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU6_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU6_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU6_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU6_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU6_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU6_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU7_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU7_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU7_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU7_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU7_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU7_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU7_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU7_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU7_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU7_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU7_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU7_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU7_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU7_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU7_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU7_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU7_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU7_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU7_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU7_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU7_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU7_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU7_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU7_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU7_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU7_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU7_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU7_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU7_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU7_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU7_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU7_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU7_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU7_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU7_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU7_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU7_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU7_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU7_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU8_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU8_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU8_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU8_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU8_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU8_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU8_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU8_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU8_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU8_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU8_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU8_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU8_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU8_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU8_TA_SQC_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU8_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC__SHIFT 0x10 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU8_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU8_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU8_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU8_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU8_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU8_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU8_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU8_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU8_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU8_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU8_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU8_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU8_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU8_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU8_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU8_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU9_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU9_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU9_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU9_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU9_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU9_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU9_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU9_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU9_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU9_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU9_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU9_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU9_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU9_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU9_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU9_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU9_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU9_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU9_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU9_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU9_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU9_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU9_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU9_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU9_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU9_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU9_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU9_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU9_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU9_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU9_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU9_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU9_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU9_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU9_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU9_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU9_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU9_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU9_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU10_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU10_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU10_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU10_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU10_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU10_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU10_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU10_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU10_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU10_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU10_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU10_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU10_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU10_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU10_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU10_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU10_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU10_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU10_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU10_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU10_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU10_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU10_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU10_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU10_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU10_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU10_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU10_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU10_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU10_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU10_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU10_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU10_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU10_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU10_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU10_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU10_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU10_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU10_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU11_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU11_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU11_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU11_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU11_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU11_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU11_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU11_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU11_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU11_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU11_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU11_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU11_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU11_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU11_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU11_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU11_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU11_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU11_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU11_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU11_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU11_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU11_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU11_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU11_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU11_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU11_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU11_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU11_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU11_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU11_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU11_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU11_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU11_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU11_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU11_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU11_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU11_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU11_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU12_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU12_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU12_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU12_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU12_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU12_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU12_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU12_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU12_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU12_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU12_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU12_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU12_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU12_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU12_TA_SQC_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU12_TA_SQC_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_MASK 0x7f0000 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC__SHIFT 0x10 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE_MASK 0x800000 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_OVERRIDE__SHIFT 0x17 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU12_TA_SQC_CTRL_REG__SQC_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU12_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU12_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU12_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU12_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU12_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU12_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU12_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU12_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU12_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU12_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU12_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU12_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU12_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU12_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU12_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU13_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU13_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU13_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU13_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU13_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU13_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU13_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU13_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU13_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU13_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU13_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU13_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU13_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU13_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU13_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU13_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU13_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU13_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU13_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU13_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU13_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU13_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU13_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU13_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU13_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU13_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU13_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU13_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU13_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU13_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU13_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU13_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU13_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU13_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU13_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU13_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU13_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU13_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU13_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU14_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU14_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU14_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU14_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU14_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU14_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU14_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU14_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU14_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU14_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU14_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU14_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU14_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU14_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU14_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU14_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU14_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU14_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU14_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU14_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU14_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU14_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU14_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU14_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU14_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU14_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU14_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU14_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU14_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU14_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU14_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU14_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU14_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU14_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU14_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU14_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU14_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU14_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU14_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU15_SP0_CTRL_REG__SP00_MASK 0x7f +#define CGTS_CU15_SP0_CTRL_REG__SP00__SHIFT 0x0 +#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE_MASK 0x80 +#define CGTS_CU15_SP0_CTRL_REG__SP00_OVERRIDE__SHIFT 0x7 +#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU15_SP0_CTRL_REG__SP00_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU15_SP0_CTRL_REG__SP00_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU15_SP0_CTRL_REG__SP00_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU15_SP0_CTRL_REG__SP01_MASK 0x7f0000 +#define CGTS_CU15_SP0_CTRL_REG__SP01__SHIFT 0x10 +#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE_MASK 0x800000 +#define CGTS_CU15_SP0_CTRL_REG__SP01_OVERRIDE__SHIFT 0x17 +#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU15_SP0_CTRL_REG__SP01_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU15_SP0_CTRL_REG__SP01_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU15_SP0_CTRL_REG__SP01_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_MASK 0x7f +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS__SHIFT 0x0 +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE_MASK 0x80 +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_OVERRIDE__SHIFT 0x7 +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU15_LDS_SQ_CTRL_REG__LDS_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_MASK 0x7f0000 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ__SHIFT 0x10 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE_MASK 0x800000 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_OVERRIDE__SHIFT 0x17 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU15_LDS_SQ_CTRL_REG__SQ_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU15_TA_CTRL_REG__TA_MASK 0x7f +#define CGTS_CU15_TA_CTRL_REG__TA__SHIFT 0x0 +#define CGTS_CU15_TA_CTRL_REG__TA_OVERRIDE_MASK 0x80 +#define CGTS_CU15_TA_CTRL_REG__TA_OVERRIDE__SHIFT 0x7 +#define CGTS_CU15_TA_CTRL_REG__TA_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU15_TA_CTRL_REG__TA_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU15_TA_CTRL_REG__TA_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU15_TA_CTRL_REG__TA_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU15_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU15_TA_CTRL_REG__TA_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU15_SP1_CTRL_REG__SP10_MASK 0x7f +#define CGTS_CU15_SP1_CTRL_REG__SP10__SHIFT 0x0 +#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE_MASK 0x80 +#define CGTS_CU15_SP1_CTRL_REG__SP10_OVERRIDE__SHIFT 0x7 +#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU15_SP1_CTRL_REG__SP10_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU15_SP1_CTRL_REG__SP10_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU15_SP1_CTRL_REG__SP10_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU15_SP1_CTRL_REG__SP11_MASK 0x7f0000 +#define CGTS_CU15_SP1_CTRL_REG__SP11__SHIFT 0x10 +#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE_MASK 0x800000 +#define CGTS_CU15_SP1_CTRL_REG__SP11_OVERRIDE__SHIFT 0x17 +#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU15_SP1_CTRL_REG__SP11_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU15_SP1_CTRL_REG__SP11_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU15_SP1_CTRL_REG__SP11_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_MASK 0x7f +#define CGTS_CU15_TD_TCP_CTRL_REG__TD__SHIFT 0x0 +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE_MASK 0x80 +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_OVERRIDE__SHIFT 0x7 +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE_MASK 0x300 +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_BUSY_OVERRIDE__SHIFT 0x8 +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE_MASK 0x400 +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_LS_OVERRIDE__SHIFT 0xa +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE_MASK 0x800 +#define CGTS_CU15_TD_TCP_CTRL_REG__TD_SIMDBUSY_OVERRIDE__SHIFT 0xb +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_MASK 0x7f0000 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP__SHIFT 0x10 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_OVERRIDE_MASK 0x800000 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_OVERRIDE__SHIFT 0x17 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE_MASK 0x3000000 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_BUSY_OVERRIDE__SHIFT 0x18 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE_MASK 0x4000000 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_LS_OVERRIDE__SHIFT 0x1a +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE_MASK 0x8000000 +#define CGTS_CU15_TD_TCP_CTRL_REG__TCP_SIMDBUSY_OVERRIDE__SHIFT 0x1b +#define CGTT_SPI_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_SPI_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST_MASK 0xfc0000 +#define CGTT_SPI_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT 0x12 +#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE_MASK 0x1000000 +#define CGTT_SPI_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT 0x18 +#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE_MASK 0x4000000 +#define CGTT_SPI_CLK_CTRL__ALL_CLK_ON_OVERRIDE__SHIFT 0x1a +#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE_MASK 0x8000000 +#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE__SHIFT 0x1b +#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE_MASK 0x10000000 +#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE__SHIFT 0x1c +#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE_MASK 0x20000000 +#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE__SHIFT 0x1d +#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE_MASK 0x40000000 +#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE__SHIFT 0x1e +#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_PC_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_PC_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST_MASK 0xfc0000 +#define CGTT_PC_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT 0x12 +#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE_MASK 0x1000000 +#define CGTT_PC_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT 0x18 +#define CGTT_PC_CLK_CTRL__BACK_CLK_ON_OVERRIDE_MASK 0x2000000 +#define CGTT_PC_CLK_CTRL__BACK_CLK_ON_OVERRIDE__SHIFT 0x19 +#define CGTT_PC_CLK_CTRL__FRONT_CLK_ON_OVERRIDE_MASK 0x4000000 +#define CGTT_PC_CLK_CTRL__FRONT_CLK_ON_OVERRIDE__SHIFT 0x1a +#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE_MASK 0x8000000 +#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x1b +#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000 +#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x1c +#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000 +#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x1d +#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000 +#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x1e +#define CGTT_PC_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_PC_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_BCI_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_BCI_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_BCI_CLK_CTRL__RESERVED_MASK 0xfff000 +#define CGTT_BCI_CLK_CTRL__RESERVED__SHIFT 0xc +#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE_MASK 0x1000000 +#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE__SHIFT 0x18 +#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE_MASK 0x2000000 +#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE__SHIFT 0x19 +#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE_MASK 0x4000000 +#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE__SHIFT 0x1a +#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE_MASK 0x8000000 +#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE__SHIFT 0x1b +#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE_MASK 0x10000000 +#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE__SHIFT 0x1c +#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE_MASK 0x20000000 +#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE__SHIFT 0x1d +#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE_MASK 0x40000000 +#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE__SHIFT 0x1e +#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD_MASK 0xf +#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD__SHIFT 0x0 +#define SPI_WF_LIFETIME_CNTL__EN_MASK 0x10 +#define SPI_WF_LIFETIME_CNTL__EN__SHIFT 0x4 +#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_6__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_6__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_7__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_7__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_8__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_8__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_LIMIT_9__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN_MASK 0x80000000 +#define SPI_WF_LIFETIME_LIMIT_9__EN_WARN__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_0__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_0__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_1__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_1__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_1__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_2__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_2__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_3__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_3__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_3__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_4__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_4__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_5__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_5__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_5__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_6__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_6__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_7__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_7__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_8__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_8__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_8__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_9__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_9__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_10__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_10__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_10__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_11__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_11__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_12__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_12__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_12__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_13__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_13__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_14__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_14__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_15__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_15__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_16__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_16__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_17__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_17__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_18__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_18__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_19__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_19__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT_MASK 0x7fffffff +#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT__SHIFT 0x0 +#define SPI_WF_LIFETIME_STATUS_20__INT_SENT_MASK 0x80000000 +#define SPI_WF_LIFETIME_STATUS_20__INT_SENT__SHIFT 0x1f +#define SPI_WF_LIFETIME_DEBUG__START_VALUE_MASK 0x7fffffff +#define SPI_WF_LIFETIME_DEBUG__START_VALUE__SHIFT 0x0 +#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN_MASK 0x80000000 +#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN__SHIFT 0x1f +#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY_MASK 0x1 +#define SPI_SLAVE_DEBUG_BUSY__LS_VTX_BUSY__SHIFT 0x0 +#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY_MASK 0x2 +#define SPI_SLAVE_DEBUG_BUSY__HS_VTX_BUSY__SHIFT 0x1 +#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY_MASK 0x4 +#define SPI_SLAVE_DEBUG_BUSY__ES_VTX_BUSY__SHIFT 0x2 +#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY_MASK 0x8 +#define SPI_SLAVE_DEBUG_BUSY__GS_VTX_BUSY__SHIFT 0x3 +#define SPI_SLAVE_DEBUG_BUSY__VS_VTX_BUSY_MASK 0x10 +#define SPI_SLAVE_DEBUG_BUSY__VS_VTX_BUSY__SHIFT 0x4 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY_MASK 0x20 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC00_BUSY__SHIFT 0x5 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY_MASK 0x40 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC01_BUSY__SHIFT 0x6 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC10_BUSY_MASK 0x80 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC10_BUSY__SHIFT 0x7 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC11_BUSY_MASK 0x100 +#define SPI_SLAVE_DEBUG_BUSY__VGPR_WC11_BUSY__SHIFT 0x8 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY_MASK 0x200 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC00_BUSY__SHIFT 0x9 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY_MASK 0x400 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC01_BUSY__SHIFT 0xa +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC02_BUSY_MASK 0x800 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC02_BUSY__SHIFT 0xb +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC03_BUSY_MASK 0x1000 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC03_BUSY__SHIFT 0xc +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC10_BUSY_MASK 0x2000 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC10_BUSY__SHIFT 0xd +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC11_BUSY_MASK 0x4000 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC11_BUSY__SHIFT 0xe +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC12_BUSY_MASK 0x8000 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC12_BUSY__SHIFT 0xf +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC13_BUSY_MASK 0x10000 +#define SPI_SLAVE_DEBUG_BUSY__SGPR_WC13_BUSY__SHIFT 0x10 +#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER0_BUSY_MASK 0x20000 +#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER0_BUSY__SHIFT 0x11 +#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER1_BUSY_MASK 0x40000 +#define SPI_SLAVE_DEBUG_BUSY__WAVEBUFFER1_BUSY__SHIFT 0x12 +#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC0_BUSY_MASK 0x80000 +#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC0_BUSY__SHIFT 0x13 +#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC1_BUSY_MASK 0x100000 +#define SPI_SLAVE_DEBUG_BUSY__WAVE_WC1_BUSY__SHIFT 0x14 +#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY_MASK 0x200000 +#define SPI_SLAVE_DEBUG_BUSY__EVENT_CNTL_BUSY__SHIFT 0x15 +#define SPI_LB_CTR_CTRL__LOAD_MASK 0x1 +#define SPI_LB_CTR_CTRL__LOAD__SHIFT 0x0 +#define SPI_LB_CU_MASK__CU_MASK_MASK 0xffff +#define SPI_LB_CU_MASK__CU_MASK__SHIFT 0x0 +#define SPI_LB_DATA_REG__CNT_DATA_MASK 0xffffffff +#define SPI_LB_DATA_REG__CNT_DATA__SHIFT 0x0 +#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK_MASK 0xffff +#define SPI_PG_ENABLE_STATIC_CU_MASK__CU_MASK__SHIFT 0x0 +#define SPI_GDS_CREDITS__DS_DATA_CREDITS_MASK 0xff +#define SPI_GDS_CREDITS__DS_DATA_CREDITS__SHIFT 0x0 +#define SPI_GDS_CREDITS__DS_CMD_CREDITS_MASK 0xff00 +#define SPI_GDS_CREDITS__DS_CMD_CREDITS__SHIFT 0x8 +#define SPI_GDS_CREDITS__UNUSED_MASK 0xffff0000 +#define SPI_GDS_CREDITS__UNUSED__SHIFT 0x10 +#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE_MASK 0xffff +#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE__SHIFT 0x0 +#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE_MASK 0xffff0000 +#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE__SHIFT 0x10 +#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE_MASK 0xffff +#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE__SHIFT 0x0 +#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE_MASK 0xffff0000 +#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE__SHIFT 0x10 +#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE_MASK 0xffffffff +#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_4__COUNT__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_5__COUNT__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_6__COUNT__SHIFT 0x0 +#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT_MASK 0x7ff +#define SPI_CSQ_WF_ACTIVE_COUNT_7__COUNT__SHIFT 0x0 +#define BCI_DEBUG_READ__DATA_MASK 0xffffff +#define BCI_DEBUG_READ__DATA__SHIFT 0x0 +#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK 0xffffffff +#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT 0x0 +#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK 0xff +#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT 0x0 +#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK 0xffffffff +#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT 0x0 +#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK 0xff +#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT 0x0 +#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK 0x3f +#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT 0x0 +#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK 0x3c0 +#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT 0x6 +#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK 0xffffffff +#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT 0x0 +#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK 0xff +#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT 0x0 +#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK 0xffffffff +#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT 0x0 +#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK 0xff +#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT 0x0 +#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK 0x3f +#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT 0x0 +#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK 0x3c0 +#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT 0x6 +#define SPI_SHADER_TBA_LO_PS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TBA_LO_PS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TBA_HI_PS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TBA_HI_PS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_LO_PS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TMA_LO_PS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_HI_PS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TMA_HI_PS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_LO_PS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_PGM_LO_PS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_HI_PS__MEM_BASE_MASK 0xff +#define SPI_SHADER_PGM_HI_PS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_PS__VGPRS_MASK 0x3f +#define SPI_SHADER_PGM_RSRC1_PS__VGPRS__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_PS__SGPRS_MASK 0x3c0 +#define SPI_SHADER_PGM_RSRC1_PS__SGPRS__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY_MASK 0xc00 +#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY__SHIFT 0xa +#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE_MASK 0xff000 +#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE__SHIFT 0xc +#define SPI_SHADER_PGM_RSRC1_PS__PRIV_MASK 0x100000 +#define SPI_SHADER_PGM_RSRC1_PS__PRIV__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP_MASK 0x200000 +#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP__SHIFT 0x15 +#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE_MASK 0x400000 +#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE__SHIFT 0x16 +#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE_MASK 0x800000 +#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE__SHIFT 0x17 +#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE_MASK 0x1000000 +#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE__SHIFT 0x18 +#define SPI_SHADER_PGM_RSRC1_PS__CACHE_CTL_MASK 0xe000000 +#define SPI_SHADER_PGM_RSRC1_PS__CACHE_CTL__SHIFT 0x19 +#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER_MASK 0x10000000 +#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER__SHIFT 0x1c +#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN_MASK 0x80 +#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE_MASK 0xff00 +#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE__SHIFT 0x8 +#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN_MASK 0x1ff0000 +#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN__SHIFT 0x10 +#define SPI_SHADER_PGM_RSRC3_PS__CU_EN_MASK 0xffff +#define SPI_SHADER_PGM_RSRC3_PS__CU_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT_MASK 0x3f0000 +#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT__SHIFT 0x10 +#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD_MASK 0x3c00000 +#define SPI_SHADER_PGM_RSRC3_PS__LOCK_LOW_THRESHOLD__SHIFT 0x16 +#define SPI_SHADER_USER_DATA_PS_0__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_0__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_1__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_1__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_2__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_2__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_3__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_3__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_4__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_4__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_5__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_5__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_6__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_6__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_7__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_7__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_8__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_8__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_9__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_9__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_10__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_10__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_11__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_11__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_12__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_12__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_13__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_13__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_14__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_14__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_PS_15__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_PS_15__DATA__SHIFT 0x0 +#define SPI_SHADER_TBA_LO_VS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TBA_LO_VS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TBA_HI_VS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TBA_HI_VS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_LO_VS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TMA_LO_VS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_HI_VS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TMA_HI_VS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_LO_VS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_PGM_LO_VS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_HI_VS__MEM_BASE_MASK 0xff +#define SPI_SHADER_PGM_HI_VS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_VS__VGPRS_MASK 0x3f +#define SPI_SHADER_PGM_RSRC1_VS__VGPRS__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_VS__SGPRS_MASK 0x3c0 +#define SPI_SHADER_PGM_RSRC1_VS__SGPRS__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY_MASK 0xc00 +#define SPI_SHADER_PGM_RSRC1_VS__PRIORITY__SHIFT 0xa +#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE_MASK 0xff000 +#define SPI_SHADER_PGM_RSRC1_VS__FLOAT_MODE__SHIFT 0xc +#define SPI_SHADER_PGM_RSRC1_VS__PRIV_MASK 0x100000 +#define SPI_SHADER_PGM_RSRC1_VS__PRIV__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP_MASK 0x200000 +#define SPI_SHADER_PGM_RSRC1_VS__DX10_CLAMP__SHIFT 0x15 +#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE_MASK 0x400000 +#define SPI_SHADER_PGM_RSRC1_VS__DEBUG_MODE__SHIFT 0x16 +#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE_MASK 0x800000 +#define SPI_SHADER_PGM_RSRC1_VS__IEEE_MODE__SHIFT 0x17 +#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT_MASK 0x3000000 +#define SPI_SHADER_PGM_RSRC1_VS__VGPR_COMP_CNT__SHIFT 0x18 +#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE_MASK 0x4000000 +#define SPI_SHADER_PGM_RSRC1_VS__CU_GROUP_ENABLE__SHIFT 0x1a +#define SPI_SHADER_PGM_RSRC1_VS__CACHE_CTL_MASK 0x38000000 +#define SPI_SHADER_PGM_RSRC1_VS__CACHE_CTL__SHIFT 0x1b +#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER_MASK 0x40000000 +#define SPI_SHADER_PGM_RSRC1_VS__CDBG_USER__SHIFT 0x1e +#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_VS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_VS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_VS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN_MASK 0x80 +#define SPI_SHADER_PGM_RSRC2_VS__OC_LDS_EN__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN_MASK 0x100 +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE0_EN__SHIFT 0x8 +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN_MASK 0x200 +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE1_EN__SHIFT 0x9 +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN_MASK 0x400 +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE2_EN__SHIFT 0xa +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN_MASK 0x800 +#define SPI_SHADER_PGM_RSRC2_VS__SO_BASE3_EN__SHIFT 0xb +#define SPI_SHADER_PGM_RSRC2_VS__SO_EN_MASK 0x1000 +#define SPI_SHADER_PGM_RSRC2_VS__SO_EN__SHIFT 0xc +#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN_MASK 0x3fe000 +#define SPI_SHADER_PGM_RSRC2_VS__EXCP_EN__SHIFT 0xd +#define SPI_SHADER_PGM_RSRC3_VS__CU_EN_MASK 0xffff +#define SPI_SHADER_PGM_RSRC3_VS__CU_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT_MASK 0x3f0000 +#define SPI_SHADER_PGM_RSRC3_VS__WAVE_LIMIT__SHIFT 0x10 +#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD_MASK 0x3c00000 +#define SPI_SHADER_PGM_RSRC3_VS__LOCK_LOW_THRESHOLD__SHIFT 0x16 +#define SPI_SHADER_LATE_ALLOC_VS__LIMIT_MASK 0x3f +#define SPI_SHADER_LATE_ALLOC_VS__LIMIT__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_0__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_0__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_1__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_1__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_2__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_2__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_3__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_3__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_4__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_4__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_5__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_5__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_6__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_6__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_7__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_7__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_8__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_8__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_9__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_9__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_10__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_10__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_11__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_11__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_12__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_12__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_13__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_13__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_14__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_14__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_VS_15__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_VS_15__DATA__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_ES_VS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_ES_VS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_ES_VS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_ES_VS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_ES_VS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_ES_VS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_ES_VS__OC_LDS_EN_MASK 0x80 +#define SPI_SHADER_PGM_RSRC2_ES_VS__OC_LDS_EN__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_ES_VS__EXCP_EN_MASK 0x1ff00 +#define SPI_SHADER_PGM_RSRC2_ES_VS__EXCP_EN__SHIFT 0x8 +#define SPI_SHADER_PGM_RSRC2_ES_VS__LDS_SIZE_MASK 0x1ff00000 +#define SPI_SHADER_PGM_RSRC2_ES_VS__LDS_SIZE__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC2_LS_VS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_LS_VS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_LS_VS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_LS_VS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_LS_VS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_LS_VS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_LS_VS__LDS_SIZE_MASK 0xff80 +#define SPI_SHADER_PGM_RSRC2_LS_VS__LDS_SIZE__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_LS_VS__EXCP_EN_MASK 0x1ff0000 +#define SPI_SHADER_PGM_RSRC2_LS_VS__EXCP_EN__SHIFT 0x10 +#define SPI_SHADER_TBA_LO_GS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TBA_LO_GS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TBA_HI_GS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TBA_HI_GS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_LO_GS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TMA_LO_GS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_HI_GS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TMA_HI_GS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_LO_GS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_PGM_LO_GS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_HI_GS__MEM_BASE_MASK 0xff +#define SPI_SHADER_PGM_HI_GS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_GS__VGPRS_MASK 0x3f +#define SPI_SHADER_PGM_RSRC1_GS__VGPRS__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_GS__SGPRS_MASK 0x3c0 +#define SPI_SHADER_PGM_RSRC1_GS__SGPRS__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY_MASK 0xc00 +#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY__SHIFT 0xa +#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE_MASK 0xff000 +#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE__SHIFT 0xc +#define SPI_SHADER_PGM_RSRC1_GS__PRIV_MASK 0x100000 +#define SPI_SHADER_PGM_RSRC1_GS__PRIV__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP_MASK 0x200000 +#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP__SHIFT 0x15 +#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE_MASK 0x400000 +#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE__SHIFT 0x16 +#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE_MASK 0x800000 +#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE__SHIFT 0x17 +#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE_MASK 0x1000000 +#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE__SHIFT 0x18 +#define SPI_SHADER_PGM_RSRC1_GS__CACHE_CTL_MASK 0xe000000 +#define SPI_SHADER_PGM_RSRC1_GS__CACHE_CTL__SHIFT 0x19 +#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER_MASK 0x10000000 +#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER__SHIFT 0x1c +#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN_MASK 0xff80 +#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC3_GS__CU_EN_MASK 0xffff +#define SPI_SHADER_PGM_RSRC3_GS__CU_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT_MASK 0x3f0000 +#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT__SHIFT 0x10 +#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD_MASK 0x3c00000 +#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD__SHIFT 0x16 +#define SPI_SHADER_USER_DATA_GS_0__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_0__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_1__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_1__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_2__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_2__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_3__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_3__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_4__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_4__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_5__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_5__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_6__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_6__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_7__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_7__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_8__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_8__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_9__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_9__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_10__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_10__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_11__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_11__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_12__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_12__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_13__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_13__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_14__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_14__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_GS_15__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_GS_15__DATA__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_ES_GS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_ES_GS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_ES_GS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_ES_GS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_ES_GS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_ES_GS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_ES_GS__OC_LDS_EN_MASK 0x80 +#define SPI_SHADER_PGM_RSRC2_ES_GS__OC_LDS_EN__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_ES_GS__EXCP_EN_MASK 0x1ff00 +#define SPI_SHADER_PGM_RSRC2_ES_GS__EXCP_EN__SHIFT 0x8 +#define SPI_SHADER_PGM_RSRC2_ES_GS__LDS_SIZE_MASK 0x1ff00000 +#define SPI_SHADER_PGM_RSRC2_ES_GS__LDS_SIZE__SHIFT 0x14 +#define SPI_SHADER_TBA_LO_ES__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TBA_LO_ES__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TBA_HI_ES__MEM_BASE_MASK 0xff +#define SPI_SHADER_TBA_HI_ES__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_LO_ES__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TMA_LO_ES__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_HI_ES__MEM_BASE_MASK 0xff +#define SPI_SHADER_TMA_HI_ES__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_LO_ES__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_PGM_LO_ES__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_HI_ES__MEM_BASE_MASK 0xff +#define SPI_SHADER_PGM_HI_ES__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_ES__VGPRS_MASK 0x3f +#define SPI_SHADER_PGM_RSRC1_ES__VGPRS__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_ES__SGPRS_MASK 0x3c0 +#define SPI_SHADER_PGM_RSRC1_ES__SGPRS__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC1_ES__PRIORITY_MASK 0xc00 +#define SPI_SHADER_PGM_RSRC1_ES__PRIORITY__SHIFT 0xa +#define SPI_SHADER_PGM_RSRC1_ES__FLOAT_MODE_MASK 0xff000 +#define SPI_SHADER_PGM_RSRC1_ES__FLOAT_MODE__SHIFT 0xc +#define SPI_SHADER_PGM_RSRC1_ES__PRIV_MASK 0x100000 +#define SPI_SHADER_PGM_RSRC1_ES__PRIV__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC1_ES__DX10_CLAMP_MASK 0x200000 +#define SPI_SHADER_PGM_RSRC1_ES__DX10_CLAMP__SHIFT 0x15 +#define SPI_SHADER_PGM_RSRC1_ES__DEBUG_MODE_MASK 0x400000 +#define SPI_SHADER_PGM_RSRC1_ES__DEBUG_MODE__SHIFT 0x16 +#define SPI_SHADER_PGM_RSRC1_ES__IEEE_MODE_MASK 0x800000 +#define SPI_SHADER_PGM_RSRC1_ES__IEEE_MODE__SHIFT 0x17 +#define SPI_SHADER_PGM_RSRC1_ES__VGPR_COMP_CNT_MASK 0x3000000 +#define SPI_SHADER_PGM_RSRC1_ES__VGPR_COMP_CNT__SHIFT 0x18 +#define SPI_SHADER_PGM_RSRC1_ES__CU_GROUP_ENABLE_MASK 0x4000000 +#define SPI_SHADER_PGM_RSRC1_ES__CU_GROUP_ENABLE__SHIFT 0x1a +#define SPI_SHADER_PGM_RSRC1_ES__CACHE_CTL_MASK 0x38000000 +#define SPI_SHADER_PGM_RSRC1_ES__CACHE_CTL__SHIFT 0x1b +#define SPI_SHADER_PGM_RSRC1_ES__CDBG_USER_MASK 0x40000000 +#define SPI_SHADER_PGM_RSRC1_ES__CDBG_USER__SHIFT 0x1e +#define SPI_SHADER_PGM_RSRC2_ES__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_ES__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_ES__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_ES__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_ES__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_ES__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_ES__OC_LDS_EN_MASK 0x80 +#define SPI_SHADER_PGM_RSRC2_ES__OC_LDS_EN__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_ES__EXCP_EN_MASK 0x1ff00 +#define SPI_SHADER_PGM_RSRC2_ES__EXCP_EN__SHIFT 0x8 +#define SPI_SHADER_PGM_RSRC2_ES__LDS_SIZE_MASK 0x1ff00000 +#define SPI_SHADER_PGM_RSRC2_ES__LDS_SIZE__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC3_ES__CU_EN_MASK 0xffff +#define SPI_SHADER_PGM_RSRC3_ES__CU_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC3_ES__WAVE_LIMIT_MASK 0x3f0000 +#define SPI_SHADER_PGM_RSRC3_ES__WAVE_LIMIT__SHIFT 0x10 +#define SPI_SHADER_PGM_RSRC3_ES__LOCK_LOW_THRESHOLD_MASK 0x3c00000 +#define SPI_SHADER_PGM_RSRC3_ES__LOCK_LOW_THRESHOLD__SHIFT 0x16 +#define SPI_SHADER_USER_DATA_ES_0__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_0__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_1__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_1__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_2__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_2__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_3__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_3__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_4__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_4__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_5__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_5__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_6__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_6__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_7__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_7__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_8__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_8__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_9__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_9__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_10__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_10__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_11__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_11__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_12__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_12__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_13__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_13__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_14__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_14__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_ES_15__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_ES_15__DATA__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_LS_ES__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_LS_ES__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_LS_ES__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_LS_ES__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_LS_ES__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_LS_ES__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_LS_ES__LDS_SIZE_MASK 0xff80 +#define SPI_SHADER_PGM_RSRC2_LS_ES__LDS_SIZE__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_LS_ES__EXCP_EN_MASK 0x1ff0000 +#define SPI_SHADER_PGM_RSRC2_LS_ES__EXCP_EN__SHIFT 0x10 +#define SPI_SHADER_TBA_LO_HS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TBA_LO_HS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TBA_HI_HS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TBA_HI_HS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_LO_HS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TMA_LO_HS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_HI_HS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TMA_HI_HS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_LO_HS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_PGM_LO_HS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_HI_HS__MEM_BASE_MASK 0xff +#define SPI_SHADER_PGM_HI_HS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_HS__VGPRS_MASK 0x3f +#define SPI_SHADER_PGM_RSRC1_HS__VGPRS__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_HS__SGPRS_MASK 0x3c0 +#define SPI_SHADER_PGM_RSRC1_HS__SGPRS__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY_MASK 0xc00 +#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY__SHIFT 0xa +#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE_MASK 0xff000 +#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE__SHIFT 0xc +#define SPI_SHADER_PGM_RSRC1_HS__PRIV_MASK 0x100000 +#define SPI_SHADER_PGM_RSRC1_HS__PRIV__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP_MASK 0x200000 +#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP__SHIFT 0x15 +#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE_MASK 0x400000 +#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE__SHIFT 0x16 +#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE_MASK 0x800000 +#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE__SHIFT 0x17 +#define SPI_SHADER_PGM_RSRC1_HS__CACHE_CTL_MASK 0x7000000 +#define SPI_SHADER_PGM_RSRC1_HS__CACHE_CTL__SHIFT 0x18 +#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER_MASK 0x8000000 +#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER__SHIFT 0x1b +#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN_MASK 0x80 +#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN_MASK 0x100 +#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN__SHIFT 0x8 +#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN_MASK 0x3fe00 +#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN__SHIFT 0x9 +#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT_MASK 0x3f +#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD_MASK 0x3c0 +#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD__SHIFT 0x6 +#define SPI_SHADER_USER_DATA_HS_0__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_0__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_1__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_1__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_2__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_2__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_3__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_3__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_4__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_4__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_5__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_5__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_6__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_6__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_7__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_7__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_8__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_8__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_9__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_9__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_10__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_10__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_11__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_11__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_12__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_12__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_13__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_13__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_14__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_14__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_HS_15__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_HS_15__DATA__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_LS_HS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_LS_HS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_LS_HS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_LS_HS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_LS_HS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_LS_HS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_LS_HS__LDS_SIZE_MASK 0xff80 +#define SPI_SHADER_PGM_RSRC2_LS_HS__LDS_SIZE__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_LS_HS__EXCP_EN_MASK 0x1ff0000 +#define SPI_SHADER_PGM_RSRC2_LS_HS__EXCP_EN__SHIFT 0x10 +#define SPI_SHADER_TBA_LO_LS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TBA_LO_LS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TBA_HI_LS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TBA_HI_LS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_LO_LS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_TMA_LO_LS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_TMA_HI_LS__MEM_BASE_MASK 0xff +#define SPI_SHADER_TMA_HI_LS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_LO_LS__MEM_BASE_MASK 0xffffffff +#define SPI_SHADER_PGM_LO_LS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_HI_LS__MEM_BASE_MASK 0xff +#define SPI_SHADER_PGM_HI_LS__MEM_BASE__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_LS__VGPRS_MASK 0x3f +#define SPI_SHADER_PGM_RSRC1_LS__VGPRS__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC1_LS__SGPRS_MASK 0x3c0 +#define SPI_SHADER_PGM_RSRC1_LS__SGPRS__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC1_LS__PRIORITY_MASK 0xc00 +#define SPI_SHADER_PGM_RSRC1_LS__PRIORITY__SHIFT 0xa +#define SPI_SHADER_PGM_RSRC1_LS__FLOAT_MODE_MASK 0xff000 +#define SPI_SHADER_PGM_RSRC1_LS__FLOAT_MODE__SHIFT 0xc +#define SPI_SHADER_PGM_RSRC1_LS__PRIV_MASK 0x100000 +#define SPI_SHADER_PGM_RSRC1_LS__PRIV__SHIFT 0x14 +#define SPI_SHADER_PGM_RSRC1_LS__DX10_CLAMP_MASK 0x200000 +#define SPI_SHADER_PGM_RSRC1_LS__DX10_CLAMP__SHIFT 0x15 +#define SPI_SHADER_PGM_RSRC1_LS__DEBUG_MODE_MASK 0x400000 +#define SPI_SHADER_PGM_RSRC1_LS__DEBUG_MODE__SHIFT 0x16 +#define SPI_SHADER_PGM_RSRC1_LS__IEEE_MODE_MASK 0x800000 +#define SPI_SHADER_PGM_RSRC1_LS__IEEE_MODE__SHIFT 0x17 +#define SPI_SHADER_PGM_RSRC1_LS__VGPR_COMP_CNT_MASK 0x3000000 +#define SPI_SHADER_PGM_RSRC1_LS__VGPR_COMP_CNT__SHIFT 0x18 +#define SPI_SHADER_PGM_RSRC1_LS__CACHE_CTL_MASK 0x1c000000 +#define SPI_SHADER_PGM_RSRC1_LS__CACHE_CTL__SHIFT 0x1a +#define SPI_SHADER_PGM_RSRC1_LS__CDBG_USER_MASK 0x20000000 +#define SPI_SHADER_PGM_RSRC1_LS__CDBG_USER__SHIFT 0x1d +#define SPI_SHADER_PGM_RSRC2_LS__SCRATCH_EN_MASK 0x1 +#define SPI_SHADER_PGM_RSRC2_LS__SCRATCH_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC2_LS__USER_SGPR_MASK 0x3e +#define SPI_SHADER_PGM_RSRC2_LS__USER_SGPR__SHIFT 0x1 +#define SPI_SHADER_PGM_RSRC2_LS__TRAP_PRESENT_MASK 0x40 +#define SPI_SHADER_PGM_RSRC2_LS__TRAP_PRESENT__SHIFT 0x6 +#define SPI_SHADER_PGM_RSRC2_LS__LDS_SIZE_MASK 0xff80 +#define SPI_SHADER_PGM_RSRC2_LS__LDS_SIZE__SHIFT 0x7 +#define SPI_SHADER_PGM_RSRC2_LS__EXCP_EN_MASK 0x1ff0000 +#define SPI_SHADER_PGM_RSRC2_LS__EXCP_EN__SHIFT 0x10 +#define SPI_SHADER_PGM_RSRC3_LS__CU_EN_MASK 0xffff +#define SPI_SHADER_PGM_RSRC3_LS__CU_EN__SHIFT 0x0 +#define SPI_SHADER_PGM_RSRC3_LS__WAVE_LIMIT_MASK 0x3f0000 +#define SPI_SHADER_PGM_RSRC3_LS__WAVE_LIMIT__SHIFT 0x10 +#define SPI_SHADER_PGM_RSRC3_LS__LOCK_LOW_THRESHOLD_MASK 0x3c00000 +#define SPI_SHADER_PGM_RSRC3_LS__LOCK_LOW_THRESHOLD__SHIFT 0x16 +#define SPI_SHADER_USER_DATA_LS_0__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_0__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_1__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_1__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_2__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_2__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_3__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_3__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_4__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_4__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_5__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_5__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_6__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_6__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_7__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_7__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_8__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_8__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_9__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_9__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_10__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_10__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_11__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_11__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_12__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_12__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_13__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_13__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_14__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_14__DATA__SHIFT 0x0 +#define SPI_SHADER_USER_DATA_LS_15__DATA_MASK 0xffffffff +#define SPI_SHADER_USER_DATA_LS_15__DATA__SHIFT 0x0 +#define SQ_CONFIG__UNUSED_MASK 0xff +#define SQ_CONFIG__UNUSED__SHIFT 0x0 +#define SQ_CONFIG__DEBUG_EN_MASK 0x100 +#define SQ_CONFIG__DEBUG_EN__SHIFT 0x8 +#define SQ_CONFIG__DISABLE_SCA_BYPASS_MASK 0x200 +#define SQ_CONFIG__DISABLE_SCA_BYPASS__SHIFT 0x9 +#define SQ_CONFIG__DISABLE_IB_DEP_CHECK_MASK 0x400 +#define SQ_CONFIG__DISABLE_IB_DEP_CHECK__SHIFT 0xa +#define SQ_CONFIG__ENABLE_SOFT_CLAUSE_MASK 0x800 +#define SQ_CONFIG__ENABLE_SOFT_CLAUSE__SHIFT 0xb +#define SQ_CONFIG__EARLY_TA_DONE_DISABLE_MASK 0x1000 +#define SQ_CONFIG__EARLY_TA_DONE_DISABLE__SHIFT 0xc +#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE_MASK 0x2000 +#define SQ_CONFIG__DUA_FLAT_LOCK_ENABLE__SHIFT 0xd +#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE_MASK 0x4000 +#define SQ_CONFIG__DUA_LDS_BYPASS_DISABLE__SHIFT 0xe +#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE_MASK 0x8000 +#define SQ_CONFIG__DUA_FLAT_LDS_PINGPONG_DISABLE__SHIFT 0xf +#define SQC_CONFIG__INST_CACHE_SIZE_MASK 0x3 +#define SQC_CONFIG__INST_CACHE_SIZE__SHIFT 0x0 +#define SQC_CONFIG__DATA_CACHE_SIZE_MASK 0xc +#define SQC_CONFIG__DATA_CACHE_SIZE__SHIFT 0x2 +#define SQC_CONFIG__MISS_FIFO_DEPTH_MASK 0x30 +#define SQC_CONFIG__MISS_FIFO_DEPTH__SHIFT 0x4 +#define SQC_CONFIG__HIT_FIFO_DEPTH_MASK 0x40 +#define SQC_CONFIG__HIT_FIFO_DEPTH__SHIFT 0x6 +#define SQC_CONFIG__FORCE_ALWAYS_MISS_MASK 0x80 +#define SQC_CONFIG__FORCE_ALWAYS_MISS__SHIFT 0x7 +#define SQC_CONFIG__FORCE_IN_ORDER_MASK 0x100 +#define SQC_CONFIG__FORCE_IN_ORDER__SHIFT 0x8 +#define SQC_CONFIG__IDENTITY_HASH_BANK_MASK 0x200 +#define SQC_CONFIG__IDENTITY_HASH_BANK__SHIFT 0x9 +#define SQC_CONFIG__IDENTITY_HASH_SET_MASK 0x400 +#define SQC_CONFIG__IDENTITY_HASH_SET__SHIFT 0xa +#define SQC_CONFIG__PER_VMID_INV_DISABLE_MASK 0x800 +#define SQC_CONFIG__PER_VMID_INV_DISABLE__SHIFT 0xb +#define SQC_CACHES__INST_INVALIDATE_MASK 0x1 +#define SQC_CACHES__INST_INVALIDATE__SHIFT 0x0 +#define SQC_CACHES__DATA_INVALIDATE_MASK 0x2 +#define SQC_CACHES__DATA_INVALIDATE__SHIFT 0x1 +#define SQC_CACHES__INVALIDATE_VOLATILE_MASK 0x4 +#define SQC_CACHES__INVALIDATE_VOLATILE__SHIFT 0x2 +#define SQ_RANDOM_WAVE_PRI__RET_MASK 0x7f +#define SQ_RANDOM_WAVE_PRI__RET__SHIFT 0x0 +#define SQ_RANDOM_WAVE_PRI__RUI_MASK 0x380 +#define SQ_RANDOM_WAVE_PRI__RUI__SHIFT 0x7 +#define SQ_RANDOM_WAVE_PRI__RNG_MASK 0x1ffc00 +#define SQ_RANDOM_WAVE_PRI__RNG__SHIFT 0xa +#define SQ_REG_CREDITS__SRBM_CREDITS_MASK 0x3f +#define SQ_REG_CREDITS__SRBM_CREDITS__SHIFT 0x0 +#define SQ_REG_CREDITS__CMD_CREDITS_MASK 0xf00 +#define SQ_REG_CREDITS__CMD_CREDITS__SHIFT 0x8 +#define SQ_REG_CREDITS__REG_BUSY_MASK 0x10000000 +#define SQ_REG_CREDITS__REG_BUSY__SHIFT 0x1c +#define SQ_REG_CREDITS__SRBM_OVERFLOW_MASK 0x20000000 +#define SQ_REG_CREDITS__SRBM_OVERFLOW__SHIFT 0x1d +#define SQ_REG_CREDITS__IMMED_OVERFLOW_MASK 0x40000000 +#define SQ_REG_CREDITS__IMMED_OVERFLOW__SHIFT 0x1e +#define SQ_REG_CREDITS__CMD_OVERFLOW_MASK 0x80000000 +#define SQ_REG_CREDITS__CMD_OVERFLOW__SHIFT 0x1f +#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE_MASK 0xf +#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE__SHIFT 0x0 +#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE_MASK 0xf00 +#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE__SHIFT 0x8 +#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE_MASK 0x30000 +#define SQ_FIFO_SIZES__EXPORT_BUF_SIZE__SHIFT 0x10 +#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE_MASK 0xc0000 +#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE__SHIFT 0x12 +#define SQ_INTERRUPT_AUTO_MASK__MASK_MASK 0xffffff +#define SQ_INTERRUPT_AUTO_MASK__MASK__SHIFT 0x0 +#define SQ_INTERRUPT_MSG_CTRL__STALL_MASK 0x1 +#define SQ_INTERRUPT_MSG_CTRL__STALL__SHIFT 0x0 +#define SQ_PERFCOUNTER_CTRL__PS_EN_MASK 0x1 +#define SQ_PERFCOUNTER_CTRL__PS_EN__SHIFT 0x0 +#define SQ_PERFCOUNTER_CTRL__VS_EN_MASK 0x2 +#define SQ_PERFCOUNTER_CTRL__VS_EN__SHIFT 0x1 +#define SQ_PERFCOUNTER_CTRL__GS_EN_MASK 0x4 +#define SQ_PERFCOUNTER_CTRL__GS_EN__SHIFT 0x2 +#define SQ_PERFCOUNTER_CTRL__ES_EN_MASK 0x8 +#define SQ_PERFCOUNTER_CTRL__ES_EN__SHIFT 0x3 +#define SQ_PERFCOUNTER_CTRL__HS_EN_MASK 0x10 +#define SQ_PERFCOUNTER_CTRL__HS_EN__SHIFT 0x4 +#define SQ_PERFCOUNTER_CTRL__LS_EN_MASK 0x20 +#define SQ_PERFCOUNTER_CTRL__LS_EN__SHIFT 0x5 +#define SQ_PERFCOUNTER_CTRL__CS_EN_MASK 0x40 +#define SQ_PERFCOUNTER_CTRL__CS_EN__SHIFT 0x6 +#define SQ_PERFCOUNTER_CTRL__CNTR_RATE_MASK 0x1f00 +#define SQ_PERFCOUNTER_CTRL__CNTR_RATE__SHIFT 0x8 +#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH_MASK 0x2000 +#define SQ_PERFCOUNTER_CTRL__DISABLE_FLUSH__SHIFT 0xd +#define SQ_PERFCOUNTER_MASK__SH0_MASK_MASK 0xffff +#define SQ_PERFCOUNTER_MASK__SH0_MASK__SHIFT 0x0 +#define SQ_PERFCOUNTER_MASK__SH1_MASK_MASK 0xffff0000 +#define SQ_PERFCOUNTER_MASK__SH1_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER_CTRL2__FORCE_EN_MASK 0x1 +#define SQ_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT 0x0 +#define CC_SQC_BANK_DISABLE__SQC0_BANK_DISABLE_MASK 0xf0000 +#define CC_SQC_BANK_DISABLE__SQC0_BANK_DISABLE__SHIFT 0x10 +#define CC_SQC_BANK_DISABLE__SQC1_BANK_DISABLE_MASK 0xf00000 +#define CC_SQC_BANK_DISABLE__SQC1_BANK_DISABLE__SHIFT 0x14 +#define CC_SQC_BANK_DISABLE__SQC2_BANK_DISABLE_MASK 0xf000000 +#define CC_SQC_BANK_DISABLE__SQC2_BANK_DISABLE__SHIFT 0x18 +#define CC_SQC_BANK_DISABLE__SQC3_BANK_DISABLE_MASK 0xf0000000 +#define CC_SQC_BANK_DISABLE__SQC3_BANK_DISABLE__SHIFT 0x1c +#define USER_SQC_BANK_DISABLE__SQC0_BANK_DISABLE_MASK 0xf0000 +#define USER_SQC_BANK_DISABLE__SQC0_BANK_DISABLE__SHIFT 0x10 +#define USER_SQC_BANK_DISABLE__SQC1_BANK_DISABLE_MASK 0xf00000 +#define USER_SQC_BANK_DISABLE__SQC1_BANK_DISABLE__SHIFT 0x14 +#define USER_SQC_BANK_DISABLE__SQC2_BANK_DISABLE_MASK 0xf000000 +#define USER_SQC_BANK_DISABLE__SQC2_BANK_DISABLE__SHIFT 0x18 +#define USER_SQC_BANK_DISABLE__SQC3_BANK_DISABLE_MASK 0xf0000000 +#define USER_SQC_BANK_DISABLE__SQC3_BANK_DISABLE__SHIFT 0x1c +#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER8_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER9_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER10_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER11_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER12_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER13_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER14_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SQ_PERFCOUNTER15_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER8_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER9_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER10_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER11_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER12_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER13_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER14_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SQ_PERFCOUNTER15_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SQ_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER0_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER0_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER0_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER0_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER1_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER1_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER1_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER1_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER2_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER2_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER2_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER2_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER3_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER3_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER3_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER3_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER4_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER4_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER4_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER4_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER4_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER4_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER5_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER5_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER5_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER5_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER5_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER5_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER6_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER6_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER6_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER6_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER6_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER6_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER7_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER7_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER7_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER7_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER7_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER7_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER8_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER8_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER8_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER8_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER8_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER8_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER8_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER8_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER8_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER9_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER9_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER9_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER9_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER9_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER9_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER9_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER9_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER9_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER10_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER10_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER10_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER10_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER10_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER10_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER10_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER10_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER10_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER11_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER11_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER11_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER11_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER11_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER11_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER11_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER11_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER11_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER12_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER12_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER12_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER12_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER12_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER12_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER12_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER12_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER12_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER13_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER13_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER13_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER13_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER13_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER13_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER13_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER13_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER13_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER14_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER14_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER14_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER14_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER14_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER14_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER14_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER14_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER14_SELECT__PERF_MODE__SHIFT 0x1c +#define SQ_PERFCOUNTER15_SELECT__PERF_SEL_MASK 0xff +#define SQ_PERFCOUNTER15_SELECT__PERF_SEL__SHIFT 0x0 +#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK_MASK 0xf000 +#define SQ_PERFCOUNTER15_SELECT__SQC_BANK_MASK__SHIFT 0xc +#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK_MASK 0xf0000 +#define SQ_PERFCOUNTER15_SELECT__SQC_CLIENT_MASK__SHIFT 0x10 +#define SQ_PERFCOUNTER15_SELECT__SPM_MODE_MASK 0xf00000 +#define SQ_PERFCOUNTER15_SELECT__SPM_MODE__SHIFT 0x14 +#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK_MASK 0xf000000 +#define SQ_PERFCOUNTER15_SELECT__SIMD_MASK__SHIFT 0x18 +#define SQ_PERFCOUNTER15_SELECT__PERF_MODE_MASK 0xf0000000 +#define SQ_PERFCOUNTER15_SELECT__PERF_MODE__SHIFT 0x1c +#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000 +#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e +#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000 +#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e +#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0xffff +#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x0 +#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000 +#define SQ_ALU_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x10 +#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0xffff +#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x0 +#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000 +#define SQ_TEX_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x10 +#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0_MASK 0xffff +#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH0__SHIFT 0x0 +#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1_MASK 0xffff0000 +#define SQ_LDS_CLK_CTRL__FORCE_CU_ON_SH1__SHIFT 0x10 +#define SQ_POWER_THROTTLE__MIN_POWER_MASK 0x3fff +#define SQ_POWER_THROTTLE__MIN_POWER__SHIFT 0x0 +#define SQ_POWER_THROTTLE__MAX_POWER_MASK 0x3fff0000 +#define SQ_POWER_THROTTLE__MAX_POWER__SHIFT 0x10 +#define SQ_POWER_THROTTLE__PHASE_OFFSET_MASK 0xc0000000 +#define SQ_POWER_THROTTLE__PHASE_OFFSET__SHIFT 0x1e +#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA_MASK 0x3fff +#define SQ_POWER_THROTTLE2__MAX_POWER_DELTA__SHIFT 0x0 +#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000 +#define SQ_POWER_THROTTLE2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10 +#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000 +#define SQ_POWER_THROTTLE2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b +#define SQ_POWER_THROTTLE2__USE_REF_CLOCK_MASK 0x80000000 +#define SQ_POWER_THROTTLE2__USE_REF_CLOCK__SHIFT 0x1f +#define SQ_TIME_HI__TIME_MASK 0xffffffff +#define SQ_TIME_HI__TIME__SHIFT 0x0 +#define SQ_TIME_LO__TIME_MASK 0xffffffff +#define SQ_TIME_LO__TIME__SHIFT 0x0 +#define SQ_THREAD_TRACE_BASE__ADDR_MASK 0xffffffff +#define SQ_THREAD_TRACE_BASE__ADDR__SHIFT 0x0 +#define SQ_THREAD_TRACE_BASE2__ADDR_HI_MASK 0xf +#define SQ_THREAD_TRACE_BASE2__ADDR_HI__SHIFT 0x0 +#define SQ_THREAD_TRACE_BASE2__ATC_MASK 0x10 +#define SQ_THREAD_TRACE_BASE2__ATC__SHIFT 0x4 +#define SQ_THREAD_TRACE_SIZE__SIZE_MASK 0x3fffff +#define SQ_THREAD_TRACE_SIZE__SIZE__SHIFT 0x0 +#define SQ_THREAD_TRACE_MASK__CU_SEL_MASK 0x1f +#define SQ_THREAD_TRACE_MASK__CU_SEL__SHIFT 0x0 +#define SQ_THREAD_TRACE_MASK__SH_SEL_MASK 0x20 +#define SQ_THREAD_TRACE_MASK__SH_SEL__SHIFT 0x5 +#define SQ_THREAD_TRACE_MASK__REG_STALL_EN_MASK 0x80 +#define SQ_THREAD_TRACE_MASK__REG_STALL_EN__SHIFT 0x7 +#define SQ_THREAD_TRACE_MASK__SIMD_EN_MASK 0xf00 +#define SQ_THREAD_TRACE_MASK__SIMD_EN__SHIFT 0x8 +#define SQ_THREAD_TRACE_MASK__VM_ID_MASK_MASK 0x3000 +#define SQ_THREAD_TRACE_MASK__VM_ID_MASK__SHIFT 0xc +#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN_MASK 0x4000 +#define SQ_THREAD_TRACE_MASK__SPI_STALL_EN__SHIFT 0xe +#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN_MASK 0x8000 +#define SQ_THREAD_TRACE_MASK__SQ_STALL_EN__SHIFT 0xf +#define SQ_THREAD_TRACE_MASK__RANDOM_SEED_MASK 0xffff0000 +#define SQ_THREAD_TRACE_MASK__RANDOM_SEED__SHIFT 0x10 +#define SQ_THREAD_TRACE_USERDATA_0__DATA_MASK 0xffffffff +#define SQ_THREAD_TRACE_USERDATA_0__DATA__SHIFT 0x0 +#define SQ_THREAD_TRACE_USERDATA_1__DATA_MASK 0xffffffff +#define SQ_THREAD_TRACE_USERDATA_1__DATA__SHIFT 0x0 +#define SQ_THREAD_TRACE_USERDATA_2__DATA_MASK 0xffffffff +#define SQ_THREAD_TRACE_USERDATA_2__DATA__SHIFT 0x0 +#define SQ_THREAD_TRACE_USERDATA_3__DATA_MASK 0xffffffff +#define SQ_THREAD_TRACE_USERDATA_3__DATA__SHIFT 0x0 +#define SQ_THREAD_TRACE_MODE__MASK_PS_MASK 0x7 +#define SQ_THREAD_TRACE_MODE__MASK_PS__SHIFT 0x0 +#define SQ_THREAD_TRACE_MODE__MASK_VS_MASK 0x38 +#define SQ_THREAD_TRACE_MODE__MASK_VS__SHIFT 0x3 +#define SQ_THREAD_TRACE_MODE__MASK_GS_MASK 0x1c0 +#define SQ_THREAD_TRACE_MODE__MASK_GS__SHIFT 0x6 +#define SQ_THREAD_TRACE_MODE__MASK_ES_MASK 0xe00 +#define SQ_THREAD_TRACE_MODE__MASK_ES__SHIFT 0x9 +#define SQ_THREAD_TRACE_MODE__MASK_HS_MASK 0x7000 +#define SQ_THREAD_TRACE_MODE__MASK_HS__SHIFT 0xc +#define SQ_THREAD_TRACE_MODE__MASK_LS_MASK 0x38000 +#define SQ_THREAD_TRACE_MODE__MASK_LS__SHIFT 0xf +#define SQ_THREAD_TRACE_MODE__MASK_CS_MASK 0x1c0000 +#define SQ_THREAD_TRACE_MODE__MASK_CS__SHIFT 0x12 +#define SQ_THREAD_TRACE_MODE__MODE_MASK 0x600000 +#define SQ_THREAD_TRACE_MODE__MODE__SHIFT 0x15 +#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE_MASK 0x1800000 +#define SQ_THREAD_TRACE_MODE__CAPTURE_MODE__SHIFT 0x17 +#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN_MASK 0x2000000 +#define SQ_THREAD_TRACE_MODE__AUTOFLUSH_EN__SHIFT 0x19 +#define SQ_THREAD_TRACE_MODE__PRIV_MASK 0x4000000 +#define SQ_THREAD_TRACE_MODE__PRIV__SHIFT 0x1a +#define SQ_THREAD_TRACE_MODE__ISSUE_MASK_MASK 0x18000000 +#define SQ_THREAD_TRACE_MODE__ISSUE_MASK__SHIFT 0x1b +#define SQ_THREAD_TRACE_MODE__TEST_MODE_MASK 0x20000000 +#define SQ_THREAD_TRACE_MODE__TEST_MODE__SHIFT 0x1d +#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN_MASK 0x40000000 +#define SQ_THREAD_TRACE_MODE__INTERRUPT_EN__SHIFT 0x1e +#define SQ_THREAD_TRACE_MODE__WRAP_MASK 0x80000000 +#define SQ_THREAD_TRACE_MODE__WRAP__SHIFT 0x1f +#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER_MASK 0x80000000 +#define SQ_THREAD_TRACE_CTRL__RESET_BUFFER__SHIFT 0x1f +#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK_MASK 0xffff +#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_MASK__SHIFT 0x0 +#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK_MASK 0xff0000 +#define SQ_THREAD_TRACE_TOKEN_MASK__REG_MASK__SHIFT 0x10 +#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL_MASK 0x1000000 +#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DROP_ON_STALL__SHIFT 0x18 +#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK_MASK 0xffff +#define SQ_THREAD_TRACE_TOKEN_MASK2__INST_MASK__SHIFT 0x0 +#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK_MASK 0xffff +#define SQ_THREAD_TRACE_PERF_MASK__SH0_MASK__SHIFT 0x0 +#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK_MASK 0xffff0000 +#define SQ_THREAD_TRACE_PERF_MASK__SH1_MASK__SHIFT 0x10 +#define SQ_THREAD_TRACE_WPTR__WPTR_MASK 0x3fffffff +#define SQ_THREAD_TRACE_WPTR__WPTR__SHIFT 0x0 +#define SQ_THREAD_TRACE_WPTR__READ_OFFSET_MASK 0xc0000000 +#define SQ_THREAD_TRACE_WPTR__READ_OFFSET__SHIFT 0x1e +#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK 0x3ff +#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING__SHIFT 0x0 +#define SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK 0x3ff0000 +#define SQ_THREAD_TRACE_STATUS__FINISH_DONE__SHIFT 0x10 +#define SQ_THREAD_TRACE_STATUS__NEW_BUF_MASK 0x20000000 +#define SQ_THREAD_TRACE_STATUS__NEW_BUF__SHIFT 0x1d +#define SQ_THREAD_TRACE_STATUS__BUSY_MASK 0x40000000 +#define SQ_THREAD_TRACE_STATUS__BUSY__SHIFT 0x1e +#define SQ_THREAD_TRACE_STATUS__FULL_MASK 0x80000000 +#define SQ_THREAD_TRACE_STATUS__FULL__SHIFT 0x1f +#define SQ_THREAD_TRACE_CNTR__CNTR_MASK 0xffffffff +#define SQ_THREAD_TRACE_CNTR__CNTR__SHIFT 0x0 +#define SQ_THREAD_TRACE_HIWATER__HIWATER_MASK 0x7 +#define SQ_THREAD_TRACE_HIWATER__HIWATER__SHIFT 0x0 +#define SQ_LB_CTR_CTRL__START_MASK 0x1 +#define SQ_LB_CTR_CTRL__START__SHIFT 0x0 +#define SQ_LB_CTR_CTRL__LOAD_MASK 0x2 +#define SQ_LB_CTR_CTRL__LOAD__SHIFT 0x1 +#define SQ_LB_CTR_CTRL__CLEAR_MASK 0x4 +#define SQ_LB_CTR_CTRL__CLEAR__SHIFT 0x2 +#define SQ_LB_DATA_ALU_CYCLES__DATA_MASK 0xffffffff +#define SQ_LB_DATA_ALU_CYCLES__DATA__SHIFT 0x0 +#define SQ_LB_DATA_TEX_CYCLES__DATA_MASK 0xffffffff +#define SQ_LB_DATA_TEX_CYCLES__DATA__SHIFT 0x0 +#define SQ_LB_DATA_ALU_STALLS__DATA_MASK 0xffffffff +#define SQ_LB_DATA_ALU_STALLS__DATA__SHIFT 0x0 +#define SQ_LB_DATA_TEX_STALLS__DATA_MASK 0xffffffff +#define SQ_LB_DATA_TEX_STALLS__DATA__SHIFT 0x0 +#define SQC_SECDED_CNT__INST_SEC_MASK 0xff +#define SQC_SECDED_CNT__INST_SEC__SHIFT 0x0 +#define SQC_SECDED_CNT__INST_DED_MASK 0xff00 +#define SQC_SECDED_CNT__INST_DED__SHIFT 0x8 +#define SQC_SECDED_CNT__DATA_SEC_MASK 0xff0000 +#define SQC_SECDED_CNT__DATA_SEC__SHIFT 0x10 +#define SQC_SECDED_CNT__DATA_DED_MASK 0xff000000 +#define SQC_SECDED_CNT__DATA_DED__SHIFT 0x18 +#define SQ_SEC_CNT__LDS_SEC_MASK 0x3f +#define SQ_SEC_CNT__LDS_SEC__SHIFT 0x0 +#define SQ_SEC_CNT__SGPR_SEC_MASK 0x1f00 +#define SQ_SEC_CNT__SGPR_SEC__SHIFT 0x8 +#define SQ_SEC_CNT__VGPR_SEC_MASK 0x1ff0000 +#define SQ_SEC_CNT__VGPR_SEC__SHIFT 0x10 +#define SQ_DED_CNT__LDS_DED_MASK 0x3f +#define SQ_DED_CNT__LDS_DED__SHIFT 0x0 +#define SQ_DED_CNT__SGPR_DED_MASK 0x1f00 +#define SQ_DED_CNT__SGPR_DED__SHIFT 0x8 +#define SQ_DED_CNT__VGPR_DED_MASK 0x1ff0000 +#define SQ_DED_CNT__VGPR_DED__SHIFT 0x10 +#define SQ_DED_INFO__WAVE_ID_MASK 0xf +#define SQ_DED_INFO__WAVE_ID__SHIFT 0x0 +#define SQ_DED_INFO__SIMD_ID_MASK 0x30 +#define SQ_DED_INFO__SIMD_ID__SHIFT 0x4 +#define SQ_DED_INFO__SOURCE_MASK 0x1c0 +#define SQ_DED_INFO__SOURCE__SHIFT 0x6 +#define SQ_DED_INFO__VM_ID_MASK 0x1e00 +#define SQ_DED_INFO__VM_ID__SHIFT 0x9 +#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS_MASK 0xffffffff +#define SQ_BUF_RSRC_WORD0__BASE_ADDRESS__SHIFT 0x0 +#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI_MASK 0xffff +#define SQ_BUF_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT 0x0 +#define SQ_BUF_RSRC_WORD1__STRIDE_MASK 0x3fff0000 +#define SQ_BUF_RSRC_WORD1__STRIDE__SHIFT 0x10 +#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE_MASK 0x40000000 +#define SQ_BUF_RSRC_WORD1__CACHE_SWIZZLE__SHIFT 0x1e +#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE_MASK 0x80000000 +#define SQ_BUF_RSRC_WORD1__SWIZZLE_ENABLE__SHIFT 0x1f +#define SQ_BUF_RSRC_WORD2__NUM_RECORDS_MASK 0xffffffff +#define SQ_BUF_RSRC_WORD2__NUM_RECORDS__SHIFT 0x0 +#define SQ_BUF_RSRC_WORD3__DST_SEL_X_MASK 0x7 +#define SQ_BUF_RSRC_WORD3__DST_SEL_X__SHIFT 0x0 +#define SQ_BUF_RSRC_WORD3__DST_SEL_Y_MASK 0x38 +#define SQ_BUF_RSRC_WORD3__DST_SEL_Y__SHIFT 0x3 +#define SQ_BUF_RSRC_WORD3__DST_SEL_Z_MASK 0x1c0 +#define SQ_BUF_RSRC_WORD3__DST_SEL_Z__SHIFT 0x6 +#define SQ_BUF_RSRC_WORD3__DST_SEL_W_MASK 0xe00 +#define SQ_BUF_RSRC_WORD3__DST_SEL_W__SHIFT 0x9 +#define SQ_BUF_RSRC_WORD3__NUM_FORMAT_MASK 0x7000 +#define SQ_BUF_RSRC_WORD3__NUM_FORMAT__SHIFT 0xc +#define SQ_BUF_RSRC_WORD3__DATA_FORMAT_MASK 0x78000 +#define SQ_BUF_RSRC_WORD3__DATA_FORMAT__SHIFT 0xf +#define SQ_BUF_RSRC_WORD3__ELEMENT_SIZE_MASK 0x180000 +#define SQ_BUF_RSRC_WORD3__ELEMENT_SIZE__SHIFT 0x13 +#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE_MASK 0x600000 +#define SQ_BUF_RSRC_WORD3__INDEX_STRIDE__SHIFT 0x15 +#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE_MASK 0x800000 +#define SQ_BUF_RSRC_WORD3__ADD_TID_ENABLE__SHIFT 0x17 +#define SQ_BUF_RSRC_WORD3__ATC_MASK 0x1000000 +#define SQ_BUF_RSRC_WORD3__ATC__SHIFT 0x18 +#define SQ_BUF_RSRC_WORD3__HASH_ENABLE_MASK 0x2000000 +#define SQ_BUF_RSRC_WORD3__HASH_ENABLE__SHIFT 0x19 +#define SQ_BUF_RSRC_WORD3__HEAP_MASK 0x4000000 +#define SQ_BUF_RSRC_WORD3__HEAP__SHIFT 0x1a +#define SQ_BUF_RSRC_WORD3__MTYPE_MASK 0x38000000 +#define SQ_BUF_RSRC_WORD3__MTYPE__SHIFT 0x1b +#define SQ_BUF_RSRC_WORD3__TYPE_MASK 0xc0000000 +#define SQ_BUF_RSRC_WORD3__TYPE__SHIFT 0x1e +#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS_MASK 0xffffffff +#define SQ_IMG_RSRC_WORD0__BASE_ADDRESS__SHIFT 0x0 +#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI_MASK 0xff +#define SQ_IMG_RSRC_WORD1__BASE_ADDRESS_HI__SHIFT 0x0 +#define SQ_IMG_RSRC_WORD1__MIN_LOD_MASK 0xfff00 +#define SQ_IMG_RSRC_WORD1__MIN_LOD__SHIFT 0x8 +#define SQ_IMG_RSRC_WORD1__DATA_FORMAT_MASK 0x3f00000 +#define SQ_IMG_RSRC_WORD1__DATA_FORMAT__SHIFT 0x14 +#define SQ_IMG_RSRC_WORD1__NUM_FORMAT_MASK 0x3c000000 +#define SQ_IMG_RSRC_WORD1__NUM_FORMAT__SHIFT 0x1a +#define SQ_IMG_RSRC_WORD1__MTYPE_MASK 0xc0000000 +#define SQ_IMG_RSRC_WORD1__MTYPE__SHIFT 0x1e +#define SQ_IMG_RSRC_WORD2__WIDTH_MASK 0x3fff +#define SQ_IMG_RSRC_WORD2__WIDTH__SHIFT 0x0 +#define SQ_IMG_RSRC_WORD2__HEIGHT_MASK 0xfffc000 +#define SQ_IMG_RSRC_WORD2__HEIGHT__SHIFT 0xe +#define SQ_IMG_RSRC_WORD2__PERF_MOD_MASK 0x70000000 +#define SQ_IMG_RSRC_WORD2__PERF_MOD__SHIFT 0x1c +#define SQ_IMG_RSRC_WORD2__INTERLACED_MASK 0x80000000 +#define SQ_IMG_RSRC_WORD2__INTERLACED__SHIFT 0x1f +#define SQ_IMG_RSRC_WORD3__DST_SEL_X_MASK 0x7 +#define SQ_IMG_RSRC_WORD3__DST_SEL_X__SHIFT 0x0 +#define SQ_IMG_RSRC_WORD3__DST_SEL_Y_MASK 0x38 +#define SQ_IMG_RSRC_WORD3__DST_SEL_Y__SHIFT 0x3 +#define SQ_IMG_RSRC_WORD3__DST_SEL_Z_MASK 0x1c0 +#define SQ_IMG_RSRC_WORD3__DST_SEL_Z__SHIFT 0x6 +#define SQ_IMG_RSRC_WORD3__DST_SEL_W_MASK 0xe00 +#define SQ_IMG_RSRC_WORD3__DST_SEL_W__SHIFT 0x9 +#define SQ_IMG_RSRC_WORD3__BASE_LEVEL_MASK 0xf000 +#define SQ_IMG_RSRC_WORD3__BASE_LEVEL__SHIFT 0xc +#define SQ_IMG_RSRC_WORD3__LAST_LEVEL_MASK 0xf0000 +#define SQ_IMG_RSRC_WORD3__LAST_LEVEL__SHIFT 0x10 +#define SQ_IMG_RSRC_WORD3__TILING_INDEX_MASK 0x1f00000 +#define SQ_IMG_RSRC_WORD3__TILING_INDEX__SHIFT 0x14 +#define SQ_IMG_RSRC_WORD3__POW2_PAD_MASK 0x2000000 +#define SQ_IMG_RSRC_WORD3__POW2_PAD__SHIFT 0x19 +#define SQ_IMG_RSRC_WORD3__MTYPE_MASK 0x4000000 +#define SQ_IMG_RSRC_WORD3__MTYPE__SHIFT 0x1a +#define SQ_IMG_RSRC_WORD3__ATC_MASK 0x8000000 +#define SQ_IMG_RSRC_WORD3__ATC__SHIFT 0x1b +#define SQ_IMG_RSRC_WORD3__TYPE_MASK 0xf0000000 +#define SQ_IMG_RSRC_WORD3__TYPE__SHIFT 0x1c +#define SQ_IMG_RSRC_WORD4__DEPTH_MASK 0x1fff +#define SQ_IMG_RSRC_WORD4__DEPTH__SHIFT 0x0 +#define SQ_IMG_RSRC_WORD4__PITCH_MASK 0x7ffe000 +#define SQ_IMG_RSRC_WORD4__PITCH__SHIFT 0xd +#define SQ_IMG_RSRC_WORD5__BASE_ARRAY_MASK 0x1fff +#define SQ_IMG_RSRC_WORD5__BASE_ARRAY__SHIFT 0x0 +#define SQ_IMG_RSRC_WORD5__LAST_ARRAY_MASK 0x3ffe000 +#define SQ_IMG_RSRC_WORD5__LAST_ARRAY__SHIFT 0xd +#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN_MASK 0xfff +#define SQ_IMG_RSRC_WORD6__MIN_LOD_WARN__SHIFT 0x0 +#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID_MASK 0xff000 +#define SQ_IMG_RSRC_WORD6__COUNTER_BANK_ID__SHIFT 0xc +#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN_MASK 0x100000 +#define SQ_IMG_RSRC_WORD6__LOD_HDW_CNT_EN__SHIFT 0x14 +#define SQ_IMG_RSRC_WORD6__UNUNSED_MASK 0xffe00000 +#define SQ_IMG_RSRC_WORD6__UNUNSED__SHIFT 0x15 +#define SQ_IMG_RSRC_WORD7__UNUNSED_MASK 0xffffffff +#define SQ_IMG_RSRC_WORD7__UNUNSED__SHIFT 0x0 +#define SQ_IMG_SAMP_WORD0__CLAMP_X_MASK 0x7 +#define SQ_IMG_SAMP_WORD0__CLAMP_X__SHIFT 0x0 +#define SQ_IMG_SAMP_WORD0__CLAMP_Y_MASK 0x38 +#define SQ_IMG_SAMP_WORD0__CLAMP_Y__SHIFT 0x3 +#define SQ_IMG_SAMP_WORD0__CLAMP_Z_MASK 0x1c0 +#define SQ_IMG_SAMP_WORD0__CLAMP_Z__SHIFT 0x6 +#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO_MASK 0xe00 +#define SQ_IMG_SAMP_WORD0__MAX_ANISO_RATIO__SHIFT 0x9 +#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC_MASK 0x7000 +#define SQ_IMG_SAMP_WORD0__DEPTH_COMPARE_FUNC__SHIFT 0xc +#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED_MASK 0x8000 +#define SQ_IMG_SAMP_WORD0__FORCE_UNNORMALIZED__SHIFT 0xf +#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD_MASK 0x70000 +#define SQ_IMG_SAMP_WORD0__ANISO_THRESHOLD__SHIFT 0x10 +#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC_MASK 0x80000 +#define SQ_IMG_SAMP_WORD0__MC_COORD_TRUNC__SHIFT 0x13 +#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA_MASK 0x100000 +#define SQ_IMG_SAMP_WORD0__FORCE_DEGAMMA__SHIFT 0x14 +#define SQ_IMG_SAMP_WORD0__ANISO_BIAS_MASK 0x7e00000 +#define SQ_IMG_SAMP_WORD0__ANISO_BIAS__SHIFT 0x15 +#define SQ_IMG_SAMP_WORD0__TRUNC_COORD_MASK 0x8000000 +#define SQ_IMG_SAMP_WORD0__TRUNC_COORD__SHIFT 0x1b +#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP_MASK 0x10000000 +#define SQ_IMG_SAMP_WORD0__DISABLE_CUBE_WRAP__SHIFT 0x1c +#define SQ_IMG_SAMP_WORD0__FILTER_MODE_MASK 0x60000000 +#define SQ_IMG_SAMP_WORD0__FILTER_MODE__SHIFT 0x1d +#define SQ_IMG_SAMP_WORD1__MIN_LOD_MASK 0xfff +#define SQ_IMG_SAMP_WORD1__MIN_LOD__SHIFT 0x0 +#define SQ_IMG_SAMP_WORD1__MAX_LOD_MASK 0xfff000 +#define SQ_IMG_SAMP_WORD1__MAX_LOD__SHIFT 0xc +#define SQ_IMG_SAMP_WORD1__PERF_MIP_MASK 0xf000000 +#define SQ_IMG_SAMP_WORD1__PERF_MIP__SHIFT 0x18 +#define SQ_IMG_SAMP_WORD1__PERF_Z_MASK 0xf0000000 +#define SQ_IMG_SAMP_WORD1__PERF_Z__SHIFT 0x1c +#define SQ_IMG_SAMP_WORD2__LOD_BIAS_MASK 0x3fff +#define SQ_IMG_SAMP_WORD2__LOD_BIAS__SHIFT 0x0 +#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC_MASK 0xfc000 +#define SQ_IMG_SAMP_WORD2__LOD_BIAS_SEC__SHIFT 0xe +#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER_MASK 0x300000 +#define SQ_IMG_SAMP_WORD2__XY_MAG_FILTER__SHIFT 0x14 +#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER_MASK 0xc00000 +#define SQ_IMG_SAMP_WORD2__XY_MIN_FILTER__SHIFT 0x16 +#define SQ_IMG_SAMP_WORD2__Z_FILTER_MASK 0x3000000 +#define SQ_IMG_SAMP_WORD2__Z_FILTER__SHIFT 0x18 +#define SQ_IMG_SAMP_WORD2__MIP_FILTER_MASK 0xc000000 +#define SQ_IMG_SAMP_WORD2__MIP_FILTER__SHIFT 0x1a +#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP_MASK 0x10000000 +#define SQ_IMG_SAMP_WORD2__MIP_POINT_PRECLAMP__SHIFT 0x1c +#define SQ_IMG_SAMP_WORD2__DISABLE_LSB_CEIL_MASK 0x20000000 +#define SQ_IMG_SAMP_WORD2__DISABLE_LSB_CEIL__SHIFT 0x1d +#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX_MASK 0x40000000 +#define SQ_IMG_SAMP_WORD2__FILTER_PREC_FIX__SHIFT 0x1e +#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR_MASK 0xfff +#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_PTR__SHIFT 0x0 +#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE_MASK 0xc0000000 +#define SQ_IMG_SAMP_WORD3__BORDER_COLOR_TYPE__SHIFT 0x1e +#define SQ_FLAT_SCRATCH_WORD0__SIZE_MASK 0x7ffff +#define SQ_FLAT_SCRATCH_WORD0__SIZE__SHIFT 0x0 +#define SQ_FLAT_SCRATCH_WORD1__OFFSET_MASK 0xffffff +#define SQ_FLAT_SCRATCH_WORD1__OFFSET__SHIFT 0x0 +#define SQ_IND_INDEX__WAVE_ID_MASK 0xf +#define SQ_IND_INDEX__WAVE_ID__SHIFT 0x0 +#define SQ_IND_INDEX__SIMD_ID_MASK 0x30 +#define SQ_IND_INDEX__SIMD_ID__SHIFT 0x4 +#define SQ_IND_INDEX__THREAD_ID_MASK 0xfc0 +#define SQ_IND_INDEX__THREAD_ID__SHIFT 0x6 +#define SQ_IND_INDEX__AUTO_INCR_MASK 0x1000 +#define SQ_IND_INDEX__AUTO_INCR__SHIFT 0xc +#define SQ_IND_INDEX__FORCE_READ_MASK 0x2000 +#define SQ_IND_INDEX__FORCE_READ__SHIFT 0xd +#define SQ_IND_INDEX__READ_TIMEOUT_MASK 0x4000 +#define SQ_IND_INDEX__READ_TIMEOUT__SHIFT 0xe +#define SQ_IND_INDEX__UNINDEXED_MASK 0x8000 +#define SQ_IND_INDEX__UNINDEXED__SHIFT 0xf +#define SQ_IND_INDEX__INDEX_MASK 0xffff0000 +#define SQ_IND_INDEX__INDEX__SHIFT 0x10 +#define SQ_CMD__CMD_MASK 0x7 +#define SQ_CMD__CMD__SHIFT 0x0 +#define SQ_CMD__MODE_MASK 0x70 +#define SQ_CMD__MODE__SHIFT 0x4 +#define SQ_CMD__CHECK_VMID_MASK 0x80 +#define SQ_CMD__CHECK_VMID__SHIFT 0x7 +#define SQ_CMD__TRAP_ID_MASK 0x700 +#define SQ_CMD__TRAP_ID__SHIFT 0x8 +#define SQ_CMD__WAVE_ID_MASK 0xf0000 +#define SQ_CMD__WAVE_ID__SHIFT 0x10 +#define SQ_CMD__SIMD_ID_MASK 0x300000 +#define SQ_CMD__SIMD_ID__SHIFT 0x14 +#define SQ_CMD__QUEUE_ID_MASK 0x7000000 +#define SQ_CMD__QUEUE_ID__SHIFT 0x18 +#define SQ_CMD__VM_ID_MASK 0xf0000000 +#define SQ_CMD__VM_ID__SHIFT 0x1c +#define SQ_IND_DATA__DATA_MASK 0xffffffff +#define SQ_IND_DATA__DATA__SHIFT 0x0 +#define SQ_REG_TIMESTAMP__TIMESTAMP_MASK 0xff +#define SQ_REG_TIMESTAMP__TIMESTAMP__SHIFT 0x0 +#define SQ_CMD_TIMESTAMP__TIMESTAMP_MASK 0xff +#define SQ_CMD_TIMESTAMP__TIMESTAMP__SHIFT 0x0 +#define SQ_HV_VMID_CTRL__DEFAULT_VMID_MASK 0xf +#define SQ_HV_VMID_CTRL__DEFAULT_VMID__SHIFT 0x0 +#define SQ_HV_VMID_CTRL__ALLOWED_VMID_MASK_MASK 0xffff0 +#define SQ_HV_VMID_CTRL__ALLOWED_VMID_MASK__SHIFT 0x4 +#define SQ_WAVE_INST_DW0__INST_DW0_MASK 0xffffffff +#define SQ_WAVE_INST_DW0__INST_DW0__SHIFT 0x0 +#define SQ_WAVE_INST_DW1__INST_DW1_MASK 0xffffffff +#define SQ_WAVE_INST_DW1__INST_DW1__SHIFT 0x0 +#define SQ_WAVE_PC_LO__PC_LO_MASK 0xffffffff +#define SQ_WAVE_PC_LO__PC_LO__SHIFT 0x0 +#define SQ_WAVE_PC_HI__PC_HI_MASK 0xff +#define SQ_WAVE_PC_HI__PC_HI__SHIFT 0x0 +#define SQ_WAVE_IB_DBG0__IBUF_ST_MASK 0x7 +#define SQ_WAVE_IB_DBG0__IBUF_ST__SHIFT 0x0 +#define SQ_WAVE_IB_DBG0__PC_INVALID_MASK 0x8 +#define SQ_WAVE_IB_DBG0__PC_INVALID__SHIFT 0x3 +#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW_MASK 0x10 +#define SQ_WAVE_IB_DBG0__NEED_NEXT_DW__SHIFT 0x4 +#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT_MASK 0xe0 +#define SQ_WAVE_IB_DBG0__NO_PREFETCH_CNT__SHIFT 0x5 +#define SQ_WAVE_IB_DBG0__IBUF_RPTR_MASK 0x300 +#define SQ_WAVE_IB_DBG0__IBUF_RPTR__SHIFT 0x8 +#define SQ_WAVE_IB_DBG0__IBUF_WPTR_MASK 0xc00 +#define SQ_WAVE_IB_DBG0__IBUF_WPTR__SHIFT 0xa +#define SQ_WAVE_IB_DBG0__INST_STR_ST_MASK 0x70000 +#define SQ_WAVE_IB_DBG0__INST_STR_ST__SHIFT 0x10 +#define SQ_WAVE_IB_DBG0__MISC_CNT_MASK 0x380000 +#define SQ_WAVE_IB_DBG0__MISC_CNT__SHIFT 0x13 +#define SQ_WAVE_IB_DBG0__ECC_ST_MASK 0xc00000 +#define SQ_WAVE_IB_DBG0__ECC_ST__SHIFT 0x16 +#define SQ_WAVE_IB_DBG0__IS_HYB_MASK 0x1000000 +#define SQ_WAVE_IB_DBG0__IS_HYB__SHIFT 0x18 +#define SQ_WAVE_IB_DBG0__HYB_CNT_MASK 0x6000000 +#define SQ_WAVE_IB_DBG0__HYB_CNT__SHIFT 0x19 +#define SQ_WAVE_IB_DBG0__KILL_MASK 0x8000000 +#define SQ_WAVE_IB_DBG0__KILL__SHIFT 0x1b +#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH_MASK 0x10000000 +#define SQ_WAVE_IB_DBG0__NEED_KILL_IFETCH__SHIFT 0x1c +#define SQ_WAVE_EXEC_LO__EXEC_LO_MASK 0xffffffff +#define SQ_WAVE_EXEC_LO__EXEC_LO__SHIFT 0x0 +#define SQ_WAVE_EXEC_HI__EXEC_HI_MASK 0xffffffff +#define SQ_WAVE_EXEC_HI__EXEC_HI__SHIFT 0x0 +#define SQ_WAVE_STATUS__SCC_MASK 0x1 +#define SQ_WAVE_STATUS__SCC__SHIFT 0x0 +#define SQ_WAVE_STATUS__SPI_PRIO_MASK 0x6 +#define SQ_WAVE_STATUS__SPI_PRIO__SHIFT 0x1 +#define SQ_WAVE_STATUS__WAVE_PRIO_MASK 0x18 +#define SQ_WAVE_STATUS__WAVE_PRIO__SHIFT 0x3 +#define SQ_WAVE_STATUS__PRIV_MASK 0x20 +#define SQ_WAVE_STATUS__PRIV__SHIFT 0x5 +#define SQ_WAVE_STATUS__TRAP_EN_MASK 0x40 +#define SQ_WAVE_STATUS__TRAP_EN__SHIFT 0x6 +#define SQ_WAVE_STATUS__TTRACE_EN_MASK 0x80 +#define SQ_WAVE_STATUS__TTRACE_EN__SHIFT 0x7 +#define SQ_WAVE_STATUS__EXPORT_RDY_MASK 0x100 +#define SQ_WAVE_STATUS__EXPORT_RDY__SHIFT 0x8 +#define SQ_WAVE_STATUS__EXECZ_MASK 0x200 +#define SQ_WAVE_STATUS__EXECZ__SHIFT 0x9 +#define SQ_WAVE_STATUS__VCCZ_MASK 0x400 +#define SQ_WAVE_STATUS__VCCZ__SHIFT 0xa +#define SQ_WAVE_STATUS__IN_TG_MASK 0x800 +#define SQ_WAVE_STATUS__IN_TG__SHIFT 0xb +#define SQ_WAVE_STATUS__IN_BARRIER_MASK 0x1000 +#define SQ_WAVE_STATUS__IN_BARRIER__SHIFT 0xc +#define SQ_WAVE_STATUS__HALT_MASK 0x2000 +#define SQ_WAVE_STATUS__HALT__SHIFT 0xd +#define SQ_WAVE_STATUS__TRAP_MASK 0x4000 +#define SQ_WAVE_STATUS__TRAP__SHIFT 0xe +#define SQ_WAVE_STATUS__TTRACE_CU_EN_MASK 0x8000 +#define SQ_WAVE_STATUS__TTRACE_CU_EN__SHIFT 0xf +#define SQ_WAVE_STATUS__VALID_MASK 0x10000 +#define SQ_WAVE_STATUS__VALID__SHIFT 0x10 +#define SQ_WAVE_STATUS__ECC_ERR_MASK 0x20000 +#define SQ_WAVE_STATUS__ECC_ERR__SHIFT 0x11 +#define SQ_WAVE_STATUS__SKIP_EXPORT_MASK 0x40000 +#define SQ_WAVE_STATUS__SKIP_EXPORT__SHIFT 0x12 +#define SQ_WAVE_STATUS__PERF_EN_MASK 0x80000 +#define SQ_WAVE_STATUS__PERF_EN__SHIFT 0x13 +#define SQ_WAVE_STATUS__COND_DBG_USER_MASK 0x100000 +#define SQ_WAVE_STATUS__COND_DBG_USER__SHIFT 0x14 +#define SQ_WAVE_STATUS__COND_DBG_SYS_MASK 0x200000 +#define SQ_WAVE_STATUS__COND_DBG_SYS__SHIFT 0x15 +#define SQ_WAVE_STATUS__DATA_ATC_MASK 0x400000 +#define SQ_WAVE_STATUS__DATA_ATC__SHIFT 0x16 +#define SQ_WAVE_STATUS__INST_ATC_MASK 0x800000 +#define SQ_WAVE_STATUS__INST_ATC__SHIFT 0x17 +#define SQ_WAVE_STATUS__DISPATCH_CACHE_CTRL_MASK 0x7000000 +#define SQ_WAVE_STATUS__DISPATCH_CACHE_CTRL__SHIFT 0x18 +#define SQ_WAVE_STATUS__MUST_EXPORT_MASK 0x8000000 +#define SQ_WAVE_STATUS__MUST_EXPORT__SHIFT 0x1b +#define SQ_WAVE_MODE__FP_ROUND_MASK 0xf +#define SQ_WAVE_MODE__FP_ROUND__SHIFT 0x0 +#define SQ_WAVE_MODE__FP_DENORM_MASK 0xf0 +#define SQ_WAVE_MODE__FP_DENORM__SHIFT 0x4 +#define SQ_WAVE_MODE__DX10_CLAMP_MASK 0x100 +#define SQ_WAVE_MODE__DX10_CLAMP__SHIFT 0x8 +#define SQ_WAVE_MODE__IEEE_MASK 0x200 +#define SQ_WAVE_MODE__IEEE__SHIFT 0x9 +#define SQ_WAVE_MODE__LOD_CLAMPED_MASK 0x400 +#define SQ_WAVE_MODE__LOD_CLAMPED__SHIFT 0xa +#define SQ_WAVE_MODE__DEBUG_EN_MASK 0x800 +#define SQ_WAVE_MODE__DEBUG_EN__SHIFT 0xb +#define SQ_WAVE_MODE__EXCP_EN_MASK 0x1ff000 +#define SQ_WAVE_MODE__EXCP_EN__SHIFT 0xc +#define SQ_WAVE_MODE__VSKIP_MASK 0x10000000 +#define SQ_WAVE_MODE__VSKIP__SHIFT 0x1c +#define SQ_WAVE_MODE__CSP_MASK 0xe0000000 +#define SQ_WAVE_MODE__CSP__SHIFT 0x1d +#define SQ_WAVE_TRAPSTS__EXCP_MASK 0x1ff +#define SQ_WAVE_TRAPSTS__EXCP__SHIFT 0x0 +#define SQ_WAVE_TRAPSTS__EXCP_CYCLE_MASK 0x3f0000 +#define SQ_WAVE_TRAPSTS__EXCP_CYCLE__SHIFT 0x10 +#define SQ_WAVE_TRAPSTS__DP_RATE_MASK 0xe0000000 +#define SQ_WAVE_TRAPSTS__DP_RATE__SHIFT 0x1d +#define SQ_WAVE_HW_ID__WAVE_ID_MASK 0xf +#define SQ_WAVE_HW_ID__WAVE_ID__SHIFT 0x0 +#define SQ_WAVE_HW_ID__SIMD_ID_MASK 0x30 +#define SQ_WAVE_HW_ID__SIMD_ID__SHIFT 0x4 +#define SQ_WAVE_HW_ID__PIPE_ID_MASK 0xc0 +#define SQ_WAVE_HW_ID__PIPE_ID__SHIFT 0x6 +#define SQ_WAVE_HW_ID__CU_ID_MASK 0xf00 +#define SQ_WAVE_HW_ID__CU_ID__SHIFT 0x8 +#define SQ_WAVE_HW_ID__SH_ID_MASK 0x1000 +#define SQ_WAVE_HW_ID__SH_ID__SHIFT 0xc +#define SQ_WAVE_HW_ID__SE_ID_MASK 0x6000 +#define SQ_WAVE_HW_ID__SE_ID__SHIFT 0xd +#define SQ_WAVE_HW_ID__TG_ID_MASK 0xf0000 +#define SQ_WAVE_HW_ID__TG_ID__SHIFT 0x10 +#define SQ_WAVE_HW_ID__VM_ID_MASK 0xf00000 +#define SQ_WAVE_HW_ID__VM_ID__SHIFT 0x14 +#define SQ_WAVE_HW_ID__QUEUE_ID_MASK 0x7000000 +#define SQ_WAVE_HW_ID__QUEUE_ID__SHIFT 0x18 +#define SQ_WAVE_HW_ID__STATE_ID_MASK 0x38000000 +#define SQ_WAVE_HW_ID__STATE_ID__SHIFT 0x1b +#define SQ_WAVE_HW_ID__ME_ID_MASK 0xc0000000 +#define SQ_WAVE_HW_ID__ME_ID__SHIFT 0x1e +#define SQ_WAVE_GPR_ALLOC__VGPR_BASE_MASK 0x3f +#define SQ_WAVE_GPR_ALLOC__VGPR_BASE__SHIFT 0x0 +#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE_MASK 0x3f00 +#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE__SHIFT 0x8 +#define SQ_WAVE_GPR_ALLOC__SGPR_BASE_MASK 0x3f0000 +#define SQ_WAVE_GPR_ALLOC__SGPR_BASE__SHIFT 0x10 +#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE_MASK 0xf000000 +#define SQ_WAVE_GPR_ALLOC__SGPR_SIZE__SHIFT 0x18 +#define SQ_WAVE_LDS_ALLOC__LDS_BASE_MASK 0xff +#define SQ_WAVE_LDS_ALLOC__LDS_BASE__SHIFT 0x0 +#define SQ_WAVE_LDS_ALLOC__LDS_SIZE_MASK 0x1ff000 +#define SQ_WAVE_LDS_ALLOC__LDS_SIZE__SHIFT 0xc +#define SQ_WAVE_IB_STS__VM_CNT_MASK 0xf +#define SQ_WAVE_IB_STS__VM_CNT__SHIFT 0x0 +#define SQ_WAVE_IB_STS__EXP_CNT_MASK 0x70 +#define SQ_WAVE_IB_STS__EXP_CNT__SHIFT 0x4 +#define SQ_WAVE_IB_STS__LGKM_CNT_MASK 0xf00 +#define SQ_WAVE_IB_STS__LGKM_CNT__SHIFT 0x8 +#define SQ_WAVE_IB_STS__VALU_CNT_MASK 0x7000 +#define SQ_WAVE_IB_STS__VALU_CNT__SHIFT 0xc +#define SQ_WAVE_M0__M0_MASK 0xffffffff +#define SQ_WAVE_M0__M0__SHIFT 0x0 +#define SQ_WAVE_TBA_LO__ADDR_LO_MASK 0xffffffff +#define SQ_WAVE_TBA_LO__ADDR_LO__SHIFT 0x0 +#define SQ_WAVE_TBA_HI__ADDR_HI_MASK 0xff +#define SQ_WAVE_TBA_HI__ADDR_HI__SHIFT 0x0 +#define SQ_WAVE_TMA_LO__ADDR_LO_MASK 0xffffffff +#define SQ_WAVE_TMA_LO__ADDR_LO__SHIFT 0x0 +#define SQ_WAVE_TMA_HI__ADDR_HI_MASK 0xff +#define SQ_WAVE_TMA_HI__ADDR_HI__SHIFT 0x0 +#define SQ_WAVE_TTMP0__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP0__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP1__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP1__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP2__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP2__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP3__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP3__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP4__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP4__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP5__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP5__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP6__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP6__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP7__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP7__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP8__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP8__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP9__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP9__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP10__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP10__DATA__SHIFT 0x0 +#define SQ_WAVE_TTMP11__DATA_MASK 0xffffffff +#define SQ_WAVE_TTMP11__DATA__SHIFT 0x0 +#define SQ_DEBUG_STS_GLOBAL__BUSY_MASK 0x1 +#define SQ_DEBUG_STS_GLOBAL__BUSY__SHIFT 0x0 +#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY_MASK 0x2 +#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_MSG_BUSY__SHIFT 0x1 +#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0_MASK 0xfff0 +#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH0__SHIFT 0x4 +#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1_MASK 0xfff0000 +#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SH1__SHIFT 0x10 +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0_MASK 0xff +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX0__SHIFT 0x0 +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1_MASK 0xff00 +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_GFX1__SHIFT 0x8 +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED_MASK 0xff0000 +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_IMMED__SHIFT 0x10 +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST_MASK 0xff000000 +#define SQ_DEBUG_STS_GLOBAL2__FIFO_LEVEL_HOST__SHIFT 0x18 +#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD_MASK 0xf +#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_CMD__SHIFT 0x0 +#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG_MASK 0xf0 +#define SQ_DEBUG_STS_GLOBAL3__FIFO_LEVEL_HOST_REG__SHIFT 0x4 +#define SQ_DEBUG_STS_LOCAL__BUSY_MASK 0x1 +#define SQ_DEBUG_STS_LOCAL__BUSY__SHIFT 0x0 +#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL_MASK 0x3f0 +#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL__SHIFT 0x4 +#define SQ_DEBUG_CTRL_LOCAL__UNUSED_MASK 0xff +#define SQ_DEBUG_CTRL_LOCAL__UNUSED__SHIFT 0x0 +#define SH_MEM_BASES__PRIVATE_BASE_MASK 0xffff +#define SH_MEM_BASES__PRIVATE_BASE__SHIFT 0x0 +#define SH_MEM_BASES__SHARED_BASE_MASK 0xffff0000 +#define SH_MEM_BASES__SHARED_BASE__SHIFT 0x10 +#define SH_MEM_APE1_BASE__BASE_MASK 0xffffffff +#define SH_MEM_APE1_BASE__BASE__SHIFT 0x0 +#define SH_MEM_APE1_LIMIT__LIMIT_MASK 0xffffffff +#define SH_MEM_APE1_LIMIT__LIMIT__SHIFT 0x0 +#define SH_MEM_CONFIG__PTR32_MASK 0x1 +#define SH_MEM_CONFIG__PTR32__SHIFT 0x0 +#define SH_MEM_CONFIG__PRIVATE_ATC_MASK 0x2 +#define SH_MEM_CONFIG__PRIVATE_ATC__SHIFT 0x1 +#define SH_MEM_CONFIG__ALIGNMENT_MODE_MASK 0xc +#define SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT 0x2 +#define SH_MEM_CONFIG__DEFAULT_MTYPE_MASK 0x70 +#define SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT 0x4 +#define SH_MEM_CONFIG__APE1_MTYPE_MASK 0x380 +#define SH_MEM_CONFIG__APE1_MTYPE__SHIFT 0x7 +#define SQC_POLICY__DATA_L1_POLICY_0_MASK 0x1 +#define SQC_POLICY__DATA_L1_POLICY_0__SHIFT 0x0 +#define SQC_POLICY__DATA_L1_POLICY_1_MASK 0x2 +#define SQC_POLICY__DATA_L1_POLICY_1__SHIFT 0x1 +#define SQC_POLICY__DATA_L1_POLICY_2_MASK 0x4 +#define SQC_POLICY__DATA_L1_POLICY_2__SHIFT 0x2 +#define SQC_POLICY__DATA_L1_POLICY_3_MASK 0x8 +#define SQC_POLICY__DATA_L1_POLICY_3__SHIFT 0x3 +#define SQC_POLICY__DATA_L1_POLICY_4_MASK 0x10 +#define SQC_POLICY__DATA_L1_POLICY_4__SHIFT 0x4 +#define SQC_POLICY__DATA_L1_POLICY_5_MASK 0x20 +#define SQC_POLICY__DATA_L1_POLICY_5__SHIFT 0x5 +#define SQC_POLICY__DATA_L1_POLICY_6_MASK 0x40 +#define SQC_POLICY__DATA_L1_POLICY_6__SHIFT 0x6 +#define SQC_POLICY__DATA_L1_POLICY_7_MASK 0x80 +#define SQC_POLICY__DATA_L1_POLICY_7__SHIFT 0x7 +#define SQC_POLICY__DATA_L2_POLICY_0_MASK 0x300 +#define SQC_POLICY__DATA_L2_POLICY_0__SHIFT 0x8 +#define SQC_POLICY__DATA_L2_POLICY_1_MASK 0xc00 +#define SQC_POLICY__DATA_L2_POLICY_1__SHIFT 0xa +#define SQC_POLICY__DATA_L2_POLICY_2_MASK 0x3000 +#define SQC_POLICY__DATA_L2_POLICY_2__SHIFT 0xc +#define SQC_POLICY__DATA_L2_POLICY_3_MASK 0xc000 +#define SQC_POLICY__DATA_L2_POLICY_3__SHIFT 0xe +#define SQC_POLICY__DATA_L2_POLICY_4_MASK 0x30000 +#define SQC_POLICY__DATA_L2_POLICY_4__SHIFT 0x10 +#define SQC_POLICY__DATA_L2_POLICY_5_MASK 0xc0000 +#define SQC_POLICY__DATA_L2_POLICY_5__SHIFT 0x12 +#define SQC_POLICY__DATA_L2_POLICY_6_MASK 0x300000 +#define SQC_POLICY__DATA_L2_POLICY_6__SHIFT 0x14 +#define SQC_POLICY__DATA_L2_POLICY_7_MASK 0xc00000 +#define SQC_POLICY__DATA_L2_POLICY_7__SHIFT 0x16 +#define SQC_POLICY__INST_L2_POLICY_MASK 0x3000000 +#define SQC_POLICY__INST_L2_POLICY__SHIFT 0x18 +#define SQC_VOLATILE__DATA_L1_MASK 0xf +#define SQC_VOLATILE__DATA_L1__SHIFT 0x0 +#define SQC_VOLATILE__DATA_L2_MASK 0xf0 +#define SQC_VOLATILE__DATA_L2__SHIFT 0x4 +#define SQC_VOLATILE__INST_L2_MASK 0x100 +#define SQC_VOLATILE__INST_L2__SHIFT 0x8 +#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_CMN__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_CMN__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_INST__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_INST__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID_MASK 0x1e0 +#define SQ_THREAD_TRACE_WORD_INST__WAVE_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID_MASK 0x600 +#define SQ_THREAD_TRACE_WORD_INST__SIMD_ID__SHIFT 0x9 +#define SQ_THREAD_TRACE_WORD_INST__SIZE_MASK 0x800 +#define SQ_THREAD_TRACE_WORD_INST__SIZE__SHIFT 0xb +#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE_MASK 0xf000 +#define SQ_THREAD_TRACE_WORD_INST__INST_TYPE__SHIFT 0xc +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID_MASK 0x1e0 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__WAVE_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID_MASK 0x600 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__SIMD_ID__SHIFT 0x9 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO_MASK 0xffff0000 +#define SQ_THREAD_TRACE_WORD_INST_PC_1_OF_2__PC_LO__SHIFT 0x10 +#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI_MASK 0xffffff +#define SQ_THREAD_TRACE_WORD_INST_PC_2_OF_2__PC_HI__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID_MASK 0x20 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SH_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID_MASK 0x3c0 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__CU_ID__SHIFT 0x6 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID_MASK 0x3c00 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__WAVE_ID__SHIFT 0xa +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID_MASK 0xc000 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__SIMD_ID__SHIFT 0xe +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO_MASK 0xffff0000 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_1_OF_2__DATA_LO__SHIFT 0x10 +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI_MASK 0xffff +#define SQ_THREAD_TRACE_WORD_INST_USERDATA_2_OF_2__DATA_HI__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO_MASK 0xffff0000 +#define SQ_THREAD_TRACE_WORD_TIMESTAMP_1_OF_2__TIME_LO__SHIFT 0x10 +#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI_MASK 0xffffffff +#define SQ_THREAD_TRACE_WORD_TIMESTAMP_2_OF_2__TIME_HI__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_WAVE__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_WAVE__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID_MASK 0x20 +#define SQ_THREAD_TRACE_WORD_WAVE__SH_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID_MASK 0x3c0 +#define SQ_THREAD_TRACE_WORD_WAVE__CU_ID__SHIFT 0x6 +#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID_MASK 0x3c00 +#define SQ_THREAD_TRACE_WORD_WAVE__WAVE_ID__SHIFT 0xa +#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID_MASK 0xc000 +#define SQ_THREAD_TRACE_WORD_WAVE__SIMD_ID__SHIFT 0xe +#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_MISC__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA_MASK 0xff0 +#define SQ_THREAD_TRACE_WORD_MISC__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_MISC__SH_ID_MASK 0x1000 +#define SQ_THREAD_TRACE_WORD_MISC__SH_ID__SHIFT 0xc +#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE_MASK 0xe000 +#define SQ_THREAD_TRACE_WORD_MISC__MISC_TOKEN_TYPE__SHIFT 0xd +#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_WAVE_START__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_WAVE_START__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID_MASK 0x20 +#define SQ_THREAD_TRACE_WORD_WAVE_START__SH_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID_MASK 0x3c0 +#define SQ_THREAD_TRACE_WORD_WAVE_START__CU_ID__SHIFT 0x6 +#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID_MASK 0x3c00 +#define SQ_THREAD_TRACE_WORD_WAVE_START__WAVE_ID__SHIFT 0xa +#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID_MASK 0xc000 +#define SQ_THREAD_TRACE_WORD_WAVE_START__SIMD_ID__SHIFT 0xe +#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER_MASK 0x1f0000 +#define SQ_THREAD_TRACE_WORD_WAVE_START__DISPATCHER__SHIFT 0x10 +#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED_MASK 0x200000 +#define SQ_THREAD_TRACE_WORD_WAVE_START__VS_NO_ALLOC_OR_GROUPED__SHIFT 0x15 +#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT_MASK 0x1fc00000 +#define SQ_THREAD_TRACE_WORD_WAVE_START__COUNT__SHIFT 0x16 +#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID_MASK 0xe0000000 +#define SQ_THREAD_TRACE_WORD_WAVE_START__TG_ID__SHIFT 0x1d +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID_MASK 0x60 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__PIPE_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID_MASK 0x180 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__ME_ID__SHIFT 0x7 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV_MASK 0x200 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_DROPPED_PREV__SHIFT 0x9 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE_MASK 0x1c00 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_TYPE__SHIFT 0xa +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV_MASK 0x4000 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_PRIV__SHIFT 0xe +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP_MASK 0x8000 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_OP__SHIFT 0xf +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR_MASK 0xffff0000 +#define SQ_THREAD_TRACE_WORD_REG_1_OF_2__REG_ADDR__SHIFT 0x10 +#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA_MASK 0xffffffff +#define SQ_THREAD_TRACE_WORD_REG_2_OF_2__DATA__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID_MASK 0x60 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__PIPE_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID_MASK 0x180 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__ME_ID__SHIFT 0x7 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR_MASK 0xfe00 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__REG_ADDR__SHIFT 0x9 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO_MASK 0xffff0000 +#define SQ_THREAD_TRACE_WORD_REG_CS_1_OF_2__DATA_LO__SHIFT 0x10 +#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI_MASK 0xffff +#define SQ_THREAD_TRACE_WORD_REG_CS_2_OF_2__DATA_HI__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_EVENT__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_EVENT__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID_MASK 0x20 +#define SQ_THREAD_TRACE_WORD_EVENT__SH_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_EVENT__STAGE_MASK 0x1c0 +#define SQ_THREAD_TRACE_WORD_EVENT__STAGE__SHIFT 0x6 +#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE_MASK 0xfc00 +#define SQ_THREAD_TRACE_WORD_EVENT__EVENT_TYPE__SHIFT 0xa +#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_ISSUE__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_ISSUE__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID_MASK 0x60 +#define SQ_THREAD_TRACE_WORD_ISSUE__SIMD_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST0_MASK 0x300 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST0__SHIFT 0x8 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST1_MASK 0xc00 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST1__SHIFT 0xa +#define SQ_THREAD_TRACE_WORD_ISSUE__INST2_MASK 0x3000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST2__SHIFT 0xc +#define SQ_THREAD_TRACE_WORD_ISSUE__INST3_MASK 0xc000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST3__SHIFT 0xe +#define SQ_THREAD_TRACE_WORD_ISSUE__INST4_MASK 0x30000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST4__SHIFT 0x10 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST5_MASK 0xc0000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST5__SHIFT 0x12 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST6_MASK 0x300000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST6__SHIFT 0x14 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST7_MASK 0xc00000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST7__SHIFT 0x16 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST8_MASK 0x3000000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST8__SHIFT 0x18 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST9_MASK 0xc000000 +#define SQ_THREAD_TRACE_WORD_ISSUE__INST9__SHIFT 0x1a +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE_MASK 0xf +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TOKEN_TYPE__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA_MASK 0x10 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__TIME_DELTA__SHIFT 0x4 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID_MASK 0x20 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__SH_ID__SHIFT 0x5 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID_MASK 0x3c0 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CU_ID__SHIFT 0x6 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK_MASK 0xc00 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR_BANK__SHIFT 0xa +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0_MASK 0x1fff000 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR0__SHIFT 0xc +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO_MASK 0xfe000000 +#define SQ_THREAD_TRACE_WORD_PERF_1_OF_2__CNTR1_LO__SHIFT 0x19 +#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI_MASK 0x3f +#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR1_HI__SHIFT 0x0 +#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2_MASK 0x7ffc0 +#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR2__SHIFT 0x6 +#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3_MASK 0xfff80000 +#define SQ_THREAD_TRACE_WORD_PERF_2_OF_2__CNTR3__SHIFT 0x13 +#define SQ_INTERRUPT_WORD_CMN__SE_ID_MASK 0x3000000 +#define SQ_INTERRUPT_WORD_CMN__SE_ID__SHIFT 0x18 +#define SQ_INTERRUPT_WORD_CMN__ENCODING_MASK 0xc000000 +#define SQ_INTERRUPT_WORD_CMN__ENCODING__SHIFT 0x1a +#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_MASK 0x1 +#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE__SHIFT 0x0 +#define SQ_INTERRUPT_WORD_AUTO__WLT_MASK 0x2 +#define SQ_INTERRUPT_WORD_AUTO__WLT__SHIFT 0x1 +#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_BUF_FULL_MASK 0x4 +#define SQ_INTERRUPT_WORD_AUTO__THREAD_TRACE_BUF_FULL__SHIFT 0x2 +#define SQ_INTERRUPT_WORD_AUTO__REG_TIMESTAMP_MASK 0x8 +#define SQ_INTERRUPT_WORD_AUTO__REG_TIMESTAMP__SHIFT 0x3 +#define SQ_INTERRUPT_WORD_AUTO__CMD_TIMESTAMP_MASK 0x10 +#define SQ_INTERRUPT_WORD_AUTO__CMD_TIMESTAMP__SHIFT 0x4 +#define SQ_INTERRUPT_WORD_AUTO__HOST_CMD_OVERFLOW_MASK 0x20 +#define SQ_INTERRUPT_WORD_AUTO__HOST_CMD_OVERFLOW__SHIFT 0x5 +#define SQ_INTERRUPT_WORD_AUTO__HOST_REG_OVERFLOW_MASK 0x40 +#define SQ_INTERRUPT_WORD_AUTO__HOST_REG_OVERFLOW__SHIFT 0x6 +#define SQ_INTERRUPT_WORD_AUTO__IMMED_OVERFLOW_MASK 0x80 +#define SQ_INTERRUPT_WORD_AUTO__IMMED_OVERFLOW__SHIFT 0x7 +#define SQ_INTERRUPT_WORD_AUTO__SE_ID_MASK 0x3000000 +#define SQ_INTERRUPT_WORD_AUTO__SE_ID__SHIFT 0x18 +#define SQ_INTERRUPT_WORD_AUTO__ENCODING_MASK 0xc000000 +#define SQ_INTERRUPT_WORD_AUTO__ENCODING__SHIFT 0x1a +#define SQ_INTERRUPT_WORD_WAVE__DATA_MASK 0xff +#define SQ_INTERRUPT_WORD_WAVE__DATA__SHIFT 0x0 +#define SQ_INTERRUPT_WORD_WAVE__SH_ID_MASK 0x100 +#define SQ_INTERRUPT_WORD_WAVE__SH_ID__SHIFT 0x8 +#define SQ_INTERRUPT_WORD_WAVE__PRIV_MASK 0x200 +#define SQ_INTERRUPT_WORD_WAVE__PRIV__SHIFT 0x9 +#define SQ_INTERRUPT_WORD_WAVE__VM_ID_MASK 0x3c00 +#define SQ_INTERRUPT_WORD_WAVE__VM_ID__SHIFT 0xa +#define SQ_INTERRUPT_WORD_WAVE__WAVE_ID_MASK 0x3c000 +#define SQ_INTERRUPT_WORD_WAVE__WAVE_ID__SHIFT 0xe +#define SQ_INTERRUPT_WORD_WAVE__SIMD_ID_MASK 0xc0000 +#define SQ_INTERRUPT_WORD_WAVE__SIMD_ID__SHIFT 0x12 +#define SQ_INTERRUPT_WORD_WAVE__CU_ID_MASK 0xf00000 +#define SQ_INTERRUPT_WORD_WAVE__CU_ID__SHIFT 0x14 +#define SQ_INTERRUPT_WORD_WAVE__SE_ID_MASK 0x3000000 +#define SQ_INTERRUPT_WORD_WAVE__SE_ID__SHIFT 0x18 +#define SQ_INTERRUPT_WORD_WAVE__ENCODING_MASK 0xc000000 +#define SQ_INTERRUPT_WORD_WAVE__ENCODING__SHIFT 0x1a +#define SQ_SOP2__SSRC0_MASK 0xff +#define SQ_SOP2__SSRC0__SHIFT 0x0 +#define SQ_SOP2__SSRC1_MASK 0xff00 +#define SQ_SOP2__SSRC1__SHIFT 0x8 +#define SQ_SOP2__SDST_MASK 0x7f0000 +#define SQ_SOP2__SDST__SHIFT 0x10 +#define SQ_SOP2__OP_MASK 0x3f800000 +#define SQ_SOP2__OP__SHIFT 0x17 +#define SQ_SOP2__ENCODING_MASK 0xc0000000 +#define SQ_SOP2__ENCODING__SHIFT 0x1e +#define SQ_VOP1__SRC0_MASK 0x1ff +#define SQ_VOP1__SRC0__SHIFT 0x0 +#define SQ_VOP1__OP_MASK 0x1fe00 +#define SQ_VOP1__OP__SHIFT 0x9 +#define SQ_VOP1__VDST_MASK 0x1fe0000 +#define SQ_VOP1__VDST__SHIFT 0x11 +#define SQ_VOP1__ENCODING_MASK 0xfe000000 +#define SQ_VOP1__ENCODING__SHIFT 0x19 +#define SQ_MTBUF_1__VADDR_MASK 0xff +#define SQ_MTBUF_1__VADDR__SHIFT 0x0 +#define SQ_MTBUF_1__VDATA_MASK 0xff00 +#define SQ_MTBUF_1__VDATA__SHIFT 0x8 +#define SQ_MTBUF_1__SRSRC_MASK 0x1f0000 +#define SQ_MTBUF_1__SRSRC__SHIFT 0x10 +#define SQ_MTBUF_1__SLC_MASK 0x400000 +#define SQ_MTBUF_1__SLC__SHIFT 0x16 +#define SQ_MTBUF_1__TFE_MASK 0x800000 +#define SQ_MTBUF_1__TFE__SHIFT 0x17 +#define SQ_MTBUF_1__SOFFSET_MASK 0xff000000 +#define SQ_MTBUF_1__SOFFSET__SHIFT 0x18 +#define SQ_EXP_1__VSRC0_MASK 0xff +#define SQ_EXP_1__VSRC0__SHIFT 0x0 +#define SQ_EXP_1__VSRC1_MASK 0xff00 +#define SQ_EXP_1__VSRC1__SHIFT 0x8 +#define SQ_EXP_1__VSRC2_MASK 0xff0000 +#define SQ_EXP_1__VSRC2__SHIFT 0x10 +#define SQ_EXP_1__VSRC3_MASK 0xff000000 +#define SQ_EXP_1__VSRC3__SHIFT 0x18 +#define SQ_MUBUF_1__VADDR_MASK 0xff +#define SQ_MUBUF_1__VADDR__SHIFT 0x0 +#define SQ_MUBUF_1__VDATA_MASK 0xff00 +#define SQ_MUBUF_1__VDATA__SHIFT 0x8 +#define SQ_MUBUF_1__SRSRC_MASK 0x1f0000 +#define SQ_MUBUF_1__SRSRC__SHIFT 0x10 +#define SQ_MUBUF_1__SLC_MASK 0x400000 +#define SQ_MUBUF_1__SLC__SHIFT 0x16 +#define SQ_MUBUF_1__TFE_MASK 0x800000 +#define SQ_MUBUF_1__TFE__SHIFT 0x17 +#define SQ_MUBUF_1__SOFFSET_MASK 0xff000000 +#define SQ_MUBUF_1__SOFFSET__SHIFT 0x18 +#define SQ_INST__ENCODING_MASK 0xffffffff +#define SQ_INST__ENCODING__SHIFT 0x0 +#define SQ_EXP_0__EN_MASK 0xf +#define SQ_EXP_0__EN__SHIFT 0x0 +#define SQ_EXP_0__TGT_MASK 0x3f0 +#define SQ_EXP_0__TGT__SHIFT 0x4 +#define SQ_EXP_0__COMPR_MASK 0x400 +#define SQ_EXP_0__COMPR__SHIFT 0xa +#define SQ_EXP_0__DONE_MASK 0x800 +#define SQ_EXP_0__DONE__SHIFT 0xb +#define SQ_EXP_0__VM_MASK 0x1000 +#define SQ_EXP_0__VM__SHIFT 0xc +#define SQ_EXP_0__ENCODING_MASK 0xfc000000 +#define SQ_EXP_0__ENCODING__SHIFT 0x1a +#define SQ_MUBUF_0__OFFSET_MASK 0xfff +#define SQ_MUBUF_0__OFFSET__SHIFT 0x0 +#define SQ_MUBUF_0__OFFEN_MASK 0x1000 +#define SQ_MUBUF_0__OFFEN__SHIFT 0xc +#define SQ_MUBUF_0__IDXEN_MASK 0x2000 +#define SQ_MUBUF_0__IDXEN__SHIFT 0xd +#define SQ_MUBUF_0__GLC_MASK 0x4000 +#define SQ_MUBUF_0__GLC__SHIFT 0xe +#define SQ_MUBUF_0__ADDR64_MASK 0x8000 +#define SQ_MUBUF_0__ADDR64__SHIFT 0xf +#define SQ_MUBUF_0__LDS_MASK 0x10000 +#define SQ_MUBUF_0__LDS__SHIFT 0x10 +#define SQ_MUBUF_0__OP_MASK 0x1fc0000 +#define SQ_MUBUF_0__OP__SHIFT 0x12 +#define SQ_MUBUF_0__ENCODING_MASK 0xfc000000 +#define SQ_MUBUF_0__ENCODING__SHIFT 0x1a +#define SQ_VOP3_0__VDST_MASK 0xff +#define SQ_VOP3_0__VDST__SHIFT 0x0 +#define SQ_VOP3_0__ABS_MASK 0x700 +#define SQ_VOP3_0__ABS__SHIFT 0x8 +#define SQ_VOP3_0__CLAMP_MASK 0x800 +#define SQ_VOP3_0__CLAMP__SHIFT 0xb +#define SQ_VOP3_0__OP_MASK 0x3fe0000 +#define SQ_VOP3_0__OP__SHIFT 0x11 +#define SQ_VOP3_0__ENCODING_MASK 0xfc000000 +#define SQ_VOP3_0__ENCODING__SHIFT 0x1a +#define SQ_VOP2__SRC0_MASK 0x1ff +#define SQ_VOP2__SRC0__SHIFT 0x0 +#define SQ_VOP2__VSRC1_MASK 0x1fe00 +#define SQ_VOP2__VSRC1__SHIFT 0x9 +#define SQ_VOP2__VDST_MASK 0x1fe0000 +#define SQ_VOP2__VDST__SHIFT 0x11 +#define SQ_VOP2__OP_MASK 0x7e000000 +#define SQ_VOP2__OP__SHIFT 0x19 +#define SQ_VOP2__ENCODING_MASK 0x80000000 +#define SQ_VOP2__ENCODING__SHIFT 0x1f +#define SQ_MTBUF_0__OFFSET_MASK 0xfff +#define SQ_MTBUF_0__OFFSET__SHIFT 0x0 +#define SQ_MTBUF_0__OFFEN_MASK 0x1000 +#define SQ_MTBUF_0__OFFEN__SHIFT 0xc +#define SQ_MTBUF_0__IDXEN_MASK 0x2000 +#define SQ_MTBUF_0__IDXEN__SHIFT 0xd +#define SQ_MTBUF_0__GLC_MASK 0x4000 +#define SQ_MTBUF_0__GLC__SHIFT 0xe +#define SQ_MTBUF_0__ADDR64_MASK 0x8000 +#define SQ_MTBUF_0__ADDR64__SHIFT 0xf +#define SQ_MTBUF_0__OP_MASK 0x70000 +#define SQ_MTBUF_0__OP__SHIFT 0x10 +#define SQ_MTBUF_0__DFMT_MASK 0x780000 +#define SQ_MTBUF_0__DFMT__SHIFT 0x13 +#define SQ_MTBUF_0__NFMT_MASK 0x3800000 +#define SQ_MTBUF_0__NFMT__SHIFT 0x17 +#define SQ_MTBUF_0__ENCODING_MASK 0xfc000000 +#define SQ_MTBUF_0__ENCODING__SHIFT 0x1a +#define SQ_SOPP__SIMM16_MASK 0xffff +#define SQ_SOPP__SIMM16__SHIFT 0x0 +#define SQ_SOPP__OP_MASK 0x7f0000 +#define SQ_SOPP__OP__SHIFT 0x10 +#define SQ_SOPP__ENCODING_MASK 0xff800000 +#define SQ_SOPP__ENCODING__SHIFT 0x17 +#define SQ_FLAT_0__GLC_MASK 0x10000 +#define SQ_FLAT_0__GLC__SHIFT 0x10 +#define SQ_FLAT_0__SLC_MASK 0x20000 +#define SQ_FLAT_0__SLC__SHIFT 0x11 +#define SQ_FLAT_0__OP_MASK 0x1fc0000 +#define SQ_FLAT_0__OP__SHIFT 0x12 +#define SQ_FLAT_0__ENCODING_MASK 0xfc000000 +#define SQ_FLAT_0__ENCODING__SHIFT 0x1a +#define SQ_VOP3_0_SDST_ENC__VDST_MASK 0xff +#define SQ_VOP3_0_SDST_ENC__VDST__SHIFT 0x0 +#define SQ_VOP3_0_SDST_ENC__SDST_MASK 0x7f00 +#define SQ_VOP3_0_SDST_ENC__SDST__SHIFT 0x8 +#define SQ_VOP3_0_SDST_ENC__OP_MASK 0x3fe0000 +#define SQ_VOP3_0_SDST_ENC__OP__SHIFT 0x11 +#define SQ_VOP3_0_SDST_ENC__ENCODING_MASK 0xfc000000 +#define SQ_VOP3_0_SDST_ENC__ENCODING__SHIFT 0x1a +#define SQ_MIMG_1__VADDR_MASK 0xff +#define SQ_MIMG_1__VADDR__SHIFT 0x0 +#define SQ_MIMG_1__VDATA_MASK 0xff00 +#define SQ_MIMG_1__VDATA__SHIFT 0x8 +#define SQ_MIMG_1__SRSRC_MASK 0x1f0000 +#define SQ_MIMG_1__SRSRC__SHIFT 0x10 +#define SQ_MIMG_1__SSAMP_MASK 0x3e00000 +#define SQ_MIMG_1__SSAMP__SHIFT 0x15 +#define SQ_SMRD__OFFSET_MASK 0xff +#define SQ_SMRD__OFFSET__SHIFT 0x0 +#define SQ_SMRD__IMM_MASK 0x100 +#define SQ_SMRD__IMM__SHIFT 0x8 +#define SQ_SMRD__SBASE_MASK 0x7e00 +#define SQ_SMRD__SBASE__SHIFT 0x9 +#define SQ_SMRD__SDST_MASK 0x3f8000 +#define SQ_SMRD__SDST__SHIFT 0xf +#define SQ_SMRD__OP_MASK 0x7c00000 +#define SQ_SMRD__OP__SHIFT 0x16 +#define SQ_SMRD__ENCODING_MASK 0xf8000000 +#define SQ_SMRD__ENCODING__SHIFT 0x1b +#define SQ_SOP1__SSRC0_MASK 0xff +#define SQ_SOP1__SSRC0__SHIFT 0x0 +#define SQ_SOP1__OP_MASK 0xff00 +#define SQ_SOP1__OP__SHIFT 0x8 +#define SQ_SOP1__SDST_MASK 0x7f0000 +#define SQ_SOP1__SDST__SHIFT 0x10 +#define SQ_SOP1__ENCODING_MASK 0xff800000 +#define SQ_SOP1__ENCODING__SHIFT 0x17 +#define SQ_SOPC__SSRC0_MASK 0xff +#define SQ_SOPC__SSRC0__SHIFT 0x0 +#define SQ_SOPC__SSRC1_MASK 0xff00 +#define SQ_SOPC__SSRC1__SHIFT 0x8 +#define SQ_SOPC__OP_MASK 0x7f0000 +#define SQ_SOPC__OP__SHIFT 0x10 +#define SQ_SOPC__ENCODING_MASK 0xff800000 +#define SQ_SOPC__ENCODING__SHIFT 0x17 +#define SQ_FLAT_1__ADDR_MASK 0xff +#define SQ_FLAT_1__ADDR__SHIFT 0x0 +#define SQ_FLAT_1__DATA_MASK 0xff00 +#define SQ_FLAT_1__DATA__SHIFT 0x8 +#define SQ_FLAT_1__TFE_MASK 0x800000 +#define SQ_FLAT_1__TFE__SHIFT 0x17 +#define SQ_FLAT_1__VDST_MASK 0xff000000 +#define SQ_FLAT_1__VDST__SHIFT 0x18 +#define SQ_DS_1__ADDR_MASK 0xff +#define SQ_DS_1__ADDR__SHIFT 0x0 +#define SQ_DS_1__DATA0_MASK 0xff00 +#define SQ_DS_1__DATA0__SHIFT 0x8 +#define SQ_DS_1__DATA1_MASK 0xff0000 +#define SQ_DS_1__DATA1__SHIFT 0x10 +#define SQ_DS_1__VDST_MASK 0xff000000 +#define SQ_DS_1__VDST__SHIFT 0x18 +#define SQ_VOP3_1__SRC0_MASK 0x1ff +#define SQ_VOP3_1__SRC0__SHIFT 0x0 +#define SQ_VOP3_1__SRC1_MASK 0x3fe00 +#define SQ_VOP3_1__SRC1__SHIFT 0x9 +#define SQ_VOP3_1__SRC2_MASK 0x7fc0000 +#define SQ_VOP3_1__SRC2__SHIFT 0x12 +#define SQ_VOP3_1__OMOD_MASK 0x18000000 +#define SQ_VOP3_1__OMOD__SHIFT 0x1b +#define SQ_VOP3_1__NEG_MASK 0xe0000000 +#define SQ_VOP3_1__NEG__SHIFT 0x1d +#define SQ_MIMG_0__DMASK_MASK 0xf00 +#define SQ_MIMG_0__DMASK__SHIFT 0x8 +#define SQ_MIMG_0__UNORM_MASK 0x1000 +#define SQ_MIMG_0__UNORM__SHIFT 0xc +#define SQ_MIMG_0__GLC_MASK 0x2000 +#define SQ_MIMG_0__GLC__SHIFT 0xd +#define SQ_MIMG_0__DA_MASK 0x4000 +#define SQ_MIMG_0__DA__SHIFT 0xe +#define SQ_MIMG_0__R128_MASK 0x8000 +#define SQ_MIMG_0__R128__SHIFT 0xf +#define SQ_MIMG_0__TFE_MASK 0x10000 +#define SQ_MIMG_0__TFE__SHIFT 0x10 +#define SQ_MIMG_0__LWE_MASK 0x20000 +#define SQ_MIMG_0__LWE__SHIFT 0x11 +#define SQ_MIMG_0__OP_MASK 0x1fc0000 +#define SQ_MIMG_0__OP__SHIFT 0x12 +#define SQ_MIMG_0__SLC_MASK 0x2000000 +#define SQ_MIMG_0__SLC__SHIFT 0x19 +#define SQ_MIMG_0__ENCODING_MASK 0xfc000000 +#define SQ_MIMG_0__ENCODING__SHIFT 0x1a +#define SQ_SOPK__SIMM16_MASK 0xffff +#define SQ_SOPK__SIMM16__SHIFT 0x0 +#define SQ_SOPK__SDST_MASK 0x7f0000 +#define SQ_SOPK__SDST__SHIFT 0x10 +#define SQ_SOPK__OP_MASK 0xf800000 +#define SQ_SOPK__OP__SHIFT 0x17 +#define SQ_SOPK__ENCODING_MASK 0xf0000000 +#define SQ_SOPK__ENCODING__SHIFT 0x1c +#define SQ_DS_0__OFFSET0_MASK 0xff +#define SQ_DS_0__OFFSET0__SHIFT 0x0 +#define SQ_DS_0__OFFSET1_MASK 0xff00 +#define SQ_DS_0__OFFSET1__SHIFT 0x8 +#define SQ_DS_0__GDS_MASK 0x20000 +#define SQ_DS_0__GDS__SHIFT 0x11 +#define SQ_DS_0__OP_MASK 0x3fc0000 +#define SQ_DS_0__OP__SHIFT 0x12 +#define SQ_DS_0__ENCODING_MASK 0xfc000000 +#define SQ_DS_0__ENCODING__SHIFT 0x1a +#define SQ_VOPC__SRC0_MASK 0x1ff +#define SQ_VOPC__SRC0__SHIFT 0x0 +#define SQ_VOPC__VSRC1_MASK 0x1fe00 +#define SQ_VOPC__VSRC1__SHIFT 0x9 +#define SQ_VOPC__OP_MASK 0x1fe0000 +#define SQ_VOPC__OP__SHIFT 0x11 +#define SQ_VOPC__ENCODING_MASK 0xfe000000 +#define SQ_VOPC__ENCODING__SHIFT 0x19 +#define SQ_VINTRP__VSRC_MASK 0xff +#define SQ_VINTRP__VSRC__SHIFT 0x0 +#define SQ_VINTRP__ATTRCHAN_MASK 0x300 +#define SQ_VINTRP__ATTRCHAN__SHIFT 0x8 +#define SQ_VINTRP__ATTR_MASK 0xfc00 +#define SQ_VINTRP__ATTR__SHIFT 0xa +#define SQ_VINTRP__OP_MASK 0x30000 +#define SQ_VINTRP__OP__SHIFT 0x10 +#define SQ_VINTRP__VDST_MASK 0x3fc0000 +#define SQ_VINTRP__VDST__SHIFT 0x12 +#define SQ_VINTRP__ENCODING_MASK 0xfc000000 +#define SQ_VINTRP__ENCODING__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL0__ON_DELAY_MASK 0xf +#define CGTT_SX_CLK_CTRL0__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL0__RESERVED_MASK 0xfff000 +#define CGTT_SX_CLK_CTRL0__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL1__ON_DELAY_MASK 0xf +#define CGTT_SX_CLK_CTRL1__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL1__RESERVED_MASK 0xfff000 +#define CGTT_SX_CLK_CTRL1__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL2__ON_DELAY_MASK 0xf +#define CGTT_SX_CLK_CTRL2__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL2__RESERVED_MASK 0xfff000 +#define CGTT_SX_CLK_CTRL2__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL3__ON_DELAY_MASK 0xf +#define CGTT_SX_CLK_CTRL3__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL3__RESERVED_MASK 0xfff000 +#define CGTT_SX_CLK_CTRL3__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_SX_CLK_CTRL4__ON_DELAY_MASK 0xf +#define CGTT_SX_CLK_CTRL4__ON_DELAY__SHIFT 0x0 +#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_SX_CLK_CTRL4__RESERVED_MASK 0xfff000 +#define CGTT_SX_CLK_CTRL4__RESERVED__SHIFT 0xc +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0__SHIFT 0x1f +#define SX_DEBUG_BUSY__POS_FREE_OR_VALIDS_MASK 0x1 +#define SX_DEBUG_BUSY__POS_FREE_OR_VALIDS__SHIFT 0x0 +#define SX_DEBUG_BUSY__POS_REQUESTER_BUSY_MASK 0x2 +#define SX_DEBUG_BUSY__POS_REQUESTER_BUSY__SHIFT 0x1 +#define SX_DEBUG_BUSY__PA_SX_BUSY_MASK 0x4 +#define SX_DEBUG_BUSY__PA_SX_BUSY__SHIFT 0x2 +#define SX_DEBUG_BUSY__POS_SCBD_BUSY_MASK 0x8 +#define SX_DEBUG_BUSY__POS_SCBD_BUSY__SHIFT 0x3 +#define SX_DEBUG_BUSY__POS_BANK3VAL3_BUSY_MASK 0x10 +#define SX_DEBUG_BUSY__POS_BANK3VAL3_BUSY__SHIFT 0x4 +#define SX_DEBUG_BUSY__POS_BANK3VAL2_BUSY_MASK 0x20 +#define SX_DEBUG_BUSY__POS_BANK3VAL2_BUSY__SHIFT 0x5 +#define SX_DEBUG_BUSY__POS_BANK3VAL1_BUSY_MASK 0x40 +#define SX_DEBUG_BUSY__POS_BANK3VAL1_BUSY__SHIFT 0x6 +#define SX_DEBUG_BUSY__POS_BANK3VAL0_BUSY_MASK 0x80 +#define SX_DEBUG_BUSY__POS_BANK3VAL0_BUSY__SHIFT 0x7 +#define SX_DEBUG_BUSY__POS_BANK2VAL3_BUSY_MASK 0x100 +#define SX_DEBUG_BUSY__POS_BANK2VAL3_BUSY__SHIFT 0x8 +#define SX_DEBUG_BUSY__POS_BANK2VAL2_BUSY_MASK 0x200 +#define SX_DEBUG_BUSY__POS_BANK2VAL2_BUSY__SHIFT 0x9 +#define SX_DEBUG_BUSY__POS_BANK2VAL1_BUSY_MASK 0x400 +#define SX_DEBUG_BUSY__POS_BANK2VAL1_BUSY__SHIFT 0xa +#define SX_DEBUG_BUSY__POS_BANK2VAL0_BUSY_MASK 0x800 +#define SX_DEBUG_BUSY__POS_BANK2VAL0_BUSY__SHIFT 0xb +#define SX_DEBUG_BUSY__POS_BANK1VAL3_BUSY_MASK 0x1000 +#define SX_DEBUG_BUSY__POS_BANK1VAL3_BUSY__SHIFT 0xc +#define SX_DEBUG_BUSY__POS_BANK1VAL2_BUSY_MASK 0x2000 +#define SX_DEBUG_BUSY__POS_BANK1VAL2_BUSY__SHIFT 0xd +#define SX_DEBUG_BUSY__POS_BANK1VAL1_BUSY_MASK 0x4000 +#define SX_DEBUG_BUSY__POS_BANK1VAL1_BUSY__SHIFT 0xe +#define SX_DEBUG_BUSY__POS_BANK1VAL0_BUSY_MASK 0x8000 +#define SX_DEBUG_BUSY__POS_BANK1VAL0_BUSY__SHIFT 0xf +#define SX_DEBUG_BUSY__POS_BANK0VAL3_BUSY_MASK 0x10000 +#define SX_DEBUG_BUSY__POS_BANK0VAL3_BUSY__SHIFT 0x10 +#define SX_DEBUG_BUSY__POS_BANK0VAL2_BUSY_MASK 0x20000 +#define SX_DEBUG_BUSY__POS_BANK0VAL2_BUSY__SHIFT 0x11 +#define SX_DEBUG_BUSY__POS_BANK0VAL1_BUSY_MASK 0x40000 +#define SX_DEBUG_BUSY__POS_BANK0VAL1_BUSY__SHIFT 0x12 +#define SX_DEBUG_BUSY__POS_BANK0VAL0_BUSY_MASK 0x80000 +#define SX_DEBUG_BUSY__POS_BANK0VAL0_BUSY__SHIFT 0x13 +#define SX_DEBUG_BUSY__POS_INMUX_VALID_MASK 0x100000 +#define SX_DEBUG_BUSY__POS_INMUX_VALID__SHIFT 0x14 +#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ3_MASK 0x200000 +#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ3__SHIFT 0x15 +#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ2_MASK 0x400000 +#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ2__SHIFT 0x16 +#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ1_MASK 0x800000 +#define SX_DEBUG_BUSY__WRCTRL1_VALIDQ1__SHIFT 0x17 +#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ3_MASK 0x1000000 +#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ3__SHIFT 0x18 +#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ2_MASK 0x2000000 +#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ2__SHIFT 0x19 +#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ1_MASK 0x4000000 +#define SX_DEBUG_BUSY__WRCTRL0_VALIDQ1__SHIFT 0x1a +#define SX_DEBUG_BUSY__PCCMD_VALID_MASK 0x8000000 +#define SX_DEBUG_BUSY__PCCMD_VALID__SHIFT 0x1b +#define SX_DEBUG_BUSY__VDATA1_VALID_MASK 0x10000000 +#define SX_DEBUG_BUSY__VDATA1_VALID__SHIFT 0x1c +#define SX_DEBUG_BUSY__VDATA0_VALID_MASK 0x20000000 +#define SX_DEBUG_BUSY__VDATA0_VALID__SHIFT 0x1d +#define SX_DEBUG_BUSY__CMD_BUSYORVAL_MASK 0x40000000 +#define SX_DEBUG_BUSY__CMD_BUSYORVAL__SHIFT 0x1e +#define SX_DEBUG_BUSY__ADDR_BUSYORVAL_MASK 0x80000000 +#define SX_DEBUG_BUSY__ADDR_BUSYORVAL__SHIFT 0x1f +#define SX_DEBUG_BUSY_2__COL_SCBD_BUSY_MASK 0x1 +#define SX_DEBUG_BUSY_2__COL_SCBD_BUSY__SHIFT 0x0 +#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0_MASK 0x2 +#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0__SHIFT 0x1 +#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE_MASK 0x4 +#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE__SHIFT 0x2 +#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY_MASK 0x8 +#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY__SHIFT 0x3 +#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0_MASK 0x10 +#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0__SHIFT 0x4 +#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE_MASK 0x20 +#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE__SHIFT 0x5 +#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY_MASK 0x40 +#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY__SHIFT 0x6 +#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0_MASK 0x80 +#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0__SHIFT 0x7 +#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE_MASK 0x100 +#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE__SHIFT 0x8 +#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY_MASK 0x200 +#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY__SHIFT 0x9 +#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0_MASK 0x400 +#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0__SHIFT 0xa +#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE_MASK 0x800 +#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE__SHIFT 0xb +#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY_MASK 0x1000 +#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY__SHIFT 0xc +#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY_MASK 0x2000 +#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY__SHIFT 0xd +#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY_MASK 0x4000 +#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY__SHIFT 0xe +#define SX_DEBUG_BUSY_2__COL_DBIF3_READ_VALID_MASK 0x8000 +#define SX_DEBUG_BUSY_2__COL_DBIF3_READ_VALID__SHIFT 0xf +#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY_MASK 0x10000 +#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY__SHIFT 0x10 +#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY_MASK 0x20000 +#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY__SHIFT 0x11 +#define SX_DEBUG_BUSY_2__COL_DBIF2_READ_VALID_MASK 0x40000 +#define SX_DEBUG_BUSY_2__COL_DBIF2_READ_VALID__SHIFT 0x12 +#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY_MASK 0x80000 +#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY__SHIFT 0x13 +#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY_MASK 0x100000 +#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY__SHIFT 0x14 +#define SX_DEBUG_BUSY_2__COL_DBIF1_READ_VALID_MASK 0x200000 +#define SX_DEBUG_BUSY_2__COL_DBIF1_READ_VALID__SHIFT 0x15 +#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY_MASK 0x400000 +#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY__SHIFT 0x16 +#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY_MASK 0x800000 +#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY__SHIFT 0x17 +#define SX_DEBUG_BUSY_2__COL_DBIF0_READ_VALID_MASK 0x1000000 +#define SX_DEBUG_BUSY_2__COL_DBIF0_READ_VALID__SHIFT 0x18 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY_MASK 0x2000000 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY__SHIFT 0x19 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY_MASK 0x4000000 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY__SHIFT 0x1a +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY_MASK 0x8000000 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY__SHIFT 0x1b +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY_MASK 0x10000000 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY__SHIFT 0x1c +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY_MASK 0x20000000 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY__SHIFT 0x1d +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY_MASK 0x40000000 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY__SHIFT 0x1e +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY_MASK 0x80000000 +#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY__SHIFT 0x1f +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY_MASK 0x1 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY__SHIFT 0x0 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY_MASK 0x2 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY__SHIFT 0x1 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY_MASK 0x4 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY__SHIFT 0x2 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY_MASK 0x8 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY__SHIFT 0x3 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY_MASK 0x10 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY__SHIFT 0x4 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY_MASK 0x20 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY__SHIFT 0x5 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY_MASK 0x40 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY__SHIFT 0x6 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY_MASK 0x80 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY__SHIFT 0x7 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY_MASK 0x100 +#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY__SHIFT 0x8 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY_MASK 0x200 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY__SHIFT 0x9 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY_MASK 0x400 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY__SHIFT 0xa +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY_MASK 0x800 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY__SHIFT 0xb +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY_MASK 0x1000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY__SHIFT 0xc +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY_MASK 0x2000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY__SHIFT 0xd +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY_MASK 0x4000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY__SHIFT 0xe +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY_MASK 0x8000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY__SHIFT 0xf +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY_MASK 0x10000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY__SHIFT 0x10 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY_MASK 0x20000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY__SHIFT 0x11 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY_MASK 0x40000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY__SHIFT 0x12 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY_MASK 0x80000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY__SHIFT 0x13 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY_MASK 0x100000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY__SHIFT 0x14 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY_MASK 0x200000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY__SHIFT 0x15 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY_MASK 0x400000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY__SHIFT 0x16 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY_MASK 0x800000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY__SHIFT 0x17 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY_MASK 0x1000000 +#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY__SHIFT 0x18 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY_MASK 0x2000000 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY__SHIFT 0x19 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY_MASK 0x4000000 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY__SHIFT 0x1a +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY_MASK 0x8000000 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY__SHIFT 0x1b +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY_MASK 0x10000000 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY__SHIFT 0x1c +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY_MASK 0x20000000 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY__SHIFT 0x1d +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY_MASK 0x40000000 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY__SHIFT 0x1e +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY_MASK 0x80000000 +#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY__SHIFT 0x1f +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY_MASK 0x1 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY__SHIFT 0x0 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY_MASK 0x2 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY__SHIFT 0x1 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY_MASK 0x4 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY__SHIFT 0x2 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY_MASK 0x8 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY__SHIFT 0x3 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY_MASK 0x10 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY__SHIFT 0x4 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY_MASK 0x20 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY__SHIFT 0x5 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY_MASK 0x40 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY__SHIFT 0x6 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY_MASK 0x80 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY__SHIFT 0x7 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY_MASK 0x100 +#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY__SHIFT 0x8 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY_MASK 0x200 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY__SHIFT 0x9 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY_MASK 0x400 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY__SHIFT 0xa +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY_MASK 0x800 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY__SHIFT 0xb +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY_MASK 0x1000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY__SHIFT 0xc +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY_MASK 0x2000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY__SHIFT 0xd +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY_MASK 0x4000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY__SHIFT 0xe +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY_MASK 0x8000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY__SHIFT 0xf +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY_MASK 0x10000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY__SHIFT 0x10 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY_MASK 0x20000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY__SHIFT 0x11 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY_MASK 0x40000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY__SHIFT 0x12 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY_MASK 0x80000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY__SHIFT 0x13 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY_MASK 0x100000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY__SHIFT 0x14 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY_MASK 0x200000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY__SHIFT 0x15 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY_MASK 0x400000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY__SHIFT 0x16 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY_MASK 0x800000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY__SHIFT 0x17 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY_MASK 0x1000000 +#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY__SHIFT 0x18 +#define SX_DEBUG_BUSY_4__RESERVED_MASK 0xfe000000 +#define SX_DEBUG_BUSY_4__RESERVED__SHIFT 0x19 +#define SX_DEBUG_1__SX_DB_QUAD_CREDIT_MASK 0x7f +#define SX_DEBUG_1__SX_DB_QUAD_CREDIT__SHIFT 0x0 +#define SX_DEBUG_1__DEBUG_DATA_MASK 0xffffff80 +#define SX_DEBUG_1__DEBUG_DATA__SHIFT 0x7 +#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define SX_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define SX_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define SX_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define SX_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define SX_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define SX_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define SX_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define SX_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define SX_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define SX_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define SX_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define SX_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK 0x3ff +#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x0 +#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK 0xffc00 +#define SX_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0xa +#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2_MASK 0x3ff +#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x0 +#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3_MASK 0xffc00 +#define SX_PERFCOUNTER1_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0xa +#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCC_CTRL__CACHE_SIZE_MASK 0x3 +#define TCC_CTRL__CACHE_SIZE__SHIFT 0x0 +#define TCC_CTRL__RATE_MASK 0xc +#define TCC_CTRL__RATE__SHIFT 0x2 +#define TCC_CTRL__WRITEBACK_MARGIN_MASK 0xf0 +#define TCC_CTRL__WRITEBACK_MARGIN__SHIFT 0x4 +#define TCC_CTRL__SRC_FIFO_SIZE_MASK 0xf000 +#define TCC_CTRL__SRC_FIFO_SIZE__SHIFT 0xc +#define TCC_CTRL__LATENCY_FIFO_SIZE_MASK 0xf0000 +#define TCC_CTRL__LATENCY_FIFO_SIZE__SHIFT 0x10 +#define TCC_CTRL__WB_OR_INV_ALL_VMIDS_MASK 0x100000 +#define TCC_CTRL__WB_OR_INV_ALL_VMIDS__SHIFT 0x14 +#define TCC_EDC_COUNTER__SEC_COUNT_MASK 0xf +#define TCC_EDC_COUNTER__SEC_COUNT__SHIFT 0x0 +#define TCC_EDC_COUNTER__DED_COUNT_MASK 0xf0000 +#define TCC_EDC_COUNTER__DED_COUNT__SHIFT 0x10 +#define TCC_REDUNDANCY__MC_SEL0_MASK 0x1 +#define TCC_REDUNDANCY__MC_SEL0__SHIFT 0x0 +#define TCC_REDUNDANCY__MC_SEL1_MASK 0x2 +#define TCC_REDUNDANCY__MC_SEL1__SHIFT 0x1 +#define TCC_CGTT_SCLK_CTRL__ON_DELAY_MASK 0xf +#define TCC_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define TCC_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define TCC_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define TCA_CGTT_SCLK_CTRL__ON_DELAY_MASK 0xf +#define TCA_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define TCA_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define TCA_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define TCS_CGTT_SCLK_CTRL__ON_DELAY_MASK 0xf +#define TCS_CGTT_SCLK_CTRL__ON_DELAY__SHIFT 0x0 +#define TCS_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define TCS_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define TCS_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define TCC_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define TCC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define TCC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define TCC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define TCC_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define TCC_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define TCC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00 +#define TCC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000 +#define TCC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18 +#define TCC_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define TCC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf000000 +#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x18 +#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf0000000 +#define TCC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x1c +#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff +#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00 +#define TCC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa +#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf000000 +#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x18 +#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf0000000 +#define TCC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x1c +#define TCC_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define TCC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCC_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define TCC_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define TCC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCC_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCA_CTRL__HOLE_TIMEOUT_MASK 0xf +#define TCA_CTRL__HOLE_TIMEOUT__SHIFT 0x0 +#define TCA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define TCA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define TCA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define TCA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define TCA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define TCA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define TCA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00 +#define TCA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000 +#define TCA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18 +#define TCA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define TCA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf000000 +#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x18 +#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf0000000 +#define TCA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x1c +#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff +#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00 +#define TCA_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa +#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf000000 +#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x18 +#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf0000000 +#define TCA_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x1c +#define TCA_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define TCA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCA_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define TCA_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define TCA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCA_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCS_CTRL__RATE_MASK 0x3 +#define TCS_CTRL__RATE__SHIFT 0x0 +#define TCS_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define TCS_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define TCS_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define TCS_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define TCS_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCS_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCS_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define TCS_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define TCS_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCS_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define TCS_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define TCS_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TCS_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define TCS_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define TCS_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf000000 +#define TCS_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x18 +#define TCS_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf0000000 +#define TCS_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x1c +#define TCS_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define TCS_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define TCS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCS_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCS_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define TCS_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define TCS_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define TCS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCS_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCS_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define TCS_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define TCS_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define TCS_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCS_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCS_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCS_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define TCS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCS_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCS_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCS_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCS_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TA_BC_BASE_ADDR__ADDRESS_MASK 0xffffffff +#define TA_BC_BASE_ADDR__ADDRESS__SHIFT 0x0 +#define TA_BC_BASE_ADDR_HI__ADDRESS_MASK 0xff +#define TA_BC_BASE_ADDR_HI__ADDRESS__SHIFT 0x0 +#define TD_CNTL__SYNC_PHASE_SH_MASK 0x3 +#define TD_CNTL__SYNC_PHASE_SH__SHIFT 0x0 +#define TD_CNTL__SYNC_PHASE_VC_SMX_MASK 0x30 +#define TD_CNTL__SYNC_PHASE_VC_SMX__SHIFT 0x4 +#define TD_CNTL__PAD_STALL_EN_MASK 0x100 +#define TD_CNTL__PAD_STALL_EN__SHIFT 0x8 +#define TD_CNTL__EXTEND_LDS_STALL_MASK 0x600 +#define TD_CNTL__EXTEND_LDS_STALL__SHIFT 0x9 +#define TD_CNTL__LDS_STALL_PHASE_ADJUST_MASK 0x1800 +#define TD_CNTL__LDS_STALL_PHASE_ADJUST__SHIFT 0xb +#define TD_CNTL__PRECISION_COMPATIBILITY_MASK 0x8000 +#define TD_CNTL__PRECISION_COMPATIBILITY__SHIFT 0xf +#define TD_CNTL__GATHER4_FLOAT_MODE_MASK 0x10000 +#define TD_CNTL__GATHER4_FLOAT_MODE__SHIFT 0x10 +#define TD_CNTL__LD_FLOAT_MODE_MASK 0x40000 +#define TD_CNTL__LD_FLOAT_MODE__SHIFT 0x12 +#define TD_CNTL__GATHER4_DX9_MODE_MASK 0x80000 +#define TD_CNTL__GATHER4_DX9_MODE__SHIFT 0x13 +#define TD_CNTL__DISABLE_POWER_THROTTLE_MASK 0x100000 +#define TD_CNTL__DISABLE_POWER_THROTTLE__SHIFT 0x14 +#define TD_STATUS__BUSY_MASK 0x80000000 +#define TD_STATUS__BUSY__SHIFT 0x1f +#define TD_DEBUG_INDEX__INDEX_MASK 0x1f +#define TD_DEBUG_INDEX__INDEX__SHIFT 0x0 +#define TD_DEBUG_DATA__DATA_MASK 0xffffffff +#define TD_DEBUG_DATA__DATA__SHIFT 0x0 +#define TD_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0xff +#define TD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define TD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x3fc00 +#define TD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define TD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define TD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define TD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define TD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define TD_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define TD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define TD_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff +#define TD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define TD_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x3fc00 +#define TD_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define TD_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define TD_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define TD_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000 +#define TD_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18 +#define TD_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define TD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0xff +#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x3fc00 +#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000 +#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18 +#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c +#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TD_SCRATCH__SCRATCH_MASK 0xffffffff +#define TD_SCRATCH__SCRATCH__SHIFT 0x0 +#define TA_CNTL__TC_DATA_CREDIT_MASK 0xe000 +#define TA_CNTL__TC_DATA_CREDIT__SHIFT 0xd +#define TA_CNTL__ALIGNER_CREDIT_MASK 0x1f0000 +#define TA_CNTL__ALIGNER_CREDIT__SHIFT 0x10 +#define TA_CNTL__TD_FIFO_CREDIT_MASK 0xffc00000 +#define TA_CNTL__TD_FIFO_CREDIT__SHIFT 0x16 +#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N_MASK 0x1 +#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N__SHIFT 0x0 +#define TA_CNTL_AUX__RESERVED_MASK 0xe +#define TA_CNTL_AUX__RESERVED__SHIFT 0x1 +#define TA_CNTL_AUX__ANISO_WEIGHT_MODE_MASK 0x10000 +#define TA_CNTL_AUX__ANISO_WEIGHT_MODE__SHIFT 0x10 +#define TA_RESERVED_010C__Unused_MASK 0xffffffff +#define TA_RESERVED_010C__Unused__SHIFT 0x0 +#define TA_CS_BC_BASE_ADDR__ADDRESS_MASK 0xffffffff +#define TA_CS_BC_BASE_ADDR__ADDRESS__SHIFT 0x0 +#define TA_CS_BC_BASE_ADDR_HI__ADDRESS_MASK 0xff +#define TA_CS_BC_BASE_ADDR_HI__ADDRESS__SHIFT 0x0 +#define TA_STATUS__FG_PFIFO_EMPTYB_MASK 0x1000 +#define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT 0xc +#define TA_STATUS__FG_LFIFO_EMPTYB_MASK 0x2000 +#define TA_STATUS__FG_LFIFO_EMPTYB__SHIFT 0xd +#define TA_STATUS__FG_SFIFO_EMPTYB_MASK 0x4000 +#define TA_STATUS__FG_SFIFO_EMPTYB__SHIFT 0xe +#define TA_STATUS__FL_PFIFO_EMPTYB_MASK 0x10000 +#define TA_STATUS__FL_PFIFO_EMPTYB__SHIFT 0x10 +#define TA_STATUS__FL_LFIFO_EMPTYB_MASK 0x20000 +#define TA_STATUS__FL_LFIFO_EMPTYB__SHIFT 0x11 +#define TA_STATUS__FL_SFIFO_EMPTYB_MASK 0x40000 +#define TA_STATUS__FL_SFIFO_EMPTYB__SHIFT 0x12 +#define TA_STATUS__FA_PFIFO_EMPTYB_MASK 0x100000 +#define TA_STATUS__FA_PFIFO_EMPTYB__SHIFT 0x14 +#define TA_STATUS__FA_LFIFO_EMPTYB_MASK 0x200000 +#define TA_STATUS__FA_LFIFO_EMPTYB__SHIFT 0x15 +#define TA_STATUS__FA_SFIFO_EMPTYB_MASK 0x400000 +#define TA_STATUS__FA_SFIFO_EMPTYB__SHIFT 0x16 +#define TA_STATUS__IN_BUSY_MASK 0x1000000 +#define TA_STATUS__IN_BUSY__SHIFT 0x18 +#define TA_STATUS__FG_BUSY_MASK 0x2000000 +#define TA_STATUS__FG_BUSY__SHIFT 0x19 +#define TA_STATUS__LA_BUSY_MASK 0x4000000 +#define TA_STATUS__LA_BUSY__SHIFT 0x1a +#define TA_STATUS__FL_BUSY_MASK 0x8000000 +#define TA_STATUS__FL_BUSY__SHIFT 0x1b +#define TA_STATUS__TA_BUSY_MASK 0x10000000 +#define TA_STATUS__TA_BUSY__SHIFT 0x1c +#define TA_STATUS__FA_BUSY_MASK 0x20000000 +#define TA_STATUS__FA_BUSY__SHIFT 0x1d +#define TA_STATUS__AL_BUSY_MASK 0x40000000 +#define TA_STATUS__AL_BUSY__SHIFT 0x1e +#define TA_STATUS__BUSY_MASK 0x80000000 +#define TA_STATUS__BUSY__SHIFT 0x1f +#define TA_DEBUG_INDEX__INDEX_MASK 0x1f +#define TA_DEBUG_INDEX__INDEX__SHIFT 0x0 +#define TA_DEBUG_DATA__DATA_MASK 0xffffffff +#define TA_DEBUG_DATA__DATA__SHIFT 0x0 +#define TA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0xff +#define TA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define TA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0x3fc00 +#define TA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define TA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define TA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define TA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define TA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define TA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define TA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define TA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff +#define TA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define TA_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0x3fc00 +#define TA_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define TA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define TA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define TA_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000 +#define TA_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18 +#define TA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define TA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0xff +#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0x3fc00 +#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000 +#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18 +#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c +#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TA_SCRATCH__SCRATCH_MASK 0xffffffff +#define TA_SCRATCH__SCRATCH__SHIFT 0x0 +#define SH_HIDDEN_PRIVATE_BASE_VMID__ADDRESS_MASK 0xffffffff +#define SH_HIDDEN_PRIVATE_BASE_VMID__ADDRESS__SHIFT 0x0 +#define SH_STATIC_MEM_CONFIG__SWIZZLE_ENABLE_MASK 0x1 +#define SH_STATIC_MEM_CONFIG__SWIZZLE_ENABLE__SHIFT 0x0 +#define SH_STATIC_MEM_CONFIG__ELEMENT_SIZE_MASK 0x6 +#define SH_STATIC_MEM_CONFIG__ELEMENT_SIZE__SHIFT 0x1 +#define SH_STATIC_MEM_CONFIG__INDEX_STRIDE_MASK 0x18 +#define SH_STATIC_MEM_CONFIG__INDEX_STRIDE__SHIFT 0x3 +#define SH_STATIC_MEM_CONFIG__PRIVATE_MTYPE_MASK 0xe0 +#define SH_STATIC_MEM_CONFIG__PRIVATE_MTYPE__SHIFT 0x5 +#define SH_STATIC_MEM_CONFIG__READ_ONLY_CNTL_MASK 0xff00 +#define SH_STATIC_MEM_CONFIG__READ_ONLY_CNTL__SHIFT 0x8 +#define TCP_INVALIDATE__START_MASK 0x1 +#define TCP_INVALIDATE__START__SHIFT 0x0 +#define TCP_STATUS__TCP_BUSY_MASK 0x1 +#define TCP_STATUS__TCP_BUSY__SHIFT 0x0 +#define TCP_CNTL__FORCE_HIT_MASK 0x1 +#define TCP_CNTL__FORCE_HIT__SHIFT 0x0 +#define TCP_CNTL__FORCE_MISS_MASK 0x2 +#define TCP_CNTL__FORCE_MISS__SHIFT 0x1 +#define TCP_CNTL__L1_SIZE_MASK 0xc +#define TCP_CNTL__L1_SIZE__SHIFT 0x2 +#define TCP_CNTL__FLAT_BUF_HASH_ENABLE_MASK 0x10 +#define TCP_CNTL__FLAT_BUF_HASH_ENABLE__SHIFT 0x4 +#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE_MASK 0x20 +#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE__SHIFT 0x5 +#define TCP_CNTL__FORCE_EOW_TOTAL_CNT_MASK 0x1f8000 +#define TCP_CNTL__FORCE_EOW_TOTAL_CNT__SHIFT 0xf +#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT_MASK 0xfc00000 +#define TCP_CNTL__FORCE_EOW_TAGRAM_CNT__SHIFT 0x16 +#define TCP_CNTL__DISABLE_Z_MAP_MASK 0x10000000 +#define TCP_CNTL__DISABLE_Z_MAP__SHIFT 0x1c +#define TCP_CNTL__INV_ALL_VMIDS_MASK 0x20000000 +#define TCP_CNTL__INV_ALL_VMIDS__SHIFT 0x1d +#define TCP_CHAN_STEER_LO__CHAN0_MASK 0xf +#define TCP_CHAN_STEER_LO__CHAN0__SHIFT 0x0 +#define TCP_CHAN_STEER_LO__CHAN1_MASK 0xf0 +#define TCP_CHAN_STEER_LO__CHAN1__SHIFT 0x4 +#define TCP_CHAN_STEER_LO__CHAN2_MASK 0xf00 +#define TCP_CHAN_STEER_LO__CHAN2__SHIFT 0x8 +#define TCP_CHAN_STEER_LO__CHAN3_MASK 0xf000 +#define TCP_CHAN_STEER_LO__CHAN3__SHIFT 0xc +#define TCP_CHAN_STEER_LO__CHAN4_MASK 0xf0000 +#define TCP_CHAN_STEER_LO__CHAN4__SHIFT 0x10 +#define TCP_CHAN_STEER_LO__CHAN5_MASK 0xf00000 +#define TCP_CHAN_STEER_LO__CHAN5__SHIFT 0x14 +#define TCP_CHAN_STEER_LO__CHAN6_MASK 0xf000000 +#define TCP_CHAN_STEER_LO__CHAN6__SHIFT 0x18 +#define TCP_CHAN_STEER_LO__CHAN7_MASK 0xf0000000 +#define TCP_CHAN_STEER_LO__CHAN7__SHIFT 0x1c +#define TCP_CHAN_STEER_HI__CHAN8_MASK 0xf +#define TCP_CHAN_STEER_HI__CHAN8__SHIFT 0x0 +#define TCP_CHAN_STEER_HI__CHAN9_MASK 0xf0 +#define TCP_CHAN_STEER_HI__CHAN9__SHIFT 0x4 +#define TCP_CHAN_STEER_HI__CHANA_MASK 0xf00 +#define TCP_CHAN_STEER_HI__CHANA__SHIFT 0x8 +#define TCP_CHAN_STEER_HI__CHANB_MASK 0xf000 +#define TCP_CHAN_STEER_HI__CHANB__SHIFT 0xc +#define TCP_CHAN_STEER_HI__CHANC_MASK 0xf0000 +#define TCP_CHAN_STEER_HI__CHANC__SHIFT 0x10 +#define TCP_CHAN_STEER_HI__CHAND_MASK 0xf00000 +#define TCP_CHAN_STEER_HI__CHAND__SHIFT 0x14 +#define TCP_CHAN_STEER_HI__CHANE_MASK 0xf000000 +#define TCP_CHAN_STEER_HI__CHANE__SHIFT 0x18 +#define TCP_CHAN_STEER_HI__CHANF_MASK 0xf0000000 +#define TCP_CHAN_STEER_HI__CHANF__SHIFT 0x1c +#define TCP_ADDR_CONFIG__NUM_TCC_BANKS_MASK 0xf +#define TCP_ADDR_CONFIG__NUM_TCC_BANKS__SHIFT 0x0 +#define TCP_ADDR_CONFIG__NUM_BANKS_MASK 0x30 +#define TCP_ADDR_CONFIG__NUM_BANKS__SHIFT 0x4 +#define TCP_ADDR_CONFIG__COLHI_WIDTH_MASK 0x1c0 +#define TCP_ADDR_CONFIG__COLHI_WIDTH__SHIFT 0x6 +#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI_MASK 0x200 +#define TCP_ADDR_CONFIG__RB_SPLIT_COLHI__SHIFT 0x9 +#define TCP_CREDIT__LFIFO_CREDIT_MASK 0x3ff +#define TCP_CREDIT__LFIFO_CREDIT__SHIFT 0x0 +#define TCP_CREDIT__REQ_FIFO_CREDIT_MASK 0x7f0000 +#define TCP_CREDIT__REQ_FIFO_CREDIT__SHIFT 0x10 +#define TCP_CREDIT__TD_CREDIT_MASK 0xe0000000 +#define TCP_CREDIT__TD_CREDIT__SHIFT 0x1d +#define TCP_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define TCP_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define TCP_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCP_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define TCP_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define TCP_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00 +#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000 +#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18 +#define TCP_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCP_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000 +#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18 +#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c +#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff +#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0 +#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00 +#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa +#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf000000 +#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x18 +#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x1c +#define TCP_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0x3ff +#define TCP_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCP_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCP_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define TCP_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0x3ff +#define TCP_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define TCP_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define TCP_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS_MASK 0x7 +#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_BITS__SHIFT 0x0 +#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS_MASK 0x700 +#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_BITS__SHIFT 0x8 +#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT_MASK 0x70000 +#define TCP_BUFFER_ADDR_HASH_CNTL__CHANNEL_XOR_COUNT__SHIFT 0x10 +#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT_MASK 0x7000000 +#define TCP_BUFFER_ADDR_HASH_CNTL__BANK_XOR_COUNT__SHIFT 0x18 +#define TCP_EDC_COUNTER__SEC_COUNT_MASK 0xf +#define TCP_EDC_COUNTER__SEC_COUNT__SHIFT 0x0 +#define TCP_EDC_COUNTER__DED_COUNT_MASK 0xf0000 +#define TCP_EDC_COUNTER__DED_COUNT__SHIFT 0x10 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_0_MASK 0x3 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_0__SHIFT 0x0 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_1_MASK 0xc +#define TC_CFG_L1_LOAD_POLICY0__POLICY_1__SHIFT 0x2 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_2_MASK 0x30 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_2__SHIFT 0x4 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_3_MASK 0xc0 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_3__SHIFT 0x6 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_4_MASK 0x300 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_4__SHIFT 0x8 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_5_MASK 0xc00 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_5__SHIFT 0xa +#define TC_CFG_L1_LOAD_POLICY0__POLICY_6_MASK 0x3000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_6__SHIFT 0xc +#define TC_CFG_L1_LOAD_POLICY0__POLICY_7_MASK 0xc000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_7__SHIFT 0xe +#define TC_CFG_L1_LOAD_POLICY0__POLICY_8_MASK 0x30000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_8__SHIFT 0x10 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_9_MASK 0xc0000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_9__SHIFT 0x12 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_10_MASK 0x300000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_10__SHIFT 0x14 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_11_MASK 0xc00000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_11__SHIFT 0x16 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_12_MASK 0x3000000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_12__SHIFT 0x18 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_13_MASK 0xc000000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_13__SHIFT 0x1a +#define TC_CFG_L1_LOAD_POLICY0__POLICY_14_MASK 0x30000000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_14__SHIFT 0x1c +#define TC_CFG_L1_LOAD_POLICY0__POLICY_15_MASK 0xc0000000 +#define TC_CFG_L1_LOAD_POLICY0__POLICY_15__SHIFT 0x1e +#define TC_CFG_L1_LOAD_POLICY1__POLICY_16_MASK 0x3 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_16__SHIFT 0x0 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_17_MASK 0xc +#define TC_CFG_L1_LOAD_POLICY1__POLICY_17__SHIFT 0x2 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_18_MASK 0x30 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_18__SHIFT 0x4 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_19_MASK 0xc0 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_19__SHIFT 0x6 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_20_MASK 0x300 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_20__SHIFT 0x8 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_21_MASK 0xc00 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_21__SHIFT 0xa +#define TC_CFG_L1_LOAD_POLICY1__POLICY_22_MASK 0x3000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_22__SHIFT 0xc +#define TC_CFG_L1_LOAD_POLICY1__POLICY_23_MASK 0xc000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_23__SHIFT 0xe +#define TC_CFG_L1_LOAD_POLICY1__POLICY_24_MASK 0x30000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_24__SHIFT 0x10 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_25_MASK 0xc0000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_25__SHIFT 0x12 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_26_MASK 0x300000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_26__SHIFT 0x14 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_27_MASK 0xc00000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_27__SHIFT 0x16 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_28_MASK 0x3000000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_28__SHIFT 0x18 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_29_MASK 0xc000000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_29__SHIFT 0x1a +#define TC_CFG_L1_LOAD_POLICY1__POLICY_30_MASK 0x30000000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_30__SHIFT 0x1c +#define TC_CFG_L1_LOAD_POLICY1__POLICY_31_MASK 0xc0000000 +#define TC_CFG_L1_LOAD_POLICY1__POLICY_31__SHIFT 0x1e +#define TC_CFG_L1_STORE_POLICY__POLICY_0_MASK 0x1 +#define TC_CFG_L1_STORE_POLICY__POLICY_0__SHIFT 0x0 +#define TC_CFG_L1_STORE_POLICY__POLICY_1_MASK 0x2 +#define TC_CFG_L1_STORE_POLICY__POLICY_1__SHIFT 0x1 +#define TC_CFG_L1_STORE_POLICY__POLICY_2_MASK 0x4 +#define TC_CFG_L1_STORE_POLICY__POLICY_2__SHIFT 0x2 +#define TC_CFG_L1_STORE_POLICY__POLICY_3_MASK 0x8 +#define TC_CFG_L1_STORE_POLICY__POLICY_3__SHIFT 0x3 +#define TC_CFG_L1_STORE_POLICY__POLICY_4_MASK 0x10 +#define TC_CFG_L1_STORE_POLICY__POLICY_4__SHIFT 0x4 +#define TC_CFG_L1_STORE_POLICY__POLICY_5_MASK 0x20 +#define TC_CFG_L1_STORE_POLICY__POLICY_5__SHIFT 0x5 +#define TC_CFG_L1_STORE_POLICY__POLICY_6_MASK 0x40 +#define TC_CFG_L1_STORE_POLICY__POLICY_6__SHIFT 0x6 +#define TC_CFG_L1_STORE_POLICY__POLICY_7_MASK 0x80 +#define TC_CFG_L1_STORE_POLICY__POLICY_7__SHIFT 0x7 +#define TC_CFG_L1_STORE_POLICY__POLICY_8_MASK 0x100 +#define TC_CFG_L1_STORE_POLICY__POLICY_8__SHIFT 0x8 +#define TC_CFG_L1_STORE_POLICY__POLICY_9_MASK 0x200 +#define TC_CFG_L1_STORE_POLICY__POLICY_9__SHIFT 0x9 +#define TC_CFG_L1_STORE_POLICY__POLICY_10_MASK 0x400 +#define TC_CFG_L1_STORE_POLICY__POLICY_10__SHIFT 0xa +#define TC_CFG_L1_STORE_POLICY__POLICY_11_MASK 0x800 +#define TC_CFG_L1_STORE_POLICY__POLICY_11__SHIFT 0xb +#define TC_CFG_L1_STORE_POLICY__POLICY_12_MASK 0x1000 +#define TC_CFG_L1_STORE_POLICY__POLICY_12__SHIFT 0xc +#define TC_CFG_L1_STORE_POLICY__POLICY_13_MASK 0x2000 +#define TC_CFG_L1_STORE_POLICY__POLICY_13__SHIFT 0xd +#define TC_CFG_L1_STORE_POLICY__POLICY_14_MASK 0x4000 +#define TC_CFG_L1_STORE_POLICY__POLICY_14__SHIFT 0xe +#define TC_CFG_L1_STORE_POLICY__POLICY_15_MASK 0x8000 +#define TC_CFG_L1_STORE_POLICY__POLICY_15__SHIFT 0xf +#define TC_CFG_L1_STORE_POLICY__POLICY_16_MASK 0x10000 +#define TC_CFG_L1_STORE_POLICY__POLICY_16__SHIFT 0x10 +#define TC_CFG_L1_STORE_POLICY__POLICY_17_MASK 0x20000 +#define TC_CFG_L1_STORE_POLICY__POLICY_17__SHIFT 0x11 +#define TC_CFG_L1_STORE_POLICY__POLICY_18_MASK 0x40000 +#define TC_CFG_L1_STORE_POLICY__POLICY_18__SHIFT 0x12 +#define TC_CFG_L1_STORE_POLICY__POLICY_19_MASK 0x80000 +#define TC_CFG_L1_STORE_POLICY__POLICY_19__SHIFT 0x13 +#define TC_CFG_L1_STORE_POLICY__POLICY_20_MASK 0x100000 +#define TC_CFG_L1_STORE_POLICY__POLICY_20__SHIFT 0x14 +#define TC_CFG_L1_STORE_POLICY__POLICY_21_MASK 0x200000 +#define TC_CFG_L1_STORE_POLICY__POLICY_21__SHIFT 0x15 +#define TC_CFG_L1_STORE_POLICY__POLICY_22_MASK 0x400000 +#define TC_CFG_L1_STORE_POLICY__POLICY_22__SHIFT 0x16 +#define TC_CFG_L1_STORE_POLICY__POLICY_23_MASK 0x800000 +#define TC_CFG_L1_STORE_POLICY__POLICY_23__SHIFT 0x17 +#define TC_CFG_L1_STORE_POLICY__POLICY_24_MASK 0x1000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_24__SHIFT 0x18 +#define TC_CFG_L1_STORE_POLICY__POLICY_25_MASK 0x2000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_25__SHIFT 0x19 +#define TC_CFG_L1_STORE_POLICY__POLICY_26_MASK 0x4000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_26__SHIFT 0x1a +#define TC_CFG_L1_STORE_POLICY__POLICY_27_MASK 0x8000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_27__SHIFT 0x1b +#define TC_CFG_L1_STORE_POLICY__POLICY_28_MASK 0x10000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_28__SHIFT 0x1c +#define TC_CFG_L1_STORE_POLICY__POLICY_29_MASK 0x20000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_29__SHIFT 0x1d +#define TC_CFG_L1_STORE_POLICY__POLICY_30_MASK 0x40000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_30__SHIFT 0x1e +#define TC_CFG_L1_STORE_POLICY__POLICY_31_MASK 0x80000000 +#define TC_CFG_L1_STORE_POLICY__POLICY_31__SHIFT 0x1f +#define TC_CFG_L2_LOAD_POLICY0__POLICY_0_MASK 0x3 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_0__SHIFT 0x0 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_1_MASK 0xc +#define TC_CFG_L2_LOAD_POLICY0__POLICY_1__SHIFT 0x2 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_2_MASK 0x30 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_2__SHIFT 0x4 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_3_MASK 0xc0 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_3__SHIFT 0x6 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_4_MASK 0x300 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_4__SHIFT 0x8 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_5_MASK 0xc00 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_5__SHIFT 0xa +#define TC_CFG_L2_LOAD_POLICY0__POLICY_6_MASK 0x3000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_6__SHIFT 0xc +#define TC_CFG_L2_LOAD_POLICY0__POLICY_7_MASK 0xc000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_7__SHIFT 0xe +#define TC_CFG_L2_LOAD_POLICY0__POLICY_8_MASK 0x30000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_8__SHIFT 0x10 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_9_MASK 0xc0000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_9__SHIFT 0x12 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_10_MASK 0x300000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_10__SHIFT 0x14 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_11_MASK 0xc00000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_11__SHIFT 0x16 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_12_MASK 0x3000000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_12__SHIFT 0x18 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_13_MASK 0xc000000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_13__SHIFT 0x1a +#define TC_CFG_L2_LOAD_POLICY0__POLICY_14_MASK 0x30000000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_14__SHIFT 0x1c +#define TC_CFG_L2_LOAD_POLICY0__POLICY_15_MASK 0xc0000000 +#define TC_CFG_L2_LOAD_POLICY0__POLICY_15__SHIFT 0x1e +#define TC_CFG_L2_LOAD_POLICY1__POLICY_16_MASK 0x3 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_16__SHIFT 0x0 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_17_MASK 0xc +#define TC_CFG_L2_LOAD_POLICY1__POLICY_17__SHIFT 0x2 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_18_MASK 0x30 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_18__SHIFT 0x4 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_19_MASK 0xc0 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_19__SHIFT 0x6 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_20_MASK 0x300 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_20__SHIFT 0x8 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_21_MASK 0xc00 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_21__SHIFT 0xa +#define TC_CFG_L2_LOAD_POLICY1__POLICY_22_MASK 0x3000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_22__SHIFT 0xc +#define TC_CFG_L2_LOAD_POLICY1__POLICY_23_MASK 0xc000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_23__SHIFT 0xe +#define TC_CFG_L2_LOAD_POLICY1__POLICY_24_MASK 0x30000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_24__SHIFT 0x10 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_25_MASK 0xc0000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_25__SHIFT 0x12 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_26_MASK 0x300000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_26__SHIFT 0x14 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_27_MASK 0xc00000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_27__SHIFT 0x16 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_28_MASK 0x3000000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_28__SHIFT 0x18 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_29_MASK 0xc000000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_29__SHIFT 0x1a +#define TC_CFG_L2_LOAD_POLICY1__POLICY_30_MASK 0x30000000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_30__SHIFT 0x1c +#define TC_CFG_L2_LOAD_POLICY1__POLICY_31_MASK 0xc0000000 +#define TC_CFG_L2_LOAD_POLICY1__POLICY_31__SHIFT 0x1e +#define TC_CFG_L2_STORE_POLICY0__POLICY_0_MASK 0x3 +#define TC_CFG_L2_STORE_POLICY0__POLICY_0__SHIFT 0x0 +#define TC_CFG_L2_STORE_POLICY0__POLICY_1_MASK 0xc +#define TC_CFG_L2_STORE_POLICY0__POLICY_1__SHIFT 0x2 +#define TC_CFG_L2_STORE_POLICY0__POLICY_2_MASK 0x30 +#define TC_CFG_L2_STORE_POLICY0__POLICY_2__SHIFT 0x4 +#define TC_CFG_L2_STORE_POLICY0__POLICY_3_MASK 0xc0 +#define TC_CFG_L2_STORE_POLICY0__POLICY_3__SHIFT 0x6 +#define TC_CFG_L2_STORE_POLICY0__POLICY_4_MASK 0x300 +#define TC_CFG_L2_STORE_POLICY0__POLICY_4__SHIFT 0x8 +#define TC_CFG_L2_STORE_POLICY0__POLICY_5_MASK 0xc00 +#define TC_CFG_L2_STORE_POLICY0__POLICY_5__SHIFT 0xa +#define TC_CFG_L2_STORE_POLICY0__POLICY_6_MASK 0x3000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_6__SHIFT 0xc +#define TC_CFG_L2_STORE_POLICY0__POLICY_7_MASK 0xc000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_7__SHIFT 0xe +#define TC_CFG_L2_STORE_POLICY0__POLICY_8_MASK 0x30000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_8__SHIFT 0x10 +#define TC_CFG_L2_STORE_POLICY0__POLICY_9_MASK 0xc0000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_9__SHIFT 0x12 +#define TC_CFG_L2_STORE_POLICY0__POLICY_10_MASK 0x300000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_10__SHIFT 0x14 +#define TC_CFG_L2_STORE_POLICY0__POLICY_11_MASK 0xc00000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_11__SHIFT 0x16 +#define TC_CFG_L2_STORE_POLICY0__POLICY_12_MASK 0x3000000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_12__SHIFT 0x18 +#define TC_CFG_L2_STORE_POLICY0__POLICY_13_MASK 0xc000000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_13__SHIFT 0x1a +#define TC_CFG_L2_STORE_POLICY0__POLICY_14_MASK 0x30000000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_14__SHIFT 0x1c +#define TC_CFG_L2_STORE_POLICY0__POLICY_15_MASK 0xc0000000 +#define TC_CFG_L2_STORE_POLICY0__POLICY_15__SHIFT 0x1e +#define TC_CFG_L2_STORE_POLICY1__POLICY_16_MASK 0x3 +#define TC_CFG_L2_STORE_POLICY1__POLICY_16__SHIFT 0x0 +#define TC_CFG_L2_STORE_POLICY1__POLICY_17_MASK 0xc +#define TC_CFG_L2_STORE_POLICY1__POLICY_17__SHIFT 0x2 +#define TC_CFG_L2_STORE_POLICY1__POLICY_18_MASK 0x30 +#define TC_CFG_L2_STORE_POLICY1__POLICY_18__SHIFT 0x4 +#define TC_CFG_L2_STORE_POLICY1__POLICY_19_MASK 0xc0 +#define TC_CFG_L2_STORE_POLICY1__POLICY_19__SHIFT 0x6 +#define TC_CFG_L2_STORE_POLICY1__POLICY_20_MASK 0x300 +#define TC_CFG_L2_STORE_POLICY1__POLICY_20__SHIFT 0x8 +#define TC_CFG_L2_STORE_POLICY1__POLICY_21_MASK 0xc00 +#define TC_CFG_L2_STORE_POLICY1__POLICY_21__SHIFT 0xa +#define TC_CFG_L2_STORE_POLICY1__POLICY_22_MASK 0x3000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_22__SHIFT 0xc +#define TC_CFG_L2_STORE_POLICY1__POLICY_23_MASK 0xc000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_23__SHIFT 0xe +#define TC_CFG_L2_STORE_POLICY1__POLICY_24_MASK 0x30000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_24__SHIFT 0x10 +#define TC_CFG_L2_STORE_POLICY1__POLICY_25_MASK 0xc0000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_25__SHIFT 0x12 +#define TC_CFG_L2_STORE_POLICY1__POLICY_26_MASK 0x300000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_26__SHIFT 0x14 +#define TC_CFG_L2_STORE_POLICY1__POLICY_27_MASK 0xc00000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_27__SHIFT 0x16 +#define TC_CFG_L2_STORE_POLICY1__POLICY_28_MASK 0x3000000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_28__SHIFT 0x18 +#define TC_CFG_L2_STORE_POLICY1__POLICY_29_MASK 0xc000000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_29__SHIFT 0x1a +#define TC_CFG_L2_STORE_POLICY1__POLICY_30_MASK 0x30000000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_30__SHIFT 0x1c +#define TC_CFG_L2_STORE_POLICY1__POLICY_31_MASK 0xc0000000 +#define TC_CFG_L2_STORE_POLICY1__POLICY_31__SHIFT 0x1e +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0_MASK 0x3 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_0__SHIFT 0x0 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1_MASK 0xc +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_1__SHIFT 0x2 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2_MASK 0x30 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_2__SHIFT 0x4 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3_MASK 0xc0 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_3__SHIFT 0x6 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4_MASK 0x300 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_4__SHIFT 0x8 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5_MASK 0xc00 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_5__SHIFT 0xa +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6_MASK 0x3000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_6__SHIFT 0xc +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7_MASK 0xc000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_7__SHIFT 0xe +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8_MASK 0x30000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_8__SHIFT 0x10 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9_MASK 0xc0000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_9__SHIFT 0x12 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10_MASK 0x300000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_10__SHIFT 0x14 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11_MASK 0xc00000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_11__SHIFT 0x16 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12_MASK 0x3000000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_12__SHIFT 0x18 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13_MASK 0xc000000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_13__SHIFT 0x1a +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14_MASK 0x30000000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_14__SHIFT 0x1c +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15_MASK 0xc0000000 +#define TC_CFG_L2_ATOMIC_POLICY__POLICY_15__SHIFT 0x1e +#define TC_CFG_L1_VOLATILE__VOL_MASK 0xf +#define TC_CFG_L1_VOLATILE__VOL__SHIFT 0x0 +#define TC_CFG_L2_VOLATILE__VOL_MASK 0xf +#define TC_CFG_L2_VOLATILE__VOL__SHIFT 0x0 +#define TCP_WATCH0_ADDR_H__ADDR_MASK 0xffff +#define TCP_WATCH0_ADDR_H__ADDR__SHIFT 0x0 +#define TCP_WATCH1_ADDR_H__ADDR_MASK 0xffff +#define TCP_WATCH1_ADDR_H__ADDR__SHIFT 0x0 +#define TCP_WATCH2_ADDR_H__ADDR_MASK 0xffff +#define TCP_WATCH2_ADDR_H__ADDR__SHIFT 0x0 +#define TCP_WATCH3_ADDR_H__ADDR_MASK 0xffff +#define TCP_WATCH3_ADDR_H__ADDR__SHIFT 0x0 +#define TCP_WATCH0_ADDR_L__ADDR_MASK 0xffffffc0 +#define TCP_WATCH0_ADDR_L__ADDR__SHIFT 0x6 +#define TCP_WATCH1_ADDR_L__ADDR_MASK 0xffffffc0 +#define TCP_WATCH1_ADDR_L__ADDR__SHIFT 0x6 +#define TCP_WATCH2_ADDR_L__ADDR_MASK 0xffffffc0 +#define TCP_WATCH2_ADDR_L__ADDR__SHIFT 0x6 +#define TCP_WATCH3_ADDR_L__ADDR_MASK 0xffffffc0 +#define TCP_WATCH3_ADDR_L__ADDR__SHIFT 0x6 +#define TCP_WATCH0_CNTL__MASK_MASK 0xffffff +#define TCP_WATCH0_CNTL__MASK__SHIFT 0x0 +#define TCP_WATCH0_CNTL__VMID_MASK 0xf000000 +#define TCP_WATCH0_CNTL__VMID__SHIFT 0x18 +#define TCP_WATCH0_CNTL__MODE_MASK 0x60000000 +#define TCP_WATCH0_CNTL__MODE__SHIFT 0x1d +#define TCP_WATCH0_CNTL__VALID_MASK 0x80000000 +#define TCP_WATCH0_CNTL__VALID__SHIFT 0x1f +#define TCP_WATCH1_CNTL__MASK_MASK 0xffffff +#define TCP_WATCH1_CNTL__MASK__SHIFT 0x0 +#define TCP_WATCH1_CNTL__VMID_MASK 0xf000000 +#define TCP_WATCH1_CNTL__VMID__SHIFT 0x18 +#define TCP_WATCH1_CNTL__MODE_MASK 0x60000000 +#define TCP_WATCH1_CNTL__MODE__SHIFT 0x1d +#define TCP_WATCH1_CNTL__VALID_MASK 0x80000000 +#define TCP_WATCH1_CNTL__VALID__SHIFT 0x1f +#define TCP_WATCH2_CNTL__MASK_MASK 0xffffff +#define TCP_WATCH2_CNTL__MASK__SHIFT 0x0 +#define TCP_WATCH2_CNTL__VMID_MASK 0xf000000 +#define TCP_WATCH2_CNTL__VMID__SHIFT 0x18 +#define TCP_WATCH2_CNTL__MODE_MASK 0x60000000 +#define TCP_WATCH2_CNTL__MODE__SHIFT 0x1d +#define TCP_WATCH2_CNTL__VALID_MASK 0x80000000 +#define TCP_WATCH2_CNTL__VALID__SHIFT 0x1f +#define TCP_WATCH3_CNTL__MASK_MASK 0xffffff +#define TCP_WATCH3_CNTL__MASK__SHIFT 0x0 +#define TCP_WATCH3_CNTL__VMID_MASK 0xf000000 +#define TCP_WATCH3_CNTL__VMID__SHIFT 0x18 +#define TCP_WATCH3_CNTL__MODE_MASK 0x60000000 +#define TCP_WATCH3_CNTL__MODE__SHIFT 0x1d +#define TCP_WATCH3_CNTL__VALID_MASK 0x80000000 +#define TCP_WATCH3_CNTL__VALID__SHIFT 0x1f +#define TD_CGTT_CTRL__ON_DELAY_MASK 0xf +#define TD_CGTT_CTRL__ON_DELAY__SHIFT 0x0 +#define TD_CGTT_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define TD_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define TD_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define TD_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define TD_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define TD_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define TD_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define TD_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define TD_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define TD_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define TD_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define TA_CGTT_CTRL__ON_DELAY_MASK 0xf +#define TA_CGTT_CTRL__ON_DELAY__SHIFT 0x0 +#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_TCP_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_TCP_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_TCP_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_TCP_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_TCP_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_TCI_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_TCI_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_TCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_TCI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define TCI_STATUS__TCI_BUSY_MASK 0x1 +#define TCI_STATUS__TCI_BUSY__SHIFT 0x0 +#define TCI_CNTL_1__WBINVL1_NUM_CYCLES_MASK 0xffff +#define TCI_CNTL_1__WBINVL1_NUM_CYCLES__SHIFT 0x0 +#define TCI_CNTL_1__REQ_FIFO_DEPTH_MASK 0xff0000 +#define TCI_CNTL_1__REQ_FIFO_DEPTH__SHIFT 0x10 +#define TCI_CNTL_1__WDATA_RAM_DEPTH_MASK 0xff000000 +#define TCI_CNTL_1__WDATA_RAM_DEPTH__SHIFT 0x18 +#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2_MASK 0x1 +#define TCI_CNTL_2__L1_INVAL_ON_WBINVL2__SHIFT 0x0 +#define TCI_CNTL_2__TCA_MAX_CREDIT_MASK 0x1fe +#define TCI_CNTL_2__TCA_MAX_CREDIT__SHIFT 0x1 +#define GDS_CONFIG__SH0_GPR_PHASE_SEL_MASK 0x6 +#define GDS_CONFIG__SH0_GPR_PHASE_SEL__SHIFT 0x1 +#define GDS_CONFIG__SH1_GPR_PHASE_SEL_MASK 0x18 +#define GDS_CONFIG__SH1_GPR_PHASE_SEL__SHIFT 0x3 +#define GDS_CONFIG__SH2_GPR_PHASE_SEL_MASK 0x60 +#define GDS_CONFIG__SH2_GPR_PHASE_SEL__SHIFT 0x5 +#define GDS_CONFIG__SH3_GPR_PHASE_SEL_MASK 0x180 +#define GDS_CONFIG__SH3_GPR_PHASE_SEL__SHIFT 0x7 +#define GDS_CNTL_STATUS__GDS_BUSY_MASK 0x1 +#define GDS_CNTL_STATUS__GDS_BUSY__SHIFT 0x0 +#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY_MASK 0x2 +#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY__SHIFT 0x1 +#define GDS_CNTL_STATUS__ORD_APP_BUSY_MASK 0x4 +#define GDS_CNTL_STATUS__ORD_APP_BUSY__SHIFT 0x2 +#define GDS_CNTL_STATUS__DS_BANK_CONFLICT_MASK 0x8 +#define GDS_CNTL_STATUS__DS_BANK_CONFLICT__SHIFT 0x3 +#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT_MASK 0x10 +#define GDS_CNTL_STATUS__DS_ADDR_CONFLICT__SHIFT 0x4 +#define GDS_CNTL_STATUS__DS_WR_CLAMP_MASK 0x20 +#define GDS_CNTL_STATUS__DS_WR_CLAMP__SHIFT 0x5 +#define GDS_CNTL_STATUS__DS_RD_CLAMP_MASK 0x40 +#define GDS_CNTL_STATUS__DS_RD_CLAMP__SHIFT 0x6 +#define GDS_ENHANCE2__MISC_MASK 0xffff +#define GDS_ENHANCE2__MISC__SHIFT 0x0 +#define GDS_ENHANCE2__UNUSED_MASK 0xffff0000 +#define GDS_ENHANCE2__UNUSED__SHIFT 0x10 +#define GDS_PROTECTION_FAULT__WRITE_DIS_MASK 0x1 +#define GDS_PROTECTION_FAULT__WRITE_DIS__SHIFT 0x0 +#define GDS_PROTECTION_FAULT__FAULT_DETECTED_MASK 0x2 +#define GDS_PROTECTION_FAULT__FAULT_DETECTED__SHIFT 0x1 +#define GDS_PROTECTION_FAULT__GRBM_MASK 0x4 +#define GDS_PROTECTION_FAULT__GRBM__SHIFT 0x2 +#define GDS_PROTECTION_FAULT__SH_ID_MASK 0x38 +#define GDS_PROTECTION_FAULT__SH_ID__SHIFT 0x3 +#define GDS_PROTECTION_FAULT__CU_ID_MASK 0x3c0 +#define GDS_PROTECTION_FAULT__CU_ID__SHIFT 0x6 +#define GDS_PROTECTION_FAULT__SIMD_ID_MASK 0xc00 +#define GDS_PROTECTION_FAULT__SIMD_ID__SHIFT 0xa +#define GDS_PROTECTION_FAULT__WAVE_ID_MASK 0xf000 +#define GDS_PROTECTION_FAULT__WAVE_ID__SHIFT 0xc +#define GDS_PROTECTION_FAULT__ADDRESS_MASK 0xffff0000 +#define GDS_PROTECTION_FAULT__ADDRESS__SHIFT 0x10 +#define GDS_VM_PROTECTION_FAULT__WRITE_DIS_MASK 0x1 +#define GDS_VM_PROTECTION_FAULT__WRITE_DIS__SHIFT 0x0 +#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED_MASK 0x2 +#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED__SHIFT 0x1 +#define GDS_VM_PROTECTION_FAULT__GWS_MASK 0x4 +#define GDS_VM_PROTECTION_FAULT__GWS__SHIFT 0x2 +#define GDS_VM_PROTECTION_FAULT__OA_MASK 0x8 +#define GDS_VM_PROTECTION_FAULT__OA__SHIFT 0x3 +#define GDS_VM_PROTECTION_FAULT__GRBM_MASK 0x10 +#define GDS_VM_PROTECTION_FAULT__GRBM__SHIFT 0x4 +#define GDS_VM_PROTECTION_FAULT__VMID_MASK 0xf00 +#define GDS_VM_PROTECTION_FAULT__VMID__SHIFT 0x8 +#define GDS_VM_PROTECTION_FAULT__ADDRESS_MASK 0xffff0000 +#define GDS_VM_PROTECTION_FAULT__ADDRESS__SHIFT 0x10 +#define GDS_SECDED_CNT__DED_MASK 0xffff +#define GDS_SECDED_CNT__DED__SHIFT 0x0 +#define GDS_SECDED_CNT__SEC_MASK 0xffff0000 +#define GDS_SECDED_CNT__SEC__SHIFT 0x10 +#define GDS_GRBM_SECDED_CNT__DED_MASK 0xffff +#define GDS_GRBM_SECDED_CNT__DED__SHIFT 0x0 +#define GDS_GRBM_SECDED_CNT__SEC_MASK 0xffff0000 +#define GDS_GRBM_SECDED_CNT__SEC__SHIFT 0x10 +#define GDS_OA_DED__ME0_GFXHP3D_PIX_DED_MASK 0x1 +#define GDS_OA_DED__ME0_GFXHP3D_PIX_DED__SHIFT 0x0 +#define GDS_OA_DED__ME0_GFXHP3D_VTX_DED_MASK 0x2 +#define GDS_OA_DED__ME0_GFXHP3D_VTX_DED__SHIFT 0x1 +#define GDS_OA_DED__ME0_CS_DED_MASK 0x4 +#define GDS_OA_DED__ME0_CS_DED__SHIFT 0x2 +#define GDS_OA_DED__UNUSED0_MASK 0x8 +#define GDS_OA_DED__UNUSED0__SHIFT 0x3 +#define GDS_OA_DED__ME1_PIPE0_DED_MASK 0x10 +#define GDS_OA_DED__ME1_PIPE0_DED__SHIFT 0x4 +#define GDS_OA_DED__ME1_PIPE1_DED_MASK 0x20 +#define GDS_OA_DED__ME1_PIPE1_DED__SHIFT 0x5 +#define GDS_OA_DED__ME1_PIPE2_DED_MASK 0x40 +#define GDS_OA_DED__ME1_PIPE2_DED__SHIFT 0x6 +#define GDS_OA_DED__ME1_PIPE3_DED_MASK 0x80 +#define GDS_OA_DED__ME1_PIPE3_DED__SHIFT 0x7 +#define GDS_OA_DED__ME2_PIPE0_DED_MASK 0x100 +#define GDS_OA_DED__ME2_PIPE0_DED__SHIFT 0x8 +#define GDS_OA_DED__ME2_PIPE1_DED_MASK 0x200 +#define GDS_OA_DED__ME2_PIPE1_DED__SHIFT 0x9 +#define GDS_OA_DED__ME2_PIPE2_DED_MASK 0x400 +#define GDS_OA_DED__ME2_PIPE2_DED__SHIFT 0xa +#define GDS_OA_DED__ME2_PIPE3_DED_MASK 0x800 +#define GDS_OA_DED__ME2_PIPE3_DED__SHIFT 0xb +#define GDS_OA_DED__UNUSED1_MASK 0xfffff000 +#define GDS_OA_DED__UNUSED1__SHIFT 0xc +#define GDS_DEBUG_CNTL__GDS_DEBUG_INDX_MASK 0x1f +#define GDS_DEBUG_CNTL__GDS_DEBUG_INDX__SHIFT 0x0 +#define GDS_DEBUG_CNTL__UNUSED_MASK 0xffffffe0 +#define GDS_DEBUG_CNTL__UNUSED__SHIFT 0x5 +#define GDS_DEBUG_DATA__DATA_MASK 0xffffffff +#define GDS_DEBUG_DATA__DATA__SHIFT 0x0 +#define CGTT_GDS_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_GDS_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x2000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5_MASK 0x4000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5__SHIFT 0x1a +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0__SHIFT 0x1f +#define GDS_RD_ADDR__READ_ADDR_MASK 0xffffffff +#define GDS_RD_ADDR__READ_ADDR__SHIFT 0x0 +#define GDS_RD_DATA__READ_DATA_MASK 0xffffffff +#define GDS_RD_DATA__READ_DATA__SHIFT 0x0 +#define GDS_RD_BURST_ADDR__BURST_ADDR_MASK 0xffffffff +#define GDS_RD_BURST_ADDR__BURST_ADDR__SHIFT 0x0 +#define GDS_RD_BURST_COUNT__BURST_COUNT_MASK 0xffffffff +#define GDS_RD_BURST_COUNT__BURST_COUNT__SHIFT 0x0 +#define GDS_RD_BURST_DATA__BURST_DATA_MASK 0xffffffff +#define GDS_RD_BURST_DATA__BURST_DATA__SHIFT 0x0 +#define GDS_WR_ADDR__WRITE_ADDR_MASK 0xffffffff +#define GDS_WR_ADDR__WRITE_ADDR__SHIFT 0x0 +#define GDS_WR_DATA__WRITE_DATA_MASK 0xffffffff +#define GDS_WR_DATA__WRITE_DATA__SHIFT 0x0 +#define GDS_WR_BURST_ADDR__WRITE_ADDR_MASK 0xffffffff +#define GDS_WR_BURST_ADDR__WRITE_ADDR__SHIFT 0x0 +#define GDS_WR_BURST_DATA__WRITE_DATA_MASK 0xffffffff +#define GDS_WR_BURST_DATA__WRITE_DATA__SHIFT 0x0 +#define GDS_WRITE_COMPLETE__WRITE_COMPLETE_MASK 0xffffffff +#define GDS_WRITE_COMPLETE__WRITE_COMPLETE__SHIFT 0x0 +#define GDS_ATOM_CNTL__AINC_MASK 0x3f +#define GDS_ATOM_CNTL__AINC__SHIFT 0x0 +#define GDS_ATOM_CNTL__UNUSED1_MASK 0xc0 +#define GDS_ATOM_CNTL__UNUSED1__SHIFT 0x6 +#define GDS_ATOM_CNTL__DMODE_MASK 0x100 +#define GDS_ATOM_CNTL__DMODE__SHIFT 0x8 +#define GDS_ATOM_CNTL__UNUSED2_MASK 0xfffffe00 +#define GDS_ATOM_CNTL__UNUSED2__SHIFT 0x9 +#define GDS_ATOM_COMPLETE__COMPLETE_MASK 0x1 +#define GDS_ATOM_COMPLETE__COMPLETE__SHIFT 0x0 +#define GDS_ATOM_COMPLETE__UNUSED_MASK 0xfffffffe +#define GDS_ATOM_COMPLETE__UNUSED__SHIFT 0x1 +#define GDS_ATOM_BASE__BASE_MASK 0xffff +#define GDS_ATOM_BASE__BASE__SHIFT 0x0 +#define GDS_ATOM_BASE__UNUSED_MASK 0xffff0000 +#define GDS_ATOM_BASE__UNUSED__SHIFT 0x10 +#define GDS_ATOM_SIZE__SIZE_MASK 0xffff +#define GDS_ATOM_SIZE__SIZE__SHIFT 0x0 +#define GDS_ATOM_SIZE__UNUSED_MASK 0xffff0000 +#define GDS_ATOM_SIZE__UNUSED__SHIFT 0x10 +#define GDS_ATOM_OFFSET0__OFFSET0_MASK 0xff +#define GDS_ATOM_OFFSET0__OFFSET0__SHIFT 0x0 +#define GDS_ATOM_OFFSET0__UNUSED_MASK 0xffffff00 +#define GDS_ATOM_OFFSET0__UNUSED__SHIFT 0x8 +#define GDS_ATOM_OFFSET1__OFFSET1_MASK 0xff +#define GDS_ATOM_OFFSET1__OFFSET1__SHIFT 0x0 +#define GDS_ATOM_OFFSET1__UNUSED_MASK 0xffffff00 +#define GDS_ATOM_OFFSET1__UNUSED__SHIFT 0x8 +#define GDS_ATOM_DST__DST_MASK 0xffffffff +#define GDS_ATOM_DST__DST__SHIFT 0x0 +#define GDS_ATOM_OP__OP_MASK 0xff +#define GDS_ATOM_OP__OP__SHIFT 0x0 +#define GDS_ATOM_OP__UNUSED_MASK 0xffffff00 +#define GDS_ATOM_OP__UNUSED__SHIFT 0x8 +#define GDS_ATOM_SRC0__DATA_MASK 0xffffffff +#define GDS_ATOM_SRC0__DATA__SHIFT 0x0 +#define GDS_ATOM_SRC0_U__DATA_MASK 0xffffffff +#define GDS_ATOM_SRC0_U__DATA__SHIFT 0x0 +#define GDS_ATOM_SRC1__DATA_MASK 0xffffffff +#define GDS_ATOM_SRC1__DATA__SHIFT 0x0 +#define GDS_ATOM_SRC1_U__DATA_MASK 0xffffffff +#define GDS_ATOM_SRC1_U__DATA__SHIFT 0x0 +#define GDS_ATOM_READ0__DATA_MASK 0xffffffff +#define GDS_ATOM_READ0__DATA__SHIFT 0x0 +#define GDS_ATOM_READ0_U__DATA_MASK 0xffffffff +#define GDS_ATOM_READ0_U__DATA__SHIFT 0x0 +#define GDS_ATOM_READ1__DATA_MASK 0xffffffff +#define GDS_ATOM_READ1__DATA__SHIFT 0x0 +#define GDS_ATOM_READ1_U__DATA_MASK 0xffffffff +#define GDS_ATOM_READ1_U__DATA__SHIFT 0x0 +#define GDS_GWS_RESOURCE_CNTL__INDEX_MASK 0x3f +#define GDS_GWS_RESOURCE_CNTL__INDEX__SHIFT 0x0 +#define GDS_GWS_RESOURCE_CNTL__UNUSED_MASK 0xffffffc0 +#define GDS_GWS_RESOURCE_CNTL__UNUSED__SHIFT 0x6 +#define GDS_GWS_RESOURCE__FLAG_MASK 0x1 +#define GDS_GWS_RESOURCE__FLAG__SHIFT 0x0 +#define GDS_GWS_RESOURCE__COUNTER_MASK 0x1ffe +#define GDS_GWS_RESOURCE__COUNTER__SHIFT 0x1 +#define GDS_GWS_RESOURCE__TYPE_MASK 0x2000 +#define GDS_GWS_RESOURCE__TYPE__SHIFT 0xd +#define GDS_GWS_RESOURCE__DED_MASK 0x4000 +#define GDS_GWS_RESOURCE__DED__SHIFT 0xe +#define GDS_GWS_RESOURCE__RELEASE_ALL_MASK 0x8000 +#define GDS_GWS_RESOURCE__RELEASE_ALL__SHIFT 0xf +#define GDS_GWS_RESOURCE__HEAD_QUEUE_MASK 0x7ff0000 +#define GDS_GWS_RESOURCE__HEAD_QUEUE__SHIFT 0x10 +#define GDS_GWS_RESOURCE__HEAD_VALID_MASK 0x8000000 +#define GDS_GWS_RESOURCE__HEAD_VALID__SHIFT 0x1b +#define GDS_GWS_RESOURCE__HEAD_FLAG_MASK 0x10000000 +#define GDS_GWS_RESOURCE__HEAD_FLAG__SHIFT 0x1c +#define GDS_GWS_RESOURCE__UNUSED1_MASK 0xe0000000 +#define GDS_GWS_RESOURCE__UNUSED1__SHIFT 0x1d +#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT_MASK 0xffff +#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT__SHIFT 0x0 +#define GDS_GWS_RESOURCE_CNT__UNUSED_MASK 0xffff0000 +#define GDS_GWS_RESOURCE_CNT__UNUSED__SHIFT 0x10 +#define GDS_OA_CNTL__INDEX_MASK 0xf +#define GDS_OA_CNTL__INDEX__SHIFT 0x0 +#define GDS_OA_CNTL__UNUSED_MASK 0xfffffff0 +#define GDS_OA_CNTL__UNUSED__SHIFT 0x4 +#define GDS_OA_COUNTER__SPACE_AVAILABLE_MASK 0xffffffff +#define GDS_OA_COUNTER__SPACE_AVAILABLE__SHIFT 0x0 +#define GDS_OA_ADDRESS__DS_ADDRESS_MASK 0xffff +#define GDS_OA_ADDRESS__DS_ADDRESS__SHIFT 0x0 +#define GDS_OA_ADDRESS__CRAWLER_TYPE_MASK 0xf0000 +#define GDS_OA_ADDRESS__CRAWLER_TYPE__SHIFT 0x10 +#define GDS_OA_ADDRESS__CRAWLER_MASK 0xf00000 +#define GDS_OA_ADDRESS__CRAWLER__SHIFT 0x14 +#define GDS_OA_ADDRESS__UNUSED_MASK 0x3f000000 +#define GDS_OA_ADDRESS__UNUSED__SHIFT 0x18 +#define GDS_OA_ADDRESS__NO_ALLOC_MASK 0x40000000 +#define GDS_OA_ADDRESS__NO_ALLOC__SHIFT 0x1e +#define GDS_OA_ADDRESS__ENABLE_MASK 0x80000000 +#define GDS_OA_ADDRESS__ENABLE__SHIFT 0x1f +#define GDS_OA_INCDEC__VALUE_MASK 0x7fffffff +#define GDS_OA_INCDEC__VALUE__SHIFT 0x0 +#define GDS_OA_INCDEC__INCDEC_MASK 0x80000000 +#define GDS_OA_INCDEC__INCDEC__SHIFT 0x1f +#define GDS_OA_RING_SIZE__RING_SIZE_MASK 0xffffffff +#define GDS_OA_RING_SIZE__RING_SIZE__SHIFT 0x0 +#define GDS_DEBUG_REG0__spare1_MASK 0x3f +#define GDS_DEBUG_REG0__spare1__SHIFT 0x0 +#define GDS_DEBUG_REG0__write_buff_valid_MASK 0x40 +#define GDS_DEBUG_REG0__write_buff_valid__SHIFT 0x6 +#define GDS_DEBUG_REG0__wr_pixel_nxt_ptr_MASK 0xf80 +#define GDS_DEBUG_REG0__wr_pixel_nxt_ptr__SHIFT 0x7 +#define GDS_DEBUG_REG0__last_pixel_ptr_MASK 0x1000 +#define GDS_DEBUG_REG0__last_pixel_ptr__SHIFT 0xc +#define GDS_DEBUG_REG0__cstate_MASK 0x1e000 +#define GDS_DEBUG_REG0__cstate__SHIFT 0xd +#define GDS_DEBUG_REG0__buff_write_MASK 0x20000 +#define GDS_DEBUG_REG0__buff_write__SHIFT 0x11 +#define GDS_DEBUG_REG0__flush_request_MASK 0x40000 +#define GDS_DEBUG_REG0__flush_request__SHIFT 0x12 +#define GDS_DEBUG_REG0__wr_buffer_wr_complete_MASK 0x80000 +#define GDS_DEBUG_REG0__wr_buffer_wr_complete__SHIFT 0x13 +#define GDS_DEBUG_REG0__wbuf_fifo_empty_MASK 0x100000 +#define GDS_DEBUG_REG0__wbuf_fifo_empty__SHIFT 0x14 +#define GDS_DEBUG_REG0__wbuf_fifo_full_MASK 0x200000 +#define GDS_DEBUG_REG0__wbuf_fifo_full__SHIFT 0x15 +#define GDS_DEBUG_REG0__spare_MASK 0xffc00000 +#define GDS_DEBUG_REG0__spare__SHIFT 0x16 +#define GDS_DEBUG_REG1__tag_hit_MASK 0x1 +#define GDS_DEBUG_REG1__tag_hit__SHIFT 0x0 +#define GDS_DEBUG_REG1__tag_miss_MASK 0x2 +#define GDS_DEBUG_REG1__tag_miss__SHIFT 0x1 +#define GDS_DEBUG_REG1__pixel_addr_MASK 0x1fffc +#define GDS_DEBUG_REG1__pixel_addr__SHIFT 0x2 +#define GDS_DEBUG_REG1__pixel_vld_MASK 0x20000 +#define GDS_DEBUG_REG1__pixel_vld__SHIFT 0x11 +#define GDS_DEBUG_REG1__data_ready_MASK 0x40000 +#define GDS_DEBUG_REG1__data_ready__SHIFT 0x12 +#define GDS_DEBUG_REG1__awaiting_data_MASK 0x80000 +#define GDS_DEBUG_REG1__awaiting_data__SHIFT 0x13 +#define GDS_DEBUG_REG1__addr_fifo_full_MASK 0x100000 +#define GDS_DEBUG_REG1__addr_fifo_full__SHIFT 0x14 +#define GDS_DEBUG_REG1__addr_fifo_empty_MASK 0x200000 +#define GDS_DEBUG_REG1__addr_fifo_empty__SHIFT 0x15 +#define GDS_DEBUG_REG1__buffer_loaded_MASK 0x400000 +#define GDS_DEBUG_REG1__buffer_loaded__SHIFT 0x16 +#define GDS_DEBUG_REG1__buffer_invalid_MASK 0x800000 +#define GDS_DEBUG_REG1__buffer_invalid__SHIFT 0x17 +#define GDS_DEBUG_REG1__spare_MASK 0xff000000 +#define GDS_DEBUG_REG1__spare__SHIFT 0x18 +#define GDS_DEBUG_REG2__ds_full_MASK 0x1 +#define GDS_DEBUG_REG2__ds_full__SHIFT 0x0 +#define GDS_DEBUG_REG2__ds_credit_avail_MASK 0x2 +#define GDS_DEBUG_REG2__ds_credit_avail__SHIFT 0x1 +#define GDS_DEBUG_REG2__ord_idx_free_MASK 0x4 +#define GDS_DEBUG_REG2__ord_idx_free__SHIFT 0x2 +#define GDS_DEBUG_REG2__cmd_write_MASK 0x8 +#define GDS_DEBUG_REG2__cmd_write__SHIFT 0x3 +#define GDS_DEBUG_REG2__app_sel_MASK 0xf0 +#define GDS_DEBUG_REG2__app_sel__SHIFT 0x4 +#define GDS_DEBUG_REG2__req_MASK 0x7fff00 +#define GDS_DEBUG_REG2__req__SHIFT 0x8 +#define GDS_DEBUG_REG2__spare_MASK 0xff800000 +#define GDS_DEBUG_REG2__spare__SHIFT 0x17 +#define GDS_DEBUG_REG3__pipe_num_busy_MASK 0x7ff +#define GDS_DEBUG_REG3__pipe_num_busy__SHIFT 0x0 +#define GDS_DEBUG_REG3__pipe0_busy_num_MASK 0x7800 +#define GDS_DEBUG_REG3__pipe0_busy_num__SHIFT 0xb +#define GDS_DEBUG_REG3__spare_MASK 0xffff8000 +#define GDS_DEBUG_REG3__spare__SHIFT 0xf +#define GDS_DEBUG_REG4__gws_busy_MASK 0x1 +#define GDS_DEBUG_REG4__gws_busy__SHIFT 0x0 +#define GDS_DEBUG_REG4__gws_req_MASK 0x2 +#define GDS_DEBUG_REG4__gws_req__SHIFT 0x1 +#define GDS_DEBUG_REG4__gws_out_stall_MASK 0x4 +#define GDS_DEBUG_REG4__gws_out_stall__SHIFT 0x2 +#define GDS_DEBUG_REG4__cur_reso_MASK 0x1f8 +#define GDS_DEBUG_REG4__cur_reso__SHIFT 0x3 +#define GDS_DEBUG_REG4__cur_reso_head_valid_MASK 0x200 +#define GDS_DEBUG_REG4__cur_reso_head_valid__SHIFT 0x9 +#define GDS_DEBUG_REG4__cur_reso_head_dirty_MASK 0x400 +#define GDS_DEBUG_REG4__cur_reso_head_dirty__SHIFT 0xa +#define GDS_DEBUG_REG4__cur_reso_head_flag_MASK 0x800 +#define GDS_DEBUG_REG4__cur_reso_head_flag__SHIFT 0xb +#define GDS_DEBUG_REG4__cur_reso_fed_MASK 0x1000 +#define GDS_DEBUG_REG4__cur_reso_fed__SHIFT 0xc +#define GDS_DEBUG_REG4__cur_reso_barrier_MASK 0x2000 +#define GDS_DEBUG_REG4__cur_reso_barrier__SHIFT 0xd +#define GDS_DEBUG_REG4__cur_reso_flag_MASK 0x4000 +#define GDS_DEBUG_REG4__cur_reso_flag__SHIFT 0xe +#define GDS_DEBUG_REG4__cur_reso_cnt_gt0_MASK 0x8000 +#define GDS_DEBUG_REG4__cur_reso_cnt_gt0__SHIFT 0xf +#define GDS_DEBUG_REG4__credit_cnt_gt0_MASK 0x10000 +#define GDS_DEBUG_REG4__credit_cnt_gt0__SHIFT 0x10 +#define GDS_DEBUG_REG4__cmd_write_MASK 0x20000 +#define GDS_DEBUG_REG4__cmd_write__SHIFT 0x11 +#define GDS_DEBUG_REG4__grbm_gws_reso_wr_MASK 0x40000 +#define GDS_DEBUG_REG4__grbm_gws_reso_wr__SHIFT 0x12 +#define GDS_DEBUG_REG4__grbm_gws_reso_rd_MASK 0x80000 +#define GDS_DEBUG_REG4__grbm_gws_reso_rd__SHIFT 0x13 +#define GDS_DEBUG_REG4__ram_read_busy_MASK 0x100000 +#define GDS_DEBUG_REG4__ram_read_busy__SHIFT 0x14 +#define GDS_DEBUG_REG4__gws_bulkfree_MASK 0x200000 +#define GDS_DEBUG_REG4__gws_bulkfree__SHIFT 0x15 +#define GDS_DEBUG_REG4__ram_gws_re_MASK 0x400000 +#define GDS_DEBUG_REG4__ram_gws_re__SHIFT 0x16 +#define GDS_DEBUG_REG4__ram_gws_we_MASK 0x800000 +#define GDS_DEBUG_REG4__ram_gws_we__SHIFT 0x17 +#define GDS_DEBUG_REG4__spare_MASK 0xff000000 +#define GDS_DEBUG_REG4__spare__SHIFT 0x18 +#define GDS_DEBUG_REG5__write_dis_MASK 0x1 +#define GDS_DEBUG_REG5__write_dis__SHIFT 0x0 +#define GDS_DEBUG_REG5__dec_error_MASK 0x2 +#define GDS_DEBUG_REG5__dec_error__SHIFT 0x1 +#define GDS_DEBUG_REG5__alloc_opco_error_MASK 0x4 +#define GDS_DEBUG_REG5__alloc_opco_error__SHIFT 0x2 +#define GDS_DEBUG_REG5__dealloc_opco_error_MASK 0x8 +#define GDS_DEBUG_REG5__dealloc_opco_error__SHIFT 0x3 +#define GDS_DEBUG_REG5__wrap_opco_error_MASK 0x10 +#define GDS_DEBUG_REG5__wrap_opco_error__SHIFT 0x4 +#define GDS_DEBUG_REG5__spare_MASK 0xe0 +#define GDS_DEBUG_REG5__spare__SHIFT 0x5 +#define GDS_DEBUG_REG5__error_ds_address_MASK 0x3fff00 +#define GDS_DEBUG_REG5__error_ds_address__SHIFT 0x8 +#define GDS_DEBUG_REG5__spare1_MASK 0xffc00000 +#define GDS_DEBUG_REG5__spare1__SHIFT 0x16 +#define GDS_DEBUG_REG6__oa_busy_MASK 0x1 +#define GDS_DEBUG_REG6__oa_busy__SHIFT 0x0 +#define GDS_DEBUG_REG6__counters_enabled_MASK 0x1e +#define GDS_DEBUG_REG6__counters_enabled__SHIFT 0x1 +#define GDS_DEBUG_REG6__counters_busy_MASK 0x1fffe0 +#define GDS_DEBUG_REG6__counters_busy__SHIFT 0x5 +#define GDS_DEBUG_REG6__spare_MASK 0xffe00000 +#define GDS_DEBUG_REG6__spare__SHIFT 0x15 +#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define GDS_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define GDS_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define GDS_PERFCOUNTER2_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK 0xf00000 +#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT 0x14 +#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT_MASK 0x3ff +#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT__SHIFT 0x0 +#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1_MASK 0xffc00 +#define GDS_PERFCOUNTER3_SELECT__PERFCOUNTER_SELECT1__SHIFT 0xa +#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE_MASK 0xf00000 +#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT 0x14 +#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2_MASK 0x3ff +#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT2__SHIFT 0x0 +#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3_MASK 0xffc00 +#define GDS_PERFCOUNTER0_SELECT1__PERFCOUNTER_SELECT3__SHIFT 0xa +#define GDS_VMID0_BASE__BASE_MASK 0xffff +#define GDS_VMID0_BASE__BASE__SHIFT 0x0 +#define GDS_VMID1_BASE__BASE_MASK 0xffff +#define GDS_VMID1_BASE__BASE__SHIFT 0x0 +#define GDS_VMID2_BASE__BASE_MASK 0xffff +#define GDS_VMID2_BASE__BASE__SHIFT 0x0 +#define GDS_VMID3_BASE__BASE_MASK 0xffff +#define GDS_VMID3_BASE__BASE__SHIFT 0x0 +#define GDS_VMID4_BASE__BASE_MASK 0xffff +#define GDS_VMID4_BASE__BASE__SHIFT 0x0 +#define GDS_VMID5_BASE__BASE_MASK 0xffff +#define GDS_VMID5_BASE__BASE__SHIFT 0x0 +#define GDS_VMID6_BASE__BASE_MASK 0xffff +#define GDS_VMID6_BASE__BASE__SHIFT 0x0 +#define GDS_VMID7_BASE__BASE_MASK 0xffff +#define GDS_VMID7_BASE__BASE__SHIFT 0x0 +#define GDS_VMID8_BASE__BASE_MASK 0xffff +#define GDS_VMID8_BASE__BASE__SHIFT 0x0 +#define GDS_VMID9_BASE__BASE_MASK 0xffff +#define GDS_VMID9_BASE__BASE__SHIFT 0x0 +#define GDS_VMID10_BASE__BASE_MASK 0xffff +#define GDS_VMID10_BASE__BASE__SHIFT 0x0 +#define GDS_VMID11_BASE__BASE_MASK 0xffff +#define GDS_VMID11_BASE__BASE__SHIFT 0x0 +#define GDS_VMID12_BASE__BASE_MASK 0xffff +#define GDS_VMID12_BASE__BASE__SHIFT 0x0 +#define GDS_VMID13_BASE__BASE_MASK 0xffff +#define GDS_VMID13_BASE__BASE__SHIFT 0x0 +#define GDS_VMID14_BASE__BASE_MASK 0xffff +#define GDS_VMID14_BASE__BASE__SHIFT 0x0 +#define GDS_VMID15_BASE__BASE_MASK 0xffff +#define GDS_VMID15_BASE__BASE__SHIFT 0x0 +#define GDS_VMID0_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID0_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID1_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID1_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID2_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID2_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID3_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID3_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID4_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID4_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID5_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID5_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID6_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID6_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID7_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID7_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID8_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID8_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID9_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID9_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID10_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID10_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID11_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID11_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID12_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID12_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID13_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID13_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID14_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID14_SIZE__SIZE__SHIFT 0x0 +#define GDS_VMID15_SIZE__SIZE_MASK 0x1ffff +#define GDS_VMID15_SIZE__SIZE__SHIFT 0x0 +#define GDS_GWS_VMID0__BASE_MASK 0x3f +#define GDS_GWS_VMID0__BASE__SHIFT 0x0 +#define GDS_GWS_VMID0__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID0__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID1__BASE_MASK 0x3f +#define GDS_GWS_VMID1__BASE__SHIFT 0x0 +#define GDS_GWS_VMID1__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID1__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID2__BASE_MASK 0x3f +#define GDS_GWS_VMID2__BASE__SHIFT 0x0 +#define GDS_GWS_VMID2__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID2__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID3__BASE_MASK 0x3f +#define GDS_GWS_VMID3__BASE__SHIFT 0x0 +#define GDS_GWS_VMID3__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID3__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID4__BASE_MASK 0x3f +#define GDS_GWS_VMID4__BASE__SHIFT 0x0 +#define GDS_GWS_VMID4__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID4__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID5__BASE_MASK 0x3f +#define GDS_GWS_VMID5__BASE__SHIFT 0x0 +#define GDS_GWS_VMID5__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID5__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID6__BASE_MASK 0x3f +#define GDS_GWS_VMID6__BASE__SHIFT 0x0 +#define GDS_GWS_VMID6__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID6__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID7__BASE_MASK 0x3f +#define GDS_GWS_VMID7__BASE__SHIFT 0x0 +#define GDS_GWS_VMID7__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID7__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID8__BASE_MASK 0x3f +#define GDS_GWS_VMID8__BASE__SHIFT 0x0 +#define GDS_GWS_VMID8__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID8__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID9__BASE_MASK 0x3f +#define GDS_GWS_VMID9__BASE__SHIFT 0x0 +#define GDS_GWS_VMID9__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID9__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID10__BASE_MASK 0x3f +#define GDS_GWS_VMID10__BASE__SHIFT 0x0 +#define GDS_GWS_VMID10__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID10__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID11__BASE_MASK 0x3f +#define GDS_GWS_VMID11__BASE__SHIFT 0x0 +#define GDS_GWS_VMID11__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID11__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID12__BASE_MASK 0x3f +#define GDS_GWS_VMID12__BASE__SHIFT 0x0 +#define GDS_GWS_VMID12__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID12__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID13__BASE_MASK 0x3f +#define GDS_GWS_VMID13__BASE__SHIFT 0x0 +#define GDS_GWS_VMID13__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID13__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID14__BASE_MASK 0x3f +#define GDS_GWS_VMID14__BASE__SHIFT 0x0 +#define GDS_GWS_VMID14__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID14__SIZE__SHIFT 0x10 +#define GDS_GWS_VMID15__BASE_MASK 0x3f +#define GDS_GWS_VMID15__BASE__SHIFT 0x0 +#define GDS_GWS_VMID15__SIZE_MASK 0x7f0000 +#define GDS_GWS_VMID15__SIZE__SHIFT 0x10 +#define GDS_OA_VMID0__MASK_MASK 0xffff +#define GDS_OA_VMID0__MASK__SHIFT 0x0 +#define GDS_OA_VMID0__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID0__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID1__MASK_MASK 0xffff +#define GDS_OA_VMID1__MASK__SHIFT 0x0 +#define GDS_OA_VMID1__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID1__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID2__MASK_MASK 0xffff +#define GDS_OA_VMID2__MASK__SHIFT 0x0 +#define GDS_OA_VMID2__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID2__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID3__MASK_MASK 0xffff +#define GDS_OA_VMID3__MASK__SHIFT 0x0 +#define GDS_OA_VMID3__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID3__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID4__MASK_MASK 0xffff +#define GDS_OA_VMID4__MASK__SHIFT 0x0 +#define GDS_OA_VMID4__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID4__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID5__MASK_MASK 0xffff +#define GDS_OA_VMID5__MASK__SHIFT 0x0 +#define GDS_OA_VMID5__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID5__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID6__MASK_MASK 0xffff +#define GDS_OA_VMID6__MASK__SHIFT 0x0 +#define GDS_OA_VMID6__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID6__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID7__MASK_MASK 0xffff +#define GDS_OA_VMID7__MASK__SHIFT 0x0 +#define GDS_OA_VMID7__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID7__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID8__MASK_MASK 0xffff +#define GDS_OA_VMID8__MASK__SHIFT 0x0 +#define GDS_OA_VMID8__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID8__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID9__MASK_MASK 0xffff +#define GDS_OA_VMID9__MASK__SHIFT 0x0 +#define GDS_OA_VMID9__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID9__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID10__MASK_MASK 0xffff +#define GDS_OA_VMID10__MASK__SHIFT 0x0 +#define GDS_OA_VMID10__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID10__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID11__MASK_MASK 0xffff +#define GDS_OA_VMID11__MASK__SHIFT 0x0 +#define GDS_OA_VMID11__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID11__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID12__MASK_MASK 0xffff +#define GDS_OA_VMID12__MASK__SHIFT 0x0 +#define GDS_OA_VMID12__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID12__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID13__MASK_MASK 0xffff +#define GDS_OA_VMID13__MASK__SHIFT 0x0 +#define GDS_OA_VMID13__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID13__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID14__MASK_MASK 0xffff +#define GDS_OA_VMID14__MASK__SHIFT 0x0 +#define GDS_OA_VMID14__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID14__UNUSED__SHIFT 0x10 +#define GDS_OA_VMID15__MASK_MASK 0xffff +#define GDS_OA_VMID15__MASK__SHIFT 0x0 +#define GDS_OA_VMID15__UNUSED_MASK 0xffff0000 +#define GDS_OA_VMID15__UNUSED__SHIFT 0x10 +#define GDS_GWS_RESET0__RESOURCE0_RESET_MASK 0x1 +#define GDS_GWS_RESET0__RESOURCE0_RESET__SHIFT 0x0 +#define GDS_GWS_RESET0__RESOURCE1_RESET_MASK 0x2 +#define GDS_GWS_RESET0__RESOURCE1_RESET__SHIFT 0x1 +#define GDS_GWS_RESET0__RESOURCE2_RESET_MASK 0x4 +#define GDS_GWS_RESET0__RESOURCE2_RESET__SHIFT 0x2 +#define GDS_GWS_RESET0__RESOURCE3_RESET_MASK 0x8 +#define GDS_GWS_RESET0__RESOURCE3_RESET__SHIFT 0x3 +#define GDS_GWS_RESET0__RESOURCE4_RESET_MASK 0x10 +#define GDS_GWS_RESET0__RESOURCE4_RESET__SHIFT 0x4 +#define GDS_GWS_RESET0__RESOURCE5_RESET_MASK 0x20 +#define GDS_GWS_RESET0__RESOURCE5_RESET__SHIFT 0x5 +#define GDS_GWS_RESET0__RESOURCE6_RESET_MASK 0x40 +#define GDS_GWS_RESET0__RESOURCE6_RESET__SHIFT 0x6 +#define GDS_GWS_RESET0__RESOURCE7_RESET_MASK 0x80 +#define GDS_GWS_RESET0__RESOURCE7_RESET__SHIFT 0x7 +#define GDS_GWS_RESET0__RESOURCE8_RESET_MASK 0x100 +#define GDS_GWS_RESET0__RESOURCE8_RESET__SHIFT 0x8 +#define GDS_GWS_RESET0__RESOURCE9_RESET_MASK 0x200 +#define GDS_GWS_RESET0__RESOURCE9_RESET__SHIFT 0x9 +#define GDS_GWS_RESET0__RESOURCE10_RESET_MASK 0x400 +#define GDS_GWS_RESET0__RESOURCE10_RESET__SHIFT 0xa +#define GDS_GWS_RESET0__RESOURCE11_RESET_MASK 0x800 +#define GDS_GWS_RESET0__RESOURCE11_RESET__SHIFT 0xb +#define GDS_GWS_RESET0__RESOURCE12_RESET_MASK 0x1000 +#define GDS_GWS_RESET0__RESOURCE12_RESET__SHIFT 0xc +#define GDS_GWS_RESET0__RESOURCE13_RESET_MASK 0x2000 +#define GDS_GWS_RESET0__RESOURCE13_RESET__SHIFT 0xd +#define GDS_GWS_RESET0__RESOURCE14_RESET_MASK 0x4000 +#define GDS_GWS_RESET0__RESOURCE14_RESET__SHIFT 0xe +#define GDS_GWS_RESET0__RESOURCE15_RESET_MASK 0x8000 +#define GDS_GWS_RESET0__RESOURCE15_RESET__SHIFT 0xf +#define GDS_GWS_RESET0__RESOURCE16_RESET_MASK 0x10000 +#define GDS_GWS_RESET0__RESOURCE16_RESET__SHIFT 0x10 +#define GDS_GWS_RESET0__RESOURCE17_RESET_MASK 0x20000 +#define GDS_GWS_RESET0__RESOURCE17_RESET__SHIFT 0x11 +#define GDS_GWS_RESET0__RESOURCE18_RESET_MASK 0x40000 +#define GDS_GWS_RESET0__RESOURCE18_RESET__SHIFT 0x12 +#define GDS_GWS_RESET0__RESOURCE19_RESET_MASK 0x80000 +#define GDS_GWS_RESET0__RESOURCE19_RESET__SHIFT 0x13 +#define GDS_GWS_RESET0__RESOURCE20_RESET_MASK 0x100000 +#define GDS_GWS_RESET0__RESOURCE20_RESET__SHIFT 0x14 +#define GDS_GWS_RESET0__RESOURCE21_RESET_MASK 0x200000 +#define GDS_GWS_RESET0__RESOURCE21_RESET__SHIFT 0x15 +#define GDS_GWS_RESET0__RESOURCE22_RESET_MASK 0x400000 +#define GDS_GWS_RESET0__RESOURCE22_RESET__SHIFT 0x16 +#define GDS_GWS_RESET0__RESOURCE23_RESET_MASK 0x800000 +#define GDS_GWS_RESET0__RESOURCE23_RESET__SHIFT 0x17 +#define GDS_GWS_RESET0__RESOURCE24_RESET_MASK 0x1000000 +#define GDS_GWS_RESET0__RESOURCE24_RESET__SHIFT 0x18 +#define GDS_GWS_RESET0__RESOURCE25_RESET_MASK 0x2000000 +#define GDS_GWS_RESET0__RESOURCE25_RESET__SHIFT 0x19 +#define GDS_GWS_RESET0__RESOURCE26_RESET_MASK 0x4000000 +#define GDS_GWS_RESET0__RESOURCE26_RESET__SHIFT 0x1a +#define GDS_GWS_RESET0__RESOURCE27_RESET_MASK 0x8000000 +#define GDS_GWS_RESET0__RESOURCE27_RESET__SHIFT 0x1b +#define GDS_GWS_RESET0__RESOURCE28_RESET_MASK 0x10000000 +#define GDS_GWS_RESET0__RESOURCE28_RESET__SHIFT 0x1c +#define GDS_GWS_RESET0__RESOURCE29_RESET_MASK 0x20000000 +#define GDS_GWS_RESET0__RESOURCE29_RESET__SHIFT 0x1d +#define GDS_GWS_RESET0__RESOURCE30_RESET_MASK 0x40000000 +#define GDS_GWS_RESET0__RESOURCE30_RESET__SHIFT 0x1e +#define GDS_GWS_RESET0__RESOURCE31_RESET_MASK 0x80000000 +#define GDS_GWS_RESET0__RESOURCE31_RESET__SHIFT 0x1f +#define GDS_GWS_RESET1__RESOURCE32_RESET_MASK 0x1 +#define GDS_GWS_RESET1__RESOURCE32_RESET__SHIFT 0x0 +#define GDS_GWS_RESET1__RESOURCE33_RESET_MASK 0x2 +#define GDS_GWS_RESET1__RESOURCE33_RESET__SHIFT 0x1 +#define GDS_GWS_RESET1__RESOURCE34_RESET_MASK 0x4 +#define GDS_GWS_RESET1__RESOURCE34_RESET__SHIFT 0x2 +#define GDS_GWS_RESET1__RESOURCE35_RESET_MASK 0x8 +#define GDS_GWS_RESET1__RESOURCE35_RESET__SHIFT 0x3 +#define GDS_GWS_RESET1__RESOURCE36_RESET_MASK 0x10 +#define GDS_GWS_RESET1__RESOURCE36_RESET__SHIFT 0x4 +#define GDS_GWS_RESET1__RESOURCE37_RESET_MASK 0x20 +#define GDS_GWS_RESET1__RESOURCE37_RESET__SHIFT 0x5 +#define GDS_GWS_RESET1__RESOURCE38_RESET_MASK 0x40 +#define GDS_GWS_RESET1__RESOURCE38_RESET__SHIFT 0x6 +#define GDS_GWS_RESET1__RESOURCE39_RESET_MASK 0x80 +#define GDS_GWS_RESET1__RESOURCE39_RESET__SHIFT 0x7 +#define GDS_GWS_RESET1__RESOURCE40_RESET_MASK 0x100 +#define GDS_GWS_RESET1__RESOURCE40_RESET__SHIFT 0x8 +#define GDS_GWS_RESET1__RESOURCE41_RESET_MASK 0x200 +#define GDS_GWS_RESET1__RESOURCE41_RESET__SHIFT 0x9 +#define GDS_GWS_RESET1__RESOURCE42_RESET_MASK 0x400 +#define GDS_GWS_RESET1__RESOURCE42_RESET__SHIFT 0xa +#define GDS_GWS_RESET1__RESOURCE43_RESET_MASK 0x800 +#define GDS_GWS_RESET1__RESOURCE43_RESET__SHIFT 0xb +#define GDS_GWS_RESET1__RESOURCE44_RESET_MASK 0x1000 +#define GDS_GWS_RESET1__RESOURCE44_RESET__SHIFT 0xc +#define GDS_GWS_RESET1__RESOURCE45_RESET_MASK 0x2000 +#define GDS_GWS_RESET1__RESOURCE45_RESET__SHIFT 0xd +#define GDS_GWS_RESET1__RESOURCE46_RESET_MASK 0x4000 +#define GDS_GWS_RESET1__RESOURCE46_RESET__SHIFT 0xe +#define GDS_GWS_RESET1__RESOURCE47_RESET_MASK 0x8000 +#define GDS_GWS_RESET1__RESOURCE47_RESET__SHIFT 0xf +#define GDS_GWS_RESET1__RESOURCE48_RESET_MASK 0x10000 +#define GDS_GWS_RESET1__RESOURCE48_RESET__SHIFT 0x10 +#define GDS_GWS_RESET1__RESOURCE49_RESET_MASK 0x20000 +#define GDS_GWS_RESET1__RESOURCE49_RESET__SHIFT 0x11 +#define GDS_GWS_RESET1__RESOURCE50_RESET_MASK 0x40000 +#define GDS_GWS_RESET1__RESOURCE50_RESET__SHIFT 0x12 +#define GDS_GWS_RESET1__RESOURCE51_RESET_MASK 0x80000 +#define GDS_GWS_RESET1__RESOURCE51_RESET__SHIFT 0x13 +#define GDS_GWS_RESET1__RESOURCE52_RESET_MASK 0x100000 +#define GDS_GWS_RESET1__RESOURCE52_RESET__SHIFT 0x14 +#define GDS_GWS_RESET1__RESOURCE53_RESET_MASK 0x200000 +#define GDS_GWS_RESET1__RESOURCE53_RESET__SHIFT 0x15 +#define GDS_GWS_RESET1__RESOURCE54_RESET_MASK 0x400000 +#define GDS_GWS_RESET1__RESOURCE54_RESET__SHIFT 0x16 +#define GDS_GWS_RESET1__RESOURCE55_RESET_MASK 0x800000 +#define GDS_GWS_RESET1__RESOURCE55_RESET__SHIFT 0x17 +#define GDS_GWS_RESET1__RESOURCE56_RESET_MASK 0x1000000 +#define GDS_GWS_RESET1__RESOURCE56_RESET__SHIFT 0x18 +#define GDS_GWS_RESET1__RESOURCE57_RESET_MASK 0x2000000 +#define GDS_GWS_RESET1__RESOURCE57_RESET__SHIFT 0x19 +#define GDS_GWS_RESET1__RESOURCE58_RESET_MASK 0x4000000 +#define GDS_GWS_RESET1__RESOURCE58_RESET__SHIFT 0x1a +#define GDS_GWS_RESET1__RESOURCE59_RESET_MASK 0x8000000 +#define GDS_GWS_RESET1__RESOURCE59_RESET__SHIFT 0x1b +#define GDS_GWS_RESET1__RESOURCE60_RESET_MASK 0x10000000 +#define GDS_GWS_RESET1__RESOURCE60_RESET__SHIFT 0x1c +#define GDS_GWS_RESET1__RESOURCE61_RESET_MASK 0x20000000 +#define GDS_GWS_RESET1__RESOURCE61_RESET__SHIFT 0x1d +#define GDS_GWS_RESET1__RESOURCE62_RESET_MASK 0x40000000 +#define GDS_GWS_RESET1__RESOURCE62_RESET__SHIFT 0x1e +#define GDS_GWS_RESET1__RESOURCE63_RESET_MASK 0x80000000 +#define GDS_GWS_RESET1__RESOURCE63_RESET__SHIFT 0x1f +#define GDS_GWS_RESOURCE_RESET__RESET_MASK 0x1 +#define GDS_GWS_RESOURCE_RESET__RESET__SHIFT 0x0 +#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID_MASK 0xff00 +#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID__SHIFT 0x8 +#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID_MASK 0xfff +#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT 0x0 +#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET_MASK 0x1 +#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET__SHIFT 0x0 +#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET_MASK 0x2 +#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET__SHIFT 0x1 +#define GDS_OA_RESET_MASK__ME0_CS_RESET_MASK 0x4 +#define GDS_OA_RESET_MASK__ME0_CS_RESET__SHIFT 0x2 +#define GDS_OA_RESET_MASK__UNUSED0_MASK 0x8 +#define GDS_OA_RESET_MASK__UNUSED0__SHIFT 0x3 +#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET_MASK 0x10 +#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET__SHIFT 0x4 +#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET_MASK 0x20 +#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET__SHIFT 0x5 +#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET_MASK 0x40 +#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET__SHIFT 0x6 +#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET_MASK 0x80 +#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET__SHIFT 0x7 +#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET_MASK 0x100 +#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET__SHIFT 0x8 +#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET_MASK 0x200 +#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET__SHIFT 0x9 +#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET_MASK 0x400 +#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET__SHIFT 0xa +#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET_MASK 0x800 +#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET__SHIFT 0xb +#define GDS_OA_RESET_MASK__UNUSED1_MASK 0xfffff000 +#define GDS_OA_RESET_MASK__UNUSED1__SHIFT 0xc +#define GDS_OA_RESET__RESET_MASK 0x1 +#define GDS_OA_RESET__RESET__SHIFT 0x0 +#define GDS_OA_RESET__PIPE_ID_MASK 0xff00 +#define GDS_OA_RESET__PIPE_ID__SHIFT 0x8 +#define GDS_ENHANCE__MISC_MASK 0xffff +#define GDS_ENHANCE__MISC__SHIFT 0x0 +#define GDS_ENHANCE__AUTO_INC_INDEX_MASK 0x10000 +#define GDS_ENHANCE__AUTO_INC_INDEX__SHIFT 0x10 +#define GDS_ENHANCE__CGPG_RESTORE_MASK 0x20000 +#define GDS_ENHANCE__CGPG_RESTORE__SHIFT 0x11 +#define GDS_ENHANCE__UNUSED_MASK 0xfffc0000 +#define GDS_ENHANCE__UNUSED__SHIFT 0x12 +#define GDS_OA_CGPG_RESTORE__VMID_MASK 0xff +#define GDS_OA_CGPG_RESTORE__VMID__SHIFT 0x0 +#define GDS_OA_CGPG_RESTORE__MEID_MASK 0xf00 +#define GDS_OA_CGPG_RESTORE__MEID__SHIFT 0x8 +#define GDS_OA_CGPG_RESTORE__PIPEID_MASK 0xf000 +#define GDS_OA_CGPG_RESTORE__PIPEID__SHIFT 0xc +#define GDS_OA_CGPG_RESTORE__UNUSED_MASK 0xffff0000 +#define GDS_OA_CGPG_RESTORE__UNUSED__SHIFT 0x10 +#define CS_COPY_STATE__SRC_STATE_ID_MASK 0x7 +#define CS_COPY_STATE__SRC_STATE_ID__SHIFT 0x0 +#define GFX_COPY_STATE__SRC_STATE_ID_MASK 0x7 +#define GFX_COPY_STATE__SRC_STATE_ID__SHIFT 0x0 +#define VGT_DRAW_INITIATOR__SOURCE_SELECT_MASK 0x3 +#define VGT_DRAW_INITIATOR__SOURCE_SELECT__SHIFT 0x0 +#define VGT_DRAW_INITIATOR__MAJOR_MODE_MASK 0xc +#define VGT_DRAW_INITIATOR__MAJOR_MODE__SHIFT 0x2 +#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX_MASK 0x10 +#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX__SHIFT 0x4 +#define VGT_DRAW_INITIATOR__NOT_EOP_MASK 0x20 +#define VGT_DRAW_INITIATOR__NOT_EOP__SHIFT 0x5 +#define VGT_DRAW_INITIATOR__USE_OPAQUE_MASK 0x40 +#define VGT_DRAW_INITIATOR__USE_OPAQUE__SHIFT 0x6 +#define VGT_EVENT_INITIATOR__EVENT_TYPE_MASK 0x3f +#define VGT_EVENT_INITIATOR__EVENT_TYPE__SHIFT 0x0 +#define VGT_EVENT_INITIATOR__ADDRESS_HI_MASK 0x7fc0000 +#define VGT_EVENT_INITIATOR__ADDRESS_HI__SHIFT 0x12 +#define VGT_EVENT_INITIATOR__EXTENDED_EVENT_MASK 0x8000000 +#define VGT_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT 0x1b +#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW_MASK 0xfffffff +#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW__SHIFT 0x0 +#define VGT_DMA_BASE_HI__BASE_ADDR_MASK 0xff +#define VGT_DMA_BASE_HI__BASE_ADDR__SHIFT 0x0 +#define VGT_DMA_BASE__BASE_ADDR_MASK 0xffffffff +#define VGT_DMA_BASE__BASE_ADDR__SHIFT 0x0 +#define VGT_DMA_INDEX_TYPE__INDEX_TYPE_MASK 0x3 +#define VGT_DMA_INDEX_TYPE__INDEX_TYPE__SHIFT 0x0 +#define VGT_DMA_INDEX_TYPE__SWAP_MODE_MASK 0xc +#define VGT_DMA_INDEX_TYPE__SWAP_MODE__SHIFT 0x2 +#define VGT_DMA_INDEX_TYPE__BUF_TYPE_MASK 0x30 +#define VGT_DMA_INDEX_TYPE__BUF_TYPE__SHIFT 0x4 +#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY_MASK 0xc0 +#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY__SHIFT 0x6 +#define VGT_DMA_INDEX_TYPE__ATC_MASK 0x100 +#define VGT_DMA_INDEX_TYPE__ATC__SHIFT 0x8 +#define VGT_DMA_INDEX_TYPE__NOT_EOP_MASK 0x200 +#define VGT_DMA_INDEX_TYPE__NOT_EOP__SHIFT 0x9 +#define VGT_DMA_INDEX_TYPE__REQ_PATH_MASK 0x400 +#define VGT_DMA_INDEX_TYPE__REQ_PATH__SHIFT 0xa +#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES_MASK 0xffffffff +#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES__SHIFT 0x0 +#define IA_ENHANCE__MISC_MASK 0xffffffff +#define IA_ENHANCE__MISC__SHIFT 0x0 +#define VGT_DMA_SIZE__NUM_INDICES_MASK 0xffffffff +#define VGT_DMA_SIZE__NUM_INDICES__SHIFT 0x0 +#define VGT_DMA_MAX_SIZE__MAX_SIZE_MASK 0xffffffff +#define VGT_DMA_MAX_SIZE__MAX_SIZE__SHIFT 0x0 +#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE_MASK 0x3f +#define VGT_DMA_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT 0x0 +#define VGT_DMA_CONTROL__PRIMGROUP_SIZE_MASK 0xffff +#define VGT_DMA_CONTROL__PRIMGROUP_SIZE__SHIFT 0x0 +#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP_MASK 0x20000 +#define VGT_DMA_CONTROL__IA_SWITCH_ON_EOP__SHIFT 0x11 +#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP_MASK 0x100000 +#define VGT_DMA_CONTROL__WD_SWITCH_ON_EOP__SHIFT 0x14 +#define VGT_IMMED_DATA__DATA_MASK 0xffffffff +#define VGT_IMMED_DATA__DATA__SHIFT 0x0 +#define VGT_INDEX_TYPE__INDEX_TYPE_MASK 0x3 +#define VGT_INDEX_TYPE__INDEX_TYPE__SHIFT 0x0 +#define VGT_NUM_INDICES__NUM_INDICES_MASK 0xffffffff +#define VGT_NUM_INDICES__NUM_INDICES__SHIFT 0x0 +#define VGT_NUM_INSTANCES__NUM_INSTANCES_MASK 0xffffffff +#define VGT_NUM_INSTANCES__NUM_INSTANCES__SHIFT 0x0 +#define VGT_PRIMITIVE_TYPE__PRIM_TYPE_MASK 0x3f +#define VGT_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT 0x0 +#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN_MASK 0x1 +#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN__SHIFT 0x0 +#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI_MASK 0x2 +#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI__SHIFT 0x1 +#define VGT_PRIMITIVEID_RESET__VALUE_MASK 0xffffffff +#define VGT_PRIMITIVEID_RESET__VALUE__SHIFT 0x0 +#define VGT_VTX_CNT_EN__VTX_CNT_EN_MASK 0x1 +#define VGT_VTX_CNT_EN__VTX_CNT_EN__SHIFT 0x0 +#define VGT_REUSE_OFF__REUSE_OFF_MASK 0x1 +#define VGT_REUSE_OFF__REUSE_OFF__SHIFT 0x0 +#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE_MASK 0xffffffff +#define VGT_INSTANCE_STEP_RATE_0__STEP_RATE__SHIFT 0x0 +#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE_MASK 0xffffffff +#define VGT_INSTANCE_STEP_RATE_1__STEP_RATE__SHIFT 0x0 +#define VGT_MAX_VTX_INDX__MAX_INDX_MASK 0xffffffff +#define VGT_MAX_VTX_INDX__MAX_INDX__SHIFT 0x0 +#define VGT_MIN_VTX_INDX__MIN_INDX_MASK 0xffffffff +#define VGT_MIN_VTX_INDX__MIN_INDX__SHIFT 0x0 +#define VGT_INDX_OFFSET__INDX_OFFSET_MASK 0xffffffff +#define VGT_INDX_OFFSET__INDX_OFFSET__SHIFT 0x0 +#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH_MASK 0xff +#define VGT_VERTEX_REUSE_BLOCK_CNTL__VTX_REUSE_DEPTH__SHIFT 0x0 +#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST_MASK 0x7f +#define VGT_OUT_DEALLOC_CNTL__DEALLOC_DIST__SHIFT 0x0 +#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK 0xffffffff +#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT 0x0 +#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN_MASK 0x1 +#define VGT_MULTI_PRIM_IB_RESET_EN__RESET_EN__SHIFT 0x0 +#define VGT_ENHANCE__MISC_MASK 0xffffffff +#define VGT_ENHANCE__MISC__SHIFT 0x0 +#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT_MASK 0x7 +#define VGT_OUTPUT_PATH_CNTL__PATH_SELECT__SHIFT 0x0 +#define VGT_HOS_CNTL__TESS_MODE_MASK 0x3 +#define VGT_HOS_CNTL__TESS_MODE__SHIFT 0x0 +#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS_MASK 0xffffffff +#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS__SHIFT 0x0 +#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS_MASK 0xffffffff +#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS__SHIFT 0x0 +#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH_MASK 0xff +#define VGT_HOS_REUSE_DEPTH__REUSE_DEPTH__SHIFT 0x0 +#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE_MASK 0x1f +#define VGT_GROUP_PRIM_TYPE__PRIM_TYPE__SHIFT 0x0 +#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER_MASK 0x4000 +#define VGT_GROUP_PRIM_TYPE__RETAIN_ORDER__SHIFT 0xe +#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS_MASK 0x8000 +#define VGT_GROUP_PRIM_TYPE__RETAIN_QUADS__SHIFT 0xf +#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER_MASK 0x70000 +#define VGT_GROUP_PRIM_TYPE__PRIM_ORDER__SHIFT 0x10 +#define VGT_GROUP_FIRST_DECR__FIRST_DECR_MASK 0xf +#define VGT_GROUP_FIRST_DECR__FIRST_DECR__SHIFT 0x0 +#define VGT_GROUP_DECR__DECR_MASK 0xf +#define VGT_GROUP_DECR__DECR__SHIFT 0x0 +#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN_MASK 0x1 +#define VGT_GROUP_VECT_0_CNTL__COMP_X_EN__SHIFT 0x0 +#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN_MASK 0x2 +#define VGT_GROUP_VECT_0_CNTL__COMP_Y_EN__SHIFT 0x1 +#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN_MASK 0x4 +#define VGT_GROUP_VECT_0_CNTL__COMP_Z_EN__SHIFT 0x2 +#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN_MASK 0x8 +#define VGT_GROUP_VECT_0_CNTL__COMP_W_EN__SHIFT 0x3 +#define VGT_GROUP_VECT_0_CNTL__STRIDE_MASK 0xff00 +#define VGT_GROUP_VECT_0_CNTL__STRIDE__SHIFT 0x8 +#define VGT_GROUP_VECT_0_CNTL__SHIFT_MASK 0xff0000 +#define VGT_GROUP_VECT_0_CNTL__SHIFT__SHIFT 0x10 +#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN_MASK 0x1 +#define VGT_GROUP_VECT_1_CNTL__COMP_X_EN__SHIFT 0x0 +#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN_MASK 0x2 +#define VGT_GROUP_VECT_1_CNTL__COMP_Y_EN__SHIFT 0x1 +#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN_MASK 0x4 +#define VGT_GROUP_VECT_1_CNTL__COMP_Z_EN__SHIFT 0x2 +#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN_MASK 0x8 +#define VGT_GROUP_VECT_1_CNTL__COMP_W_EN__SHIFT 0x3 +#define VGT_GROUP_VECT_1_CNTL__STRIDE_MASK 0xff00 +#define VGT_GROUP_VECT_1_CNTL__STRIDE__SHIFT 0x8 +#define VGT_GROUP_VECT_1_CNTL__SHIFT_MASK 0xff0000 +#define VGT_GROUP_VECT_1_CNTL__SHIFT__SHIFT 0x10 +#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV_MASK 0xf +#define VGT_GROUP_VECT_0_FMT_CNTL__X_CONV__SHIFT 0x0 +#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET_MASK 0xf0 +#define VGT_GROUP_VECT_0_FMT_CNTL__X_OFFSET__SHIFT 0x4 +#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV_MASK 0xf00 +#define VGT_GROUP_VECT_0_FMT_CNTL__Y_CONV__SHIFT 0x8 +#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET_MASK 0xf000 +#define VGT_GROUP_VECT_0_FMT_CNTL__Y_OFFSET__SHIFT 0xc +#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV_MASK 0xf0000 +#define VGT_GROUP_VECT_0_FMT_CNTL__Z_CONV__SHIFT 0x10 +#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET_MASK 0xf00000 +#define VGT_GROUP_VECT_0_FMT_CNTL__Z_OFFSET__SHIFT 0x14 +#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV_MASK 0xf000000 +#define VGT_GROUP_VECT_0_FMT_CNTL__W_CONV__SHIFT 0x18 +#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET_MASK 0xf0000000 +#define VGT_GROUP_VECT_0_FMT_CNTL__W_OFFSET__SHIFT 0x1c +#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV_MASK 0xf +#define VGT_GROUP_VECT_1_FMT_CNTL__X_CONV__SHIFT 0x0 +#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET_MASK 0xf0 +#define VGT_GROUP_VECT_1_FMT_CNTL__X_OFFSET__SHIFT 0x4 +#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV_MASK 0xf00 +#define VGT_GROUP_VECT_1_FMT_CNTL__Y_CONV__SHIFT 0x8 +#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET_MASK 0xf000 +#define VGT_GROUP_VECT_1_FMT_CNTL__Y_OFFSET__SHIFT 0xc +#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV_MASK 0xf0000 +#define VGT_GROUP_VECT_1_FMT_CNTL__Z_CONV__SHIFT 0x10 +#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET_MASK 0xf00000 +#define VGT_GROUP_VECT_1_FMT_CNTL__Z_OFFSET__SHIFT 0x14 +#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV_MASK 0xf000000 +#define VGT_GROUP_VECT_1_FMT_CNTL__W_CONV__SHIFT 0x18 +#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET_MASK 0xf0000000 +#define VGT_GROUP_VECT_1_FMT_CNTL__W_OFFSET__SHIFT 0x1c +#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT_MASK 0x3ff +#define VGT_VTX_VECT_EJECT_REG__PRIM_COUNT__SHIFT 0x0 +#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH_MASK 0x1ff +#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH__SHIFT 0x0 +#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH_MASK 0x3f +#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH__SHIFT 0x0 +#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH_MASK 0x3f +#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH__SHIFT 0x0 +#define VGT_LAST_COPY_STATE__SRC_STATE_ID_MASK 0x7 +#define VGT_LAST_COPY_STATE__SRC_STATE_ID__SHIFT 0x0 +#define VGT_LAST_COPY_STATE__DST_STATE_ID_MASK 0x70000 +#define VGT_LAST_COPY_STATE__DST_STATE_ID__SHIFT 0x10 +#define CC_GC_SHADER_ARRAY_CONFIG__DPFP_RATE_MASK 0x6 +#define CC_GC_SHADER_ARRAY_CONFIG__DPFP_RATE__SHIFT 0x1 +#define CC_GC_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE_MASK 0x8 +#define CC_GC_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE__SHIFT 0x3 +#define CC_GC_SHADER_ARRAY_CONFIG__HALF_LDS_MASK 0x10 +#define CC_GC_SHADER_ARRAY_CONFIG__HALF_LDS__SHIFT 0x4 +#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK 0xffff0000 +#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT 0x10 +#define GC_USER_SHADER_ARRAY_CONFIG__DPFP_RATE_MASK 0x6 +#define GC_USER_SHADER_ARRAY_CONFIG__DPFP_RATE__SHIFT 0x1 +#define GC_USER_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE_MASK 0x8 +#define GC_USER_SHADER_ARRAY_CONFIG__SQC_BALANCE_DISABLE__SHIFT 0x3 +#define GC_USER_SHADER_ARRAY_CONFIG__HALF_LDS_MASK 0x10 +#define GC_USER_SHADER_ARRAY_CONFIG__HALF_LDS__SHIFT 0x4 +#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK 0xffff0000 +#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT 0x10 +#define VGT_GS_MODE__MODE_MASK 0x7 +#define VGT_GS_MODE__MODE__SHIFT 0x0 +#define VGT_GS_MODE__RESERVED_0_MASK 0x8 +#define VGT_GS_MODE__RESERVED_0__SHIFT 0x3 +#define VGT_GS_MODE__CUT_MODE_MASK 0x30 +#define VGT_GS_MODE__CUT_MODE__SHIFT 0x4 +#define VGT_GS_MODE__RESERVED_1_MASK 0x7c0 +#define VGT_GS_MODE__RESERVED_1__SHIFT 0x6 +#define VGT_GS_MODE__GS_C_PACK_EN_MASK 0x800 +#define VGT_GS_MODE__GS_C_PACK_EN__SHIFT 0xb +#define VGT_GS_MODE__RESERVED_2_MASK 0x1000 +#define VGT_GS_MODE__RESERVED_2__SHIFT 0xc +#define VGT_GS_MODE__ES_PASSTHRU_MASK 0x2000 +#define VGT_GS_MODE__ES_PASSTHRU__SHIFT 0xd +#define VGT_GS_MODE__COMPUTE_MODE_MASK 0x4000 +#define VGT_GS_MODE__COMPUTE_MODE__SHIFT 0xe +#define VGT_GS_MODE__FAST_COMPUTE_MODE_MASK 0x8000 +#define VGT_GS_MODE__FAST_COMPUTE_MODE__SHIFT 0xf +#define VGT_GS_MODE__ELEMENT_INFO_EN_MASK 0x10000 +#define VGT_GS_MODE__ELEMENT_INFO_EN__SHIFT 0x10 +#define VGT_GS_MODE__PARTIAL_THD_AT_EOI_MASK 0x20000 +#define VGT_GS_MODE__PARTIAL_THD_AT_EOI__SHIFT 0x11 +#define VGT_GS_MODE__SUPPRESS_CUTS_MASK 0x40000 +#define VGT_GS_MODE__SUPPRESS_CUTS__SHIFT 0x12 +#define VGT_GS_MODE__ES_WRITE_OPTIMIZE_MASK 0x80000 +#define VGT_GS_MODE__ES_WRITE_OPTIMIZE__SHIFT 0x13 +#define VGT_GS_MODE__GS_WRITE_OPTIMIZE_MASK 0x100000 +#define VGT_GS_MODE__GS_WRITE_OPTIMIZE__SHIFT 0x14 +#define VGT_GS_MODE__ONCHIP_MASK 0x600000 +#define VGT_GS_MODE__ONCHIP__SHIFT 0x15 +#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP_MASK 0x7ff +#define VGT_GS_ONCHIP_CNTL__ES_VERTS_PER_SUBGRP__SHIFT 0x0 +#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP_MASK 0x3ff800 +#define VGT_GS_ONCHIP_CNTL__GS_PRIMS_PER_SUBGRP__SHIFT 0xb +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_MASK 0x3f +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE__SHIFT 0x0 +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1_MASK 0x3f00 +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_1__SHIFT 0x8 +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2_MASK 0x3f0000 +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_2__SHIFT 0x10 +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3_MASK 0xfc00000 +#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_3__SHIFT 0x16 +#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM_MASK 0x80000000 +#define VGT_GS_OUT_PRIM_TYPE__UNIQUE_TYPE_PER_STREAM__SHIFT 0x1f +#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION_MASK 0x3 +#define VGT_CACHE_INVALIDATION__CACHE_INVALIDATION__SHIFT 0x0 +#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER_MASK 0x20 +#define VGT_CACHE_INVALIDATION__VS_NO_EXTRA_BUFFER__SHIFT 0x5 +#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN_MASK 0xc0 +#define VGT_CACHE_INVALIDATION__AUTO_INVLD_EN__SHIFT 0x6 +#define VGT_CACHE_INVALIDATION__USE_GS_DONE_MASK 0x200 +#define VGT_CACHE_INVALIDATION__USE_GS_DONE__SHIFT 0x9 +#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD_MASK 0x800 +#define VGT_CACHE_INVALIDATION__DIS_RANGE_FULL_INVLD__SHIFT 0xb +#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN_MASK 0x1000 +#define VGT_CACHE_INVALIDATION__GS_LATE_ALLOC_EN__SHIFT 0xc +#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH_MASK 0x2000 +#define VGT_CACHE_INVALIDATION__STREAMOUT_FULL_FLUSH__SHIFT 0xd +#define VGT_CACHE_INVALIDATION__ES_LIMIT_MASK 0x1f0000 +#define VGT_CACHE_INVALIDATION__ES_LIMIT__SHIFT 0x10 +#define VGT_RESET_DEBUG__GS_DISABLE_MASK 0x1 +#define VGT_RESET_DEBUG__GS_DISABLE__SHIFT 0x0 +#define VGT_RESET_DEBUG__TESS_DISABLE_MASK 0x2 +#define VGT_RESET_DEBUG__TESS_DISABLE__SHIFT 0x1 +#define VGT_RESET_DEBUG__WD_DISABLE_MASK 0x4 +#define VGT_RESET_DEBUG__WD_DISABLE__SHIFT 0x2 +#define VGT_STRMOUT_DELAY__SKIP_DELAY_MASK 0xff +#define VGT_STRMOUT_DELAY__SKIP_DELAY__SHIFT 0x0 +#define VGT_STRMOUT_DELAY__SE0_WD_DELAY_MASK 0x700 +#define VGT_STRMOUT_DELAY__SE0_WD_DELAY__SHIFT 0x8 +#define VGT_STRMOUT_DELAY__SE1_WD_DELAY_MASK 0x3800 +#define VGT_STRMOUT_DELAY__SE1_WD_DELAY__SHIFT 0xb +#define VGT_STRMOUT_DELAY__SE2_WD_DELAY_MASK 0x1c000 +#define VGT_STRMOUT_DELAY__SE2_WD_DELAY__SHIFT 0xe +#define VGT_STRMOUT_DELAY__SE3_WD_DELAY_MASK 0xe0000 +#define VGT_STRMOUT_DELAY__SE3_WD_DELAY__SHIFT 0x11 +#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH_MASK 0x7f +#define VGT_FIFO_DEPTHS__VS_DEALLOC_TBL_DEPTH__SHIFT 0x0 +#define VGT_FIFO_DEPTHS__RESERVED_0_MASK 0x80 +#define VGT_FIFO_DEPTHS__RESERVED_0__SHIFT 0x7 +#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH_MASK 0x3fff00 +#define VGT_FIFO_DEPTHS__CLIPP_FIFO_DEPTH__SHIFT 0x8 +#define VGT_FIFO_DEPTHS__RESERVED_1_MASK 0x400000 +#define VGT_FIFO_DEPTHS__RESERVED_1__SHIFT 0x16 +#define VGT_GS_PER_ES__GS_PER_ES_MASK 0x7ff +#define VGT_GS_PER_ES__GS_PER_ES__SHIFT 0x0 +#define VGT_ES_PER_GS__ES_PER_GS_MASK 0x7ff +#define VGT_ES_PER_GS__ES_PER_GS__SHIFT 0x0 +#define VGT_GS_PER_VS__GS_PER_VS_MASK 0xf +#define VGT_GS_PER_VS__GS_PER_VS__SHIFT 0x0 +#define VGT_GS_VERTEX_REUSE__VERT_REUSE_MASK 0x1f +#define VGT_GS_VERTEX_REUSE__VERT_REUSE__SHIFT 0x0 +#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES_MASK 0x3 +#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES__SHIFT 0x0 +#define IA_CNTL_STATUS__IA_BUSY_MASK 0x1 +#define IA_CNTL_STATUS__IA_BUSY__SHIFT 0x0 +#define IA_CNTL_STATUS__IA_DMA_BUSY_MASK 0x2 +#define IA_CNTL_STATUS__IA_DMA_BUSY__SHIFT 0x1 +#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY_MASK 0x4 +#define IA_CNTL_STATUS__IA_DMA_REQ_BUSY__SHIFT 0x2 +#define IA_CNTL_STATUS__IA_GRP_BUSY_MASK 0x8 +#define IA_CNTL_STATUS__IA_GRP_BUSY__SHIFT 0x3 +#define IA_CNTL_STATUS__IA_ADC_BUSY_MASK 0x10 +#define IA_CNTL_STATUS__IA_ADC_BUSY__SHIFT 0x4 +#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN_MASK 0x1 +#define VGT_STRMOUT_CONFIG__STREAMOUT_0_EN__SHIFT 0x0 +#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN_MASK 0x2 +#define VGT_STRMOUT_CONFIG__STREAMOUT_1_EN__SHIFT 0x1 +#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN_MASK 0x4 +#define VGT_STRMOUT_CONFIG__STREAMOUT_2_EN__SHIFT 0x2 +#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN_MASK 0x8 +#define VGT_STRMOUT_CONFIG__STREAMOUT_3_EN__SHIFT 0x3 +#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK 0x70 +#define VGT_STRMOUT_CONFIG__RAST_STREAM__SHIFT 0x4 +#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK_MASK 0xf00 +#define VGT_STRMOUT_CONFIG__RAST_STREAM_MASK__SHIFT 0x8 +#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK_MASK 0x80000000 +#define VGT_STRMOUT_CONFIG__USE_RAST_STREAM_MASK__SHIFT 0x1f +#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_SIZE_0__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_SIZE_1__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_SIZE_2__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_SIZE_3__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_OFFSET_0__OFFSET__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_OFFSET_1__OFFSET__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_OFFSET_2__OFFSET__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_OFFSET_3__OFFSET__SHIFT 0x0 +#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE_MASK 0x3ff +#define VGT_STRMOUT_VTX_STRIDE_0__STRIDE__SHIFT 0x0 +#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE_MASK 0x3ff +#define VGT_STRMOUT_VTX_STRIDE_1__STRIDE__SHIFT 0x0 +#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE_MASK 0x3ff +#define VGT_STRMOUT_VTX_STRIDE_2__STRIDE__SHIFT 0x0 +#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE_MASK 0x3ff +#define VGT_STRMOUT_VTX_STRIDE_3__STRIDE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN_MASK 0xf +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_0_BUFFER_EN__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN_MASK 0xf0 +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_1_BUFFER_EN__SHIFT 0x4 +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN_MASK 0xf00 +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_2_BUFFER_EN__SHIFT 0x8 +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN_MASK 0xf000 +#define VGT_STRMOUT_BUFFER_CONFIG__STREAM_3_BUFFER_EN__SHIFT 0xc +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_0__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_1__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_2__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_BUFFER_FILLED_SIZE_3__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET_MASK 0xffffffff +#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET__SHIFT 0x0 +#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE_MASK 0xffffffff +#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE__SHIFT 0x0 +#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE_MASK 0x1ff +#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE__SHIFT 0x0 +#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT_MASK 0x7ff +#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT__SHIFT 0x0 +#define IA_VMID_OVERRIDE__ENABLE_MASK 0x1 +#define IA_VMID_OVERRIDE__ENABLE__SHIFT 0x0 +#define IA_VMID_OVERRIDE__VMID_MASK 0x1e +#define IA_VMID_OVERRIDE__VMID__SHIFT 0x1 +#define VGT_SHADER_STAGES_EN__LS_EN_MASK 0x3 +#define VGT_SHADER_STAGES_EN__LS_EN__SHIFT 0x0 +#define VGT_SHADER_STAGES_EN__HS_EN_MASK 0x4 +#define VGT_SHADER_STAGES_EN__HS_EN__SHIFT 0x2 +#define VGT_SHADER_STAGES_EN__ES_EN_MASK 0x18 +#define VGT_SHADER_STAGES_EN__ES_EN__SHIFT 0x3 +#define VGT_SHADER_STAGES_EN__GS_EN_MASK 0x20 +#define VGT_SHADER_STAGES_EN__GS_EN__SHIFT 0x5 +#define VGT_SHADER_STAGES_EN__VS_EN_MASK 0xc0 +#define VGT_SHADER_STAGES_EN__VS_EN__SHIFT 0x6 +#define VGT_SHADER_STAGES_EN__DYNAMIC_HS_MASK 0x100 +#define VGT_SHADER_STAGES_EN__DYNAMIC_HS__SHIFT 0x8 +#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX_MASK 0xffffffff +#define VGT_DISPATCH_DRAW_INDEX__MATCH_INDEX__SHIFT 0x0 +#define VGT_LS_HS_CONFIG__NUM_PATCHES_MASK 0xff +#define VGT_LS_HS_CONFIG__NUM_PATCHES__SHIFT 0x0 +#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK 0x3f00 +#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT 0x8 +#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP_MASK 0xfc000 +#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP__SHIFT 0xe +#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK 0x3f00 +#define VGT_DMA_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT 0x8 +#define VGT_TF_PARAM__TYPE_MASK 0x3 +#define VGT_TF_PARAM__TYPE__SHIFT 0x0 +#define VGT_TF_PARAM__PARTITIONING_MASK 0x1c +#define VGT_TF_PARAM__PARTITIONING__SHIFT 0x2 +#define VGT_TF_PARAM__TOPOLOGY_MASK 0xe0 +#define VGT_TF_PARAM__TOPOLOGY__SHIFT 0x5 +#define VGT_TF_PARAM__RESERVED_REDUC_AXIS_MASK 0x100 +#define VGT_TF_PARAM__RESERVED_REDUC_AXIS__SHIFT 0x8 +#define VGT_TF_PARAM__DEPRECATED_MASK 0x200 +#define VGT_TF_PARAM__DEPRECATED__SHIFT 0x9 +#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD_MASK 0x3c00 +#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD__SHIFT 0xa +#define VGT_TF_PARAM__DISABLE_DONUTS_MASK 0x4000 +#define VGT_TF_PARAM__DISABLE_DONUTS__SHIFT 0xe +#define VGT_TF_PARAM__RDREQ_POLICY_MASK 0x18000 +#define VGT_TF_PARAM__RDREQ_POLICY__SHIFT 0xf +#define VGT_TF_RING_SIZE__SIZE_MASK 0xffff +#define VGT_TF_RING_SIZE__SIZE__SHIFT 0x0 +#define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK 0x1 +#define VGT_SYS_CONFIG__DUAL_CORE_EN__SHIFT 0x0 +#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP_MASK 0x7e +#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP__SHIFT 0x1 +#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE_MASK 0x80 +#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE__SHIFT 0x7 +#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING_MASK 0x1ff +#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING__SHIFT 0x0 +#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY_MASK 0x600 +#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY__SHIFT 0x9 +#define VGT_TF_MEMORY_BASE__BASE_MASK 0xffffffff +#define VGT_TF_MEMORY_BASE__BASE__SHIFT 0x0 +#define VGT_GS_INSTANCE_CNT__ENABLE_MASK 0x1 +#define VGT_GS_INSTANCE_CNT__ENABLE__SHIFT 0x0 +#define VGT_GS_INSTANCE_CNT__CNT_MASK 0x1fc +#define VGT_GS_INSTANCE_CNT__CNT__SHIFT 0x2 +#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE_MASK 0xffff +#define IA_MULTI_VGT_PARAM__PRIMGROUP_SIZE__SHIFT 0x0 +#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON_MASK 0x10000 +#define IA_MULTI_VGT_PARAM__PARTIAL_VS_WAVE_ON__SHIFT 0x10 +#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP_MASK 0x20000 +#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOP__SHIFT 0x11 +#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON_MASK 0x40000 +#define IA_MULTI_VGT_PARAM__PARTIAL_ES_WAVE_ON__SHIFT 0x12 +#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI_MASK 0x80000 +#define IA_MULTI_VGT_PARAM__SWITCH_ON_EOI__SHIFT 0x13 +#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP_MASK 0x100000 +#define IA_MULTI_VGT_PARAM__WD_SWITCH_ON_EOP__SHIFT 0x14 +#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID_MASK 0xfff +#define VGT_VS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT 0x0 +#define VGT_ESGS_RING_SIZE__MEM_SIZE_MASK 0xffffffff +#define VGT_ESGS_RING_SIZE__MEM_SIZE__SHIFT 0x0 +#define VGT_GSVS_RING_SIZE__MEM_SIZE_MASK 0xffffffff +#define VGT_GSVS_RING_SIZE__MEM_SIZE__SHIFT 0x0 +#define VGT_GSVS_RING_OFFSET_1__OFFSET_MASK 0x7fff +#define VGT_GSVS_RING_OFFSET_1__OFFSET__SHIFT 0x0 +#define VGT_GSVS_RING_OFFSET_2__OFFSET_MASK 0x7fff +#define VGT_GSVS_RING_OFFSET_2__OFFSET__SHIFT 0x0 +#define VGT_GSVS_RING_OFFSET_3__OFFSET_MASK 0x7fff +#define VGT_GSVS_RING_OFFSET_3__OFFSET__SHIFT 0x0 +#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE_MASK 0x7fff +#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE__SHIFT 0x0 +#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE_MASK 0x7fff +#define VGT_GSVS_RING_ITEMSIZE__ITEMSIZE__SHIFT 0x0 +#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE_MASK 0x7fff +#define VGT_GS_VERT_ITEMSIZE__ITEMSIZE__SHIFT 0x0 +#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE_MASK 0x7fff +#define VGT_GS_VERT_ITEMSIZE_1__ITEMSIZE__SHIFT 0x0 +#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE_MASK 0x7fff +#define VGT_GS_VERT_ITEMSIZE_2__ITEMSIZE__SHIFT 0x0 +#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE_MASK 0x7fff +#define VGT_GS_VERT_ITEMSIZE_3__ITEMSIZE__SHIFT 0x0 +#define WD_CNTL_STATUS__WD_BUSY_MASK 0x1 +#define WD_CNTL_STATUS__WD_BUSY__SHIFT 0x0 +#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY_MASK 0x2 +#define WD_CNTL_STATUS__WD_SPL_DMA_BUSY__SHIFT 0x1 +#define WD_CNTL_STATUS__WD_SPL_DI_BUSY_MASK 0x4 +#define WD_CNTL_STATUS__WD_SPL_DI_BUSY__SHIFT 0x2 +#define WD_CNTL_STATUS__WD_ADC_BUSY_MASK 0x8 +#define WD_CNTL_STATUS__WD_ADC_BUSY__SHIFT 0x3 +#define WD_ENHANCE__MISC_MASK 0xffffffff +#define WD_ENHANCE__MISC__SHIFT 0x0 +#define GFX_PIPE_CONTROL__HYSTERESIS_CNT_MASK 0x1fff +#define GFX_PIPE_CONTROL__HYSTERESIS_CNT__SHIFT 0x0 +#define GFX_PIPE_CONTROL__RESERVED_MASK 0xe000 +#define GFX_PIPE_CONTROL__RESERVED__SHIFT 0xd +#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN_MASK 0x10000 +#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN__SHIFT 0x10 +#define GFX_PIPE_PRIORITY__HP_PIPE_SELECT_MASK 0x1 +#define GFX_PIPE_PRIORITY__HP_PIPE_SELECT__SHIFT 0x0 +#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK 0x2000000 +#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT 0x19 +#define CGTT_VGT_CLK_CTRL__DBG_ENABLE_MASK 0x4000000 +#define CGTT_VGT_CLK_CTRL__DBG_ENABLE__SHIFT 0x1a +#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_VGT_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE_MASK 0x20000000 +#define CGTT_VGT_CLK_CTRL__GS_OVERRIDE__SHIFT 0x1d +#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000 +#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e +#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK 0x2000000 +#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT 0x19 +#define CGTT_IA_CLK_CTRL__DBG_ENABLE_MASK 0x4000000 +#define CGTT_IA_CLK_CTRL__DBG_ENABLE__SHIFT 0x1a +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000 +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000 +#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK 0x40000000 +#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1e +#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK 0xf +#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x1000000 +#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK 0x2000000 +#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT 0x19 +#define CGTT_WD_CLK_CTRL__DBG_ENABLE_MASK 0x4000000 +#define CGTT_WD_CLK_CTRL__DBG_ENABLE__SHIFT 0x1a +#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x8000000 +#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define CGTT_WD_CLK_CTRL__ADC_OVERRIDE_MASK 0x10000000 +#define CGTT_WD_CLK_CTRL__ADC_OVERRIDE__SHIFT 0x1c +#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK 0x20000000 +#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT 0x1d +#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK 0x40000000 +#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT 0x1e +#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK 0x80000000 +#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT 0x1f +#define VGT_DEBUG_CNTL__VGT_DEBUG_INDX_MASK 0x3f +#define VGT_DEBUG_CNTL__VGT_DEBUG_INDX__SHIFT 0x0 +#define VGT_DEBUG_CNTL__VGT_DEBUG_SEL_BUS_B_MASK 0x40 +#define VGT_DEBUG_CNTL__VGT_DEBUG_SEL_BUS_B__SHIFT 0x6 +#define VGT_DEBUG_DATA__DATA_MASK 0xffffffff +#define VGT_DEBUG_DATA__DATA__SHIFT 0x0 +#define IA_DEBUG_CNTL__IA_DEBUG_INDX_MASK 0x3f +#define IA_DEBUG_CNTL__IA_DEBUG_INDX__SHIFT 0x0 +#define IA_DEBUG_CNTL__IA_DEBUG_SEL_BUS_B_MASK 0x40 +#define IA_DEBUG_CNTL__IA_DEBUG_SEL_BUS_B__SHIFT 0x6 +#define IA_DEBUG_DATA__DATA_MASK 0xffffffff +#define IA_DEBUG_DATA__DATA__SHIFT 0x0 +#define VGT_CNTL_STATUS__VGT_BUSY_MASK 0x1 +#define VGT_CNTL_STATUS__VGT_BUSY__SHIFT 0x0 +#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY_MASK 0x2 +#define VGT_CNTL_STATUS__VGT_OUT_INDX_BUSY__SHIFT 0x1 +#define VGT_CNTL_STATUS__VGT_OUT_BUSY_MASK 0x4 +#define VGT_CNTL_STATUS__VGT_OUT_BUSY__SHIFT 0x2 +#define VGT_CNTL_STATUS__VGT_PT_BUSY_MASK 0x8 +#define VGT_CNTL_STATUS__VGT_PT_BUSY__SHIFT 0x3 +#define VGT_CNTL_STATUS__VGT_TE_BUSY_MASK 0x10 +#define VGT_CNTL_STATUS__VGT_TE_BUSY__SHIFT 0x4 +#define VGT_CNTL_STATUS__VGT_VR_BUSY_MASK 0x20 +#define VGT_CNTL_STATUS__VGT_VR_BUSY__SHIFT 0x5 +#define VGT_CNTL_STATUS__VGT_PI_BUSY_MASK 0x40 +#define VGT_CNTL_STATUS__VGT_PI_BUSY__SHIFT 0x6 +#define VGT_CNTL_STATUS__VGT_GS_BUSY_MASK 0x80 +#define VGT_CNTL_STATUS__VGT_GS_BUSY__SHIFT 0x7 +#define VGT_CNTL_STATUS__VGT_HS_BUSY_MASK 0x100 +#define VGT_CNTL_STATUS__VGT_HS_BUSY__SHIFT 0x8 +#define VGT_CNTL_STATUS__VGT_TE11_BUSY_MASK 0x200 +#define VGT_CNTL_STATUS__VGT_TE11_BUSY__SHIFT 0x9 +#define WD_DEBUG_CNTL__WD_DEBUG_INDX_MASK 0x3f +#define WD_DEBUG_CNTL__WD_DEBUG_INDX__SHIFT 0x0 +#define WD_DEBUG_CNTL__WD_DEBUG_SEL_BUS_B_MASK 0x40 +#define WD_DEBUG_CNTL__WD_DEBUG_SEL_BUS_B__SHIFT 0x6 +#define WD_DEBUG_DATA__DATA_MASK 0xffffffff +#define WD_DEBUG_DATA__DATA__SHIFT 0x0 +#define CC_GC_PRIM_CONFIG__INACTIVE_IA_MASK 0x30000 +#define CC_GC_PRIM_CONFIG__INACTIVE_IA__SHIFT 0x10 +#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA_MASK 0xf000000 +#define CC_GC_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT 0x18 +#define GC_USER_PRIM_CONFIG__INACTIVE_IA_MASK 0x30000 +#define GC_USER_PRIM_CONFIG__INACTIVE_IA__SHIFT 0x10 +#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA_MASK 0xf000000 +#define GC_USER_PRIM_CONFIG__INACTIVE_VGT_PA__SHIFT 0x18 +#define WD_DEBUG_REG0__wd_busy_extended_MASK 0x1 +#define WD_DEBUG_REG0__wd_busy_extended__SHIFT 0x0 +#define WD_DEBUG_REG0__wd_nodma_busy_extended_MASK 0x2 +#define WD_DEBUG_REG0__wd_nodma_busy_extended__SHIFT 0x1 +#define WD_DEBUG_REG0__wd_busy_MASK 0x4 +#define WD_DEBUG_REG0__wd_busy__SHIFT 0x2 +#define WD_DEBUG_REG0__wd_nodma_busy_MASK 0x8 +#define WD_DEBUG_REG0__wd_nodma_busy__SHIFT 0x3 +#define WD_DEBUG_REG0__rbiu_busy_MASK 0x10 +#define WD_DEBUG_REG0__rbiu_busy__SHIFT 0x4 +#define WD_DEBUG_REG0__spl_dma_busy_MASK 0x20 +#define WD_DEBUG_REG0__spl_dma_busy__SHIFT 0x5 +#define WD_DEBUG_REG0__spl_di_busy_MASK 0x40 +#define WD_DEBUG_REG0__spl_di_busy__SHIFT 0x6 +#define WD_DEBUG_REG0__vgt0_active_q_MASK 0x80 +#define WD_DEBUG_REG0__vgt0_active_q__SHIFT 0x7 +#define WD_DEBUG_REG0__vgt1_active_q_MASK 0x100 +#define WD_DEBUG_REG0__vgt1_active_q__SHIFT 0x8 +#define WD_DEBUG_REG0__spl_dma_p1_busy_MASK 0x200 +#define WD_DEBUG_REG0__spl_dma_p1_busy__SHIFT 0x9 +#define WD_DEBUG_REG0__rbiu_dr_p1_fifo_busy_MASK 0x400 +#define WD_DEBUG_REG0__rbiu_dr_p1_fifo_busy__SHIFT 0xa +#define WD_DEBUG_REG0__rbiu_di_p1_fifo_busy_MASK 0x800 +#define WD_DEBUG_REG0__rbiu_di_p1_fifo_busy__SHIFT 0xb +#define WD_DEBUG_REG0__SPARE2_MASK 0x1000 +#define WD_DEBUG_REG0__SPARE2__SHIFT 0xc +#define WD_DEBUG_REG0__rbiu_dr_fifo_busy_MASK 0x2000 +#define WD_DEBUG_REG0__rbiu_dr_fifo_busy__SHIFT 0xd +#define WD_DEBUG_REG0__rbiu_spl_dr_valid_MASK 0x4000 +#define WD_DEBUG_REG0__rbiu_spl_dr_valid__SHIFT 0xe +#define WD_DEBUG_REG0__spl_rbiu_dr_read_MASK 0x8000 +#define WD_DEBUG_REG0__spl_rbiu_dr_read__SHIFT 0xf +#define WD_DEBUG_REG0__SPARE3_MASK 0x10000 +#define WD_DEBUG_REG0__SPARE3__SHIFT 0x10 +#define WD_DEBUG_REG0__rbiu_di_fifo_busy_MASK 0x20000 +#define WD_DEBUG_REG0__rbiu_di_fifo_busy__SHIFT 0x11 +#define WD_DEBUG_REG0__rbiu_spl_di_valid_MASK 0x40000 +#define WD_DEBUG_REG0__rbiu_spl_di_valid__SHIFT 0x12 +#define WD_DEBUG_REG0__spl_rbiu_di_read_MASK 0x80000 +#define WD_DEBUG_REG0__spl_rbiu_di_read__SHIFT 0x13 +#define WD_DEBUG_REG0__se0_synced_q_MASK 0x100000 +#define WD_DEBUG_REG0__se0_synced_q__SHIFT 0x14 +#define WD_DEBUG_REG0__se1_synced_q_MASK 0x200000 +#define WD_DEBUG_REG0__se1_synced_q__SHIFT 0x15 +#define WD_DEBUG_REG0__se2_synced_q_MASK 0x400000 +#define WD_DEBUG_REG0__se2_synced_q__SHIFT 0x16 +#define WD_DEBUG_REG0__se3_synced_q_MASK 0x800000 +#define WD_DEBUG_REG0__se3_synced_q__SHIFT 0x17 +#define WD_DEBUG_REG0__reg_clk_busy_MASK 0x1000000 +#define WD_DEBUG_REG0__reg_clk_busy__SHIFT 0x18 +#define WD_DEBUG_REG0__input_clk_busy_MASK 0x2000000 +#define WD_DEBUG_REG0__input_clk_busy__SHIFT 0x19 +#define WD_DEBUG_REG0__core_clk_busy_MASK 0x4000000 +#define WD_DEBUG_REG0__core_clk_busy__SHIFT 0x1a +#define WD_DEBUG_REG0__vgt2_active_q_MASK 0x8000000 +#define WD_DEBUG_REG0__vgt2_active_q__SHIFT 0x1b +#define WD_DEBUG_REG0__sclk_reg_vld_MASK 0x10000000 +#define WD_DEBUG_REG0__sclk_reg_vld__SHIFT 0x1c +#define WD_DEBUG_REG0__sclk_input_vld_MASK 0x20000000 +#define WD_DEBUG_REG0__sclk_input_vld__SHIFT 0x1d +#define WD_DEBUG_REG0__sclk_core_vld_MASK 0x40000000 +#define WD_DEBUG_REG0__sclk_core_vld__SHIFT 0x1e +#define WD_DEBUG_REG0__vgt3_active_q_MASK 0x80000000 +#define WD_DEBUG_REG0__vgt3_active_q__SHIFT 0x1f +#define WD_DEBUG_REG1__grbm_fifo_empty_MASK 0x1 +#define WD_DEBUG_REG1__grbm_fifo_empty__SHIFT 0x0 +#define WD_DEBUG_REG1__grbm_fifo_full_MASK 0x2 +#define WD_DEBUG_REG1__grbm_fifo_full__SHIFT 0x1 +#define WD_DEBUG_REG1__grbm_fifo_we_MASK 0x4 +#define WD_DEBUG_REG1__grbm_fifo_we__SHIFT 0x2 +#define WD_DEBUG_REG1__grbm_fifo_re_MASK 0x8 +#define WD_DEBUG_REG1__grbm_fifo_re__SHIFT 0x3 +#define WD_DEBUG_REG1__draw_initiator_valid_q_MASK 0x10 +#define WD_DEBUG_REG1__draw_initiator_valid_q__SHIFT 0x4 +#define WD_DEBUG_REG1__event_initiator_valid_q_MASK 0x20 +#define WD_DEBUG_REG1__event_initiator_valid_q__SHIFT 0x5 +#define WD_DEBUG_REG1__event_addr_valid_q_MASK 0x40 +#define WD_DEBUG_REG1__event_addr_valid_q__SHIFT 0x6 +#define WD_DEBUG_REG1__dma_request_valid_q_MASK 0x80 +#define WD_DEBUG_REG1__dma_request_valid_q__SHIFT 0x7 +#define WD_DEBUG_REG1__SPARE0_MASK 0x100 +#define WD_DEBUG_REG1__SPARE0__SHIFT 0x8 +#define WD_DEBUG_REG1__min_indx_valid_q_MASK 0x200 +#define WD_DEBUG_REG1__min_indx_valid_q__SHIFT 0x9 +#define WD_DEBUG_REG1__max_indx_valid_q_MASK 0x400 +#define WD_DEBUG_REG1__max_indx_valid_q__SHIFT 0xa +#define WD_DEBUG_REG1__indx_offset_valid_q_MASK 0x800 +#define WD_DEBUG_REG1__indx_offset_valid_q__SHIFT 0xb +#define WD_DEBUG_REG1__grbm_fifo_rdata_reg_id_MASK 0x1f000 +#define WD_DEBUG_REG1__grbm_fifo_rdata_reg_id__SHIFT 0xc +#define WD_DEBUG_REG1__grbm_fifo_rdata_state_MASK 0xe0000 +#define WD_DEBUG_REG1__grbm_fifo_rdata_state__SHIFT 0x11 +#define WD_DEBUG_REG1__free_cnt_q_MASK 0x3f00000 +#define WD_DEBUG_REG1__free_cnt_q__SHIFT 0x14 +#define WD_DEBUG_REG1__rbiu_di_fifo_we_MASK 0x4000000 +#define WD_DEBUG_REG1__rbiu_di_fifo_we__SHIFT 0x1a +#define WD_DEBUG_REG1__rbiu_dr_fifo_we_MASK 0x8000000 +#define WD_DEBUG_REG1__rbiu_dr_fifo_we__SHIFT 0x1b +#define WD_DEBUG_REG1__rbiu_di_fifo_empty_MASK 0x10000000 +#define WD_DEBUG_REG1__rbiu_di_fifo_empty__SHIFT 0x1c +#define WD_DEBUG_REG1__rbiu_di_fifo_full_MASK 0x20000000 +#define WD_DEBUG_REG1__rbiu_di_fifo_full__SHIFT 0x1d +#define WD_DEBUG_REG1__rbiu_dr_fifo_empty_MASK 0x40000000 +#define WD_DEBUG_REG1__rbiu_dr_fifo_empty__SHIFT 0x1e +#define WD_DEBUG_REG1__rbiu_dr_fifo_full_MASK 0x80000000 +#define WD_DEBUG_REG1__rbiu_dr_fifo_full__SHIFT 0x1f +#define WD_DEBUG_REG2__p1_grbm_fifo_empty_MASK 0x1 +#define WD_DEBUG_REG2__p1_grbm_fifo_empty__SHIFT 0x0 +#define WD_DEBUG_REG2__p1_grbm_fifo_full_MASK 0x2 +#define WD_DEBUG_REG2__p1_grbm_fifo_full__SHIFT 0x1 +#define WD_DEBUG_REG2__p1_grbm_fifo_we_MASK 0x4 +#define WD_DEBUG_REG2__p1_grbm_fifo_we__SHIFT 0x2 +#define WD_DEBUG_REG2__p1_grbm_fifo_re_MASK 0x8 +#define WD_DEBUG_REG2__p1_grbm_fifo_re__SHIFT 0x3 +#define WD_DEBUG_REG2__p1_draw_initiator_valid_q_MASK 0x10 +#define WD_DEBUG_REG2__p1_draw_initiator_valid_q__SHIFT 0x4 +#define WD_DEBUG_REG2__p1_event_initiator_valid_q_MASK 0x20 +#define WD_DEBUG_REG2__p1_event_initiator_valid_q__SHIFT 0x5 +#define WD_DEBUG_REG2__p1_event_addr_valid_q_MASK 0x40 +#define WD_DEBUG_REG2__p1_event_addr_valid_q__SHIFT 0x6 +#define WD_DEBUG_REG2__p1_dma_request_valid_q_MASK 0x80 +#define WD_DEBUG_REG2__p1_dma_request_valid_q__SHIFT 0x7 +#define WD_DEBUG_REG2__SPARE0_MASK 0x100 +#define WD_DEBUG_REG2__SPARE0__SHIFT 0x8 +#define WD_DEBUG_REG2__p1_min_indx_valid_q_MASK 0x200 +#define WD_DEBUG_REG2__p1_min_indx_valid_q__SHIFT 0x9 +#define WD_DEBUG_REG2__p1_max_indx_valid_q_MASK 0x400 +#define WD_DEBUG_REG2__p1_max_indx_valid_q__SHIFT 0xa +#define WD_DEBUG_REG2__p1_indx_offset_valid_q_MASK 0x800 +#define WD_DEBUG_REG2__p1_indx_offset_valid_q__SHIFT 0xb +#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_reg_id_MASK 0x1f000 +#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_reg_id__SHIFT 0xc +#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_state_MASK 0xe0000 +#define WD_DEBUG_REG2__p1_grbm_fifo_rdata_state__SHIFT 0x11 +#define WD_DEBUG_REG2__p1_free_cnt_q_MASK 0x3f00000 +#define WD_DEBUG_REG2__p1_free_cnt_q__SHIFT 0x14 +#define WD_DEBUG_REG2__p1_rbiu_di_fifo_we_MASK 0x4000000 +#define WD_DEBUG_REG2__p1_rbiu_di_fifo_we__SHIFT 0x1a +#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_we_MASK 0x8000000 +#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_we__SHIFT 0x1b +#define WD_DEBUG_REG2__p1_rbiu_di_fifo_empty_MASK 0x10000000 +#define WD_DEBUG_REG2__p1_rbiu_di_fifo_empty__SHIFT 0x1c +#define WD_DEBUG_REG2__p1_rbiu_di_fifo_full_MASK 0x20000000 +#define WD_DEBUG_REG2__p1_rbiu_di_fifo_full__SHIFT 0x1d +#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_empty_MASK 0x40000000 +#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_empty__SHIFT 0x1e +#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_full_MASK 0x80000000 +#define WD_DEBUG_REG2__p1_rbiu_dr_fifo_full__SHIFT 0x1f +#define WD_DEBUG_REG3__rbiu_spl_dr_valid_MASK 0x1 +#define WD_DEBUG_REG3__rbiu_spl_dr_valid__SHIFT 0x0 +#define WD_DEBUG_REG3__SPARE0_MASK 0x2 +#define WD_DEBUG_REG3__SPARE0__SHIFT 0x1 +#define WD_DEBUG_REG3__pipe0_dr_MASK 0x4 +#define WD_DEBUG_REG3__pipe0_dr__SHIFT 0x2 +#define WD_DEBUG_REG3__pipe0_rtr_MASK 0x8 +#define WD_DEBUG_REG3__pipe0_rtr__SHIFT 0x3 +#define WD_DEBUG_REG3__pipe1_dr_MASK 0x10 +#define WD_DEBUG_REG3__pipe1_dr__SHIFT 0x4 +#define WD_DEBUG_REG3__pipe1_rtr_MASK 0x20 +#define WD_DEBUG_REG3__pipe1_rtr__SHIFT 0x5 +#define WD_DEBUG_REG3__wd_subdma_fifo_empty_MASK 0x40 +#define WD_DEBUG_REG3__wd_subdma_fifo_empty__SHIFT 0x6 +#define WD_DEBUG_REG3__wd_subdma_fifo_full_MASK 0x80 +#define WD_DEBUG_REG3__wd_subdma_fifo_full__SHIFT 0x7 +#define WD_DEBUG_REG3__dma_buf_type_p0_q_MASK 0x300 +#define WD_DEBUG_REG3__dma_buf_type_p0_q__SHIFT 0x8 +#define WD_DEBUG_REG3__dma_zero_indices_p0_q_MASK 0x400 +#define WD_DEBUG_REG3__dma_zero_indices_p0_q__SHIFT 0xa +#define WD_DEBUG_REG3__dma_req_path_p3_q_MASK 0x800 +#define WD_DEBUG_REG3__dma_req_path_p3_q__SHIFT 0xb +#define WD_DEBUG_REG3__dma_not_eop_p1_q_MASK 0x1000 +#define WD_DEBUG_REG3__dma_not_eop_p1_q__SHIFT 0xc +#define WD_DEBUG_REG3__out_of_range_p4_MASK 0x2000 +#define WD_DEBUG_REG3__out_of_range_p4__SHIFT 0xd +#define WD_DEBUG_REG3__last_sub_dma_p3_q_MASK 0x4000 +#define WD_DEBUG_REG3__last_sub_dma_p3_q__SHIFT 0xe +#define WD_DEBUG_REG3__last_rdreq_of_sub_dma_p4_MASK 0x8000 +#define WD_DEBUG_REG3__last_rdreq_of_sub_dma_p4__SHIFT 0xf +#define WD_DEBUG_REG3__WD_IA_dma_send_d_MASK 0x10000 +#define WD_DEBUG_REG3__WD_IA_dma_send_d__SHIFT 0x10 +#define WD_DEBUG_REG3__WD_IA_dma_rtr_MASK 0x20000 +#define WD_DEBUG_REG3__WD_IA_dma_rtr__SHIFT 0x11 +#define WD_DEBUG_REG3__WD_IA1_dma_send_d_MASK 0x40000 +#define WD_DEBUG_REG3__WD_IA1_dma_send_d__SHIFT 0x12 +#define WD_DEBUG_REG3__WD_IA1_dma_rtr_MASK 0x80000 +#define WD_DEBUG_REG3__WD_IA1_dma_rtr__SHIFT 0x13 +#define WD_DEBUG_REG3__last_inst_of_dma_p2_MASK 0x100000 +#define WD_DEBUG_REG3__last_inst_of_dma_p2__SHIFT 0x14 +#define WD_DEBUG_REG3__last_sd_of_inst_p2_MASK 0x200000 +#define WD_DEBUG_REG3__last_sd_of_inst_p2__SHIFT 0x15 +#define WD_DEBUG_REG3__last_sd_of_dma_p2_MASK 0x400000 +#define WD_DEBUG_REG3__last_sd_of_dma_p2__SHIFT 0x16 +#define WD_DEBUG_REG3__SPARE1_MASK 0x800000 +#define WD_DEBUG_REG3__SPARE1__SHIFT 0x17 +#define WD_DEBUG_REG3__WD_IA_dma_busy_MASK 0x1000000 +#define WD_DEBUG_REG3__WD_IA_dma_busy__SHIFT 0x18 +#define WD_DEBUG_REG3__WD_IA1_dma_busy_MASK 0x2000000 +#define WD_DEBUG_REG3__WD_IA1_dma_busy__SHIFT 0x19 +#define WD_DEBUG_REG3__send_to_ia1_p3_q_MASK 0x4000000 +#define WD_DEBUG_REG3__send_to_ia1_p3_q__SHIFT 0x1a +#define WD_DEBUG_REG3__dma_wd_switch_on_eop_p3_q_MASK 0x8000000 +#define WD_DEBUG_REG3__dma_wd_switch_on_eop_p3_q__SHIFT 0x1b +#define WD_DEBUG_REG3__pipe3_dr_MASK 0x10000000 +#define WD_DEBUG_REG3__pipe3_dr__SHIFT 0x1c +#define WD_DEBUG_REG3__pipe3_rtr_MASK 0x20000000 +#define WD_DEBUG_REG3__pipe3_rtr__SHIFT 0x1d +#define WD_DEBUG_REG3__wd_dma2draw_fifo_empty_MASK 0x40000000 +#define WD_DEBUG_REG3__wd_dma2draw_fifo_empty__SHIFT 0x1e +#define WD_DEBUG_REG3__wd_dma2draw_fifo_full_MASK 0x80000000 +#define WD_DEBUG_REG3__wd_dma2draw_fifo_full__SHIFT 0x1f +#define WD_DEBUG_REG4__rbiu_spl_di_valid_MASK 0x1 +#define WD_DEBUG_REG4__rbiu_spl_di_valid__SHIFT 0x0 +#define WD_DEBUG_REG4__spl_rbiu_di_read_MASK 0x2 +#define WD_DEBUG_REG4__spl_rbiu_di_read__SHIFT 0x1 +#define WD_DEBUG_REG4__rbiu_spl_p1_di_valid_MASK 0x4 +#define WD_DEBUG_REG4__rbiu_spl_p1_di_valid__SHIFT 0x2 +#define WD_DEBUG_REG4__spl_rbiu_p1_di_read_MASK 0x8 +#define WD_DEBUG_REG4__spl_rbiu_p1_di_read__SHIFT 0x3 +#define WD_DEBUG_REG4__pipe0_dr_MASK 0x10 +#define WD_DEBUG_REG4__pipe0_dr__SHIFT 0x4 +#define WD_DEBUG_REG4__pipe0_rtr_MASK 0x20 +#define WD_DEBUG_REG4__pipe0_rtr__SHIFT 0x5 +#define WD_DEBUG_REG4__pipe1_dr_MASK 0x40 +#define WD_DEBUG_REG4__pipe1_dr__SHIFT 0x6 +#define WD_DEBUG_REG4__pipe1_rtr_MASK 0x80 +#define WD_DEBUG_REG4__pipe1_rtr__SHIFT 0x7 +#define WD_DEBUG_REG4__pipe2_dr_MASK 0x100 +#define WD_DEBUG_REG4__pipe2_dr__SHIFT 0x8 +#define WD_DEBUG_REG4__pipe2_rtr_MASK 0x200 +#define WD_DEBUG_REG4__pipe2_rtr__SHIFT 0x9 +#define WD_DEBUG_REG4__pipe3_ld_MASK 0x400 +#define WD_DEBUG_REG4__pipe3_ld__SHIFT 0xa +#define WD_DEBUG_REG4__pipe3_rtr_MASK 0x800 +#define WD_DEBUG_REG4__pipe3_rtr__SHIFT 0xb +#define WD_DEBUG_REG4__WD_IA_draw_send_d_MASK 0x1000 +#define WD_DEBUG_REG4__WD_IA_draw_send_d__SHIFT 0xc +#define WD_DEBUG_REG4__WD_IA_draw_rtr_MASK 0x2000 +#define WD_DEBUG_REG4__WD_IA_draw_rtr__SHIFT 0xd +#define WD_DEBUG_REG4__di_type_p0_MASK 0xc000 +#define WD_DEBUG_REG4__di_type_p0__SHIFT 0xe +#define WD_DEBUG_REG4__di_state_sel_p1_q_MASK 0x70000 +#define WD_DEBUG_REG4__di_state_sel_p1_q__SHIFT 0x10 +#define WD_DEBUG_REG4__di_wd_switch_on_eop_p1_q_MASK 0x80000 +#define WD_DEBUG_REG4__di_wd_switch_on_eop_p1_q__SHIFT 0x13 +#define WD_DEBUG_REG4__rbiu_spl_pipe0_lockout_MASK 0x100000 +#define WD_DEBUG_REG4__rbiu_spl_pipe0_lockout__SHIFT 0x14 +#define WD_DEBUG_REG4__last_inst_of_di_p2_MASK 0x200000 +#define WD_DEBUG_REG4__last_inst_of_di_p2__SHIFT 0x15 +#define WD_DEBUG_REG4__last_sd_of_inst_p2_MASK 0x400000 +#define WD_DEBUG_REG4__last_sd_of_inst_p2__SHIFT 0x16 +#define WD_DEBUG_REG4__last_sd_of_di_p2_MASK 0x800000 +#define WD_DEBUG_REG4__last_sd_of_di_p2__SHIFT 0x17 +#define WD_DEBUG_REG4__not_eop_wait_p1_q_MASK 0x1000000 +#define WD_DEBUG_REG4__not_eop_wait_p1_q__SHIFT 0x18 +#define WD_DEBUG_REG4__not_eop_wait_q_MASK 0x2000000 +#define WD_DEBUG_REG4__not_eop_wait_q__SHIFT 0x19 +#define WD_DEBUG_REG4__ext_event_wait_p1_q_MASK 0x4000000 +#define WD_DEBUG_REG4__ext_event_wait_p1_q__SHIFT 0x1a +#define WD_DEBUG_REG4__ext_event_wait_q_MASK 0x8000000 +#define WD_DEBUG_REG4__ext_event_wait_q__SHIFT 0x1b +#define WD_DEBUG_REG4__WD_IA1_draw_send_d_MASK 0x10000000 +#define WD_DEBUG_REG4__WD_IA1_draw_send_d__SHIFT 0x1c +#define WD_DEBUG_REG4__WD_IA1_draw_rtr_MASK 0x20000000 +#define WD_DEBUG_REG4__WD_IA1_draw_rtr__SHIFT 0x1d +#define WD_DEBUG_REG4__send_to_ia1_q_MASK 0x40000000 +#define WD_DEBUG_REG4__send_to_ia1_q__SHIFT 0x1e +#define WD_DEBUG_REG4__dual_ia_mode_MASK 0x80000000 +#define WD_DEBUG_REG4__dual_ia_mode__SHIFT 0x1f +#define WD_DEBUG_REG5__p1_rbiu_spl_dr_valid_MASK 0x1 +#define WD_DEBUG_REG5__p1_rbiu_spl_dr_valid__SHIFT 0x0 +#define WD_DEBUG_REG5__SPARE0_MASK 0x2 +#define WD_DEBUG_REG5__SPARE0__SHIFT 0x1 +#define WD_DEBUG_REG5__p1_pipe0_dr_MASK 0x4 +#define WD_DEBUG_REG5__p1_pipe0_dr__SHIFT 0x2 +#define WD_DEBUG_REG5__p1_pipe0_rtr_MASK 0x8 +#define WD_DEBUG_REG5__p1_pipe0_rtr__SHIFT 0x3 +#define WD_DEBUG_REG5__p1_pipe1_dr_MASK 0x10 +#define WD_DEBUG_REG5__p1_pipe1_dr__SHIFT 0x4 +#define WD_DEBUG_REG5__p1_pipe1_rtr_MASK 0x20 +#define WD_DEBUG_REG5__p1_pipe1_rtr__SHIFT 0x5 +#define WD_DEBUG_REG5__p1_wd_subdma_fifo_empty_MASK 0x40 +#define WD_DEBUG_REG5__p1_wd_subdma_fifo_empty__SHIFT 0x6 +#define WD_DEBUG_REG5__p1_wd_subdma_fifo_full_MASK 0x80 +#define WD_DEBUG_REG5__p1_wd_subdma_fifo_full__SHIFT 0x7 +#define WD_DEBUG_REG5__p1_dma_buf_type_p0_q_MASK 0x300 +#define WD_DEBUG_REG5__p1_dma_buf_type_p0_q__SHIFT 0x8 +#define WD_DEBUG_REG5__p1_dma_zero_indices_p0_q_MASK 0x400 +#define WD_DEBUG_REG5__p1_dma_zero_indices_p0_q__SHIFT 0xa +#define WD_DEBUG_REG5__p1_dma_req_path_p3_q_MASK 0x800 +#define WD_DEBUG_REG5__p1_dma_req_path_p3_q__SHIFT 0xb +#define WD_DEBUG_REG5__p1_dma_not_eop_p1_q_MASK 0x1000 +#define WD_DEBUG_REG5__p1_dma_not_eop_p1_q__SHIFT 0xc +#define WD_DEBUG_REG5__p1_out_of_range_p4_MASK 0x2000 +#define WD_DEBUG_REG5__p1_out_of_range_p4__SHIFT 0xd +#define WD_DEBUG_REG5__p1_last_sub_dma_p3_q_MASK 0x4000 +#define WD_DEBUG_REG5__p1_last_sub_dma_p3_q__SHIFT 0xe +#define WD_DEBUG_REG5__p1_last_rdreq_of_sub_dma_p4_MASK 0x8000 +#define WD_DEBUG_REG5__p1_last_rdreq_of_sub_dma_p4__SHIFT 0xf +#define WD_DEBUG_REG5__p1_WD_IA_dma_send_d_MASK 0x10000 +#define WD_DEBUG_REG5__p1_WD_IA_dma_send_d__SHIFT 0x10 +#define WD_DEBUG_REG5__p1_WD_IA_dma_rtr_MASK 0x20000 +#define WD_DEBUG_REG5__p1_WD_IA_dma_rtr__SHIFT 0x11 +#define WD_DEBUG_REG5__p1_WD_IA1_dma_send_d_MASK 0x40000 +#define WD_DEBUG_REG5__p1_WD_IA1_dma_send_d__SHIFT 0x12 +#define WD_DEBUG_REG5__p1_WD_IA1_dma_rtr_MASK 0x80000 +#define WD_DEBUG_REG5__p1_WD_IA1_dma_rtr__SHIFT 0x13 +#define WD_DEBUG_REG5__p1_last_inst_of_dma_p2_MASK 0x100000 +#define WD_DEBUG_REG5__p1_last_inst_of_dma_p2__SHIFT 0x14 +#define WD_DEBUG_REG5__p1_last_sd_of_inst_p2_MASK 0x200000 +#define WD_DEBUG_REG5__p1_last_sd_of_inst_p2__SHIFT 0x15 +#define WD_DEBUG_REG5__p1_last_sd_of_dma_p2_MASK 0x400000 +#define WD_DEBUG_REG5__p1_last_sd_of_dma_p2__SHIFT 0x16 +#define WD_DEBUG_REG5__SPARE1_MASK 0x800000 +#define WD_DEBUG_REG5__SPARE1__SHIFT 0x17 +#define WD_DEBUG_REG5__p1_WD_IA_dma_busy_MASK 0x1000000 +#define WD_DEBUG_REG5__p1_WD_IA_dma_busy__SHIFT 0x18 +#define WD_DEBUG_REG5__p1_WD_IA1_dma_busy_MASK 0x2000000 +#define WD_DEBUG_REG5__p1_WD_IA1_dma_busy__SHIFT 0x19 +#define WD_DEBUG_REG5__p1_send_to_ia1_p3_q_MASK 0x4000000 +#define WD_DEBUG_REG5__p1_send_to_ia1_p3_q__SHIFT 0x1a +#define WD_DEBUG_REG5__p1_dma_wd_switch_on_eop_p3_q_MASK 0x8000000 +#define WD_DEBUG_REG5__p1_dma_wd_switch_on_eop_p3_q__SHIFT 0x1b +#define WD_DEBUG_REG5__p1_pipe3_dr_MASK 0x10000000 +#define WD_DEBUG_REG5__p1_pipe3_dr__SHIFT 0x1c +#define WD_DEBUG_REG5__p1_pipe3_rtr_MASK 0x20000000 +#define WD_DEBUG_REG5__p1_pipe3_rtr__SHIFT 0x1d +#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_empty_MASK 0x40000000 +#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_empty__SHIFT 0x1e +#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_full_MASK 0x80000000 +#define WD_DEBUG_REG5__p1_wd_dma2draw_fifo_full__SHIFT 0x1f +#define IA_DEBUG_REG0__ia_busy_extended_MASK 0x1 +#define IA_DEBUG_REG0__ia_busy_extended__SHIFT 0x0 +#define IA_DEBUG_REG0__ia_nodma_busy_extended_MASK 0x2 +#define IA_DEBUG_REG0__ia_nodma_busy_extended__SHIFT 0x1 +#define IA_DEBUG_REG0__ia_busy_MASK 0x4 +#define IA_DEBUG_REG0__ia_busy__SHIFT 0x2 +#define IA_DEBUG_REG0__ia_nodma_busy_MASK 0x8 +#define IA_DEBUG_REG0__ia_nodma_busy__SHIFT 0x3 +#define IA_DEBUG_REG0__SPARE0_MASK 0x10 +#define IA_DEBUG_REG0__SPARE0__SHIFT 0x4 +#define IA_DEBUG_REG0__dma_req_busy_MASK 0x20 +#define IA_DEBUG_REG0__dma_req_busy__SHIFT 0x5 +#define IA_DEBUG_REG0__dma_busy_MASK 0x40 +#define IA_DEBUG_REG0__dma_busy__SHIFT 0x6 +#define IA_DEBUG_REG0__mc_xl8r_busy_MASK 0x80 +#define IA_DEBUG_REG0__mc_xl8r_busy__SHIFT 0x7 +#define IA_DEBUG_REG0__grp_busy_MASK 0x100 +#define IA_DEBUG_REG0__grp_busy__SHIFT 0x8 +#define IA_DEBUG_REG0__SPARE1_MASK 0x200 +#define IA_DEBUG_REG0__SPARE1__SHIFT 0x9 +#define IA_DEBUG_REG0__dma_grp_valid_MASK 0x400 +#define IA_DEBUG_REG0__dma_grp_valid__SHIFT 0xa +#define IA_DEBUG_REG0__grp_dma_read_MASK 0x800 +#define IA_DEBUG_REG0__grp_dma_read__SHIFT 0xb +#define IA_DEBUG_REG0__dma_grp_hp_valid_MASK 0x1000 +#define IA_DEBUG_REG0__dma_grp_hp_valid__SHIFT 0xc +#define IA_DEBUG_REG0__grp_dma_hp_read_MASK 0x2000 +#define IA_DEBUG_REG0__grp_dma_hp_read__SHIFT 0xd +#define IA_DEBUG_REG0__SPARE2_MASK 0xffc000 +#define IA_DEBUG_REG0__SPARE2__SHIFT 0xe +#define IA_DEBUG_REG0__reg_clk_busy_MASK 0x1000000 +#define IA_DEBUG_REG0__reg_clk_busy__SHIFT 0x18 +#define IA_DEBUG_REG0__core_clk_busy_MASK 0x2000000 +#define IA_DEBUG_REG0__core_clk_busy__SHIFT 0x19 +#define IA_DEBUG_REG0__SPARE3_MASK 0x4000000 +#define IA_DEBUG_REG0__SPARE3__SHIFT 0x1a +#define IA_DEBUG_REG0__SPARE4_MASK 0x8000000 +#define IA_DEBUG_REG0__SPARE4__SHIFT 0x1b +#define IA_DEBUG_REG0__sclk_reg_vld_MASK 0x10000000 +#define IA_DEBUG_REG0__sclk_reg_vld__SHIFT 0x1c +#define IA_DEBUG_REG0__sclk_core_vld_MASK 0x20000000 +#define IA_DEBUG_REG0__sclk_core_vld__SHIFT 0x1d +#define IA_DEBUG_REG0__SPARE5_MASK 0x40000000 +#define IA_DEBUG_REG0__SPARE5__SHIFT 0x1e +#define IA_DEBUG_REG0__SPARE6_MASK 0x80000000 +#define IA_DEBUG_REG0__SPARE6__SHIFT 0x1f +#define IA_DEBUG_REG1__dma_input_fifo_empty_MASK 0x1 +#define IA_DEBUG_REG1__dma_input_fifo_empty__SHIFT 0x0 +#define IA_DEBUG_REG1__dma_input_fifo_full_MASK 0x2 +#define IA_DEBUG_REG1__dma_input_fifo_full__SHIFT 0x1 +#define IA_DEBUG_REG1__start_new_packet_MASK 0x4 +#define IA_DEBUG_REG1__start_new_packet__SHIFT 0x2 +#define IA_DEBUG_REG1__dma_rdreq_dr_q_MASK 0x8 +#define IA_DEBUG_REG1__dma_rdreq_dr_q__SHIFT 0x3 +#define IA_DEBUG_REG1__dma_zero_indices_q_MASK 0x10 +#define IA_DEBUG_REG1__dma_zero_indices_q__SHIFT 0x4 +#define IA_DEBUG_REG1__dma_buf_type_q_MASK 0x60 +#define IA_DEBUG_REG1__dma_buf_type_q__SHIFT 0x5 +#define IA_DEBUG_REG1__dma_req_path_q_MASK 0x80 +#define IA_DEBUG_REG1__dma_req_path_q__SHIFT 0x7 +#define IA_DEBUG_REG1__discard_1st_chunk_MASK 0x100 +#define IA_DEBUG_REG1__discard_1st_chunk__SHIFT 0x8 +#define IA_DEBUG_REG1__discard_2nd_chunk_MASK 0x200 +#define IA_DEBUG_REG1__discard_2nd_chunk__SHIFT 0x9 +#define IA_DEBUG_REG1__second_tc_ret_data_q_MASK 0x400 +#define IA_DEBUG_REG1__second_tc_ret_data_q__SHIFT 0xa +#define IA_DEBUG_REG1__dma_tc_ret_sel_q_MASK 0x800 +#define IA_DEBUG_REG1__dma_tc_ret_sel_q__SHIFT 0xb +#define IA_DEBUG_REG1__last_rdreq_in_dma_op_MASK 0x1000 +#define IA_DEBUG_REG1__last_rdreq_in_dma_op__SHIFT 0xc +#define IA_DEBUG_REG1__dma_mask_fifo_empty_MASK 0x2000 +#define IA_DEBUG_REG1__dma_mask_fifo_empty__SHIFT 0xd +#define IA_DEBUG_REG1__dma_data_fifo_empty_q_MASK 0x4000 +#define IA_DEBUG_REG1__dma_data_fifo_empty_q__SHIFT 0xe +#define IA_DEBUG_REG1__dma_data_fifo_full_MASK 0x8000 +#define IA_DEBUG_REG1__dma_data_fifo_full__SHIFT 0xf +#define IA_DEBUG_REG1__dma_req_fifo_empty_MASK 0x10000 +#define IA_DEBUG_REG1__dma_req_fifo_empty__SHIFT 0x10 +#define IA_DEBUG_REG1__dma_req_fifo_full_MASK 0x20000 +#define IA_DEBUG_REG1__dma_req_fifo_full__SHIFT 0x11 +#define IA_DEBUG_REG1__stage2_dr_MASK 0x40000 +#define IA_DEBUG_REG1__stage2_dr__SHIFT 0x12 +#define IA_DEBUG_REG1__stage2_rtr_MASK 0x80000 +#define IA_DEBUG_REG1__stage2_rtr__SHIFT 0x13 +#define IA_DEBUG_REG1__stage3_dr_MASK 0x100000 +#define IA_DEBUG_REG1__stage3_dr__SHIFT 0x14 +#define IA_DEBUG_REG1__stage3_rtr_MASK 0x200000 +#define IA_DEBUG_REG1__stage3_rtr__SHIFT 0x15 +#define IA_DEBUG_REG1__stage4_dr_MASK 0x400000 +#define IA_DEBUG_REG1__stage4_dr__SHIFT 0x16 +#define IA_DEBUG_REG1__stage4_rtr_MASK 0x800000 +#define IA_DEBUG_REG1__stage4_rtr__SHIFT 0x17 +#define IA_DEBUG_REG1__dma_skid_fifo_empty_MASK 0x1000000 +#define IA_DEBUG_REG1__dma_skid_fifo_empty__SHIFT 0x18 +#define IA_DEBUG_REG1__dma_skid_fifo_full_MASK 0x2000000 +#define IA_DEBUG_REG1__dma_skid_fifo_full__SHIFT 0x19 +#define IA_DEBUG_REG1__dma_grp_valid_MASK 0x4000000 +#define IA_DEBUG_REG1__dma_grp_valid__SHIFT 0x1a +#define IA_DEBUG_REG1__grp_dma_read_MASK 0x8000000 +#define IA_DEBUG_REG1__grp_dma_read__SHIFT 0x1b +#define IA_DEBUG_REG1__current_data_valid_MASK 0x10000000 +#define IA_DEBUG_REG1__current_data_valid__SHIFT 0x1c +#define IA_DEBUG_REG1__out_of_range_r2_q_MASK 0x20000000 +#define IA_DEBUG_REG1__out_of_range_r2_q__SHIFT 0x1d +#define IA_DEBUG_REG1__dma_mask_fifo_we_MASK 0x40000000 +#define IA_DEBUG_REG1__dma_mask_fifo_we__SHIFT 0x1e +#define IA_DEBUG_REG1__dma_ret_data_we_q_MASK 0x80000000 +#define IA_DEBUG_REG1__dma_ret_data_we_q__SHIFT 0x1f +#define IA_DEBUG_REG2__hp_dma_input_fifo_empty_MASK 0x1 +#define IA_DEBUG_REG2__hp_dma_input_fifo_empty__SHIFT 0x0 +#define IA_DEBUG_REG2__hp_dma_input_fifo_full_MASK 0x2 +#define IA_DEBUG_REG2__hp_dma_input_fifo_full__SHIFT 0x1 +#define IA_DEBUG_REG2__hp_start_new_packet_MASK 0x4 +#define IA_DEBUG_REG2__hp_start_new_packet__SHIFT 0x2 +#define IA_DEBUG_REG2__hp_dma_rdreq_dr_q_MASK 0x8 +#define IA_DEBUG_REG2__hp_dma_rdreq_dr_q__SHIFT 0x3 +#define IA_DEBUG_REG2__hp_dma_zero_indices_q_MASK 0x10 +#define IA_DEBUG_REG2__hp_dma_zero_indices_q__SHIFT 0x4 +#define IA_DEBUG_REG2__hp_dma_buf_type_q_MASK 0x60 +#define IA_DEBUG_REG2__hp_dma_buf_type_q__SHIFT 0x5 +#define IA_DEBUG_REG2__hp_dma_req_path_q_MASK 0x80 +#define IA_DEBUG_REG2__hp_dma_req_path_q__SHIFT 0x7 +#define IA_DEBUG_REG2__hp_discard_1st_chunk_MASK 0x100 +#define IA_DEBUG_REG2__hp_discard_1st_chunk__SHIFT 0x8 +#define IA_DEBUG_REG2__hp_discard_2nd_chunk_MASK 0x200 +#define IA_DEBUG_REG2__hp_discard_2nd_chunk__SHIFT 0x9 +#define IA_DEBUG_REG2__hp_second_tc_ret_data_q_MASK 0x400 +#define IA_DEBUG_REG2__hp_second_tc_ret_data_q__SHIFT 0xa +#define IA_DEBUG_REG2__hp_dma_tc_ret_sel_q_MASK 0x800 +#define IA_DEBUG_REG2__hp_dma_tc_ret_sel_q__SHIFT 0xb +#define IA_DEBUG_REG2__hp_last_rdreq_in_dma_op_MASK 0x1000 +#define IA_DEBUG_REG2__hp_last_rdreq_in_dma_op__SHIFT 0xc +#define IA_DEBUG_REG2__hp_dma_mask_fifo_empty_MASK 0x2000 +#define IA_DEBUG_REG2__hp_dma_mask_fifo_empty__SHIFT 0xd +#define IA_DEBUG_REG2__hp_dma_data_fifo_empty_q_MASK 0x4000 +#define IA_DEBUG_REG2__hp_dma_data_fifo_empty_q__SHIFT 0xe +#define IA_DEBUG_REG2__hp_dma_data_fifo_full_MASK 0x8000 +#define IA_DEBUG_REG2__hp_dma_data_fifo_full__SHIFT 0xf +#define IA_DEBUG_REG2__hp_dma_req_fifo_empty_MASK 0x10000 +#define IA_DEBUG_REG2__hp_dma_req_fifo_empty__SHIFT 0x10 +#define IA_DEBUG_REG2__hp_dma_req_fifo_full_MASK 0x20000 +#define IA_DEBUG_REG2__hp_dma_req_fifo_full__SHIFT 0x11 +#define IA_DEBUG_REG2__hp_stage2_dr_MASK 0x40000 +#define IA_DEBUG_REG2__hp_stage2_dr__SHIFT 0x12 +#define IA_DEBUG_REG2__hp_stage2_rtr_MASK 0x80000 +#define IA_DEBUG_REG2__hp_stage2_rtr__SHIFT 0x13 +#define IA_DEBUG_REG2__hp_stage3_dr_MASK 0x100000 +#define IA_DEBUG_REG2__hp_stage3_dr__SHIFT 0x14 +#define IA_DEBUG_REG2__hp_stage3_rtr_MASK 0x200000 +#define IA_DEBUG_REG2__hp_stage3_rtr__SHIFT 0x15 +#define IA_DEBUG_REG2__hp_stage4_dr_MASK 0x400000 +#define IA_DEBUG_REG2__hp_stage4_dr__SHIFT 0x16 +#define IA_DEBUG_REG2__hp_stage4_rtr_MASK 0x800000 +#define IA_DEBUG_REG2__hp_stage4_rtr__SHIFT 0x17 +#define IA_DEBUG_REG2__hp_dma_skid_fifo_empty_MASK 0x1000000 +#define IA_DEBUG_REG2__hp_dma_skid_fifo_empty__SHIFT 0x18 +#define IA_DEBUG_REG2__hp_dma_skid_fifo_full_MASK 0x2000000 +#define IA_DEBUG_REG2__hp_dma_skid_fifo_full__SHIFT 0x19 +#define IA_DEBUG_REG2__hp_dma_grp_valid_MASK 0x4000000 +#define IA_DEBUG_REG2__hp_dma_grp_valid__SHIFT 0x1a +#define IA_DEBUG_REG2__hp_grp_dma_read_MASK 0x8000000 +#define IA_DEBUG_REG2__hp_grp_dma_read__SHIFT 0x1b +#define IA_DEBUG_REG2__hp_current_data_valid_MASK 0x10000000 +#define IA_DEBUG_REG2__hp_current_data_valid__SHIFT 0x1c +#define IA_DEBUG_REG2__hp_out_of_range_r2_q_MASK 0x20000000 +#define IA_DEBUG_REG2__hp_out_of_range_r2_q__SHIFT 0x1d +#define IA_DEBUG_REG2__hp_dma_mask_fifo_we_MASK 0x40000000 +#define IA_DEBUG_REG2__hp_dma_mask_fifo_we__SHIFT 0x1e +#define IA_DEBUG_REG2__hp_dma_ret_data_we_q_MASK 0x80000000 +#define IA_DEBUG_REG2__hp_dma_ret_data_we_q__SHIFT 0x1f +#define IA_DEBUG_REG3__dma_pipe0_rdreq_valid_MASK 0x1 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_valid__SHIFT 0x0 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_read_MASK 0x2 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_read__SHIFT 0x1 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_null_out_MASK 0x4 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_null_out__SHIFT 0x2 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_eop_out_MASK 0x8 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_eop_out__SHIFT 0x3 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_use_tc_out_MASK 0x10 +#define IA_DEBUG_REG3__dma_pipe0_rdreq_use_tc_out__SHIFT 0x4 +#define IA_DEBUG_REG3__grp_dma_draw_is_pipe0_MASK 0x20 +#define IA_DEBUG_REG3__grp_dma_draw_is_pipe0__SHIFT 0x5 +#define IA_DEBUG_REG3__must_service_pipe0_req_MASK 0x40 +#define IA_DEBUG_REG3__must_service_pipe0_req__SHIFT 0x6 +#define IA_DEBUG_REG3__send_pipe1_req_MASK 0x80 +#define IA_DEBUG_REG3__send_pipe1_req__SHIFT 0x7 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_valid_MASK 0x100 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_valid__SHIFT 0x8 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_read_MASK 0x200 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_read__SHIFT 0x9 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_null_out_MASK 0x400 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_null_out__SHIFT 0xa +#define IA_DEBUG_REG3__dma_pipe1_rdreq_eop_out_MASK 0x800 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_eop_out__SHIFT 0xb +#define IA_DEBUG_REG3__dma_pipe1_rdreq_use_tc_out_MASK 0x1000 +#define IA_DEBUG_REG3__dma_pipe1_rdreq_use_tc_out__SHIFT 0xc +#define IA_DEBUG_REG3__ia_mc_rdreq_rtr_q_MASK 0x2000 +#define IA_DEBUG_REG3__ia_mc_rdreq_rtr_q__SHIFT 0xd +#define IA_DEBUG_REG3__mc_out_rtr_MASK 0x4000 +#define IA_DEBUG_REG3__mc_out_rtr__SHIFT 0xe +#define IA_DEBUG_REG3__dma_rdreq_send_out_MASK 0x8000 +#define IA_DEBUG_REG3__dma_rdreq_send_out__SHIFT 0xf +#define IA_DEBUG_REG3__pipe0_dr_MASK 0x10000 +#define IA_DEBUG_REG3__pipe0_dr__SHIFT 0x10 +#define IA_DEBUG_REG3__pipe0_rtr_MASK 0x20000 +#define IA_DEBUG_REG3__pipe0_rtr__SHIFT 0x11 +#define IA_DEBUG_REG3__ia_tc_rdreq_rtr_q_MASK 0x40000 +#define IA_DEBUG_REG3__ia_tc_rdreq_rtr_q__SHIFT 0x12 +#define IA_DEBUG_REG3__tc_out_rtr_MASK 0x80000 +#define IA_DEBUG_REG3__tc_out_rtr__SHIFT 0x13 +#define IA_DEBUG_REG3__pair0_valid_p1_MASK 0x100000 +#define IA_DEBUG_REG3__pair0_valid_p1__SHIFT 0x14 +#define IA_DEBUG_REG3__pair1_valid_p1_MASK 0x200000 +#define IA_DEBUG_REG3__pair1_valid_p1__SHIFT 0x15 +#define IA_DEBUG_REG3__pair2_valid_p1_MASK 0x400000 +#define IA_DEBUG_REG3__pair2_valid_p1__SHIFT 0x16 +#define IA_DEBUG_REG3__pair3_valid_p1_MASK 0x800000 +#define IA_DEBUG_REG3__pair3_valid_p1__SHIFT 0x17 +#define IA_DEBUG_REG3__tc_req_count_q_MASK 0x3000000 +#define IA_DEBUG_REG3__tc_req_count_q__SHIFT 0x18 +#define IA_DEBUG_REG3__discard_1st_chunk_MASK 0x4000000 +#define IA_DEBUG_REG3__discard_1st_chunk__SHIFT 0x1a +#define IA_DEBUG_REG3__discard_2nd_chunk_MASK 0x8000000 +#define IA_DEBUG_REG3__discard_2nd_chunk__SHIFT 0x1b +#define IA_DEBUG_REG3__last_tc_req_p1_MASK 0x10000000 +#define IA_DEBUG_REG3__last_tc_req_p1__SHIFT 0x1c +#define IA_DEBUG_REG3__IA_TC_rdreq_send_out_MASK 0x20000000 +#define IA_DEBUG_REG3__IA_TC_rdreq_send_out__SHIFT 0x1d +#define IA_DEBUG_REG3__TC_IA_rdret_valid_in_MASK 0x40000000 +#define IA_DEBUG_REG3__TC_IA_rdret_valid_in__SHIFT 0x1e +#define IA_DEBUG_REG3__TAP_IA_rdret_vld_in_MASK 0x80000000 +#define IA_DEBUG_REG3__TAP_IA_rdret_vld_in__SHIFT 0x1f +#define IA_DEBUG_REG4__pipe0_dr_MASK 0x1 +#define IA_DEBUG_REG4__pipe0_dr__SHIFT 0x0 +#define IA_DEBUG_REG4__pipe1_dr_MASK 0x2 +#define IA_DEBUG_REG4__pipe1_dr__SHIFT 0x1 +#define IA_DEBUG_REG4__pipe2_dr_MASK 0x4 +#define IA_DEBUG_REG4__pipe2_dr__SHIFT 0x2 +#define IA_DEBUG_REG4__pipe3_dr_MASK 0x8 +#define IA_DEBUG_REG4__pipe3_dr__SHIFT 0x3 +#define IA_DEBUG_REG4__pipe4_dr_MASK 0x10 +#define IA_DEBUG_REG4__pipe4_dr__SHIFT 0x4 +#define IA_DEBUG_REG4__pipe5_dr_MASK 0x20 +#define IA_DEBUG_REG4__pipe5_dr__SHIFT 0x5 +#define IA_DEBUG_REG4__grp_se0_fifo_empty_MASK 0x40 +#define IA_DEBUG_REG4__grp_se0_fifo_empty__SHIFT 0x6 +#define IA_DEBUG_REG4__grp_se0_fifo_full_MASK 0x80 +#define IA_DEBUG_REG4__grp_se0_fifo_full__SHIFT 0x7 +#define IA_DEBUG_REG4__pipe0_rtr_MASK 0x100 +#define IA_DEBUG_REG4__pipe0_rtr__SHIFT 0x8 +#define IA_DEBUG_REG4__pipe1_rtr_MASK 0x200 +#define IA_DEBUG_REG4__pipe1_rtr__SHIFT 0x9 +#define IA_DEBUG_REG4__pipe2_rtr_MASK 0x400 +#define IA_DEBUG_REG4__pipe2_rtr__SHIFT 0xa +#define IA_DEBUG_REG4__pipe3_rtr_MASK 0x800 +#define IA_DEBUG_REG4__pipe3_rtr__SHIFT 0xb +#define IA_DEBUG_REG4__pipe4_rtr_MASK 0x1000 +#define IA_DEBUG_REG4__pipe4_rtr__SHIFT 0xc +#define IA_DEBUG_REG4__pipe5_rtr_MASK 0x2000 +#define IA_DEBUG_REG4__pipe5_rtr__SHIFT 0xd +#define IA_DEBUG_REG4__ia_vgt_prim_rtr_q_MASK 0x4000 +#define IA_DEBUG_REG4__ia_vgt_prim_rtr_q__SHIFT 0xe +#define IA_DEBUG_REG4__ia_se1vgt_prim_rtr_q_MASK 0x8000 +#define IA_DEBUG_REG4__ia_se1vgt_prim_rtr_q__SHIFT 0xf +#define IA_DEBUG_REG4__di_major_mode_p1_q_MASK 0x10000 +#define IA_DEBUG_REG4__di_major_mode_p1_q__SHIFT 0x10 +#define IA_DEBUG_REG4__gs_mode_p1_q_MASK 0xe0000 +#define IA_DEBUG_REG4__gs_mode_p1_q__SHIFT 0x11 +#define IA_DEBUG_REG4__di_event_flag_p1_q_MASK 0x100000 +#define IA_DEBUG_REG4__di_event_flag_p1_q__SHIFT 0x14 +#define IA_DEBUG_REG4__di_state_sel_p1_q_MASK 0xe00000 +#define IA_DEBUG_REG4__di_state_sel_p1_q__SHIFT 0x15 +#define IA_DEBUG_REG4__draw_opaq_en_p1_q_MASK 0x1000000 +#define IA_DEBUG_REG4__draw_opaq_en_p1_q__SHIFT 0x18 +#define IA_DEBUG_REG4__draw_opaq_active_q_MASK 0x2000000 +#define IA_DEBUG_REG4__draw_opaq_active_q__SHIFT 0x19 +#define IA_DEBUG_REG4__di_source_select_p1_q_MASK 0xc000000 +#define IA_DEBUG_REG4__di_source_select_p1_q__SHIFT 0x1a +#define IA_DEBUG_REG4__ready_to_read_di_MASK 0x10000000 +#define IA_DEBUG_REG4__ready_to_read_di__SHIFT 0x1c +#define IA_DEBUG_REG4__di_first_group_of_draw_q_MASK 0x20000000 +#define IA_DEBUG_REG4__di_first_group_of_draw_q__SHIFT 0x1d +#define IA_DEBUG_REG4__last_shift_of_draw_MASK 0x40000000 +#define IA_DEBUG_REG4__last_shift_of_draw__SHIFT 0x1e +#define IA_DEBUG_REG4__current_shift_is_vect1_q_MASK 0x80000000 +#define IA_DEBUG_REG4__current_shift_is_vect1_q__SHIFT 0x1f +#define IA_DEBUG_REG5__di_index_counter_q_15_0_MASK 0xffff +#define IA_DEBUG_REG5__di_index_counter_q_15_0__SHIFT 0x0 +#define IA_DEBUG_REG5__instanceid_13_0_MASK 0x3fff0000 +#define IA_DEBUG_REG5__instanceid_13_0__SHIFT 0x10 +#define IA_DEBUG_REG5__draw_input_fifo_full_MASK 0x40000000 +#define IA_DEBUG_REG5__draw_input_fifo_full__SHIFT 0x1e +#define IA_DEBUG_REG5__draw_input_fifo_empty_MASK 0x80000000 +#define IA_DEBUG_REG5__draw_input_fifo_empty__SHIFT 0x1f +#define IA_DEBUG_REG6__current_shift_q_MASK 0xf +#define IA_DEBUG_REG6__current_shift_q__SHIFT 0x0 +#define IA_DEBUG_REG6__current_stride_pre_MASK 0xf0 +#define IA_DEBUG_REG6__current_stride_pre__SHIFT 0x4 +#define IA_DEBUG_REG6__current_stride_q_MASK 0x1f00 +#define IA_DEBUG_REG6__current_stride_q__SHIFT 0x8 +#define IA_DEBUG_REG6__first_group_partial_MASK 0x2000 +#define IA_DEBUG_REG6__first_group_partial__SHIFT 0xd +#define IA_DEBUG_REG6__second_group_partial_MASK 0x4000 +#define IA_DEBUG_REG6__second_group_partial__SHIFT 0xe +#define IA_DEBUG_REG6__curr_prim_partial_MASK 0x8000 +#define IA_DEBUG_REG6__curr_prim_partial__SHIFT 0xf +#define IA_DEBUG_REG6__next_stride_q_MASK 0x1f0000 +#define IA_DEBUG_REG6__next_stride_q__SHIFT 0x10 +#define IA_DEBUG_REG6__next_group_partial_MASK 0x200000 +#define IA_DEBUG_REG6__next_group_partial__SHIFT 0x15 +#define IA_DEBUG_REG6__after_group_partial_MASK 0x400000 +#define IA_DEBUG_REG6__after_group_partial__SHIFT 0x16 +#define IA_DEBUG_REG6__extract_group_MASK 0x800000 +#define IA_DEBUG_REG6__extract_group__SHIFT 0x17 +#define IA_DEBUG_REG6__grp_shift_debug_data_MASK 0xff000000 +#define IA_DEBUG_REG6__grp_shift_debug_data__SHIFT 0x18 +#define IA_DEBUG_REG7__reset_indx_state_q_MASK 0xf +#define IA_DEBUG_REG7__reset_indx_state_q__SHIFT 0x0 +#define IA_DEBUG_REG7__shift_vect_valid_p2_q_MASK 0xf0 +#define IA_DEBUG_REG7__shift_vect_valid_p2_q__SHIFT 0x4 +#define IA_DEBUG_REG7__shift_vect1_valid_p2_q_MASK 0xf00 +#define IA_DEBUG_REG7__shift_vect1_valid_p2_q__SHIFT 0x8 +#define IA_DEBUG_REG7__shift_vect0_reset_match_p2_q_MASK 0xf000 +#define IA_DEBUG_REG7__shift_vect0_reset_match_p2_q__SHIFT 0xc +#define IA_DEBUG_REG7__shift_vect1_reset_match_p2_q_MASK 0xf0000 +#define IA_DEBUG_REG7__shift_vect1_reset_match_p2_q__SHIFT 0x10 +#define IA_DEBUG_REG7__num_indx_in_group_p2_q_MASK 0x700000 +#define IA_DEBUG_REG7__num_indx_in_group_p2_q__SHIFT 0x14 +#define IA_DEBUG_REG7__last_group_of_draw_p2_q_MASK 0x800000 +#define IA_DEBUG_REG7__last_group_of_draw_p2_q__SHIFT 0x17 +#define IA_DEBUG_REG7__shift_event_flag_p2_q_MASK 0x1000000 +#define IA_DEBUG_REG7__shift_event_flag_p2_q__SHIFT 0x18 +#define IA_DEBUG_REG7__indx_shift_is_one_p2_q_MASK 0x2000000 +#define IA_DEBUG_REG7__indx_shift_is_one_p2_q__SHIFT 0x19 +#define IA_DEBUG_REG7__indx_shift_is_two_p2_q_MASK 0x4000000 +#define IA_DEBUG_REG7__indx_shift_is_two_p2_q__SHIFT 0x1a +#define IA_DEBUG_REG7__indx_stride_is_four_p2_q_MASK 0x8000000 +#define IA_DEBUG_REG7__indx_stride_is_four_p2_q__SHIFT 0x1b +#define IA_DEBUG_REG7__shift_prim1_reset_p3_q_MASK 0x10000000 +#define IA_DEBUG_REG7__shift_prim1_reset_p3_q__SHIFT 0x1c +#define IA_DEBUG_REG7__shift_prim1_partial_p3_q_MASK 0x20000000 +#define IA_DEBUG_REG7__shift_prim1_partial_p3_q__SHIFT 0x1d +#define IA_DEBUG_REG7__shift_prim0_reset_p3_q_MASK 0x40000000 +#define IA_DEBUG_REG7__shift_prim0_reset_p3_q__SHIFT 0x1e +#define IA_DEBUG_REG7__shift_prim0_partial_p3_q_MASK 0x80000000 +#define IA_DEBUG_REG7__shift_prim0_partial_p3_q__SHIFT 0x1f +#define IA_DEBUG_REG8__di_prim_type_p1_q_MASK 0x1f +#define IA_DEBUG_REG8__di_prim_type_p1_q__SHIFT 0x0 +#define IA_DEBUG_REG8__two_cycle_xfer_p1_q_MASK 0x20 +#define IA_DEBUG_REG8__two_cycle_xfer_p1_q__SHIFT 0x5 +#define IA_DEBUG_REG8__two_prim_input_p1_q_MASK 0x40 +#define IA_DEBUG_REG8__two_prim_input_p1_q__SHIFT 0x6 +#define IA_DEBUG_REG8__shift_vect_end_of_packet_p5_q_MASK 0x80 +#define IA_DEBUG_REG8__shift_vect_end_of_packet_p5_q__SHIFT 0x7 +#define IA_DEBUG_REG8__last_group_of_inst_p5_q_MASK 0x100 +#define IA_DEBUG_REG8__last_group_of_inst_p5_q__SHIFT 0x8 +#define IA_DEBUG_REG8__shift_prim1_null_flag_p5_q_MASK 0x200 +#define IA_DEBUG_REG8__shift_prim1_null_flag_p5_q__SHIFT 0x9 +#define IA_DEBUG_REG8__shift_prim0_null_flag_p5_q_MASK 0x400 +#define IA_DEBUG_REG8__shift_prim0_null_flag_p5_q__SHIFT 0xa +#define IA_DEBUG_REG8__grp_continued_MASK 0x800 +#define IA_DEBUG_REG8__grp_continued__SHIFT 0xb +#define IA_DEBUG_REG8__grp_state_sel_MASK 0x7000 +#define IA_DEBUG_REG8__grp_state_sel__SHIFT 0xc +#define IA_DEBUG_REG8__grp_sub_prim_type_MASK 0x1f8000 +#define IA_DEBUG_REG8__grp_sub_prim_type__SHIFT 0xf +#define IA_DEBUG_REG8__grp_output_path_MASK 0xe00000 +#define IA_DEBUG_REG8__grp_output_path__SHIFT 0x15 +#define IA_DEBUG_REG8__grp_null_primitive_MASK 0x1000000 +#define IA_DEBUG_REG8__grp_null_primitive__SHIFT 0x18 +#define IA_DEBUG_REG8__grp_eop_MASK 0x2000000 +#define IA_DEBUG_REG8__grp_eop__SHIFT 0x19 +#define IA_DEBUG_REG8__grp_eopg_MASK 0x4000000 +#define IA_DEBUG_REG8__grp_eopg__SHIFT 0x1a +#define IA_DEBUG_REG8__grp_event_flag_MASK 0x8000000 +#define IA_DEBUG_REG8__grp_event_flag__SHIFT 0x1b +#define IA_DEBUG_REG8__grp_components_valid_MASK 0xf0000000 +#define IA_DEBUG_REG8__grp_components_valid__SHIFT 0x1c +#define IA_DEBUG_REG9__send_to_se1_p6_MASK 0x1 +#define IA_DEBUG_REG9__send_to_se1_p6__SHIFT 0x0 +#define IA_DEBUG_REG9__gfx_se_switch_p6_MASK 0x2 +#define IA_DEBUG_REG9__gfx_se_switch_p6__SHIFT 0x1 +#define IA_DEBUG_REG9__null_eoi_xfer_prim1_p6_MASK 0x4 +#define IA_DEBUG_REG9__null_eoi_xfer_prim1_p6__SHIFT 0x2 +#define IA_DEBUG_REG9__null_eoi_xfer_prim0_p6_MASK 0x8 +#define IA_DEBUG_REG9__null_eoi_xfer_prim0_p6__SHIFT 0x3 +#define IA_DEBUG_REG9__prim1_eoi_p6_MASK 0x10 +#define IA_DEBUG_REG9__prim1_eoi_p6__SHIFT 0x4 +#define IA_DEBUG_REG9__prim0_eoi_p6_MASK 0x20 +#define IA_DEBUG_REG9__prim0_eoi_p6__SHIFT 0x5 +#define IA_DEBUG_REG9__prim1_valid_eopg_p6_MASK 0x40 +#define IA_DEBUG_REG9__prim1_valid_eopg_p6__SHIFT 0x6 +#define IA_DEBUG_REG9__prim0_valid_eopg_p6_MASK 0x80 +#define IA_DEBUG_REG9__prim0_valid_eopg_p6__SHIFT 0x7 +#define IA_DEBUG_REG9__prim1_to_other_se_p6_MASK 0x100 +#define IA_DEBUG_REG9__prim1_to_other_se_p6__SHIFT 0x8 +#define IA_DEBUG_REG9__eopg_on_last_prim_p6_MASK 0x200 +#define IA_DEBUG_REG9__eopg_on_last_prim_p6__SHIFT 0x9 +#define IA_DEBUG_REG9__eopg_between_prims_p6_MASK 0x400 +#define IA_DEBUG_REG9__eopg_between_prims_p6__SHIFT 0xa +#define IA_DEBUG_REG9__prim_count_eq_group_size_p6_MASK 0x800 +#define IA_DEBUG_REG9__prim_count_eq_group_size_p6__SHIFT 0xb +#define IA_DEBUG_REG9__prim_count_gt_group_size_p6_MASK 0x1000 +#define IA_DEBUG_REG9__prim_count_gt_group_size_p6__SHIFT 0xc +#define IA_DEBUG_REG9__two_prim_output_p5_q_MASK 0x2000 +#define IA_DEBUG_REG9__two_prim_output_p5_q__SHIFT 0xd +#define IA_DEBUG_REG9__SPARE0_MASK 0x4000 +#define IA_DEBUG_REG9__SPARE0__SHIFT 0xe +#define IA_DEBUG_REG9__SPARE1_MASK 0x8000 +#define IA_DEBUG_REG9__SPARE1__SHIFT 0xf +#define IA_DEBUG_REG9__shift_vect_end_of_packet_p5_q_MASK 0x10000 +#define IA_DEBUG_REG9__shift_vect_end_of_packet_p5_q__SHIFT 0x10 +#define IA_DEBUG_REG9__prim1_xfer_p6_MASK 0x20000 +#define IA_DEBUG_REG9__prim1_xfer_p6__SHIFT 0x11 +#define IA_DEBUG_REG9__grp_se1_fifo_empty_MASK 0x40000 +#define IA_DEBUG_REG9__grp_se1_fifo_empty__SHIFT 0x12 +#define IA_DEBUG_REG9__grp_se1_fifo_full_MASK 0x80000 +#define IA_DEBUG_REG9__grp_se1_fifo_full__SHIFT 0x13 +#define IA_DEBUG_REG9__prim_counter_q_MASK 0xfff00000 +#define IA_DEBUG_REG9__prim_counter_q__SHIFT 0x14 +#define VGT_DEBUG_REG0__vgt_busy_extended_MASK 0x1 +#define VGT_DEBUG_REG0__vgt_busy_extended__SHIFT 0x0 +#define VGT_DEBUG_REG0__SPARE9_MASK 0x2 +#define VGT_DEBUG_REG0__SPARE9__SHIFT 0x1 +#define VGT_DEBUG_REG0__vgt_busy_MASK 0x4 +#define VGT_DEBUG_REG0__vgt_busy__SHIFT 0x2 +#define VGT_DEBUG_REG0__SPARE8_MASK 0x8 +#define VGT_DEBUG_REG0__SPARE8__SHIFT 0x3 +#define VGT_DEBUG_REG0__SPARE7_MASK 0x10 +#define VGT_DEBUG_REG0__SPARE7__SHIFT 0x4 +#define VGT_DEBUG_REG0__SPARE6_MASK 0x20 +#define VGT_DEBUG_REG0__SPARE6__SHIFT 0x5 +#define VGT_DEBUG_REG0__SPARE5_MASK 0x40 +#define VGT_DEBUG_REG0__SPARE5__SHIFT 0x6 +#define VGT_DEBUG_REG0__SPARE4_MASK 0x80 +#define VGT_DEBUG_REG0__SPARE4__SHIFT 0x7 +#define VGT_DEBUG_REG0__pi_busy_MASK 0x100 +#define VGT_DEBUG_REG0__pi_busy__SHIFT 0x8 +#define VGT_DEBUG_REG0__vr_pi_busy_MASK 0x200 +#define VGT_DEBUG_REG0__vr_pi_busy__SHIFT 0x9 +#define VGT_DEBUG_REG0__pt_pi_busy_MASK 0x400 +#define VGT_DEBUG_REG0__pt_pi_busy__SHIFT 0xa +#define VGT_DEBUG_REG0__te_pi_busy_MASK 0x800 +#define VGT_DEBUG_REG0__te_pi_busy__SHIFT 0xb +#define VGT_DEBUG_REG0__gs_busy_MASK 0x1000 +#define VGT_DEBUG_REG0__gs_busy__SHIFT 0xc +#define VGT_DEBUG_REG0__rcm_busy_MASK 0x2000 +#define VGT_DEBUG_REG0__rcm_busy__SHIFT 0xd +#define VGT_DEBUG_REG0__tm_busy_MASK 0x4000 +#define VGT_DEBUG_REG0__tm_busy__SHIFT 0xe +#define VGT_DEBUG_REG0__cm_busy_MASK 0x8000 +#define VGT_DEBUG_REG0__cm_busy__SHIFT 0xf +#define VGT_DEBUG_REG0__gog_busy_MASK 0x10000 +#define VGT_DEBUG_REG0__gog_busy__SHIFT 0x10 +#define VGT_DEBUG_REG0__frmt_busy_MASK 0x20000 +#define VGT_DEBUG_REG0__frmt_busy__SHIFT 0x11 +#define VGT_DEBUG_REG0__SPARE10_MASK 0x40000 +#define VGT_DEBUG_REG0__SPARE10__SHIFT 0x12 +#define VGT_DEBUG_REG0__te11_pi_busy_MASK 0x80000 +#define VGT_DEBUG_REG0__te11_pi_busy__SHIFT 0x13 +#define VGT_DEBUG_REG0__SPARE3_MASK 0x100000 +#define VGT_DEBUG_REG0__SPARE3__SHIFT 0x14 +#define VGT_DEBUG_REG0__combined_out_busy_MASK 0x200000 +#define VGT_DEBUG_REG0__combined_out_busy__SHIFT 0x15 +#define VGT_DEBUG_REG0__spi_vs_interfaces_busy_MASK 0x400000 +#define VGT_DEBUG_REG0__spi_vs_interfaces_busy__SHIFT 0x16 +#define VGT_DEBUG_REG0__pa_interfaces_busy_MASK 0x800000 +#define VGT_DEBUG_REG0__pa_interfaces_busy__SHIFT 0x17 +#define VGT_DEBUG_REG0__reg_clk_busy_MASK 0x1000000 +#define VGT_DEBUG_REG0__reg_clk_busy__SHIFT 0x18 +#define VGT_DEBUG_REG0__SPARE2_MASK 0x2000000 +#define VGT_DEBUG_REG0__SPARE2__SHIFT 0x19 +#define VGT_DEBUG_REG0__core_clk_busy_MASK 0x4000000 +#define VGT_DEBUG_REG0__core_clk_busy__SHIFT 0x1a +#define VGT_DEBUG_REG0__gs_clk_busy_MASK 0x8000000 +#define VGT_DEBUG_REG0__gs_clk_busy__SHIFT 0x1b +#define VGT_DEBUG_REG0__SPARE1_MASK 0x10000000 +#define VGT_DEBUG_REG0__SPARE1__SHIFT 0x1c +#define VGT_DEBUG_REG0__sclk_core_vld_MASK 0x20000000 +#define VGT_DEBUG_REG0__sclk_core_vld__SHIFT 0x1d +#define VGT_DEBUG_REG0__sclk_gs_vld_MASK 0x40000000 +#define VGT_DEBUG_REG0__sclk_gs_vld__SHIFT 0x1e +#define VGT_DEBUG_REG0__SPARE0_MASK 0x80000000 +#define VGT_DEBUG_REG0__SPARE0__SHIFT 0x1f +#define VGT_DEBUG_REG1__SPARE9_MASK 0x1 +#define VGT_DEBUG_REG1__SPARE9__SHIFT 0x0 +#define VGT_DEBUG_REG1__SPARE8_MASK 0x2 +#define VGT_DEBUG_REG1__SPARE8__SHIFT 0x1 +#define VGT_DEBUG_REG1__SPARE7_MASK 0x4 +#define VGT_DEBUG_REG1__SPARE7__SHIFT 0x2 +#define VGT_DEBUG_REG1__SPARE6_MASK 0x8 +#define VGT_DEBUG_REG1__SPARE6__SHIFT 0x3 +#define VGT_DEBUG_REG1__SPARE5_MASK 0x10 +#define VGT_DEBUG_REG1__SPARE5__SHIFT 0x4 +#define VGT_DEBUG_REG1__SPARE4_MASK 0x20 +#define VGT_DEBUG_REG1__SPARE4__SHIFT 0x5 +#define VGT_DEBUG_REG1__SPARE3_MASK 0x40 +#define VGT_DEBUG_REG1__SPARE3__SHIFT 0x6 +#define VGT_DEBUG_REG1__SPARE2_MASK 0x80 +#define VGT_DEBUG_REG1__SPARE2__SHIFT 0x7 +#define VGT_DEBUG_REG1__SPARE1_MASK 0x100 +#define VGT_DEBUG_REG1__SPARE1__SHIFT 0x8 +#define VGT_DEBUG_REG1__SPARE0_MASK 0x200 +#define VGT_DEBUG_REG1__SPARE0__SHIFT 0x9 +#define VGT_DEBUG_REG1__pi_vr_valid_MASK 0x400 +#define VGT_DEBUG_REG1__pi_vr_valid__SHIFT 0xa +#define VGT_DEBUG_REG1__vr_pi_read_MASK 0x800 +#define VGT_DEBUG_REG1__vr_pi_read__SHIFT 0xb +#define VGT_DEBUG_REG1__pi_pt_valid_MASK 0x1000 +#define VGT_DEBUG_REG1__pi_pt_valid__SHIFT 0xc +#define VGT_DEBUG_REG1__pt_pi_read_MASK 0x2000 +#define VGT_DEBUG_REG1__pt_pi_read__SHIFT 0xd +#define VGT_DEBUG_REG1__pi_te_valid_MASK 0x4000 +#define VGT_DEBUG_REG1__pi_te_valid__SHIFT 0xe +#define VGT_DEBUG_REG1__te_grp_read_MASK 0x8000 +#define VGT_DEBUG_REG1__te_grp_read__SHIFT 0xf +#define VGT_DEBUG_REG1__vr_out_indx_valid_MASK 0x10000 +#define VGT_DEBUG_REG1__vr_out_indx_valid__SHIFT 0x10 +#define VGT_DEBUG_REG1__SPARE12_MASK 0x20000 +#define VGT_DEBUG_REG1__SPARE12__SHIFT 0x11 +#define VGT_DEBUG_REG1__vr_out_prim_valid_MASK 0x40000 +#define VGT_DEBUG_REG1__vr_out_prim_valid__SHIFT 0x12 +#define VGT_DEBUG_REG1__SPARE11_MASK 0x80000 +#define VGT_DEBUG_REG1__SPARE11__SHIFT 0x13 +#define VGT_DEBUG_REG1__pt_out_indx_valid_MASK 0x100000 +#define VGT_DEBUG_REG1__pt_out_indx_valid__SHIFT 0x14 +#define VGT_DEBUG_REG1__SPARE10_MASK 0x200000 +#define VGT_DEBUG_REG1__SPARE10__SHIFT 0x15 +#define VGT_DEBUG_REG1__pt_out_prim_valid_MASK 0x400000 +#define VGT_DEBUG_REG1__pt_out_prim_valid__SHIFT 0x16 +#define VGT_DEBUG_REG1__SPARE23_MASK 0x800000 +#define VGT_DEBUG_REG1__SPARE23__SHIFT 0x17 +#define VGT_DEBUG_REG1__te_out_data_valid_MASK 0x1000000 +#define VGT_DEBUG_REG1__te_out_data_valid__SHIFT 0x18 +#define VGT_DEBUG_REG1__SPARE25_MASK 0x2000000 +#define VGT_DEBUG_REG1__SPARE25__SHIFT 0x19 +#define VGT_DEBUG_REG1__pi_gs_valid_MASK 0x4000000 +#define VGT_DEBUG_REG1__pi_gs_valid__SHIFT 0x1a +#define VGT_DEBUG_REG1__gs_pi_read_MASK 0x8000000 +#define VGT_DEBUG_REG1__gs_pi_read__SHIFT 0x1b +#define VGT_DEBUG_REG1__gog_out_indx_valid_MASK 0x10000000 +#define VGT_DEBUG_REG1__gog_out_indx_valid__SHIFT 0x1c +#define VGT_DEBUG_REG1__out_indx_read_MASK 0x20000000 +#define VGT_DEBUG_REG1__out_indx_read__SHIFT 0x1d +#define VGT_DEBUG_REG1__gog_out_prim_valid_MASK 0x40000000 +#define VGT_DEBUG_REG1__gog_out_prim_valid__SHIFT 0x1e +#define VGT_DEBUG_REG1__out_prim_read_MASK 0x80000000 +#define VGT_DEBUG_REG1__out_prim_read__SHIFT 0x1f +#define VGT_DEBUG_REG2__hs_grp_busy_MASK 0x1 +#define VGT_DEBUG_REG2__hs_grp_busy__SHIFT 0x0 +#define VGT_DEBUG_REG2__hs_noif_busy_MASK 0x2 +#define VGT_DEBUG_REG2__hs_noif_busy__SHIFT 0x1 +#define VGT_DEBUG_REG2__tfmmIsBusy_MASK 0x4 +#define VGT_DEBUG_REG2__tfmmIsBusy__SHIFT 0x2 +#define VGT_DEBUG_REG2__lsVertIfBusy_0_MASK 0x8 +#define VGT_DEBUG_REG2__lsVertIfBusy_0__SHIFT 0x3 +#define VGT_DEBUG_REG2__te11_hs_tess_input_rtr_MASK 0x10 +#define VGT_DEBUG_REG2__te11_hs_tess_input_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG2__lsWaveIfBusy_0_MASK 0x20 +#define VGT_DEBUG_REG2__lsWaveIfBusy_0__SHIFT 0x5 +#define VGT_DEBUG_REG2__hs_te11_tess_input_rts_MASK 0x40 +#define VGT_DEBUG_REG2__hs_te11_tess_input_rts__SHIFT 0x6 +#define VGT_DEBUG_REG2__grpModBusy_MASK 0x80 +#define VGT_DEBUG_REG2__grpModBusy__SHIFT 0x7 +#define VGT_DEBUG_REG2__lsVertFifoEmpty_MASK 0x100 +#define VGT_DEBUG_REG2__lsVertFifoEmpty__SHIFT 0x8 +#define VGT_DEBUG_REG2__lsWaveFifoEmpty_MASK 0x200 +#define VGT_DEBUG_REG2__lsWaveFifoEmpty__SHIFT 0x9 +#define VGT_DEBUG_REG2__hsVertFifoEmpty_MASK 0x400 +#define VGT_DEBUG_REG2__hsVertFifoEmpty__SHIFT 0xa +#define VGT_DEBUG_REG2__hsWaveFifoEmpty_MASK 0x800 +#define VGT_DEBUG_REG2__hsWaveFifoEmpty__SHIFT 0xb +#define VGT_DEBUG_REG2__hsInputFifoEmpty_MASK 0x1000 +#define VGT_DEBUG_REG2__hsInputFifoEmpty__SHIFT 0xc +#define VGT_DEBUG_REG2__hsTifFifoEmpty_MASK 0x2000 +#define VGT_DEBUG_REG2__hsTifFifoEmpty__SHIFT 0xd +#define VGT_DEBUG_REG2__lsVertFifoFull_MASK 0x4000 +#define VGT_DEBUG_REG2__lsVertFifoFull__SHIFT 0xe +#define VGT_DEBUG_REG2__lsWaveFifoFull_MASK 0x8000 +#define VGT_DEBUG_REG2__lsWaveFifoFull__SHIFT 0xf +#define VGT_DEBUG_REG2__hsVertFifoFull_MASK 0x10000 +#define VGT_DEBUG_REG2__hsVertFifoFull__SHIFT 0x10 +#define VGT_DEBUG_REG2__hsWaveFifoFull_MASK 0x20000 +#define VGT_DEBUG_REG2__hsWaveFifoFull__SHIFT 0x11 +#define VGT_DEBUG_REG2__hsInputFifoFull_MASK 0x40000 +#define VGT_DEBUG_REG2__hsInputFifoFull__SHIFT 0x12 +#define VGT_DEBUG_REG2__hsTifFifoFull_MASK 0x80000 +#define VGT_DEBUG_REG2__hsTifFifoFull__SHIFT 0x13 +#define VGT_DEBUG_REG2__p0_rtr_MASK 0x100000 +#define VGT_DEBUG_REG2__p0_rtr__SHIFT 0x14 +#define VGT_DEBUG_REG2__p1_rtr_MASK 0x200000 +#define VGT_DEBUG_REG2__p1_rtr__SHIFT 0x15 +#define VGT_DEBUG_REG2__p0_dr_MASK 0x400000 +#define VGT_DEBUG_REG2__p0_dr__SHIFT 0x16 +#define VGT_DEBUG_REG2__p1_dr_MASK 0x800000 +#define VGT_DEBUG_REG2__p1_dr__SHIFT 0x17 +#define VGT_DEBUG_REG2__p0_rts_MASK 0x1000000 +#define VGT_DEBUG_REG2__p0_rts__SHIFT 0x18 +#define VGT_DEBUG_REG2__p1_rts_MASK 0x2000000 +#define VGT_DEBUG_REG2__p1_rts__SHIFT 0x19 +#define VGT_DEBUG_REG2__ls_sh_id_MASK 0x4000000 +#define VGT_DEBUG_REG2__ls_sh_id__SHIFT 0x1a +#define VGT_DEBUG_REG2__lsFwaveFlag_MASK 0x8000000 +#define VGT_DEBUG_REG2__lsFwaveFlag__SHIFT 0x1b +#define VGT_DEBUG_REG2__lsWaveSendFlush_MASK 0x10000000 +#define VGT_DEBUG_REG2__lsWaveSendFlush__SHIFT 0x1c +#define VGT_DEBUG_REG2__SPARE_MASK 0xe0000000 +#define VGT_DEBUG_REG2__SPARE__SHIFT 0x1d +#define VGT_DEBUG_REG3__lsTgRelInd_MASK 0xfff +#define VGT_DEBUG_REG3__lsTgRelInd__SHIFT 0x0 +#define VGT_DEBUG_REG3__lsWaveRelInd_MASK 0x3f000 +#define VGT_DEBUG_REG3__lsWaveRelInd__SHIFT 0xc +#define VGT_DEBUG_REG3__lsPatchCnt_MASK 0x3fc0000 +#define VGT_DEBUG_REG3__lsPatchCnt__SHIFT 0x12 +#define VGT_DEBUG_REG3__hsWaveRelInd_MASK 0xfc000000 +#define VGT_DEBUG_REG3__hsWaveRelInd__SHIFT 0x1a +#define VGT_DEBUG_REG4__hsPatchCnt_MASK 0xff +#define VGT_DEBUG_REG4__hsPatchCnt__SHIFT 0x0 +#define VGT_DEBUG_REG4__hsPrimId_15_0_MASK 0xffff00 +#define VGT_DEBUG_REG4__hsPrimId_15_0__SHIFT 0x8 +#define VGT_DEBUG_REG4__hsCpCnt_MASK 0x1f000000 +#define VGT_DEBUG_REG4__hsCpCnt__SHIFT 0x18 +#define VGT_DEBUG_REG4__hsWaveSendFlush_MASK 0x20000000 +#define VGT_DEBUG_REG4__hsWaveSendFlush__SHIFT 0x1d +#define VGT_DEBUG_REG4__hsFwaveFlag_MASK 0x40000000 +#define VGT_DEBUG_REG4__hsFwaveFlag__SHIFT 0x1e +#define VGT_DEBUG_REG4__SPARE_MASK 0x80000000 +#define VGT_DEBUG_REG4__SPARE__SHIFT 0x1f +#define VGT_DEBUG_REG5__SPARE4_MASK 0x7 +#define VGT_DEBUG_REG5__SPARE4__SHIFT 0x0 +#define VGT_DEBUG_REG5__hsWaveCreditCnt_0_MASK 0xf8 +#define VGT_DEBUG_REG5__hsWaveCreditCnt_0__SHIFT 0x3 +#define VGT_DEBUG_REG5__SPARE3_MASK 0x700 +#define VGT_DEBUG_REG5__SPARE3__SHIFT 0x8 +#define VGT_DEBUG_REG5__hsVertCreditCnt_0_MASK 0xf800 +#define VGT_DEBUG_REG5__hsVertCreditCnt_0__SHIFT 0xb +#define VGT_DEBUG_REG5__SPARE2_MASK 0x70000 +#define VGT_DEBUG_REG5__SPARE2__SHIFT 0x10 +#define VGT_DEBUG_REG5__lsWaveCreditCnt_0_MASK 0xf80000 +#define VGT_DEBUG_REG5__lsWaveCreditCnt_0__SHIFT 0x13 +#define VGT_DEBUG_REG5__SPARE1_MASK 0x7000000 +#define VGT_DEBUG_REG5__SPARE1__SHIFT 0x18 +#define VGT_DEBUG_REG5__lsVertCreditCnt_0_MASK 0xf8000000 +#define VGT_DEBUG_REG5__lsVertCreditCnt_0__SHIFT 0x1b +#define VGT_DEBUG_REG6__debug_BASE_MASK 0xffff +#define VGT_DEBUG_REG6__debug_BASE__SHIFT 0x0 +#define VGT_DEBUG_REG6__debug_SIZE_MASK 0xffff0000 +#define VGT_DEBUG_REG6__debug_SIZE__SHIFT 0x10 +#define VGT_DEBUG_REG7__debug_tfmmFifoEmpty_MASK 0x1 +#define VGT_DEBUG_REG7__debug_tfmmFifoEmpty__SHIFT 0x0 +#define VGT_DEBUG_REG7__debug_tfmmFifoFull_MASK 0x2 +#define VGT_DEBUG_REG7__debug_tfmmFifoFull__SHIFT 0x1 +#define VGT_DEBUG_REG7__hs_pipe0_dr_MASK 0x4 +#define VGT_DEBUG_REG7__hs_pipe0_dr__SHIFT 0x2 +#define VGT_DEBUG_REG7__hs_pipe0_rtr_MASK 0x8 +#define VGT_DEBUG_REG7__hs_pipe0_rtr__SHIFT 0x3 +#define VGT_DEBUG_REG7__hs_pipe1_rtr_MASK 0x10 +#define VGT_DEBUG_REG7__hs_pipe1_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG7__SPARE_MASK 0xffe0 +#define VGT_DEBUG_REG7__SPARE__SHIFT 0x5 +#define VGT_DEBUG_REG7__TF_addr_MASK 0xffff0000 +#define VGT_DEBUG_REG7__TF_addr__SHIFT 0x10 +#define VGT_DEBUG_REG8__rcm_busy_q_MASK 0x1 +#define VGT_DEBUG_REG8__rcm_busy_q__SHIFT 0x0 +#define VGT_DEBUG_REG8__rcm_noif_busy_q_MASK 0x2 +#define VGT_DEBUG_REG8__rcm_noif_busy_q__SHIFT 0x1 +#define VGT_DEBUG_REG8__r1_inst_rtr_MASK 0x4 +#define VGT_DEBUG_REG8__r1_inst_rtr__SHIFT 0x2 +#define VGT_DEBUG_REG8__spi_gsprim_fifo_busy_q_MASK 0x8 +#define VGT_DEBUG_REG8__spi_gsprim_fifo_busy_q__SHIFT 0x3 +#define VGT_DEBUG_REG8__spi_esvert_fifo_busy_q_MASK 0x10 +#define VGT_DEBUG_REG8__spi_esvert_fifo_busy_q__SHIFT 0x4 +#define VGT_DEBUG_REG8__gs_tbl_valid_r3_q_MASK 0x20 +#define VGT_DEBUG_REG8__gs_tbl_valid_r3_q__SHIFT 0x5 +#define VGT_DEBUG_REG8__valid_r0_q_MASK 0x40 +#define VGT_DEBUG_REG8__valid_r0_q__SHIFT 0x6 +#define VGT_DEBUG_REG8__valid_r1_q_MASK 0x80 +#define VGT_DEBUG_REG8__valid_r1_q__SHIFT 0x7 +#define VGT_DEBUG_REG8__valid_r2_MASK 0x100 +#define VGT_DEBUG_REG8__valid_r2__SHIFT 0x8 +#define VGT_DEBUG_REG8__valid_r2_q_MASK 0x200 +#define VGT_DEBUG_REG8__valid_r2_q__SHIFT 0x9 +#define VGT_DEBUG_REG8__r0_rtr_MASK 0x400 +#define VGT_DEBUG_REG8__r0_rtr__SHIFT 0xa +#define VGT_DEBUG_REG8__r1_rtr_MASK 0x800 +#define VGT_DEBUG_REG8__r1_rtr__SHIFT 0xb +#define VGT_DEBUG_REG8__r2_indx_rtr_MASK 0x1000 +#define VGT_DEBUG_REG8__r2_indx_rtr__SHIFT 0xc +#define VGT_DEBUG_REG8__r2_rtr_MASK 0x2000 +#define VGT_DEBUG_REG8__r2_rtr__SHIFT 0xd +#define VGT_DEBUG_REG8__es_gs_rtr_MASK 0x4000 +#define VGT_DEBUG_REG8__es_gs_rtr__SHIFT 0xe +#define VGT_DEBUG_REG8__gs_event_fifo_rtr_MASK 0x8000 +#define VGT_DEBUG_REG8__gs_event_fifo_rtr__SHIFT 0xf +#define VGT_DEBUG_REG8__tm_rcm_gs_event_rtr_MASK 0x10000 +#define VGT_DEBUG_REG8__tm_rcm_gs_event_rtr__SHIFT 0x10 +#define VGT_DEBUG_REG8__gs_tbl_r3_rtr_MASK 0x20000 +#define VGT_DEBUG_REG8__gs_tbl_r3_rtr__SHIFT 0x11 +#define VGT_DEBUG_REG8__prim_skid_fifo_empty_MASK 0x40000 +#define VGT_DEBUG_REG8__prim_skid_fifo_empty__SHIFT 0x12 +#define VGT_DEBUG_REG8__VGT_SPI_gsprim_rtr_q_MASK 0x80000 +#define VGT_DEBUG_REG8__VGT_SPI_gsprim_rtr_q__SHIFT 0x13 +#define VGT_DEBUG_REG8__tm_rcm_gs_tbl_rtr_MASK 0x100000 +#define VGT_DEBUG_REG8__tm_rcm_gs_tbl_rtr__SHIFT 0x14 +#define VGT_DEBUG_REG8__tm_rcm_es_tbl_rtr_MASK 0x200000 +#define VGT_DEBUG_REG8__tm_rcm_es_tbl_rtr__SHIFT 0x15 +#define VGT_DEBUG_REG8__VGT_SPI_esvert_rtr_q_MASK 0x400000 +#define VGT_DEBUG_REG8__VGT_SPI_esvert_rtr_q__SHIFT 0x16 +#define VGT_DEBUG_REG8__r2_no_bp_rtr_MASK 0x800000 +#define VGT_DEBUG_REG8__r2_no_bp_rtr__SHIFT 0x17 +#define VGT_DEBUG_REG8__hold_for_es_flush_MASK 0x1000000 +#define VGT_DEBUG_REG8__hold_for_es_flush__SHIFT 0x18 +#define VGT_DEBUG_REG8__gs_event_fifo_empty_MASK 0x2000000 +#define VGT_DEBUG_REG8__gs_event_fifo_empty__SHIFT 0x19 +#define VGT_DEBUG_REG8__gsprim_buff_empty_q_MASK 0x4000000 +#define VGT_DEBUG_REG8__gsprim_buff_empty_q__SHIFT 0x1a +#define VGT_DEBUG_REG8__gsprim_buff_full_q_MASK 0x8000000 +#define VGT_DEBUG_REG8__gsprim_buff_full_q__SHIFT 0x1b +#define VGT_DEBUG_REG8__te_prim_fifo_empty_MASK 0x10000000 +#define VGT_DEBUG_REG8__te_prim_fifo_empty__SHIFT 0x1c +#define VGT_DEBUG_REG8__te_prim_fifo_full_MASK 0x20000000 +#define VGT_DEBUG_REG8__te_prim_fifo_full__SHIFT 0x1d +#define VGT_DEBUG_REG8__te_vert_fifo_empty_MASK 0x40000000 +#define VGT_DEBUG_REG8__te_vert_fifo_empty__SHIFT 0x1e +#define VGT_DEBUG_REG8__te_vert_fifo_full_MASK 0x80000000 +#define VGT_DEBUG_REG8__te_vert_fifo_full__SHIFT 0x1f +#define VGT_DEBUG_REG9__indices_to_send_r2_q_MASK 0x3 +#define VGT_DEBUG_REG9__indices_to_send_r2_q__SHIFT 0x0 +#define VGT_DEBUG_REG9__valid_indices_r3_MASK 0x4 +#define VGT_DEBUG_REG9__valid_indices_r3__SHIFT 0x2 +#define VGT_DEBUG_REG9__gs_eov_r3_MASK 0x8 +#define VGT_DEBUG_REG9__gs_eov_r3__SHIFT 0x3 +#define VGT_DEBUG_REG9__eop_indx_r3_MASK 0x10 +#define VGT_DEBUG_REG9__eop_indx_r3__SHIFT 0x4 +#define VGT_DEBUG_REG9__eop_prim_r3_MASK 0x20 +#define VGT_DEBUG_REG9__eop_prim_r3__SHIFT 0x5 +#define VGT_DEBUG_REG9__es_eov_r3_MASK 0x40 +#define VGT_DEBUG_REG9__es_eov_r3__SHIFT 0x6 +#define VGT_DEBUG_REG9__es_tbl_state_r3_q_0_MASK 0x80 +#define VGT_DEBUG_REG9__es_tbl_state_r3_q_0__SHIFT 0x7 +#define VGT_DEBUG_REG9__pending_es_send_r3_q_MASK 0x100 +#define VGT_DEBUG_REG9__pending_es_send_r3_q__SHIFT 0x8 +#define VGT_DEBUG_REG9__pending_es_flush_r3_MASK 0x200 +#define VGT_DEBUG_REG9__pending_es_flush_r3__SHIFT 0x9 +#define VGT_DEBUG_REG9__gs_tbl_num_es_per_gs_r3_q_not_0_MASK 0x400 +#define VGT_DEBUG_REG9__gs_tbl_num_es_per_gs_r3_q_not_0__SHIFT 0xa +#define VGT_DEBUG_REG9__gs_tbl_prim_cnt_r3_q_MASK 0x3f800 +#define VGT_DEBUG_REG9__gs_tbl_prim_cnt_r3_q__SHIFT 0xb +#define VGT_DEBUG_REG9__gs_tbl_eop_r3_q_MASK 0x40000 +#define VGT_DEBUG_REG9__gs_tbl_eop_r3_q__SHIFT 0x12 +#define VGT_DEBUG_REG9__gs_tbl_state_r3_q_MASK 0x380000 +#define VGT_DEBUG_REG9__gs_tbl_state_r3_q__SHIFT 0x13 +#define VGT_DEBUG_REG9__gs_pending_state_r3_q_MASK 0x400000 +#define VGT_DEBUG_REG9__gs_pending_state_r3_q__SHIFT 0x16 +#define VGT_DEBUG_REG9__invalidate_rb_roll_over_q_MASK 0x800000 +#define VGT_DEBUG_REG9__invalidate_rb_roll_over_q__SHIFT 0x17 +#define VGT_DEBUG_REG9__gs_instancing_state_q_MASK 0x1000000 +#define VGT_DEBUG_REG9__gs_instancing_state_q__SHIFT 0x18 +#define VGT_DEBUG_REG9__es_per_gs_vert_cnt_r3_q_not_0_MASK 0x2000000 +#define VGT_DEBUG_REG9__es_per_gs_vert_cnt_r3_q_not_0__SHIFT 0x19 +#define VGT_DEBUG_REG9__gs_prim_per_es_ctr_r3_q_not_0_MASK 0x4000000 +#define VGT_DEBUG_REG9__gs_prim_per_es_ctr_r3_q_not_0__SHIFT 0x1a +#define VGT_DEBUG_REG9__pre_r0_rtr_MASK 0x8000000 +#define VGT_DEBUG_REG9__pre_r0_rtr__SHIFT 0x1b +#define VGT_DEBUG_REG9__valid_r3_q_MASK 0x10000000 +#define VGT_DEBUG_REG9__valid_r3_q__SHIFT 0x1c +#define VGT_DEBUG_REG9__valid_pre_r0_q_MASK 0x20000000 +#define VGT_DEBUG_REG9__valid_pre_r0_q__SHIFT 0x1d +#define VGT_DEBUG_REG9__SPARE0_MASK 0x40000000 +#define VGT_DEBUG_REG9__SPARE0__SHIFT 0x1e +#define VGT_DEBUG_REG9__off_chip_hs_r2_q_MASK 0x80000000 +#define VGT_DEBUG_REG9__off_chip_hs_r2_q__SHIFT 0x1f +#define VGT_DEBUG_REG10__index_buffer_depth_r1_q_MASK 0x1f +#define VGT_DEBUG_REG10__index_buffer_depth_r1_q__SHIFT 0x0 +#define VGT_DEBUG_REG10__eopg_r2_q_MASK 0x20 +#define VGT_DEBUG_REG10__eopg_r2_q__SHIFT 0x5 +#define VGT_DEBUG_REG10__eotg_r2_q_MASK 0x40 +#define VGT_DEBUG_REG10__eotg_r2_q__SHIFT 0x6 +#define VGT_DEBUG_REG10__onchip_gs_en_r0_q_MASK 0x180 +#define VGT_DEBUG_REG10__onchip_gs_en_r0_q__SHIFT 0x7 +#define VGT_DEBUG_REG10__SPARE2_MASK 0x600 +#define VGT_DEBUG_REG10__SPARE2__SHIFT 0x9 +#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_qq_MASK 0x800 +#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_qq__SHIFT 0xb +#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_q_MASK 0x1000 +#define VGT_DEBUG_REG10__rcm_mem_gsprim_re_q__SHIFT 0xc +#define VGT_DEBUG_REG10__gs_rb_space_avail_r3_q_9_0_MASK 0x7fe000 +#define VGT_DEBUG_REG10__gs_rb_space_avail_r3_q_9_0__SHIFT 0xd +#define VGT_DEBUG_REG10__es_rb_space_avail_r2_q_8_0_MASK 0xff800000 +#define VGT_DEBUG_REG10__es_rb_space_avail_r2_q_8_0__SHIFT 0x17 +#define VGT_DEBUG_REG11__tm_busy_q_MASK 0x1 +#define VGT_DEBUG_REG11__tm_busy_q__SHIFT 0x0 +#define VGT_DEBUG_REG11__tm_noif_busy_q_MASK 0x2 +#define VGT_DEBUG_REG11__tm_noif_busy_q__SHIFT 0x1 +#define VGT_DEBUG_REG11__tm_out_busy_q_MASK 0x4 +#define VGT_DEBUG_REG11__tm_out_busy_q__SHIFT 0x2 +#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_busy_MASK 0x8 +#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_busy__SHIFT 0x3 +#define VGT_DEBUG_REG11__vs_dealloc_tbl_busy_MASK 0x10 +#define VGT_DEBUG_REG11__vs_dealloc_tbl_busy__SHIFT 0x4 +#define VGT_DEBUG_REG11__SPARE1_MASK 0x20 +#define VGT_DEBUG_REG11__SPARE1__SHIFT 0x5 +#define VGT_DEBUG_REG11__spi_gsthread_fifo_busy_MASK 0x40 +#define VGT_DEBUG_REG11__spi_gsthread_fifo_busy__SHIFT 0x6 +#define VGT_DEBUG_REG11__spi_esthread_fifo_busy_MASK 0x80 +#define VGT_DEBUG_REG11__spi_esthread_fifo_busy__SHIFT 0x7 +#define VGT_DEBUG_REG11__hold_eswave_MASK 0x100 +#define VGT_DEBUG_REG11__hold_eswave__SHIFT 0x8 +#define VGT_DEBUG_REG11__es_rb_roll_over_r3_MASK 0x200 +#define VGT_DEBUG_REG11__es_rb_roll_over_r3__SHIFT 0x9 +#define VGT_DEBUG_REG11__counters_busy_r0_MASK 0x400 +#define VGT_DEBUG_REG11__counters_busy_r0__SHIFT 0xa +#define VGT_DEBUG_REG11__counters_avail_r0_MASK 0x800 +#define VGT_DEBUG_REG11__counters_avail_r0__SHIFT 0xb +#define VGT_DEBUG_REG11__counters_available_r0_MASK 0x1000 +#define VGT_DEBUG_REG11__counters_available_r0__SHIFT 0xc +#define VGT_DEBUG_REG11__vs_event_fifo_rtr_MASK 0x2000 +#define VGT_DEBUG_REG11__vs_event_fifo_rtr__SHIFT 0xd +#define VGT_DEBUG_REG11__VGT_SPI_gsthread_rtr_q_MASK 0x4000 +#define VGT_DEBUG_REG11__VGT_SPI_gsthread_rtr_q__SHIFT 0xe +#define VGT_DEBUG_REG11__VGT_SPI_esthread_rtr_q_MASK 0x8000 +#define VGT_DEBUG_REG11__VGT_SPI_esthread_rtr_q__SHIFT 0xf +#define VGT_DEBUG_REG11__gs_issue_rtr_MASK 0x10000 +#define VGT_DEBUG_REG11__gs_issue_rtr__SHIFT 0x10 +#define VGT_DEBUG_REG11__tm_pt_event_rtr_MASK 0x20000 +#define VGT_DEBUG_REG11__tm_pt_event_rtr__SHIFT 0x11 +#define VGT_DEBUG_REG11__SPARE0_MASK 0x40000 +#define VGT_DEBUG_REG11__SPARE0__SHIFT 0x12 +#define VGT_DEBUG_REG11__gs_r0_rtr_MASK 0x80000 +#define VGT_DEBUG_REG11__gs_r0_rtr__SHIFT 0x13 +#define VGT_DEBUG_REG11__es_r0_rtr_MASK 0x100000 +#define VGT_DEBUG_REG11__es_r0_rtr__SHIFT 0x14 +#define VGT_DEBUG_REG11__gog_tm_vs_event_rtr_MASK 0x200000 +#define VGT_DEBUG_REG11__gog_tm_vs_event_rtr__SHIFT 0x15 +#define VGT_DEBUG_REG11__tm_rcm_gs_event_rtr_MASK 0x400000 +#define VGT_DEBUG_REG11__tm_rcm_gs_event_rtr__SHIFT 0x16 +#define VGT_DEBUG_REG11__tm_rcm_gs_tbl_rtr_MASK 0x800000 +#define VGT_DEBUG_REG11__tm_rcm_gs_tbl_rtr__SHIFT 0x17 +#define VGT_DEBUG_REG11__tm_rcm_es_tbl_rtr_MASK 0x1000000 +#define VGT_DEBUG_REG11__tm_rcm_es_tbl_rtr__SHIFT 0x18 +#define VGT_DEBUG_REG11__vs_event_fifo_empty_MASK 0x2000000 +#define VGT_DEBUG_REG11__vs_event_fifo_empty__SHIFT 0x19 +#define VGT_DEBUG_REG11__vs_event_fifo_full_MASK 0x4000000 +#define VGT_DEBUG_REG11__vs_event_fifo_full__SHIFT 0x1a +#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_full_MASK 0x8000000 +#define VGT_DEBUG_REG11__es_rb_dealloc_fifo_full__SHIFT 0x1b +#define VGT_DEBUG_REG11__vs_dealloc_tbl_full_MASK 0x10000000 +#define VGT_DEBUG_REG11__vs_dealloc_tbl_full__SHIFT 0x1c +#define VGT_DEBUG_REG11__send_event_q_MASK 0x20000000 +#define VGT_DEBUG_REG11__send_event_q__SHIFT 0x1d +#define VGT_DEBUG_REG11__es_tbl_empty_MASK 0x40000000 +#define VGT_DEBUG_REG11__es_tbl_empty__SHIFT 0x1e +#define VGT_DEBUG_REG11__no_active_states_r0_MASK 0x80000000 +#define VGT_DEBUG_REG11__no_active_states_r0__SHIFT 0x1f +#define VGT_DEBUG_REG12__gs_state0_r0_q_MASK 0x7 +#define VGT_DEBUG_REG12__gs_state0_r0_q__SHIFT 0x0 +#define VGT_DEBUG_REG12__gs_state1_r0_q_MASK 0x38 +#define VGT_DEBUG_REG12__gs_state1_r0_q__SHIFT 0x3 +#define VGT_DEBUG_REG12__gs_state2_r0_q_MASK 0x1c0 +#define VGT_DEBUG_REG12__gs_state2_r0_q__SHIFT 0x6 +#define VGT_DEBUG_REG12__gs_state3_r0_q_MASK 0xe00 +#define VGT_DEBUG_REG12__gs_state3_r0_q__SHIFT 0x9 +#define VGT_DEBUG_REG12__gs_state4_r0_q_MASK 0x7000 +#define VGT_DEBUG_REG12__gs_state4_r0_q__SHIFT 0xc +#define VGT_DEBUG_REG12__gs_state5_r0_q_MASK 0x38000 +#define VGT_DEBUG_REG12__gs_state5_r0_q__SHIFT 0xf +#define VGT_DEBUG_REG12__gs_state6_r0_q_MASK 0x1c0000 +#define VGT_DEBUG_REG12__gs_state6_r0_q__SHIFT 0x12 +#define VGT_DEBUG_REG12__gs_state7_r0_q_MASK 0xe00000 +#define VGT_DEBUG_REG12__gs_state7_r0_q__SHIFT 0x15 +#define VGT_DEBUG_REG12__gs_state8_r0_q_MASK 0x7000000 +#define VGT_DEBUG_REG12__gs_state8_r0_q__SHIFT 0x18 +#define VGT_DEBUG_REG12__gs_state9_r0_q_MASK 0x38000000 +#define VGT_DEBUG_REG12__gs_state9_r0_q__SHIFT 0x1b +#define VGT_DEBUG_REG12__hold_eswave_eop_MASK 0x40000000 +#define VGT_DEBUG_REG12__hold_eswave_eop__SHIFT 0x1e +#define VGT_DEBUG_REG12__SPARE0_MASK 0x80000000 +#define VGT_DEBUG_REG12__SPARE0__SHIFT 0x1f +#define VGT_DEBUG_REG13__gs_state10_r0_q_MASK 0x7 +#define VGT_DEBUG_REG13__gs_state10_r0_q__SHIFT 0x0 +#define VGT_DEBUG_REG13__gs_state11_r0_q_MASK 0x38 +#define VGT_DEBUG_REG13__gs_state11_r0_q__SHIFT 0x3 +#define VGT_DEBUG_REG13__gs_state12_r0_q_MASK 0x1c0 +#define VGT_DEBUG_REG13__gs_state12_r0_q__SHIFT 0x6 +#define VGT_DEBUG_REG13__gs_state13_r0_q_MASK 0xe00 +#define VGT_DEBUG_REG13__gs_state13_r0_q__SHIFT 0x9 +#define VGT_DEBUG_REG13__gs_state14_r0_q_MASK 0x7000 +#define VGT_DEBUG_REG13__gs_state14_r0_q__SHIFT 0xc +#define VGT_DEBUG_REG13__gs_state15_r0_q_MASK 0x38000 +#define VGT_DEBUG_REG13__gs_state15_r0_q__SHIFT 0xf +#define VGT_DEBUG_REG13__gs_tbl_wrptr_r0_q_3_0_MASK 0x3c0000 +#define VGT_DEBUG_REG13__gs_tbl_wrptr_r0_q_3_0__SHIFT 0x12 +#define VGT_DEBUG_REG13__gsfetch_done_fifo_cnt_q_not_0_MASK 0x400000 +#define VGT_DEBUG_REG13__gsfetch_done_fifo_cnt_q_not_0__SHIFT 0x16 +#define VGT_DEBUG_REG13__gsfetch_done_cnt_q_not_0_MASK 0x800000 +#define VGT_DEBUG_REG13__gsfetch_done_cnt_q_not_0__SHIFT 0x17 +#define VGT_DEBUG_REG13__es_tbl_full_MASK 0x1000000 +#define VGT_DEBUG_REG13__es_tbl_full__SHIFT 0x18 +#define VGT_DEBUG_REG13__SPARE1_MASK 0x2000000 +#define VGT_DEBUG_REG13__SPARE1__SHIFT 0x19 +#define VGT_DEBUG_REG13__SPARE0_MASK 0x4000000 +#define VGT_DEBUG_REG13__SPARE0__SHIFT 0x1a +#define VGT_DEBUG_REG13__active_cm_sm_r0_q_MASK 0xf8000000 +#define VGT_DEBUG_REG13__active_cm_sm_r0_q__SHIFT 0x1b +#define VGT_DEBUG_REG14__SPARE3_MASK 0xf +#define VGT_DEBUG_REG14__SPARE3__SHIFT 0x0 +#define VGT_DEBUG_REG14__gsfetch_done_fifo_full_MASK 0x10 +#define VGT_DEBUG_REG14__gsfetch_done_fifo_full__SHIFT 0x4 +#define VGT_DEBUG_REG14__gs_rb_space_avail_r0_MASK 0x20 +#define VGT_DEBUG_REG14__gs_rb_space_avail_r0__SHIFT 0x5 +#define VGT_DEBUG_REG14__smx_es_done_cnt_r0_q_not_0_MASK 0x40 +#define VGT_DEBUG_REG14__smx_es_done_cnt_r0_q_not_0__SHIFT 0x6 +#define VGT_DEBUG_REG14__SPARE8_MASK 0x180 +#define VGT_DEBUG_REG14__SPARE8__SHIFT 0x7 +#define VGT_DEBUG_REG14__vs_done_cnt_q_not_0_MASK 0x200 +#define VGT_DEBUG_REG14__vs_done_cnt_q_not_0__SHIFT 0x9 +#define VGT_DEBUG_REG14__es_flush_cnt_busy_q_MASK 0x400 +#define VGT_DEBUG_REG14__es_flush_cnt_busy_q__SHIFT 0xa +#define VGT_DEBUG_REG14__gs_tbl_full_r0_MASK 0x800 +#define VGT_DEBUG_REG14__gs_tbl_full_r0__SHIFT 0xb +#define VGT_DEBUG_REG14__SPARE2_MASK 0x1ff000 +#define VGT_DEBUG_REG14__SPARE2__SHIFT 0xc +#define VGT_DEBUG_REG14__se1spi_gsthread_fifo_busy_MASK 0x200000 +#define VGT_DEBUG_REG14__se1spi_gsthread_fifo_busy__SHIFT 0x15 +#define VGT_DEBUG_REG14__SPARE_MASK 0x1c00000 +#define VGT_DEBUG_REG14__SPARE__SHIFT 0x16 +#define VGT_DEBUG_REG14__VGT_SE1SPI_gsthread_rtr_q_MASK 0x2000000 +#define VGT_DEBUG_REG14__VGT_SE1SPI_gsthread_rtr_q__SHIFT 0x19 +#define VGT_DEBUG_REG14__smx1_es_done_cnt_r0_q_not_0_MASK 0x4000000 +#define VGT_DEBUG_REG14__smx1_es_done_cnt_r0_q_not_0__SHIFT 0x1a +#define VGT_DEBUG_REG14__se1spi_esthread_fifo_busy_MASK 0x8000000 +#define VGT_DEBUG_REG14__se1spi_esthread_fifo_busy__SHIFT 0x1b +#define VGT_DEBUG_REG14__SPARE1_MASK 0x10000000 +#define VGT_DEBUG_REG14__SPARE1__SHIFT 0x1c +#define VGT_DEBUG_REG14__gsfetch_done_se1_cnt_q_not_0_MASK 0x20000000 +#define VGT_DEBUG_REG14__gsfetch_done_se1_cnt_q_not_0__SHIFT 0x1d +#define VGT_DEBUG_REG14__SPARE0_MASK 0x40000000 +#define VGT_DEBUG_REG14__SPARE0__SHIFT 0x1e +#define VGT_DEBUG_REG14__VGT_SE1SPI_esthread_rtr_q_MASK 0x80000000 +#define VGT_DEBUG_REG14__VGT_SE1SPI_esthread_rtr_q__SHIFT 0x1f +#define VGT_DEBUG_REG15__cm_busy_q_MASK 0x1 +#define VGT_DEBUG_REG15__cm_busy_q__SHIFT 0x0 +#define VGT_DEBUG_REG15__counters_busy_q_MASK 0x2 +#define VGT_DEBUG_REG15__counters_busy_q__SHIFT 0x1 +#define VGT_DEBUG_REG15__output_fifo_empty_MASK 0x4 +#define VGT_DEBUG_REG15__output_fifo_empty__SHIFT 0x2 +#define VGT_DEBUG_REG15__output_fifo_full_MASK 0x8 +#define VGT_DEBUG_REG15__output_fifo_full__SHIFT 0x3 +#define VGT_DEBUG_REG15__counters_full_MASK 0x10 +#define VGT_DEBUG_REG15__counters_full__SHIFT 0x4 +#define VGT_DEBUG_REG15__active_sm_q_MASK 0x3e0 +#define VGT_DEBUG_REG15__active_sm_q__SHIFT 0x5 +#define VGT_DEBUG_REG15__entry_rdptr_q_MASK 0x7c00 +#define VGT_DEBUG_REG15__entry_rdptr_q__SHIFT 0xa +#define VGT_DEBUG_REG15__cntr_tbl_wrptr_q_MASK 0xf8000 +#define VGT_DEBUG_REG15__cntr_tbl_wrptr_q__SHIFT 0xf +#define VGT_DEBUG_REG15__SPARE25_MASK 0x3f00000 +#define VGT_DEBUG_REG15__SPARE25__SHIFT 0x14 +#define VGT_DEBUG_REG15__st_cut_mode_q_MASK 0xc000000 +#define VGT_DEBUG_REG15__st_cut_mode_q__SHIFT 0x1a +#define VGT_DEBUG_REG15__gs_done_array_q_not_0_MASK 0x10000000 +#define VGT_DEBUG_REG15__gs_done_array_q_not_0__SHIFT 0x1c +#define VGT_DEBUG_REG15__SPARE31_MASK 0xe0000000 +#define VGT_DEBUG_REG15__SPARE31__SHIFT 0x1d +#define VGT_DEBUG_REG16__gog_busy_MASK 0x1 +#define VGT_DEBUG_REG16__gog_busy__SHIFT 0x0 +#define VGT_DEBUG_REG16__gog_state_q_MASK 0xe +#define VGT_DEBUG_REG16__gog_state_q__SHIFT 0x1 +#define VGT_DEBUG_REG16__r0_rtr_MASK 0x10 +#define VGT_DEBUG_REG16__r0_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG16__r1_rtr_MASK 0x20 +#define VGT_DEBUG_REG16__r1_rtr__SHIFT 0x5 +#define VGT_DEBUG_REG16__r1_upstream_rtr_MASK 0x40 +#define VGT_DEBUG_REG16__r1_upstream_rtr__SHIFT 0x6 +#define VGT_DEBUG_REG16__r2_vs_tbl_rtr_MASK 0x80 +#define VGT_DEBUG_REG16__r2_vs_tbl_rtr__SHIFT 0x7 +#define VGT_DEBUG_REG16__r2_prim_rtr_MASK 0x100 +#define VGT_DEBUG_REG16__r2_prim_rtr__SHIFT 0x8 +#define VGT_DEBUG_REG16__r2_indx_rtr_MASK 0x200 +#define VGT_DEBUG_REG16__r2_indx_rtr__SHIFT 0x9 +#define VGT_DEBUG_REG16__r2_rtr_MASK 0x400 +#define VGT_DEBUG_REG16__r2_rtr__SHIFT 0xa +#define VGT_DEBUG_REG16__gog_tm_vs_event_rtr_MASK 0x800 +#define VGT_DEBUG_REG16__gog_tm_vs_event_rtr__SHIFT 0xb +#define VGT_DEBUG_REG16__r3_force_vs_tbl_we_rtr_MASK 0x1000 +#define VGT_DEBUG_REG16__r3_force_vs_tbl_we_rtr__SHIFT 0xc +#define VGT_DEBUG_REG16__indx_valid_r2_q_MASK 0x2000 +#define VGT_DEBUG_REG16__indx_valid_r2_q__SHIFT 0xd +#define VGT_DEBUG_REG16__prim_valid_r2_q_MASK 0x4000 +#define VGT_DEBUG_REG16__prim_valid_r2_q__SHIFT 0xe +#define VGT_DEBUG_REG16__valid_r2_q_MASK 0x8000 +#define VGT_DEBUG_REG16__valid_r2_q__SHIFT 0xf +#define VGT_DEBUG_REG16__prim_valid_r1_q_MASK 0x10000 +#define VGT_DEBUG_REG16__prim_valid_r1_q__SHIFT 0x10 +#define VGT_DEBUG_REG16__indx_valid_r1_q_MASK 0x20000 +#define VGT_DEBUG_REG16__indx_valid_r1_q__SHIFT 0x11 +#define VGT_DEBUG_REG16__valid_r1_q_MASK 0x40000 +#define VGT_DEBUG_REG16__valid_r1_q__SHIFT 0x12 +#define VGT_DEBUG_REG16__indx_valid_r0_q_MASK 0x80000 +#define VGT_DEBUG_REG16__indx_valid_r0_q__SHIFT 0x13 +#define VGT_DEBUG_REG16__prim_valid_r0_q_MASK 0x100000 +#define VGT_DEBUG_REG16__prim_valid_r0_q__SHIFT 0x14 +#define VGT_DEBUG_REG16__valid_r0_q_MASK 0x200000 +#define VGT_DEBUG_REG16__valid_r0_q__SHIFT 0x15 +#define VGT_DEBUG_REG16__send_event_q_MASK 0x400000 +#define VGT_DEBUG_REG16__send_event_q__SHIFT 0x16 +#define VGT_DEBUG_REG16__SPARE24_MASK 0x800000 +#define VGT_DEBUG_REG16__SPARE24__SHIFT 0x17 +#define VGT_DEBUG_REG16__vert_seen_since_sopg_r2_q_MASK 0x1000000 +#define VGT_DEBUG_REG16__vert_seen_since_sopg_r2_q__SHIFT 0x18 +#define VGT_DEBUG_REG16__gog_out_prim_state_sel_MASK 0xe000000 +#define VGT_DEBUG_REG16__gog_out_prim_state_sel__SHIFT 0x19 +#define VGT_DEBUG_REG16__multiple_streams_en_r1_q_MASK 0x10000000 +#define VGT_DEBUG_REG16__multiple_streams_en_r1_q__SHIFT 0x1c +#define VGT_DEBUG_REG16__vs_vert_count_r2_q_not_0_MASK 0x20000000 +#define VGT_DEBUG_REG16__vs_vert_count_r2_q_not_0__SHIFT 0x1d +#define VGT_DEBUG_REG16__num_gs_r2_q_not_0_MASK 0x40000000 +#define VGT_DEBUG_REG16__num_gs_r2_q_not_0__SHIFT 0x1e +#define VGT_DEBUG_REG16__new_vs_thread_r2_MASK 0x80000000 +#define VGT_DEBUG_REG16__new_vs_thread_r2__SHIFT 0x1f +#define VGT_DEBUG_REG17__gog_out_prim_rel_indx2_5_0_MASK 0x3f +#define VGT_DEBUG_REG17__gog_out_prim_rel_indx2_5_0__SHIFT 0x0 +#define VGT_DEBUG_REG17__gog_out_prim_rel_indx1_5_0_MASK 0xfc0 +#define VGT_DEBUG_REG17__gog_out_prim_rel_indx1_5_0__SHIFT 0x6 +#define VGT_DEBUG_REG17__gog_out_prim_rel_indx0_5_0_MASK 0x3f000 +#define VGT_DEBUG_REG17__gog_out_prim_rel_indx0_5_0__SHIFT 0xc +#define VGT_DEBUG_REG17__gog_out_indx_13_0_MASK 0xfffc0000 +#define VGT_DEBUG_REG17__gog_out_indx_13_0__SHIFT 0x12 +#define VGT_DEBUG_REG18__grp_vr_valid_MASK 0x1 +#define VGT_DEBUG_REG18__grp_vr_valid__SHIFT 0x0 +#define VGT_DEBUG_REG18__pipe0_dr_MASK 0x2 +#define VGT_DEBUG_REG18__pipe0_dr__SHIFT 0x1 +#define VGT_DEBUG_REG18__pipe1_dr_MASK 0x4 +#define VGT_DEBUG_REG18__pipe1_dr__SHIFT 0x2 +#define VGT_DEBUG_REG18__vr_grp_read_MASK 0x8 +#define VGT_DEBUG_REG18__vr_grp_read__SHIFT 0x3 +#define VGT_DEBUG_REG18__pipe0_rtr_MASK 0x10 +#define VGT_DEBUG_REG18__pipe0_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG18__pipe1_rtr_MASK 0x20 +#define VGT_DEBUG_REG18__pipe1_rtr__SHIFT 0x5 +#define VGT_DEBUG_REG18__out_vr_indx_read_MASK 0x40 +#define VGT_DEBUG_REG18__out_vr_indx_read__SHIFT 0x6 +#define VGT_DEBUG_REG18__out_vr_prim_read_MASK 0x80 +#define VGT_DEBUG_REG18__out_vr_prim_read__SHIFT 0x7 +#define VGT_DEBUG_REG18__indices_to_send_q_MASK 0x700 +#define VGT_DEBUG_REG18__indices_to_send_q__SHIFT 0x8 +#define VGT_DEBUG_REG18__valid_indices_MASK 0x800 +#define VGT_DEBUG_REG18__valid_indices__SHIFT 0xb +#define VGT_DEBUG_REG18__last_indx_of_prim_MASK 0x1000 +#define VGT_DEBUG_REG18__last_indx_of_prim__SHIFT 0xc +#define VGT_DEBUG_REG18__indx0_new_d_MASK 0x2000 +#define VGT_DEBUG_REG18__indx0_new_d__SHIFT 0xd +#define VGT_DEBUG_REG18__indx1_new_d_MASK 0x4000 +#define VGT_DEBUG_REG18__indx1_new_d__SHIFT 0xe +#define VGT_DEBUG_REG18__indx2_new_d_MASK 0x8000 +#define VGT_DEBUG_REG18__indx2_new_d__SHIFT 0xf +#define VGT_DEBUG_REG18__indx2_hit_d_MASK 0x10000 +#define VGT_DEBUG_REG18__indx2_hit_d__SHIFT 0x10 +#define VGT_DEBUG_REG18__indx1_hit_d_MASK 0x20000 +#define VGT_DEBUG_REG18__indx1_hit_d__SHIFT 0x11 +#define VGT_DEBUG_REG18__indx0_hit_d_MASK 0x40000 +#define VGT_DEBUG_REG18__indx0_hit_d__SHIFT 0x12 +#define VGT_DEBUG_REG18__st_vertex_reuse_off_r0_q_MASK 0x80000 +#define VGT_DEBUG_REG18__st_vertex_reuse_off_r0_q__SHIFT 0x13 +#define VGT_DEBUG_REG18__last_group_of_instance_r0_q_MASK 0x100000 +#define VGT_DEBUG_REG18__last_group_of_instance_r0_q__SHIFT 0x14 +#define VGT_DEBUG_REG18__null_primitive_r0_q_MASK 0x200000 +#define VGT_DEBUG_REG18__null_primitive_r0_q__SHIFT 0x15 +#define VGT_DEBUG_REG18__eop_r0_q_MASK 0x400000 +#define VGT_DEBUG_REG18__eop_r0_q__SHIFT 0x16 +#define VGT_DEBUG_REG18__eject_vtx_vect_r1_d_MASK 0x800000 +#define VGT_DEBUG_REG18__eject_vtx_vect_r1_d__SHIFT 0x17 +#define VGT_DEBUG_REG18__sub_prim_type_r0_q_MASK 0x7000000 +#define VGT_DEBUG_REG18__sub_prim_type_r0_q__SHIFT 0x18 +#define VGT_DEBUG_REG18__gs_scenario_a_r0_q_MASK 0x8000000 +#define VGT_DEBUG_REG18__gs_scenario_a_r0_q__SHIFT 0x1b +#define VGT_DEBUG_REG18__gs_scenario_b_r0_q_MASK 0x10000000 +#define VGT_DEBUG_REG18__gs_scenario_b_r0_q__SHIFT 0x1c +#define VGT_DEBUG_REG18__components_valid_r0_q_MASK 0xe0000000 +#define VGT_DEBUG_REG18__components_valid_r0_q__SHIFT 0x1d +#define VGT_DEBUG_REG19__separate_out_busy_q_MASK 0x1 +#define VGT_DEBUG_REG19__separate_out_busy_q__SHIFT 0x0 +#define VGT_DEBUG_REG19__separate_out_indx_busy_q_MASK 0x2 +#define VGT_DEBUG_REG19__separate_out_indx_busy_q__SHIFT 0x1 +#define VGT_DEBUG_REG19__prim_buffer_empty_MASK 0x4 +#define VGT_DEBUG_REG19__prim_buffer_empty__SHIFT 0x2 +#define VGT_DEBUG_REG19__prim_buffer_full_MASK 0x8 +#define VGT_DEBUG_REG19__prim_buffer_full__SHIFT 0x3 +#define VGT_DEBUG_REG19__pa_clips_fifo_busy_q_MASK 0x10 +#define VGT_DEBUG_REG19__pa_clips_fifo_busy_q__SHIFT 0x4 +#define VGT_DEBUG_REG19__pa_clipp_fifo_busy_q_MASK 0x20 +#define VGT_DEBUG_REG19__pa_clipp_fifo_busy_q__SHIFT 0x5 +#define VGT_DEBUG_REG19__VGT_PA_clips_rtr_q_MASK 0x40 +#define VGT_DEBUG_REG19__VGT_PA_clips_rtr_q__SHIFT 0x6 +#define VGT_DEBUG_REG19__VGT_PA_clipp_rtr_q_MASK 0x80 +#define VGT_DEBUG_REG19__VGT_PA_clipp_rtr_q__SHIFT 0x7 +#define VGT_DEBUG_REG19__spi_vsthread_fifo_busy_q_MASK 0x100 +#define VGT_DEBUG_REG19__spi_vsthread_fifo_busy_q__SHIFT 0x8 +#define VGT_DEBUG_REG19__spi_vsvert_fifo_busy_q_MASK 0x200 +#define VGT_DEBUG_REG19__spi_vsvert_fifo_busy_q__SHIFT 0x9 +#define VGT_DEBUG_REG19__pa_clipv_fifo_busy_q_MASK 0x400 +#define VGT_DEBUG_REG19__pa_clipv_fifo_busy_q__SHIFT 0xa +#define VGT_DEBUG_REG19__hold_prim_MASK 0x800 +#define VGT_DEBUG_REG19__hold_prim__SHIFT 0xb +#define VGT_DEBUG_REG19__VGT_SPI_vsthread_rtr_q_MASK 0x1000 +#define VGT_DEBUG_REG19__VGT_SPI_vsthread_rtr_q__SHIFT 0xc +#define VGT_DEBUG_REG19__VGT_SPI_vsvert_rtr_q_MASK 0x2000 +#define VGT_DEBUG_REG19__VGT_SPI_vsvert_rtr_q__SHIFT 0xd +#define VGT_DEBUG_REG19__VGT_PA_clipv_rtr_q_MASK 0x4000 +#define VGT_DEBUG_REG19__VGT_PA_clipv_rtr_q__SHIFT 0xe +#define VGT_DEBUG_REG19__new_packet_q_MASK 0x8000 +#define VGT_DEBUG_REG19__new_packet_q__SHIFT 0xf +#define VGT_DEBUG_REG19__buffered_prim_event_MASK 0x10000 +#define VGT_DEBUG_REG19__buffered_prim_event__SHIFT 0x10 +#define VGT_DEBUG_REG19__buffered_prim_null_primitive_MASK 0x20000 +#define VGT_DEBUG_REG19__buffered_prim_null_primitive__SHIFT 0x11 +#define VGT_DEBUG_REG19__buffered_prim_eop_MASK 0x40000 +#define VGT_DEBUG_REG19__buffered_prim_eop__SHIFT 0x12 +#define VGT_DEBUG_REG19__buffered_prim_eject_vtx_vect_MASK 0x80000 +#define VGT_DEBUG_REG19__buffered_prim_eject_vtx_vect__SHIFT 0x13 +#define VGT_DEBUG_REG19__buffered_prim_type_event_MASK 0x3f00000 +#define VGT_DEBUG_REG19__buffered_prim_type_event__SHIFT 0x14 +#define VGT_DEBUG_REG19__VGT_SE1SPI_vswave_rtr_q_MASK 0x4000000 +#define VGT_DEBUG_REG19__VGT_SE1SPI_vswave_rtr_q__SHIFT 0x1a +#define VGT_DEBUG_REG19__VGT_SE1SPI_vsvert_rtr_q_MASK 0x8000000 +#define VGT_DEBUG_REG19__VGT_SE1SPI_vsvert_rtr_q__SHIFT 0x1b +#define VGT_DEBUG_REG19__num_new_unique_rel_indx_MASK 0x30000000 +#define VGT_DEBUG_REG19__num_new_unique_rel_indx__SHIFT 0x1c +#define VGT_DEBUG_REG19__null_terminate_vtx_vector_MASK 0x40000000 +#define VGT_DEBUG_REG19__null_terminate_vtx_vector__SHIFT 0x1e +#define VGT_DEBUG_REG19__filter_event_MASK 0x80000000 +#define VGT_DEBUG_REG19__filter_event__SHIFT 0x1f +#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexindex_MASK 0xffff +#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexindex__SHIFT 0x0 +#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexcount_not_0_MASK 0x10000 +#define VGT_DEBUG_REG20__dbg_VGT_SPI_vsthread_sovertexcount_not_0__SHIFT 0x10 +#define VGT_DEBUG_REG20__SPARE17_MASK 0x20000 +#define VGT_DEBUG_REG20__SPARE17__SHIFT 0x11 +#define VGT_DEBUG_REG20__alloc_counter_q_MASK 0x3c0000 +#define VGT_DEBUG_REG20__alloc_counter_q__SHIFT 0x12 +#define VGT_DEBUG_REG20__curr_dealloc_distance_q_MASK 0x1fc00000 +#define VGT_DEBUG_REG20__curr_dealloc_distance_q__SHIFT 0x16 +#define VGT_DEBUG_REG20__new_allocate_q_MASK 0x20000000 +#define VGT_DEBUG_REG20__new_allocate_q__SHIFT 0x1d +#define VGT_DEBUG_REG20__curr_slot_in_vtx_vect_q_not_0_MASK 0x40000000 +#define VGT_DEBUG_REG20__curr_slot_in_vtx_vect_q_not_0__SHIFT 0x1e +#define VGT_DEBUG_REG20__int_vtx_counter_q_not_0_MASK 0x80000000 +#define VGT_DEBUG_REG20__int_vtx_counter_q_not_0__SHIFT 0x1f +#define VGT_DEBUG_REG21__out_indx_fifo_empty_MASK 0x1 +#define VGT_DEBUG_REG21__out_indx_fifo_empty__SHIFT 0x0 +#define VGT_DEBUG_REG21__indx_side_fifo_empty_MASK 0x2 +#define VGT_DEBUG_REG21__indx_side_fifo_empty__SHIFT 0x1 +#define VGT_DEBUG_REG21__pipe0_dr_MASK 0x4 +#define VGT_DEBUG_REG21__pipe0_dr__SHIFT 0x2 +#define VGT_DEBUG_REG21__pipe1_dr_MASK 0x8 +#define VGT_DEBUG_REG21__pipe1_dr__SHIFT 0x3 +#define VGT_DEBUG_REG21__pipe2_dr_MASK 0x10 +#define VGT_DEBUG_REG21__pipe2_dr__SHIFT 0x4 +#define VGT_DEBUG_REG21__vsthread_buff_empty_MASK 0x20 +#define VGT_DEBUG_REG21__vsthread_buff_empty__SHIFT 0x5 +#define VGT_DEBUG_REG21__out_indx_fifo_full_MASK 0x40 +#define VGT_DEBUG_REG21__out_indx_fifo_full__SHIFT 0x6 +#define VGT_DEBUG_REG21__indx_side_fifo_full_MASK 0x80 +#define VGT_DEBUG_REG21__indx_side_fifo_full__SHIFT 0x7 +#define VGT_DEBUG_REG21__pipe0_rtr_MASK 0x100 +#define VGT_DEBUG_REG21__pipe0_rtr__SHIFT 0x8 +#define VGT_DEBUG_REG21__pipe1_rtr_MASK 0x200 +#define VGT_DEBUG_REG21__pipe1_rtr__SHIFT 0x9 +#define VGT_DEBUG_REG21__pipe2_rtr_MASK 0x400 +#define VGT_DEBUG_REG21__pipe2_rtr__SHIFT 0xa +#define VGT_DEBUG_REG21__vsthread_buff_full_MASK 0x800 +#define VGT_DEBUG_REG21__vsthread_buff_full__SHIFT 0xb +#define VGT_DEBUG_REG21__interfaces_rtr_MASK 0x1000 +#define VGT_DEBUG_REG21__interfaces_rtr__SHIFT 0xc +#define VGT_DEBUG_REG21__indx_count_q_not_0_MASK 0x2000 +#define VGT_DEBUG_REG21__indx_count_q_not_0__SHIFT 0xd +#define VGT_DEBUG_REG21__wait_for_external_eopg_q_MASK 0x4000 +#define VGT_DEBUG_REG21__wait_for_external_eopg_q__SHIFT 0xe +#define VGT_DEBUG_REG21__full_state_p1_q_MASK 0x8000 +#define VGT_DEBUG_REG21__full_state_p1_q__SHIFT 0xf +#define VGT_DEBUG_REG21__indx_side_indx_valid_MASK 0x10000 +#define VGT_DEBUG_REG21__indx_side_indx_valid__SHIFT 0x10 +#define VGT_DEBUG_REG21__stateid_p0_q_MASK 0xe0000 +#define VGT_DEBUG_REG21__stateid_p0_q__SHIFT 0x11 +#define VGT_DEBUG_REG21__is_event_p0_q_MASK 0x100000 +#define VGT_DEBUG_REG21__is_event_p0_q__SHIFT 0x14 +#define VGT_DEBUG_REG21__lshs_dealloc_p1_MASK 0x200000 +#define VGT_DEBUG_REG21__lshs_dealloc_p1__SHIFT 0x15 +#define VGT_DEBUG_REG21__stream_id_r2_q_MASK 0x400000 +#define VGT_DEBUG_REG21__stream_id_r2_q__SHIFT 0x16 +#define VGT_DEBUG_REG21__vtx_vect_counter_q_not_0_MASK 0x800000 +#define VGT_DEBUG_REG21__vtx_vect_counter_q_not_0__SHIFT 0x17 +#define VGT_DEBUG_REG21__buff_full_p1_MASK 0x1000000 +#define VGT_DEBUG_REG21__buff_full_p1__SHIFT 0x18 +#define VGT_DEBUG_REG21__strmout_valid_p1_MASK 0x2000000 +#define VGT_DEBUG_REG21__strmout_valid_p1__SHIFT 0x19 +#define VGT_DEBUG_REG21__eotg_r2_q_MASK 0x4000000 +#define VGT_DEBUG_REG21__eotg_r2_q__SHIFT 0x1a +#define VGT_DEBUG_REG21__null_r2_q_MASK 0x8000000 +#define VGT_DEBUG_REG21__null_r2_q__SHIFT 0x1b +#define VGT_DEBUG_REG21__p0_dr_MASK 0x10000000 +#define VGT_DEBUG_REG21__p0_dr__SHIFT 0x1c +#define VGT_DEBUG_REG21__p0_rtr_MASK 0x20000000 +#define VGT_DEBUG_REG21__p0_rtr__SHIFT 0x1d +#define VGT_DEBUG_REG21__eopg_p0_q_MASK 0x40000000 +#define VGT_DEBUG_REG21__eopg_p0_q__SHIFT 0x1e +#define VGT_DEBUG_REG21__p0_nobp_MASK 0x80000000 +#define VGT_DEBUG_REG21__p0_nobp__SHIFT 0x1f +#define VGT_DEBUG_REG22__cm_state16_MASK 0x3 +#define VGT_DEBUG_REG22__cm_state16__SHIFT 0x0 +#define VGT_DEBUG_REG22__cm_state17_MASK 0xc +#define VGT_DEBUG_REG22__cm_state17__SHIFT 0x2 +#define VGT_DEBUG_REG22__cm_state18_MASK 0x30 +#define VGT_DEBUG_REG22__cm_state18__SHIFT 0x4 +#define VGT_DEBUG_REG22__cm_state19_MASK 0xc0 +#define VGT_DEBUG_REG22__cm_state19__SHIFT 0x6 +#define VGT_DEBUG_REG22__cm_state20_MASK 0x300 +#define VGT_DEBUG_REG22__cm_state20__SHIFT 0x8 +#define VGT_DEBUG_REG22__cm_state21_MASK 0xc00 +#define VGT_DEBUG_REG22__cm_state21__SHIFT 0xa +#define VGT_DEBUG_REG22__cm_state22_MASK 0x3000 +#define VGT_DEBUG_REG22__cm_state22__SHIFT 0xc +#define VGT_DEBUG_REG22__cm_state23_MASK 0xc000 +#define VGT_DEBUG_REG22__cm_state23__SHIFT 0xe +#define VGT_DEBUG_REG22__cm_state24_MASK 0x30000 +#define VGT_DEBUG_REG22__cm_state24__SHIFT 0x10 +#define VGT_DEBUG_REG22__cm_state25_MASK 0xc0000 +#define VGT_DEBUG_REG22__cm_state25__SHIFT 0x12 +#define VGT_DEBUG_REG22__cm_state26_MASK 0x300000 +#define VGT_DEBUG_REG22__cm_state26__SHIFT 0x14 +#define VGT_DEBUG_REG22__cm_state27_MASK 0xc00000 +#define VGT_DEBUG_REG22__cm_state27__SHIFT 0x16 +#define VGT_DEBUG_REG22__cm_state28_MASK 0x3000000 +#define VGT_DEBUG_REG22__cm_state28__SHIFT 0x18 +#define VGT_DEBUG_REG22__cm_state29_MASK 0xc000000 +#define VGT_DEBUG_REG22__cm_state29__SHIFT 0x1a +#define VGT_DEBUG_REG22__cm_state30_MASK 0x30000000 +#define VGT_DEBUG_REG22__cm_state30__SHIFT 0x1c +#define VGT_DEBUG_REG22__cm_state31_MASK 0xc0000000 +#define VGT_DEBUG_REG22__cm_state31__SHIFT 0x1e +#define VGT_DEBUG_REG23__frmt_busy_MASK 0x1 +#define VGT_DEBUG_REG23__frmt_busy__SHIFT 0x0 +#define VGT_DEBUG_REG23__rcm_frmt_vert_rtr_MASK 0x2 +#define VGT_DEBUG_REG23__rcm_frmt_vert_rtr__SHIFT 0x1 +#define VGT_DEBUG_REG23__rcm_frmt_prim_rtr_MASK 0x4 +#define VGT_DEBUG_REG23__rcm_frmt_prim_rtr__SHIFT 0x2 +#define VGT_DEBUG_REG23__prim_r3_rtr_MASK 0x8 +#define VGT_DEBUG_REG23__prim_r3_rtr__SHIFT 0x3 +#define VGT_DEBUG_REG23__prim_r2_rtr_MASK 0x10 +#define VGT_DEBUG_REG23__prim_r2_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG23__vert_r3_rtr_MASK 0x20 +#define VGT_DEBUG_REG23__vert_r3_rtr__SHIFT 0x5 +#define VGT_DEBUG_REG23__vert_r2_rtr_MASK 0x40 +#define VGT_DEBUG_REG23__vert_r2_rtr__SHIFT 0x6 +#define VGT_DEBUG_REG23__vert_r1_rtr_MASK 0x80 +#define VGT_DEBUG_REG23__vert_r1_rtr__SHIFT 0x7 +#define VGT_DEBUG_REG23__vert_r0_rtr_MASK 0x100 +#define VGT_DEBUG_REG23__vert_r0_rtr__SHIFT 0x8 +#define VGT_DEBUG_REG23__prim_fifo_empty_MASK 0x200 +#define VGT_DEBUG_REG23__prim_fifo_empty__SHIFT 0x9 +#define VGT_DEBUG_REG23__prim_fifo_full_MASK 0x400 +#define VGT_DEBUG_REG23__prim_fifo_full__SHIFT 0xa +#define VGT_DEBUG_REG23__vert_dr_r2_q_MASK 0x800 +#define VGT_DEBUG_REG23__vert_dr_r2_q__SHIFT 0xb +#define VGT_DEBUG_REG23__prim_dr_r2_q_MASK 0x1000 +#define VGT_DEBUG_REG23__prim_dr_r2_q__SHIFT 0xc +#define VGT_DEBUG_REG23__vert_dr_r1_q_MASK 0x2000 +#define VGT_DEBUG_REG23__vert_dr_r1_q__SHIFT 0xd +#define VGT_DEBUG_REG23__vert_dr_r0_q_MASK 0x4000 +#define VGT_DEBUG_REG23__vert_dr_r0_q__SHIFT 0xe +#define VGT_DEBUG_REG23__new_verts_r2_q_MASK 0x18000 +#define VGT_DEBUG_REG23__new_verts_r2_q__SHIFT 0xf +#define VGT_DEBUG_REG23__verts_sent_r2_q_MASK 0x1e0000 +#define VGT_DEBUG_REG23__verts_sent_r2_q__SHIFT 0x11 +#define VGT_DEBUG_REG23__prim_state_sel_r2_q_MASK 0xe00000 +#define VGT_DEBUG_REG23__prim_state_sel_r2_q__SHIFT 0x15 +#define VGT_DEBUG_REG23__SPARE_MASK 0xff000000 +#define VGT_DEBUG_REG23__SPARE__SHIFT 0x18 +#define VGT_DEBUG_REG24__avail_es_rb_space_r0_q_23_0_MASK 0xffffff +#define VGT_DEBUG_REG24__avail_es_rb_space_r0_q_23_0__SHIFT 0x0 +#define VGT_DEBUG_REG24__dependent_st_cut_mode_q_MASK 0x3000000 +#define VGT_DEBUG_REG24__dependent_st_cut_mode_q__SHIFT 0x18 +#define VGT_DEBUG_REG24__SPARE31_MASK 0xfc000000 +#define VGT_DEBUG_REG24__SPARE31__SHIFT 0x1a +#define VGT_DEBUG_REG25__avail_gs_rb_space_r0_q_25_0_MASK 0x3ffffff +#define VGT_DEBUG_REG25__avail_gs_rb_space_r0_q_25_0__SHIFT 0x0 +#define VGT_DEBUG_REG25__active_sm_r0_q_MASK 0x3c000000 +#define VGT_DEBUG_REG25__active_sm_r0_q__SHIFT 0x1a +#define VGT_DEBUG_REG25__add_gs_rb_space_r1_q_MASK 0x40000000 +#define VGT_DEBUG_REG25__add_gs_rb_space_r1_q__SHIFT 0x1e +#define VGT_DEBUG_REG25__add_gs_rb_space_r0_q_MASK 0x80000000 +#define VGT_DEBUG_REG25__add_gs_rb_space_r0_q__SHIFT 0x1f +#define VGT_DEBUG_REG26__cm_state0_MASK 0x3 +#define VGT_DEBUG_REG26__cm_state0__SHIFT 0x0 +#define VGT_DEBUG_REG26__cm_state1_MASK 0xc +#define VGT_DEBUG_REG26__cm_state1__SHIFT 0x2 +#define VGT_DEBUG_REG26__cm_state2_MASK 0x30 +#define VGT_DEBUG_REG26__cm_state2__SHIFT 0x4 +#define VGT_DEBUG_REG26__cm_state3_MASK 0xc0 +#define VGT_DEBUG_REG26__cm_state3__SHIFT 0x6 +#define VGT_DEBUG_REG26__cm_state4_MASK 0x300 +#define VGT_DEBUG_REG26__cm_state4__SHIFT 0x8 +#define VGT_DEBUG_REG26__cm_state5_MASK 0xc00 +#define VGT_DEBUG_REG26__cm_state5__SHIFT 0xa +#define VGT_DEBUG_REG26__cm_state6_MASK 0x3000 +#define VGT_DEBUG_REG26__cm_state6__SHIFT 0xc +#define VGT_DEBUG_REG26__cm_state7_MASK 0xc000 +#define VGT_DEBUG_REG26__cm_state7__SHIFT 0xe +#define VGT_DEBUG_REG26__cm_state8_MASK 0x30000 +#define VGT_DEBUG_REG26__cm_state8__SHIFT 0x10 +#define VGT_DEBUG_REG26__cm_state9_MASK 0xc0000 +#define VGT_DEBUG_REG26__cm_state9__SHIFT 0x12 +#define VGT_DEBUG_REG26__cm_state10_MASK 0x300000 +#define VGT_DEBUG_REG26__cm_state10__SHIFT 0x14 +#define VGT_DEBUG_REG26__cm_state11_MASK 0xc00000 +#define VGT_DEBUG_REG26__cm_state11__SHIFT 0x16 +#define VGT_DEBUG_REG26__cm_state12_MASK 0x3000000 +#define VGT_DEBUG_REG26__cm_state12__SHIFT 0x18 +#define VGT_DEBUG_REG26__cm_state13_MASK 0xc000000 +#define VGT_DEBUG_REG26__cm_state13__SHIFT 0x1a +#define VGT_DEBUG_REG26__cm_state14_MASK 0x30000000 +#define VGT_DEBUG_REG26__cm_state14__SHIFT 0x1c +#define VGT_DEBUG_REG26__cm_state15_MASK 0xc0000000 +#define VGT_DEBUG_REG26__cm_state15__SHIFT 0x1e +#define VGT_DEBUG_REG27__pipe0_dr_MASK 0x1 +#define VGT_DEBUG_REG27__pipe0_dr__SHIFT 0x0 +#define VGT_DEBUG_REG27__gsc0_dr_MASK 0x2 +#define VGT_DEBUG_REG27__gsc0_dr__SHIFT 0x1 +#define VGT_DEBUG_REG27__pipe1_dr_MASK 0x4 +#define VGT_DEBUG_REG27__pipe1_dr__SHIFT 0x2 +#define VGT_DEBUG_REG27__tm_pt_event_rtr_MASK 0x8 +#define VGT_DEBUG_REG27__tm_pt_event_rtr__SHIFT 0x3 +#define VGT_DEBUG_REG27__pipe0_rtr_MASK 0x10 +#define VGT_DEBUG_REG27__pipe0_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG27__gsc0_rtr_MASK 0x20 +#define VGT_DEBUG_REG27__gsc0_rtr__SHIFT 0x5 +#define VGT_DEBUG_REG27__pipe1_rtr_MASK 0x40 +#define VGT_DEBUG_REG27__pipe1_rtr__SHIFT 0x6 +#define VGT_DEBUG_REG27__last_indx_of_prim_p1_q_MASK 0x80 +#define VGT_DEBUG_REG27__last_indx_of_prim_p1_q__SHIFT 0x7 +#define VGT_DEBUG_REG27__indices_to_send_p0_q_MASK 0x300 +#define VGT_DEBUG_REG27__indices_to_send_p0_q__SHIFT 0x8 +#define VGT_DEBUG_REG27__event_flag_p1_q_MASK 0x400 +#define VGT_DEBUG_REG27__event_flag_p1_q__SHIFT 0xa +#define VGT_DEBUG_REG27__eop_p1_q_MASK 0x800 +#define VGT_DEBUG_REG27__eop_p1_q__SHIFT 0xb +#define VGT_DEBUG_REG27__gs_out_prim_type_p0_q_MASK 0x3000 +#define VGT_DEBUG_REG27__gs_out_prim_type_p0_q__SHIFT 0xc +#define VGT_DEBUG_REG27__gsc_null_primitive_p0_q_MASK 0x4000 +#define VGT_DEBUG_REG27__gsc_null_primitive_p0_q__SHIFT 0xe +#define VGT_DEBUG_REG27__gsc_eop_p0_q_MASK 0x8000 +#define VGT_DEBUG_REG27__gsc_eop_p0_q__SHIFT 0xf +#define VGT_DEBUG_REG27__gsc_2cycle_output_MASK 0x10000 +#define VGT_DEBUG_REG27__gsc_2cycle_output__SHIFT 0x10 +#define VGT_DEBUG_REG27__gsc_2nd_cycle_p0_q_MASK 0x20000 +#define VGT_DEBUG_REG27__gsc_2nd_cycle_p0_q__SHIFT 0x11 +#define VGT_DEBUG_REG27__last_indx_of_vsprim_MASK 0x40000 +#define VGT_DEBUG_REG27__last_indx_of_vsprim__SHIFT 0x12 +#define VGT_DEBUG_REG27__first_vsprim_of_gsprim_p0_q_MASK 0x80000 +#define VGT_DEBUG_REG27__first_vsprim_of_gsprim_p0_q__SHIFT 0x13 +#define VGT_DEBUG_REG27__gsc_indx_count_p0_q_MASK 0x7ff00000 +#define VGT_DEBUG_REG27__gsc_indx_count_p0_q__SHIFT 0x14 +#define VGT_DEBUG_REG27__last_vsprim_of_gsprim_MASK 0x80000000 +#define VGT_DEBUG_REG27__last_vsprim_of_gsprim__SHIFT 0x1f +#define VGT_DEBUG_REG28__con_state_q_MASK 0xf +#define VGT_DEBUG_REG28__con_state_q__SHIFT 0x0 +#define VGT_DEBUG_REG28__second_cycle_q_MASK 0x10 +#define VGT_DEBUG_REG28__second_cycle_q__SHIFT 0x4 +#define VGT_DEBUG_REG28__process_tri_middle_p0_q_MASK 0x20 +#define VGT_DEBUG_REG28__process_tri_middle_p0_q__SHIFT 0x5 +#define VGT_DEBUG_REG28__process_tri_1st_2nd_half_p0_q_MASK 0x40 +#define VGT_DEBUG_REG28__process_tri_1st_2nd_half_p0_q__SHIFT 0x6 +#define VGT_DEBUG_REG28__process_tri_center_poly_p0_q_MASK 0x80 +#define VGT_DEBUG_REG28__process_tri_center_poly_p0_q__SHIFT 0x7 +#define VGT_DEBUG_REG28__pipe0_patch_dr_MASK 0x100 +#define VGT_DEBUG_REG28__pipe0_patch_dr__SHIFT 0x8 +#define VGT_DEBUG_REG28__pipe0_edge_dr_MASK 0x200 +#define VGT_DEBUG_REG28__pipe0_edge_dr__SHIFT 0x9 +#define VGT_DEBUG_REG28__pipe1_dr_MASK 0x400 +#define VGT_DEBUG_REG28__pipe1_dr__SHIFT 0xa +#define VGT_DEBUG_REG28__pipe0_patch_rtr_MASK 0x800 +#define VGT_DEBUG_REG28__pipe0_patch_rtr__SHIFT 0xb +#define VGT_DEBUG_REG28__pipe0_edge_rtr_MASK 0x1000 +#define VGT_DEBUG_REG28__pipe0_edge_rtr__SHIFT 0xc +#define VGT_DEBUG_REG28__pipe1_rtr_MASK 0x2000 +#define VGT_DEBUG_REG28__pipe1_rtr__SHIFT 0xd +#define VGT_DEBUG_REG28__outer_parity_p0_q_MASK 0x4000 +#define VGT_DEBUG_REG28__outer_parity_p0_q__SHIFT 0xe +#define VGT_DEBUG_REG28__parallel_parity_p0_q_MASK 0x8000 +#define VGT_DEBUG_REG28__parallel_parity_p0_q__SHIFT 0xf +#define VGT_DEBUG_REG28__first_ring_of_patch_p0_q_MASK 0x10000 +#define VGT_DEBUG_REG28__first_ring_of_patch_p0_q__SHIFT 0x10 +#define VGT_DEBUG_REG28__last_ring_of_patch_p0_q_MASK 0x20000 +#define VGT_DEBUG_REG28__last_ring_of_patch_p0_q__SHIFT 0x11 +#define VGT_DEBUG_REG28__last_edge_of_outer_ring_p0_q_MASK 0x40000 +#define VGT_DEBUG_REG28__last_edge_of_outer_ring_p0_q__SHIFT 0x12 +#define VGT_DEBUG_REG28__last_point_of_outer_ring_p1_MASK 0x80000 +#define VGT_DEBUG_REG28__last_point_of_outer_ring_p1__SHIFT 0x13 +#define VGT_DEBUG_REG28__last_point_of_inner_ring_p1_MASK 0x100000 +#define VGT_DEBUG_REG28__last_point_of_inner_ring_p1__SHIFT 0x14 +#define VGT_DEBUG_REG28__outer_edge_tf_eq_one_p0_q_MASK 0x200000 +#define VGT_DEBUG_REG28__outer_edge_tf_eq_one_p0_q__SHIFT 0x15 +#define VGT_DEBUG_REG28__advance_outer_point_p1_MASK 0x400000 +#define VGT_DEBUG_REG28__advance_outer_point_p1__SHIFT 0x16 +#define VGT_DEBUG_REG28__advance_inner_point_p1_MASK 0x800000 +#define VGT_DEBUG_REG28__advance_inner_point_p1__SHIFT 0x17 +#define VGT_DEBUG_REG28__next_ring_is_rect_p0_q_MASK 0x1000000 +#define VGT_DEBUG_REG28__next_ring_is_rect_p0_q__SHIFT 0x18 +#define VGT_DEBUG_REG28__pipe1_outer1_rtr_MASK 0x2000000 +#define VGT_DEBUG_REG28__pipe1_outer1_rtr__SHIFT 0x19 +#define VGT_DEBUG_REG28__pipe1_outer2_rtr_MASK 0x4000000 +#define VGT_DEBUG_REG28__pipe1_outer2_rtr__SHIFT 0x1a +#define VGT_DEBUG_REG28__pipe1_inner1_rtr_MASK 0x8000000 +#define VGT_DEBUG_REG28__pipe1_inner1_rtr__SHIFT 0x1b +#define VGT_DEBUG_REG28__pipe1_inner2_rtr_MASK 0x10000000 +#define VGT_DEBUG_REG28__pipe1_inner2_rtr__SHIFT 0x1c +#define VGT_DEBUG_REG28__pipe1_patch_rtr_MASK 0x20000000 +#define VGT_DEBUG_REG28__pipe1_patch_rtr__SHIFT 0x1d +#define VGT_DEBUG_REG28__pipe1_edge_rtr_MASK 0x40000000 +#define VGT_DEBUG_REG28__pipe1_edge_rtr__SHIFT 0x1e +#define VGT_DEBUG_REG28__use_stored_inner_q_ring2_MASK 0x80000000 +#define VGT_DEBUG_REG28__use_stored_inner_q_ring2__SHIFT 0x1f +#define VGT_DEBUG_REG29__con_state_q_MASK 0xf +#define VGT_DEBUG_REG29__con_state_q__SHIFT 0x0 +#define VGT_DEBUG_REG29__second_cycle_q_MASK 0x10 +#define VGT_DEBUG_REG29__second_cycle_q__SHIFT 0x4 +#define VGT_DEBUG_REG29__process_tri_middle_p0_q_MASK 0x20 +#define VGT_DEBUG_REG29__process_tri_middle_p0_q__SHIFT 0x5 +#define VGT_DEBUG_REG29__process_tri_1st_2nd_half_p0_q_MASK 0x40 +#define VGT_DEBUG_REG29__process_tri_1st_2nd_half_p0_q__SHIFT 0x6 +#define VGT_DEBUG_REG29__process_tri_center_poly_p0_q_MASK 0x80 +#define VGT_DEBUG_REG29__process_tri_center_poly_p0_q__SHIFT 0x7 +#define VGT_DEBUG_REG29__pipe0_patch_dr_MASK 0x100 +#define VGT_DEBUG_REG29__pipe0_patch_dr__SHIFT 0x8 +#define VGT_DEBUG_REG29__pipe0_edge_dr_MASK 0x200 +#define VGT_DEBUG_REG29__pipe0_edge_dr__SHIFT 0x9 +#define VGT_DEBUG_REG29__pipe1_dr_MASK 0x400 +#define VGT_DEBUG_REG29__pipe1_dr__SHIFT 0xa +#define VGT_DEBUG_REG29__pipe0_patch_rtr_MASK 0x800 +#define VGT_DEBUG_REG29__pipe0_patch_rtr__SHIFT 0xb +#define VGT_DEBUG_REG29__pipe0_edge_rtr_MASK 0x1000 +#define VGT_DEBUG_REG29__pipe0_edge_rtr__SHIFT 0xc +#define VGT_DEBUG_REG29__pipe1_rtr_MASK 0x2000 +#define VGT_DEBUG_REG29__pipe1_rtr__SHIFT 0xd +#define VGT_DEBUG_REG29__outer_parity_p0_q_MASK 0x4000 +#define VGT_DEBUG_REG29__outer_parity_p0_q__SHIFT 0xe +#define VGT_DEBUG_REG29__parallel_parity_p0_q_MASK 0x8000 +#define VGT_DEBUG_REG29__parallel_parity_p0_q__SHIFT 0xf +#define VGT_DEBUG_REG29__first_ring_of_patch_p0_q_MASK 0x10000 +#define VGT_DEBUG_REG29__first_ring_of_patch_p0_q__SHIFT 0x10 +#define VGT_DEBUG_REG29__last_ring_of_patch_p0_q_MASK 0x20000 +#define VGT_DEBUG_REG29__last_ring_of_patch_p0_q__SHIFT 0x11 +#define VGT_DEBUG_REG29__last_edge_of_outer_ring_p0_q_MASK 0x40000 +#define VGT_DEBUG_REG29__last_edge_of_outer_ring_p0_q__SHIFT 0x12 +#define VGT_DEBUG_REG29__last_point_of_outer_ring_p1_MASK 0x80000 +#define VGT_DEBUG_REG29__last_point_of_outer_ring_p1__SHIFT 0x13 +#define VGT_DEBUG_REG29__last_point_of_inner_ring_p1_MASK 0x100000 +#define VGT_DEBUG_REG29__last_point_of_inner_ring_p1__SHIFT 0x14 +#define VGT_DEBUG_REG29__outer_edge_tf_eq_one_p0_q_MASK 0x200000 +#define VGT_DEBUG_REG29__outer_edge_tf_eq_one_p0_q__SHIFT 0x15 +#define VGT_DEBUG_REG29__advance_outer_point_p1_MASK 0x400000 +#define VGT_DEBUG_REG29__advance_outer_point_p1__SHIFT 0x16 +#define VGT_DEBUG_REG29__advance_inner_point_p1_MASK 0x800000 +#define VGT_DEBUG_REG29__advance_inner_point_p1__SHIFT 0x17 +#define VGT_DEBUG_REG29__next_ring_is_rect_p0_q_MASK 0x1000000 +#define VGT_DEBUG_REG29__next_ring_is_rect_p0_q__SHIFT 0x18 +#define VGT_DEBUG_REG29__pipe1_outer1_rtr_MASK 0x2000000 +#define VGT_DEBUG_REG29__pipe1_outer1_rtr__SHIFT 0x19 +#define VGT_DEBUG_REG29__pipe1_outer2_rtr_MASK 0x4000000 +#define VGT_DEBUG_REG29__pipe1_outer2_rtr__SHIFT 0x1a +#define VGT_DEBUG_REG29__pipe1_inner1_rtr_MASK 0x8000000 +#define VGT_DEBUG_REG29__pipe1_inner1_rtr__SHIFT 0x1b +#define VGT_DEBUG_REG29__pipe1_inner2_rtr_MASK 0x10000000 +#define VGT_DEBUG_REG29__pipe1_inner2_rtr__SHIFT 0x1c +#define VGT_DEBUG_REG29__pipe1_patch_rtr_MASK 0x20000000 +#define VGT_DEBUG_REG29__pipe1_patch_rtr__SHIFT 0x1d +#define VGT_DEBUG_REG29__pipe1_edge_rtr_MASK 0x40000000 +#define VGT_DEBUG_REG29__pipe1_edge_rtr__SHIFT 0x1e +#define VGT_DEBUG_REG29__use_stored_inner_q_ring3_MASK 0x80000000 +#define VGT_DEBUG_REG29__use_stored_inner_q_ring3__SHIFT 0x1f +#define VGT_DEBUG_REG30__pipe0_dr_MASK 0x1 +#define VGT_DEBUG_REG30__pipe0_dr__SHIFT 0x0 +#define VGT_DEBUG_REG30__pipe0_tf_dr_MASK 0x2 +#define VGT_DEBUG_REG30__pipe0_tf_dr__SHIFT 0x1 +#define VGT_DEBUG_REG30__pipe2_dr_MASK 0x4 +#define VGT_DEBUG_REG30__pipe2_dr__SHIFT 0x2 +#define VGT_DEBUG_REG30__event_or_null_p0_q_MASK 0x8 +#define VGT_DEBUG_REG30__event_or_null_p0_q__SHIFT 0x3 +#define VGT_DEBUG_REG30__pipe0_rtr_MASK 0x10 +#define VGT_DEBUG_REG30__pipe0_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG30__pipe1_rtr_MASK 0x20 +#define VGT_DEBUG_REG30__pipe1_rtr__SHIFT 0x5 +#define VGT_DEBUG_REG30__pipe1_tf_rtr_MASK 0x40 +#define VGT_DEBUG_REG30__pipe1_tf_rtr__SHIFT 0x6 +#define VGT_DEBUG_REG30__pipe2_rtr_MASK 0x80 +#define VGT_DEBUG_REG30__pipe2_rtr__SHIFT 0x7 +#define VGT_DEBUG_REG30__ttp_patch_fifo_full_MASK 0x100 +#define VGT_DEBUG_REG30__ttp_patch_fifo_full__SHIFT 0x8 +#define VGT_DEBUG_REG30__ttp_patch_fifo_empty_MASK 0x200 +#define VGT_DEBUG_REG30__ttp_patch_fifo_empty__SHIFT 0x9 +#define VGT_DEBUG_REG30__ttp_tf0_fifo_empty_MASK 0x400 +#define VGT_DEBUG_REG30__ttp_tf0_fifo_empty__SHIFT 0xa +#define VGT_DEBUG_REG30__ttp_tf1_fifo_empty_MASK 0x800 +#define VGT_DEBUG_REG30__ttp_tf1_fifo_empty__SHIFT 0xb +#define VGT_DEBUG_REG30__ttp_tf2_fifo_empty_MASK 0x1000 +#define VGT_DEBUG_REG30__ttp_tf2_fifo_empty__SHIFT 0xc +#define VGT_DEBUG_REG30__ttp_tf3_fifo_empty_MASK 0x2000 +#define VGT_DEBUG_REG30__ttp_tf3_fifo_empty__SHIFT 0xd +#define VGT_DEBUG_REG30__ttp_tf4_fifo_empty_MASK 0x4000 +#define VGT_DEBUG_REG30__ttp_tf4_fifo_empty__SHIFT 0xe +#define VGT_DEBUG_REG30__ttp_tf5_fifo_empty_MASK 0x8000 +#define VGT_DEBUG_REG30__ttp_tf5_fifo_empty__SHIFT 0xf +#define VGT_DEBUG_REG30__tf_fetch_state_q_MASK 0x70000 +#define VGT_DEBUG_REG30__tf_fetch_state_q__SHIFT 0x10 +#define VGT_DEBUG_REG30__last_tf_of_tg_MASK 0x80000 +#define VGT_DEBUG_REG30__last_tf_of_tg__SHIFT 0x13 +#define VGT_DEBUG_REG30__tf_pointer_p0_q_MASK 0xf00000 +#define VGT_DEBUG_REG30__tf_pointer_p0_q__SHIFT 0x14 +#define VGT_DEBUG_REG30__dynamic_hs_p0_q_MASK 0x1000000 +#define VGT_DEBUG_REG30__dynamic_hs_p0_q__SHIFT 0x18 +#define VGT_DEBUG_REG30__first_fetch_of_tg_p0_q_MASK 0x2000000 +#define VGT_DEBUG_REG30__first_fetch_of_tg_p0_q__SHIFT 0x19 +#define VGT_DEBUG_REG30__first_data_ret_of_req_p0_q_MASK 0x4000000 +#define VGT_DEBUG_REG30__first_data_ret_of_req_p0_q__SHIFT 0x1a +#define VGT_DEBUG_REG30__first_data_chunk_invalid_p0_q_MASK 0x8000000 +#define VGT_DEBUG_REG30__first_data_chunk_invalid_p0_q__SHIFT 0x1b +#define VGT_DEBUG_REG30__tf_xfer_count_p2_q_MASK 0x30000000 +#define VGT_DEBUG_REG30__tf_xfer_count_p2_q__SHIFT 0x1c +#define VGT_DEBUG_REG30__pipe4_dr_MASK 0x40000000 +#define VGT_DEBUG_REG30__pipe4_dr__SHIFT 0x1e +#define VGT_DEBUG_REG30__pipe4_rtr_MASK 0x80000000 +#define VGT_DEBUG_REG30__pipe4_rtr__SHIFT 0x1f +#define VGT_DEBUG_REG31__pipe0_dr_MASK 0x1 +#define VGT_DEBUG_REG31__pipe0_dr__SHIFT 0x0 +#define VGT_DEBUG_REG31__pipe0_rtr_MASK 0x2 +#define VGT_DEBUG_REG31__pipe0_rtr__SHIFT 0x1 +#define VGT_DEBUG_REG31__pipe1_outer_dr_MASK 0x4 +#define VGT_DEBUG_REG31__pipe1_outer_dr__SHIFT 0x2 +#define VGT_DEBUG_REG31__pipe1_inner_dr_MASK 0x8 +#define VGT_DEBUG_REG31__pipe1_inner_dr__SHIFT 0x3 +#define VGT_DEBUG_REG31__pipe2_outer_dr_MASK 0x10 +#define VGT_DEBUG_REG31__pipe2_outer_dr__SHIFT 0x4 +#define VGT_DEBUG_REG31__pipe2_inner_dr_MASK 0x20 +#define VGT_DEBUG_REG31__pipe2_inner_dr__SHIFT 0x5 +#define VGT_DEBUG_REG31__pipe3_outer_dr_MASK 0x40 +#define VGT_DEBUG_REG31__pipe3_outer_dr__SHIFT 0x6 +#define VGT_DEBUG_REG31__pipe3_inner_dr_MASK 0x80 +#define VGT_DEBUG_REG31__pipe3_inner_dr__SHIFT 0x7 +#define VGT_DEBUG_REG31__pipe4_outer_dr_MASK 0x100 +#define VGT_DEBUG_REG31__pipe4_outer_dr__SHIFT 0x8 +#define VGT_DEBUG_REG31__pipe4_inner_dr_MASK 0x200 +#define VGT_DEBUG_REG31__pipe4_inner_dr__SHIFT 0x9 +#define VGT_DEBUG_REG31__pipe5_outer_dr_MASK 0x400 +#define VGT_DEBUG_REG31__pipe5_outer_dr__SHIFT 0xa +#define VGT_DEBUG_REG31__pipe5_inner_dr_MASK 0x800 +#define VGT_DEBUG_REG31__pipe5_inner_dr__SHIFT 0xb +#define VGT_DEBUG_REG31__pipe2_outer_rtr_MASK 0x1000 +#define VGT_DEBUG_REG31__pipe2_outer_rtr__SHIFT 0xc +#define VGT_DEBUG_REG31__pipe2_inner_rtr_MASK 0x2000 +#define VGT_DEBUG_REG31__pipe2_inner_rtr__SHIFT 0xd +#define VGT_DEBUG_REG31__pipe3_outer_rtr_MASK 0x4000 +#define VGT_DEBUG_REG31__pipe3_outer_rtr__SHIFT 0xe +#define VGT_DEBUG_REG31__pipe3_inner_rtr_MASK 0x8000 +#define VGT_DEBUG_REG31__pipe3_inner_rtr__SHIFT 0xf +#define VGT_DEBUG_REG31__pipe4_outer_rtr_MASK 0x10000 +#define VGT_DEBUG_REG31__pipe4_outer_rtr__SHIFT 0x10 +#define VGT_DEBUG_REG31__pipe4_inner_rtr_MASK 0x20000 +#define VGT_DEBUG_REG31__pipe4_inner_rtr__SHIFT 0x11 +#define VGT_DEBUG_REG31__pipe5_outer_rtr_MASK 0x40000 +#define VGT_DEBUG_REG31__pipe5_outer_rtr__SHIFT 0x12 +#define VGT_DEBUG_REG31__pipe5_inner_rtr_MASK 0x80000 +#define VGT_DEBUG_REG31__pipe5_inner_rtr__SHIFT 0x13 +#define VGT_DEBUG_REG31__pg_con_outer_point1_rts_MASK 0x100000 +#define VGT_DEBUG_REG31__pg_con_outer_point1_rts__SHIFT 0x14 +#define VGT_DEBUG_REG31__pg_con_outer_point2_rts_MASK 0x200000 +#define VGT_DEBUG_REG31__pg_con_outer_point2_rts__SHIFT 0x15 +#define VGT_DEBUG_REG31__pg_con_inner_point1_rts_MASK 0x400000 +#define VGT_DEBUG_REG31__pg_con_inner_point1_rts__SHIFT 0x16 +#define VGT_DEBUG_REG31__pg_con_inner_point2_rts_MASK 0x800000 +#define VGT_DEBUG_REG31__pg_con_inner_point2_rts__SHIFT 0x17 +#define VGT_DEBUG_REG31__pg_patch_fifo_empty_MASK 0x1000000 +#define VGT_DEBUG_REG31__pg_patch_fifo_empty__SHIFT 0x18 +#define VGT_DEBUG_REG31__pg_edge_fifo_empty_MASK 0x2000000 +#define VGT_DEBUG_REG31__pg_edge_fifo_empty__SHIFT 0x19 +#define VGT_DEBUG_REG31__pg_inner3_perp_fifo_empty_MASK 0x4000000 +#define VGT_DEBUG_REG31__pg_inner3_perp_fifo_empty__SHIFT 0x1a +#define VGT_DEBUG_REG31__pg_patch_fifo_full_MASK 0x8000000 +#define VGT_DEBUG_REG31__pg_patch_fifo_full__SHIFT 0x1b +#define VGT_DEBUG_REG31__pg_edge_fifo_full_MASK 0x10000000 +#define VGT_DEBUG_REG31__pg_edge_fifo_full__SHIFT 0x1c +#define VGT_DEBUG_REG31__pg_inner_perp_fifo_full_MASK 0x20000000 +#define VGT_DEBUG_REG31__pg_inner_perp_fifo_full__SHIFT 0x1d +#define VGT_DEBUG_REG31__outer_ring_done_q_MASK 0x40000000 +#define VGT_DEBUG_REG31__outer_ring_done_q__SHIFT 0x1e +#define VGT_DEBUG_REG31__inner_ring_done_q_MASK 0x80000000 +#define VGT_DEBUG_REG31__inner_ring_done_q__SHIFT 0x1f +#define VGT_DEBUG_REG32__first_ring_of_patch_MASK 0x1 +#define VGT_DEBUG_REG32__first_ring_of_patch__SHIFT 0x0 +#define VGT_DEBUG_REG32__last_ring_of_patch_MASK 0x2 +#define VGT_DEBUG_REG32__last_ring_of_patch__SHIFT 0x1 +#define VGT_DEBUG_REG32__last_edge_of_outer_ring_MASK 0x4 +#define VGT_DEBUG_REG32__last_edge_of_outer_ring__SHIFT 0x2 +#define VGT_DEBUG_REG32__last_point_of_outer_edge_MASK 0x8 +#define VGT_DEBUG_REG32__last_point_of_outer_edge__SHIFT 0x3 +#define VGT_DEBUG_REG32__last_edge_of_inner_ring_MASK 0x10 +#define VGT_DEBUG_REG32__last_edge_of_inner_ring__SHIFT 0x4 +#define VGT_DEBUG_REG32__last_point_of_inner_edge_MASK 0x20 +#define VGT_DEBUG_REG32__last_point_of_inner_edge__SHIFT 0x5 +#define VGT_DEBUG_REG32__last_patch_of_tg_p0_q_MASK 0x40 +#define VGT_DEBUG_REG32__last_patch_of_tg_p0_q__SHIFT 0x6 +#define VGT_DEBUG_REG32__event_null_special_p0_q_MASK 0x80 +#define VGT_DEBUG_REG32__event_null_special_p0_q__SHIFT 0x7 +#define VGT_DEBUG_REG32__event_flag_p5_q_MASK 0x100 +#define VGT_DEBUG_REG32__event_flag_p5_q__SHIFT 0x8 +#define VGT_DEBUG_REG32__first_point_of_patch_p5_q_MASK 0x200 +#define VGT_DEBUG_REG32__first_point_of_patch_p5_q__SHIFT 0x9 +#define VGT_DEBUG_REG32__first_point_of_edge_p5_q_MASK 0x400 +#define VGT_DEBUG_REG32__first_point_of_edge_p5_q__SHIFT 0xa +#define VGT_DEBUG_REG32__last_patch_of_tg_p5_q_MASK 0x800 +#define VGT_DEBUG_REG32__last_patch_of_tg_p5_q__SHIFT 0xb +#define VGT_DEBUG_REG32__tess_topology_p5_q_MASK 0x3000 +#define VGT_DEBUG_REG32__tess_topology_p5_q__SHIFT 0xc +#define VGT_DEBUG_REG32__pipe5_inner3_rtr_MASK 0x4000 +#define VGT_DEBUG_REG32__pipe5_inner3_rtr__SHIFT 0xe +#define VGT_DEBUG_REG32__pipe5_inner2_rtr_MASK 0x8000 +#define VGT_DEBUG_REG32__pipe5_inner2_rtr__SHIFT 0xf +#define VGT_DEBUG_REG32__pg_edge_fifo3_full_MASK 0x10000 +#define VGT_DEBUG_REG32__pg_edge_fifo3_full__SHIFT 0x10 +#define VGT_DEBUG_REG32__pg_edge_fifo2_full_MASK 0x20000 +#define VGT_DEBUG_REG32__pg_edge_fifo2_full__SHIFT 0x11 +#define VGT_DEBUG_REG32__pg_inner3_point_fifo_full_MASK 0x40000 +#define VGT_DEBUG_REG32__pg_inner3_point_fifo_full__SHIFT 0x12 +#define VGT_DEBUG_REG32__pg_outer3_point_fifo_full_MASK 0x80000 +#define VGT_DEBUG_REG32__pg_outer3_point_fifo_full__SHIFT 0x13 +#define VGT_DEBUG_REG32__pg_inner2_point_fifo_full_MASK 0x100000 +#define VGT_DEBUG_REG32__pg_inner2_point_fifo_full__SHIFT 0x14 +#define VGT_DEBUG_REG32__pg_outer2_point_fifo_full_MASK 0x200000 +#define VGT_DEBUG_REG32__pg_outer2_point_fifo_full__SHIFT 0x15 +#define VGT_DEBUG_REG32__pg_inner_point_fifo_full_MASK 0x400000 +#define VGT_DEBUG_REG32__pg_inner_point_fifo_full__SHIFT 0x16 +#define VGT_DEBUG_REG32__pg_outer_point_fifo_full_MASK 0x800000 +#define VGT_DEBUG_REG32__pg_outer_point_fifo_full__SHIFT 0x17 +#define VGT_DEBUG_REG32__inner2_fifos_rtr_MASK 0x1000000 +#define VGT_DEBUG_REG32__inner2_fifos_rtr__SHIFT 0x18 +#define VGT_DEBUG_REG32__inner_fifos_rtr_MASK 0x2000000 +#define VGT_DEBUG_REG32__inner_fifos_rtr__SHIFT 0x19 +#define VGT_DEBUG_REG32__outer_fifos_rtr_MASK 0x4000000 +#define VGT_DEBUG_REG32__outer_fifos_rtr__SHIFT 0x1a +#define VGT_DEBUG_REG32__fifos_rtr_MASK 0x8000000 +#define VGT_DEBUG_REG32__fifos_rtr__SHIFT 0x1b +#define VGT_DEBUG_REG32__SPARE_MASK 0xf0000000 +#define VGT_DEBUG_REG32__SPARE__SHIFT 0x1c +#define VGT_DEBUG_REG33__pipe0_patch_dr_MASK 0x1 +#define VGT_DEBUG_REG33__pipe0_patch_dr__SHIFT 0x0 +#define VGT_DEBUG_REG33__ring3_pipe1_dr_MASK 0x2 +#define VGT_DEBUG_REG33__ring3_pipe1_dr__SHIFT 0x1 +#define VGT_DEBUG_REG33__pipe1_dr_MASK 0x4 +#define VGT_DEBUG_REG33__pipe1_dr__SHIFT 0x2 +#define VGT_DEBUG_REG33__pipe2_dr_MASK 0x8 +#define VGT_DEBUG_REG33__pipe2_dr__SHIFT 0x3 +#define VGT_DEBUG_REG33__pipe0_patch_rtr_MASK 0x10 +#define VGT_DEBUG_REG33__pipe0_patch_rtr__SHIFT 0x4 +#define VGT_DEBUG_REG33__ring2_pipe1_dr_MASK 0x20 +#define VGT_DEBUG_REG33__ring2_pipe1_dr__SHIFT 0x5 +#define VGT_DEBUG_REG33__ring1_pipe1_dr_MASK 0x40 +#define VGT_DEBUG_REG33__ring1_pipe1_dr__SHIFT 0x6 +#define VGT_DEBUG_REG33__pipe2_rtr_MASK 0x80 +#define VGT_DEBUG_REG33__pipe2_rtr__SHIFT 0x7 +#define VGT_DEBUG_REG33__pipe3_dr_MASK 0x100 +#define VGT_DEBUG_REG33__pipe3_dr__SHIFT 0x8 +#define VGT_DEBUG_REG33__pipe3_rtr_MASK 0x200 +#define VGT_DEBUG_REG33__pipe3_rtr__SHIFT 0x9 +#define VGT_DEBUG_REG33__ring2_in_sync_q_MASK 0x400 +#define VGT_DEBUG_REG33__ring2_in_sync_q__SHIFT 0xa +#define VGT_DEBUG_REG33__ring1_in_sync_q_MASK 0x800 +#define VGT_DEBUG_REG33__ring1_in_sync_q__SHIFT 0xb +#define VGT_DEBUG_REG33__pipe1_patch_rtr_MASK 0x1000 +#define VGT_DEBUG_REG33__pipe1_patch_rtr__SHIFT 0xc +#define VGT_DEBUG_REG33__ring3_in_sync_q_MASK 0x2000 +#define VGT_DEBUG_REG33__ring3_in_sync_q__SHIFT 0xd +#define VGT_DEBUG_REG33__tm_te11_event_rtr_MASK 0x4000 +#define VGT_DEBUG_REG33__tm_te11_event_rtr__SHIFT 0xe +#define VGT_DEBUG_REG33__first_prim_of_patch_q_MASK 0x8000 +#define VGT_DEBUG_REG33__first_prim_of_patch_q__SHIFT 0xf +#define VGT_DEBUG_REG33__con_prim_fifo_full_MASK 0x10000 +#define VGT_DEBUG_REG33__con_prim_fifo_full__SHIFT 0x10 +#define VGT_DEBUG_REG33__con_vert_fifo_full_MASK 0x20000 +#define VGT_DEBUG_REG33__con_vert_fifo_full__SHIFT 0x11 +#define VGT_DEBUG_REG33__con_prim_fifo_empty_MASK 0x40000 +#define VGT_DEBUG_REG33__con_prim_fifo_empty__SHIFT 0x12 +#define VGT_DEBUG_REG33__con_vert_fifo_empty_MASK 0x80000 +#define VGT_DEBUG_REG33__con_vert_fifo_empty__SHIFT 0x13 +#define VGT_DEBUG_REG33__last_patch_of_tg_p0_q_MASK 0x100000 +#define VGT_DEBUG_REG33__last_patch_of_tg_p0_q__SHIFT 0x14 +#define VGT_DEBUG_REG33__ring3_valid_p2_MASK 0x200000 +#define VGT_DEBUG_REG33__ring3_valid_p2__SHIFT 0x15 +#define VGT_DEBUG_REG33__ring2_valid_p2_MASK 0x400000 +#define VGT_DEBUG_REG33__ring2_valid_p2__SHIFT 0x16 +#define VGT_DEBUG_REG33__ring1_valid_p2_MASK 0x800000 +#define VGT_DEBUG_REG33__ring1_valid_p2__SHIFT 0x17 +#define VGT_DEBUG_REG33__tess_type_p0_q_MASK 0x3000000 +#define VGT_DEBUG_REG33__tess_type_p0_q__SHIFT 0x18 +#define VGT_DEBUG_REG33__tess_topology_p0_q_MASK 0xc000000 +#define VGT_DEBUG_REG33__tess_topology_p0_q__SHIFT 0x1a +#define VGT_DEBUG_REG33__te11_out_vert_gs_en_MASK 0x10000000 +#define VGT_DEBUG_REG33__te11_out_vert_gs_en__SHIFT 0x1c +#define VGT_DEBUG_REG33__con_ring3_busy_MASK 0x20000000 +#define VGT_DEBUG_REG33__con_ring3_busy__SHIFT 0x1d +#define VGT_DEBUG_REG33__con_ring2_busy_MASK 0x40000000 +#define VGT_DEBUG_REG33__con_ring2_busy__SHIFT 0x1e +#define VGT_DEBUG_REG33__con_ring1_busy_MASK 0x80000000 +#define VGT_DEBUG_REG33__con_ring1_busy__SHIFT 0x1f +#define VGT_DEBUG_REG34__con_state_q_MASK 0xf +#define VGT_DEBUG_REG34__con_state_q__SHIFT 0x0 +#define VGT_DEBUG_REG34__second_cycle_q_MASK 0x10 +#define VGT_DEBUG_REG34__second_cycle_q__SHIFT 0x4 +#define VGT_DEBUG_REG34__process_tri_middle_p0_q_MASK 0x20 +#define VGT_DEBUG_REG34__process_tri_middle_p0_q__SHIFT 0x5 +#define VGT_DEBUG_REG34__process_tri_1st_2nd_half_p0_q_MASK 0x40 +#define VGT_DEBUG_REG34__process_tri_1st_2nd_half_p0_q__SHIFT 0x6 +#define VGT_DEBUG_REG34__process_tri_center_poly_p0_q_MASK 0x80 +#define VGT_DEBUG_REG34__process_tri_center_poly_p0_q__SHIFT 0x7 +#define VGT_DEBUG_REG34__pipe0_patch_dr_MASK 0x100 +#define VGT_DEBUG_REG34__pipe0_patch_dr__SHIFT 0x8 +#define VGT_DEBUG_REG34__pipe0_edge_dr_MASK 0x200 +#define VGT_DEBUG_REG34__pipe0_edge_dr__SHIFT 0x9 +#define VGT_DEBUG_REG34__pipe1_dr_MASK 0x400 +#define VGT_DEBUG_REG34__pipe1_dr__SHIFT 0xa +#define VGT_DEBUG_REG34__pipe0_patch_rtr_MASK 0x800 +#define VGT_DEBUG_REG34__pipe0_patch_rtr__SHIFT 0xb +#define VGT_DEBUG_REG34__pipe0_edge_rtr_MASK 0x1000 +#define VGT_DEBUG_REG34__pipe0_edge_rtr__SHIFT 0xc +#define VGT_DEBUG_REG34__pipe1_rtr_MASK 0x2000 +#define VGT_DEBUG_REG34__pipe1_rtr__SHIFT 0xd +#define VGT_DEBUG_REG34__outer_parity_p0_q_MASK 0x4000 +#define VGT_DEBUG_REG34__outer_parity_p0_q__SHIFT 0xe +#define VGT_DEBUG_REG34__parallel_parity_p0_q_MASK 0x8000 +#define VGT_DEBUG_REG34__parallel_parity_p0_q__SHIFT 0xf +#define VGT_DEBUG_REG34__first_ring_of_patch_p0_q_MASK 0x10000 +#define VGT_DEBUG_REG34__first_ring_of_patch_p0_q__SHIFT 0x10 +#define VGT_DEBUG_REG34__last_ring_of_patch_p0_q_MASK 0x20000 +#define VGT_DEBUG_REG34__last_ring_of_patch_p0_q__SHIFT 0x11 +#define VGT_DEBUG_REG34__last_edge_of_outer_ring_p0_q_MASK 0x40000 +#define VGT_DEBUG_REG34__last_edge_of_outer_ring_p0_q__SHIFT 0x12 +#define VGT_DEBUG_REG34__last_point_of_outer_ring_p1_MASK 0x80000 +#define VGT_DEBUG_REG34__last_point_of_outer_ring_p1__SHIFT 0x13 +#define VGT_DEBUG_REG34__last_point_of_inner_ring_p1_MASK 0x100000 +#define VGT_DEBUG_REG34__last_point_of_inner_ring_p1__SHIFT 0x14 +#define VGT_DEBUG_REG34__outer_edge_tf_eq_one_p0_q_MASK 0x200000 +#define VGT_DEBUG_REG34__outer_edge_tf_eq_one_p0_q__SHIFT 0x15 +#define VGT_DEBUG_REG34__advance_outer_point_p1_MASK 0x400000 +#define VGT_DEBUG_REG34__advance_outer_point_p1__SHIFT 0x16 +#define VGT_DEBUG_REG34__advance_inner_point_p1_MASK 0x800000 +#define VGT_DEBUG_REG34__advance_inner_point_p1__SHIFT 0x17 +#define VGT_DEBUG_REG34__next_ring_is_rect_p0_q_MASK 0x1000000 +#define VGT_DEBUG_REG34__next_ring_is_rect_p0_q__SHIFT 0x18 +#define VGT_DEBUG_REG34__pipe1_outer1_rtr_MASK 0x2000000 +#define VGT_DEBUG_REG34__pipe1_outer1_rtr__SHIFT 0x19 +#define VGT_DEBUG_REG34__pipe1_outer2_rtr_MASK 0x4000000 +#define VGT_DEBUG_REG34__pipe1_outer2_rtr__SHIFT 0x1a +#define VGT_DEBUG_REG34__pipe1_inner1_rtr_MASK 0x8000000 +#define VGT_DEBUG_REG34__pipe1_inner1_rtr__SHIFT 0x1b +#define VGT_DEBUG_REG34__pipe1_inner2_rtr_MASK 0x10000000 +#define VGT_DEBUG_REG34__pipe1_inner2_rtr__SHIFT 0x1c +#define VGT_DEBUG_REG34__pipe1_patch_rtr_MASK 0x20000000 +#define VGT_DEBUG_REG34__pipe1_patch_rtr__SHIFT 0x1d +#define VGT_DEBUG_REG34__pipe1_edge_rtr_MASK 0x40000000 +#define VGT_DEBUG_REG34__pipe1_edge_rtr__SHIFT 0x1e +#define VGT_DEBUG_REG34__use_stored_inner_q_ring1_MASK 0x80000000 +#define VGT_DEBUG_REG34__use_stored_inner_q_ring1__SHIFT 0x1f +#define VGT_DEBUG_REG35__pipe0_dr_MASK 0x1 +#define VGT_DEBUG_REG35__pipe0_dr__SHIFT 0x0 +#define VGT_DEBUG_REG35__pipe1_dr_MASK 0x2 +#define VGT_DEBUG_REG35__pipe1_dr__SHIFT 0x1 +#define VGT_DEBUG_REG35__pipe0_rtr_MASK 0x4 +#define VGT_DEBUG_REG35__pipe0_rtr__SHIFT 0x2 +#define VGT_DEBUG_REG35__pipe1_rtr_MASK 0x8 +#define VGT_DEBUG_REG35__pipe1_rtr__SHIFT 0x3 +#define VGT_DEBUG_REG35__tfreq_tg_fifo_empty_MASK 0x10 +#define VGT_DEBUG_REG35__tfreq_tg_fifo_empty__SHIFT 0x4 +#define VGT_DEBUG_REG35__tfreq_tg_fifo_full_MASK 0x20 +#define VGT_DEBUG_REG35__tfreq_tg_fifo_full__SHIFT 0x5 +#define VGT_DEBUG_REG35__tf_data_fifo_busy_q_MASK 0x40 +#define VGT_DEBUG_REG35__tf_data_fifo_busy_q__SHIFT 0x6 +#define VGT_DEBUG_REG35__tf_data_fifo_rtr_q_MASK 0x80 +#define VGT_DEBUG_REG35__tf_data_fifo_rtr_q__SHIFT 0x7 +#define VGT_DEBUG_REG35__tf_skid_fifo_empty_MASK 0x100 +#define VGT_DEBUG_REG35__tf_skid_fifo_empty__SHIFT 0x8 +#define VGT_DEBUG_REG35__tf_skid_fifo_full_MASK 0x200 +#define VGT_DEBUG_REG35__tf_skid_fifo_full__SHIFT 0x9 +#define VGT_DEBUG_REG35__vgt_tc_rdreq_rtr_q_MASK 0x400 +#define VGT_DEBUG_REG35__vgt_tc_rdreq_rtr_q__SHIFT 0xa +#define VGT_DEBUG_REG35__last_req_of_tg_p2_MASK 0x800 +#define VGT_DEBUG_REG35__last_req_of_tg_p2__SHIFT 0xb +#define VGT_DEBUG_REG35__spi_vgt_hs_done_cnt_q_MASK 0x3f000 +#define VGT_DEBUG_REG35__spi_vgt_hs_done_cnt_q__SHIFT 0xc +#define VGT_DEBUG_REG35__event_flag_p1_q_MASK 0x40000 +#define VGT_DEBUG_REG35__event_flag_p1_q__SHIFT 0x12 +#define VGT_DEBUG_REG35__null_flag_p1_q_MASK 0x80000 +#define VGT_DEBUG_REG35__null_flag_p1_q__SHIFT 0x13 +#define VGT_DEBUG_REG35__tf_data_fifo_cnt_q_MASK 0x7f00000 +#define VGT_DEBUG_REG35__tf_data_fifo_cnt_q__SHIFT 0x14 +#define VGT_DEBUG_REG35__second_tf_ret_data_q_MASK 0x8000000 +#define VGT_DEBUG_REG35__second_tf_ret_data_q__SHIFT 0x1b +#define VGT_DEBUG_REG35__first_req_of_tg_p1_q_MASK 0x10000000 +#define VGT_DEBUG_REG35__first_req_of_tg_p1_q__SHIFT 0x1c +#define VGT_DEBUG_REG35__VGT_TC_rdreq_send_out_MASK 0x20000000 +#define VGT_DEBUG_REG35__VGT_TC_rdreq_send_out__SHIFT 0x1d +#define VGT_DEBUG_REG35__VGT_TC_rdnfo_stall_out_MASK 0x40000000 +#define VGT_DEBUG_REG35__VGT_TC_rdnfo_stall_out__SHIFT 0x1e +#define VGT_DEBUG_REG35__TC_VGT_rdret_data_in_MASK 0x80000000 +#define VGT_DEBUG_REG35__TC_VGT_rdret_data_in__SHIFT 0x1f +#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK_MASK 0xff +#define VGT_PERFCOUNTER_SEID_MASK__PERF_SEID_IGNORE_MASK__SHIFT 0x0 +#define VGT_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define VGT_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define VGT_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define VGT_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define VGT_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define VGT_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define VGT_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define VGT_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0x3ff +#define VGT_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1_MASK 0xffc00 +#define VGT_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT 0xa +#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE_MASK 0xf00000 +#define VGT_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT 0x14 +#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1_MASK 0xf000000 +#define VGT_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT 0x18 +#define VGT_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define VGT_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define VGT_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0xff +#define VGT_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define VGT_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define VGT_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define VGT_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0xff +#define VGT_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define VGT_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define VGT_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define VGT_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000 +#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18 +#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define VGT_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c +#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK 0x3ff +#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT 0x0 +#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK 0xffc00 +#define VGT_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT 0xa +#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK 0xf000000 +#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT 0x18 +#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define VGT_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT 0x1c +#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define VGT_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define VGT_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define VGT_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define VGT_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define VGT_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define VGT_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define VGT_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define VGT_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define IA_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0x3ff +#define IA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define IA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK 0xffc00 +#define IA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT 0xa +#define IA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK 0xf00000 +#define IA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT 0x14 +#define IA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK 0xf000000 +#define IA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT 0x18 +#define IA_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define IA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define IA_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff +#define IA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define IA_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define IA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define IA_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0xff +#define IA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define IA_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define IA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define IA_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0xff +#define IA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define IA_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define IA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK 0x3ff +#define IA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT 0x0 +#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK 0xffc00 +#define IA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT 0xa +#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK 0xf000000 +#define IA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT 0x18 +#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK 0xf0000000 +#define IA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT 0x1c +#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define IA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define IA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define IA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define IA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define IA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define IA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define IA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define IA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define WD_PERFCOUNTER0_SELECT__PERF_SEL_MASK 0xff +#define WD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT 0x0 +#define WD_PERFCOUNTER0_SELECT__PERF_MODE_MASK 0xf0000000 +#define WD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT 0x1c +#define WD_PERFCOUNTER1_SELECT__PERF_SEL_MASK 0xff +#define WD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT 0x0 +#define WD_PERFCOUNTER1_SELECT__PERF_MODE_MASK 0xf0000000 +#define WD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT 0x1c +#define WD_PERFCOUNTER2_SELECT__PERF_SEL_MASK 0xff +#define WD_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT 0x0 +#define WD_PERFCOUNTER2_SELECT__PERF_MODE_MASK 0xf0000000 +#define WD_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT 0x1c +#define WD_PERFCOUNTER3_SELECT__PERF_SEL_MASK 0xff +#define WD_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT 0x0 +#define WD_PERFCOUNTER3_SELECT__PERF_MODE_MASK 0xf0000000 +#define WD_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT 0x1c +#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define WD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define WD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define WD_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK 0xffffffff +#define WD_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT 0x0 +#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define WD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define WD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define WD_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK 0xffffffff +#define WD_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT 0x0 +#define DIDT_IND_INDEX__DIDT_IND_INDEX_MASK 0xffffffff +#define DIDT_IND_INDEX__DIDT_IND_INDEX__SHIFT 0x0 +#define DIDT_IND_DATA__DIDT_IND_DATA_MASK 0xffffffff +#define DIDT_IND_DATA__DIDT_IND_DATA__SHIFT 0x0 +#define DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK 0x1 +#define DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT 0x0 +#define DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK 0x2 +#define DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT 0x1 +#define DIDT_SQ_CTRL0__PHASE_OFFSET_MASK 0xc +#define DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT 0x2 +#define DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK 0x10 +#define DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 +#define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 +#define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 +#define DIDT_SQ_CTRL1__MIN_POWER_MASK 0xffff +#define DIDT_SQ_CTRL1__MIN_POWER__SHIFT 0x0 +#define DIDT_SQ_CTRL1__MAX_POWER_MASK 0xffff0000 +#define DIDT_SQ_CTRL1__MAX_POWER__SHIFT 0x10 +#define DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK 0x3fff +#define DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT 0x0 +#define DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000 +#define DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10 +#define DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000 +#define DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b +#define DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK 0xff +#define DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT 0x0 +#define DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK 0xff00 +#define DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT 0x8 +#define DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK 0xff0000 +#define DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT 0x10 +#define DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK 0xff000000 +#define DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT 0x18 +#define DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK 0xff +#define DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT 0x0 +#define DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK 0xff00 +#define DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT 0x8 +#define DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK 0xff0000 +#define DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT 0x10 +#define DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK 0xff000000 +#define DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT 0x18 +#define DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK 0xff +#define DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT 0x0 +#define DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK 0xff00 +#define DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT 0x8 +#define DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK 0xff0000 +#define DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT 0x10 +#define DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK 0xff000000 +#define DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT 0x18 +#define DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK 0x1 +#define DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT 0x0 +#define DIDT_DB_CTRL0__USE_REF_CLOCK_MASK 0x2 +#define DIDT_DB_CTRL0__USE_REF_CLOCK__SHIFT 0x1 +#define DIDT_DB_CTRL0__PHASE_OFFSET_MASK 0xc +#define DIDT_DB_CTRL0__PHASE_OFFSET__SHIFT 0x2 +#define DIDT_DB_CTRL0__DIDT_CTRL_RST_MASK 0x10 +#define DIDT_DB_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 +#define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 +#define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 +#define DIDT_DB_CTRL1__MIN_POWER_MASK 0xffff +#define DIDT_DB_CTRL1__MIN_POWER__SHIFT 0x0 +#define DIDT_DB_CTRL1__MAX_POWER_MASK 0xffff0000 +#define DIDT_DB_CTRL1__MAX_POWER__SHIFT 0x10 +#define DIDT_DB_CTRL2__MAX_POWER_DELTA_MASK 0x3fff +#define DIDT_DB_CTRL2__MAX_POWER_DELTA__SHIFT 0x0 +#define DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000 +#define DIDT_DB_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10 +#define DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000 +#define DIDT_DB_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b +#define DIDT_DB_WEIGHT0_3__WEIGHT0_MASK 0xff +#define DIDT_DB_WEIGHT0_3__WEIGHT0__SHIFT 0x0 +#define DIDT_DB_WEIGHT0_3__WEIGHT1_MASK 0xff00 +#define DIDT_DB_WEIGHT0_3__WEIGHT1__SHIFT 0x8 +#define DIDT_DB_WEIGHT0_3__WEIGHT2_MASK 0xff0000 +#define DIDT_DB_WEIGHT0_3__WEIGHT2__SHIFT 0x10 +#define DIDT_DB_WEIGHT0_3__WEIGHT3_MASK 0xff000000 +#define DIDT_DB_WEIGHT0_3__WEIGHT3__SHIFT 0x18 +#define DIDT_DB_WEIGHT4_7__WEIGHT4_MASK 0xff +#define DIDT_DB_WEIGHT4_7__WEIGHT4__SHIFT 0x0 +#define DIDT_DB_WEIGHT4_7__WEIGHT5_MASK 0xff00 +#define DIDT_DB_WEIGHT4_7__WEIGHT5__SHIFT 0x8 +#define DIDT_DB_WEIGHT4_7__WEIGHT6_MASK 0xff0000 +#define DIDT_DB_WEIGHT4_7__WEIGHT6__SHIFT 0x10 +#define DIDT_DB_WEIGHT4_7__WEIGHT7_MASK 0xff000000 +#define DIDT_DB_WEIGHT4_7__WEIGHT7__SHIFT 0x18 +#define DIDT_DB_WEIGHT8_11__WEIGHT8_MASK 0xff +#define DIDT_DB_WEIGHT8_11__WEIGHT8__SHIFT 0x0 +#define DIDT_DB_WEIGHT8_11__WEIGHT9_MASK 0xff00 +#define DIDT_DB_WEIGHT8_11__WEIGHT9__SHIFT 0x8 +#define DIDT_DB_WEIGHT8_11__WEIGHT10_MASK 0xff0000 +#define DIDT_DB_WEIGHT8_11__WEIGHT10__SHIFT 0x10 +#define DIDT_DB_WEIGHT8_11__WEIGHT11_MASK 0xff000000 +#define DIDT_DB_WEIGHT8_11__WEIGHT11__SHIFT 0x18 +#define DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK 0x1 +#define DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT 0x0 +#define DIDT_TD_CTRL0__USE_REF_CLOCK_MASK 0x2 +#define DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT 0x1 +#define DIDT_TD_CTRL0__PHASE_OFFSET_MASK 0xc +#define DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT 0x2 +#define DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK 0x10 +#define DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 +#define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 +#define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 +#define DIDT_TD_CTRL1__MIN_POWER_MASK 0xffff +#define DIDT_TD_CTRL1__MIN_POWER__SHIFT 0x0 +#define DIDT_TD_CTRL1__MAX_POWER_MASK 0xffff0000 +#define DIDT_TD_CTRL1__MAX_POWER__SHIFT 0x10 +#define DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK 0x3fff +#define DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT 0x0 +#define DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000 +#define DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10 +#define DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000 +#define DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b +#define DIDT_TD_WEIGHT0_3__WEIGHT0_MASK 0xff +#define DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT 0x0 +#define DIDT_TD_WEIGHT0_3__WEIGHT1_MASK 0xff00 +#define DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT 0x8 +#define DIDT_TD_WEIGHT0_3__WEIGHT2_MASK 0xff0000 +#define DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT 0x10 +#define DIDT_TD_WEIGHT0_3__WEIGHT3_MASK 0xff000000 +#define DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT 0x18 +#define DIDT_TD_WEIGHT4_7__WEIGHT4_MASK 0xff +#define DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT 0x0 +#define DIDT_TD_WEIGHT4_7__WEIGHT5_MASK 0xff00 +#define DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT 0x8 +#define DIDT_TD_WEIGHT4_7__WEIGHT6_MASK 0xff0000 +#define DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT 0x10 +#define DIDT_TD_WEIGHT4_7__WEIGHT7_MASK 0xff000000 +#define DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT 0x18 +#define DIDT_TD_WEIGHT8_11__WEIGHT8_MASK 0xff +#define DIDT_TD_WEIGHT8_11__WEIGHT8__SHIFT 0x0 +#define DIDT_TD_WEIGHT8_11__WEIGHT9_MASK 0xff00 +#define DIDT_TD_WEIGHT8_11__WEIGHT9__SHIFT 0x8 +#define DIDT_TD_WEIGHT8_11__WEIGHT10_MASK 0xff0000 +#define DIDT_TD_WEIGHT8_11__WEIGHT10__SHIFT 0x10 +#define DIDT_TD_WEIGHT8_11__WEIGHT11_MASK 0xff000000 +#define DIDT_TD_WEIGHT8_11__WEIGHT11__SHIFT 0x18 +#define DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK 0x1 +#define DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT 0x0 +#define DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK 0x2 +#define DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT 0x1 +#define DIDT_TCP_CTRL0__PHASE_OFFSET_MASK 0xc +#define DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT 0x2 +#define DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK 0x10 +#define DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT 0x4 +#define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20 +#define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5 +#define DIDT_TCP_CTRL1__MIN_POWER_MASK 0xffff +#define DIDT_TCP_CTRL1__MIN_POWER__SHIFT 0x0 +#define DIDT_TCP_CTRL1__MAX_POWER_MASK 0xffff0000 +#define DIDT_TCP_CTRL1__MAX_POWER__SHIFT 0x10 +#define DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK 0x3fff +#define DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT 0x0 +#define DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK 0x3ff0000 +#define DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT 0x10 +#define DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK 0x78000000 +#define DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT 0x1b +#define DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK 0xff +#define DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT 0x0 +#define DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK 0xff00 +#define DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT 0x8 +#define DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK 0xff0000 +#define DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT 0x10 +#define DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK 0xff000000 +#define DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT 0x18 +#define DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK 0xff +#define DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT 0x0 +#define DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK 0xff00 +#define DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT 0x8 +#define DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK 0xff0000 +#define DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT 0x10 +#define DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK 0xff000000 +#define DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT 0x18 +#define DIDT_TCP_WEIGHT8_11__WEIGHT8_MASK 0xff +#define DIDT_TCP_WEIGHT8_11__WEIGHT8__SHIFT 0x0 +#define DIDT_TCP_WEIGHT8_11__WEIGHT9_MASK 0xff00 +#define DIDT_TCP_WEIGHT8_11__WEIGHT9__SHIFT 0x8 +#define DIDT_TCP_WEIGHT8_11__WEIGHT10_MASK 0xff0000 +#define DIDT_TCP_WEIGHT8_11__WEIGHT10__SHIFT 0x10 +#define DIDT_TCP_WEIGHT8_11__WEIGHT11_MASK 0xff000000 +#define DIDT_TCP_WEIGHT8_11__WEIGHT11__SHIFT 0x18 + +#endif /* GFX_7_2_SH_MASK_H */ diff --git a/tests/kfdtest/include/kfd_pm4_opcodes.h b/tests/kfdtest/include/kfd_pm4_opcodes.h new file mode 100644 index 0000000000..43deaeec35 --- /dev/null +++ b/tests/kfdtest/include/kfd_pm4_opcodes.h @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + + +#ifndef KFD_PM4_OPCODES_H +#define KFD_PM4_OPCODES_H + +enum it_opcode_type { + IT_NOP = 0x10, + IT_SET_BASE = 0x11, + IT_CLEAR_STATE = 0x12, + IT_INDEX_BUFFER_SIZE = 0x13, + IT_DISPATCH_DIRECT = 0x15, + IT_DISPATCH_INDIRECT = 0x16, + IT_ATOMIC_GDS = 0x1D, + IT_OCCLUSION_QUERY = 0x1F, + IT_SET_PREDICATION = 0x20, + IT_REG_RMW = 0x21, + IT_COND_EXEC = 0x22, + IT_PRED_EXEC = 0x23, + IT_DRAW_INDIRECT = 0x24, + IT_DRAW_INDEX_INDIRECT = 0x25, + IT_INDEX_BASE = 0x26, + IT_DRAW_INDEX_2 = 0x27, + IT_CONTEXT_CONTROL = 0x28, + IT_INDEX_TYPE = 0x2A, + IT_DRAW_INDIRECT_MULTI = 0x2C, + IT_DRAW_INDEX_AUTO = 0x2D, + IT_NUM_INSTANCES = 0x2F, + IT_DRAW_INDEX_MULTI_AUTO = 0x30, + IT_INDIRECT_BUFFER_CNST = 0x33, + IT_STRMOUT_BUFFER_UPDATE = 0x34, + IT_DRAW_INDEX_OFFSET_2 = 0x35, + IT_DRAW_PREAMBLE = 0x36, + IT_WRITE_DATA = 0x37, + IT_DRAW_INDEX_INDIRECT_MULTI = 0x38, + IT_MEM_SEMAPHORE = 0x39, + IT_COPY_DW = 0x3B, + IT_WAIT_REG_MEM = 0x3C, + IT_INDIRECT_BUFFER = 0x3F, + IT_COPY_DATA = 0x40, + IT_PFP_SYNC_ME = 0x42, + IT_SURFACE_SYNC = 0x43, + IT_COND_WRITE = 0x45, + IT_EVENT_WRITE = 0x46, + IT_EVENT_WRITE_EOP = 0x47, + IT_EVENT_WRITE_EOS = 0x48, + IT_RELEASE_MEM = 0x49, + IT_PREAMBLE_CNTL = 0x4A, + IT_DMA_DATA = 0x50, + IT_ACQUIRE_MEM = 0x58, + IT_REWIND = 0x59, + IT_LOAD_UCONFIG_REG = 0x5E, + IT_LOAD_SH_REG = 0x5F, + IT_LOAD_CONFIG_REG = 0x60, + IT_LOAD_CONTEXT_REG = 0x61, + IT_SET_CONFIG_REG = 0x68, + IT_SET_CONTEXT_REG = 0x69, + IT_SET_CONTEXT_REG_INDIRECT = 0x73, + IT_SET_SH_REG = 0x76, + IT_SET_SH_REG_OFFSET = 0x77, + IT_SET_QUEUE_REG = 0x78, + IT_SET_UCONFIG_REG = 0x79, + IT_SCRATCH_RAM_WRITE = 0x7D, + IT_SCRATCH_RAM_READ = 0x7E, + IT_LOAD_CONST_RAM = 0x80, + IT_WRITE_CONST_RAM = 0x81, + IT_DUMP_CONST_RAM = 0x83, + IT_INCREMENT_CE_COUNTER = 0x84, + IT_INCREMENT_DE_COUNTER = 0x85, + IT_WAIT_ON_CE_COUNTER = 0x86, + IT_WAIT_ON_DE_COUNTER_DIFF = 0x88, + IT_SWITCH_BUFFER = 0x8B, + IT_SET_RESOURCES = 0xA0, + IT_MAP_PROCESS = 0xA1, + IT_MAP_QUEUES = 0xA2, + IT_UNMAP_QUEUES = 0xA3, + IT_QUERY_STATUS = 0xA4, + IT_RUN_LIST = 0xA5, +}; + +#define PM4_TYPE_0 0 +#define PM4_TYPE_2 2 +#define PM4_TYPE_3 3 + +#endif /* KFD_PM4_OPCODES_H */ + diff --git a/tests/kfdtest/include/pm4_pkt_struct_ai.h b/tests/kfdtest/include/pm4_pkt_struct_ai.h new file mode 100644 index 0000000000..b062a06fc1 --- /dev/null +++ b/tests/kfdtest/include/pm4_pkt_struct_ai.h @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __PM4_PKT_STRUCT_AI_H__ +#define __PM4_PKT_STRUCT_AI_H__ + +#ifndef PM4_MEC_RELEASE_MEM_AI_DEFINED +#define PM4_MEC_RELEASE_MEM_AI_DEFINED + +enum AI_MEC_RELEASE_MEM_event_index_enum { + event_index__mec_release_mem__end_of_pipe = 5, + event_index__mec_release_mem__shader_done = 6 }; + +enum AI_MEC_RELEASE_MEM_cache_policy_enum { + cache_policy__mec_release_mem__lru = 0, + cache_policy__mec_release_mem__stream = 1 }; + +enum AI_MEC_RELEASE_MEM_pq_exe_status_enum { + pq_exe_status__mec_release_mem__default = 0, + pq_exe_status__mec_release_mem__phase_update = 1 }; + +enum AI_MEC_RELEASE_MEM_dst_sel_enum { + dst_sel__mec_release_mem__memory_controller = 0, + dst_sel__mec_release_mem__tc_l2 = 1, + dst_sel__mec_release_mem__queue_write_pointer_register = 2, + dst_sel__mec_release_mem__queue_write_pointer_poll_mask_bit = 3 }; + +enum AI_MEC_RELEASE_MEM_int_sel_enum { + int_sel__mec_release_mem__none = 0, + int_sel__mec_release_mem__send_interrupt_only = 1, + int_sel__mec_release_mem__send_interrupt_after_write_confirm = 2, + int_sel__mec_release_mem__send_data_after_write_confirm = 3, + int_sel__mec_release_mem__unconditionally_send_int_ctxid = 4, + int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_32_bit_compare = 5, + int_sel__mec_release_mem__conditionally_send_int_ctxid_based_on_64_bit_compare = 6 }; + +enum AI_MEC_RELEASE_MEM_data_sel_enum { + data_sel__mec_release_mem__none = 0, + data_sel__mec_release_mem__send_32_bit_low = 1, + data_sel__mec_release_mem__send_64_bit_data = 2, + data_sel__mec_release_mem__send_gpu_clock_counter = 3, + data_sel__mec_release_mem__send_cp_perfcounter_hi_lo = 4, + data_sel__mec_release_mem__store_gds_data_to_memory = 5 }; + + +typedef struct PM4_MEC_RELEASE_MEM_AI { + union { + PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int event_type:6; + unsigned int reserved1:2; + AI_MEC_RELEASE_MEM_event_index_enum event_index:4; + unsigned int tcl1_vol_action_ena:1; + unsigned int tc_vol_action_ena:1; + unsigned int reserved2:1; + unsigned int tc_wb_action_ena:1; + unsigned int tcl1_action_ena:1; + unsigned int tc_action_ena:1; + unsigned int reserved3:1; + unsigned int tc_nc_action_ena:1; + unsigned int tc_wc_action_ena:1; + unsigned int tc_md_action_ena:1; + unsigned int reserved4:3; + AI_MEC_RELEASE_MEM_cache_policy_enum cache_policy:2; + unsigned int reserved5:2; + AI_MEC_RELEASE_MEM_pq_exe_status_enum pq_exe_status:1; + unsigned int reserved6:2; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int reserved7:16; + AI_MEC_RELEASE_MEM_dst_sel_enum dst_sel:2; + unsigned int reserved8:6; + AI_MEC_RELEASE_MEM_int_sel_enum int_sel:3; + unsigned int reserved9:2; + AI_MEC_RELEASE_MEM_data_sel_enum data_sel:3; + } bitfields3; + unsigned int ordinal3; + }; + + union { + struct { + unsigned int reserved10:2; + unsigned int address_lo_32b:30; + } bitfields4a; + struct { + unsigned int reserved11:3; + unsigned int address_lo_64b:29; + } bitfields4b; + unsigned int reserved12; + + unsigned int ordinal4; + }; + + union { + unsigned int address_hi; + + unsigned int reserved13; + + unsigned int ordinal5; + }; + + union { + unsigned int data_lo; + + unsigned int cmp_data_lo; + + struct { + unsigned int dw_offset:16; + unsigned int num_dwords:16; + } bitfields6c; + unsigned int reserved14; + + unsigned int ordinal6; + }; + + union { + unsigned int data_hi; + + unsigned int cmp_data_hi; + + unsigned int reserved15; + + unsigned int reserved16; + + unsigned int ordinal7; + }; + + unsigned int int_ctxid; +} PM4MEC_RELEASE_MEM_AI, *PPM4MEC_RELEASE_MEM_AI; + +#endif // PM4_MEC_RELEASE_MEM_AI_DEFINED +#endif // __PM4_PKT_STRUCT_AI_H__ diff --git a/tests/kfdtest/include/pm4_pkt_struct_ci.h b/tests/kfdtest/include/pm4_pkt_struct_ci.h new file mode 100644 index 0000000000..5732a3dbfb --- /dev/null +++ b/tests/kfdtest/include/pm4_pkt_struct_ci.h @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2012-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __PM4_PKT_STRUCT_CI_H__ +#define __PM4_PKT_STRUCT_CI_H__ + + +enum WRITE_DATA_CI_atc_enum { atc_write_data_NOT_USE_ATC_0 = 0, atc_write_data_USE_ATC_1 = 1 }; +enum WRITE_DATA_CI_engine_sel { engine_sel_write_data_ci_MICRO_ENGINE_0 = 0, engine_sel_write_data_ci_PREFETCH_PARSER_1 = 1, engine_sel_write_data_ci_CONST_ENG_2 = 2 }; + +typedef struct _PM4WRITE_DATA_CI { + union { + PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int reserved1:8; + MEC_WRITE_DATA_dst_sel_enum dst_sel:4; + unsigned int reserved2:4; + MEC_WRITE_DATA_addr_incr_enum addr_incr:1; + unsigned int reserved3:3; + MEC_WRITE_DATA_wr_confirm_enum wr_confirm:1; + unsigned int reserved4:3; + WRITE_DATA_CI_atc_enum atc:1; + MEC_WRITE_DATA_cache_policy_enum cache_policy:2; + unsigned int volatile_setting:1; + unsigned int reserved5:2; + WRITE_DATA_CI_engine_sel engine_sel:2; + } bitfields2; + unsigned int ordinal2; + }; + + unsigned int dst_addr_lo; + + unsigned int dst_address_hi; + + unsigned int data[1]; // 1..N of these fields +} PM4WRITE_DATA_CI, *PPM4WRITE_DATA_CI; + + +enum MEC_RELEASE_MEM_CI_atc_enum { atc_mec_release_mem_ci_NOT_USE_ATC_0 = 0, atc_mec_release_mem_ci_USE_ATC_1 = 1 }; + +typedef struct _PM4_RELEASE_MEM_CI { + union { + PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int event_type:6; + unsigned int reserved1:2; + MEC_RELEASE_MEM_event_index_enum event_index:4; + unsigned int l1_vol:1; + unsigned int l2_vol:1; + unsigned int reserved:1; + unsigned int l2_wb:1; + unsigned int l1_inv:1; + unsigned int l2_inv:1; + unsigned int reserved2:6; + MEC_RELEASE_MEM_CI_atc_enum atc:1; + MEC_RELEASE_MEM_cache_policy_enum cache_policy:2; + unsigned int volatile_setting:1; + unsigned int reserved3:4; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int reserved4:16; + MEC_RELEASE_MEM_dst_sel_enum dst_sel:2; + unsigned int reserved5:6; + MEC_RELEASE_MEM_int_sel_enum int_sel:3; + unsigned int reserved6:2; + MEC_RELEASE_MEM_data_sel_enum data_sel:3; + } bitfields3; + unsigned int ordinal3; + }; + + union { + struct { + unsigned int reserved7:2; + unsigned int address_lo_dword_aligned:30; + } bitfields4a; + struct { + unsigned int reserved8:3; + unsigned int address_lo_qword_aligned:29; + } bitfields4b; + unsigned int ordinal4; + }; + + unsigned int addr_hi; + + union { + unsigned int data_lo; + struct { + unsigned int offset:16; + unsigned int num_dwords:16; + } bitfields5b; + unsigned int ordinal6; + }; + + unsigned int data_hi; +} PM4_RELEASE_MEM_CI, *PPM4_RELEASE_MEM_CI; + +#endif // __PM4_PKT_STRUCT_CI_H__ diff --git a/tests/kfdtest/include/pm4_pkt_struct_common.h b/tests/kfdtest/include/pm4_pkt_struct_common.h new file mode 100644 index 0000000000..d3b78427b9 --- /dev/null +++ b/tests/kfdtest/include/pm4_pkt_struct_common.h @@ -0,0 +1,275 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __PM4_PKT_STRUCT_COMMON_H__ +#define __PM4_PKT_STRUCT_COMMON_H__ + +#ifndef PM4_HEADER_DEFINED +#define PM4_HEADER_DEFINED +typedef union PM4_TYPE_3_HEADER +{ + struct + { + unsigned int predicate : 1; ///< predicated version of packet when set + unsigned int shaderType: 1; ///< 0: Graphics, 1: Compute Shader + unsigned int reserved1 : 6; ///< reserved + unsigned int opcode : 8; ///< IT opcode + unsigned int count : 14;///< number of DWORDs - 1 in the information body. + unsigned int type : 2; ///< packet identifier. It should be 3 for type 3 packets + }; + unsigned int u32All; +} PM4_TYPE_3_HEADER; +#endif // PM4_HEADER_DEFINED + +//--------------------DISPATCH_DIRECT-------------------- + + +typedef struct _PM4_DISPATCH_DIRECT +{ + union + { + PM4_TYPE_3_HEADER header; ///header + unsigned int ordinal1; + }; + + unsigned int dim_x; + + + unsigned int dim_y; + + + unsigned int dim_z; + + + unsigned int dispatch_initiator; + + +} PM4DISPATCH_DIRECT, *PPM4DISPATCH_DIRECT; + +//--------------------INDIRECT_BUFFER-------------------- + +enum INDIRECT_BUFFER_cache_policy_enum { cache_policy_indirect_buffer_LRU_0 = 0, cache_policy_indirect_buffer_STREAM_1 = 1, cache_policy_indirect_buffer_BYPASS_2 = 2 }; + + +//--------------------EVENT_WRITE-------------------- + +enum EVENT_WRITE_event_index_enum { event_index_event_write_OTHER_0 = 0, event_index_event_write_ZPASS_DONE_1 = 1, event_index_event_write_SAMPLE_PIPELINESTAT_2 = 2, event_index_event_write_SAMPLE_STREAMOUTSTAT_3 = 3, event_index_event_write_CS_VS_PS_PARTIAL_FLUSH_4 = 4, event_index_event_write_RESERVED_EOP_5 = 5, event_index_event_write_RESERVED_EOS_6 = 6, event_index_event_write_CACHE_FLUSH_7 = 7 }; + +typedef struct _PM4_EVENT_WRITE +{ + union + { + PM4_TYPE_3_HEADER header; ///header + unsigned int ordinal1; + }; + + union + { + struct + { + unsigned int event_type:6; + unsigned int reserved1:2; + EVENT_WRITE_event_index_enum event_index:4; + unsigned int reserved2:20; + } bitfields2; + unsigned int ordinal2; + }; + + union + { + struct + { + unsigned int reserved3:3; + unsigned int address_lo:29; + } bitfields3; + unsigned int ordinal3; + }; + + union + { + struct + { + unsigned int address_hi:16; + unsigned int reserved4:16; + } bitfields4; + unsigned int ordinal4; + }; + +} PM4EVENT_WRITE, *PPM4EVENT_WRITE; + + +//--------------------SET_SH_REG-------------------- + + +typedef struct _PM4_SET_SH_REG +{ + union + { + PM4_TYPE_3_HEADER header; ///header + unsigned int ordinal1; + }; + + union + { + struct + { + unsigned int reg_offset:16; + unsigned int reserved1:16; + } bitfields2; + unsigned int ordinal2; + }; + + unsigned int reg_data[1]; //1..N of these fields + + +} PM4SET_SH_REG, *PPM4SET_SH_REG; + + +//--------------------ACQUIRE_MEM-------------------- + +enum ACQUIRE_MEM_engine_enum { engine_acquire_mem_PFP_0 = 0, engine_acquire_mem_ME_1 = 1 }; + + +typedef struct _PM4_ACQUIRE_MEM +{ + union + { + PM4_TYPE_3_HEADER header; ///header + unsigned int ordinal1; + }; + + union + { + struct + { + unsigned int coher_cntl:31; + ACQUIRE_MEM_engine_enum engine:1; + } bitfields2; + unsigned int ordinal2; + }; + + unsigned int coher_size; + + + union + { + struct + { + unsigned int coher_size_hi:8; + unsigned int reserved1:24; + } bitfields3; + unsigned int ordinal4; + }; + + unsigned int coher_base_lo; + + + union + { + struct + { + unsigned int coher_base_hi:25; + unsigned int reserved2:7; + } bitfields4; + unsigned int ordinal6; + }; + + union + { + struct + { + unsigned int poll_interval:16; + unsigned int reserved3:16; + } bitfields5; + unsigned int ordinal7; + }; + +} PM4ACQUIRE_MEM, *PPM4ACQUIRE_MEM; + + +//--------------------MEC_INDIRECT_BUFFER-------------------- + +typedef struct _PM4_MEC_INDIRECT_BUFFER +{ + union + { + PM4_TYPE_3_HEADER header; ///header + unsigned int ordinal1; + }; + + union + { + struct + { + unsigned int swap_function:2; + unsigned int ib_base_lo:30; + } bitfields2; + unsigned int ordinal2; + }; + + union + { + struct + { + unsigned int ib_base_hi:16; + unsigned int reserved1:16; + } bitfields3; + unsigned int ordinal3; + }; + + union + { + struct + { + unsigned int ib_size:20; + unsigned int chain:1; + unsigned int offload_polling:1; + unsigned int volatile_setting:1; + unsigned int valid:1; + unsigned int vmid:4; + INDIRECT_BUFFER_cache_policy_enum cache_policy:2; + unsigned int reserved4:2; + } bitfields4; + unsigned int ordinal4; + }; + +} PM4MEC_INDIRECT_BUFFER, *PPM4MEC_INDIRECT_BUFFER; + +//--------------------MEC_WRITE_DATA-------------------- + +enum MEC_WRITE_DATA_dst_sel_enum { dst_sel_mec_write_data_MEM_MAPPED_REGISTER_0 = 0, dst_sel_mec_write_data_TC_L2_2 = 2, dst_sel_mec_write_data_GDS_3 = 3, dst_sel_mec_write_data_MEMORY_5 = 5 }; +enum MEC_WRITE_DATA_addr_incr_enum { addr_incr_mec_write_data_INCREMENT_ADDR_0 = 0, addr_incr_mec_write_data_DO_NOT_INCREMENT_ADDR_1 = 1 }; +enum MEC_WRITE_DATA_wr_confirm_enum { wr_confirm_mec_write_data_DO_NOT_WAIT_FOR_CONFIRMATION_0 = 0, wr_confirm_mec_write_data_WAIT_FOR_CONFIRMATION_1 = 1 }; +enum MEC_WRITE_DATA_cache_policy_enum { cache_policy_mec_write_data_LRU_0 = 0, cache_policy_mec_write_data_STREAM_1 = 1, cache_policy_mec_write_data_BYPASS_2 = 2 }; + +//--------------------MEC_RELEASE_MEM-------------------- + +enum MEC_RELEASE_MEM_event_index_enum { event_index_mec_release_mem_EVENT_WRITE_EOP_5 = 5, event_index_mec_release_mem_CS_Done_6 = 6 }; +enum MEC_RELEASE_MEM_cache_policy_enum { cache_policy_mec_release_mem_LRU_0 = 0, cache_policy_mec_release_mem_STREAM_1 = 1, cache_policy_mec_release_mem_BYPASS_2 = 2 }; +enum MEC_RELEASE_MEM_dst_sel_enum { dst_sel_mec_release_mem_MEMORY_CONTROLLER_0 = 0, dst_sel_mec_release_mem_TC_L2_1 = 1 }; +enum MEC_RELEASE_MEM_int_sel_enum { int_sel_mec_release_mem_NONE_0 = 0, int_sel_mec_release_mem_SEND_INTERRUPT_ONLY_1 = 1, int_sel_mec_release_mem_SEND_INTERRUPT_AFTER_WRITE_CONFIRM_2 = 2, int_sel_mec_release_mem_SEND_DATA_AFTER_WRITE_CONFIRM_3 = 3 }; +enum MEC_RELEASE_MEM_data_sel_enum { data_sel_mec_release_mem_NONE_0 = 0, data_sel_mec_release_mem_SEND_32_BIT_LOW_1 = 1, data_sel_mec_release_mem_SEND_64_BIT_DATA_2 = 2, data_sel_mec_release_mem_SEND_GPU_CLOCK_COUNTER_3 = 3, data_sel_mec_release_mem_SEND_CP_PERFCOUNTER_HI_LO_4 = 4, data_sel_mec_release_mem_STORE_GDS_DATA_TO_MEMORY_5 = 5 }; + + +#endif + diff --git a/tests/kfdtest/include/sdma_pkt_struct.h b/tests/kfdtest/include/sdma_pkt_struct.h new file mode 100644 index 0000000000..aa13006256 --- /dev/null +++ b/tests/kfdtest/include/sdma_pkt_struct.h @@ -0,0 +1,316 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __SDMA_PKT_STRUCT_H__ +#define __SDMA_PKT_STRUCT_H__ + + +const unsigned int SDMA_OP_COPY = 1; +const unsigned int SDMA_OP_WRITE = 2; + +const unsigned int SDMA_OP_FENCE = 5; +const unsigned int SDMA_OP_TRAP = 6; + +const unsigned int SDMA_OP_CONST_FILL = 11; + +const unsigned int SDMA_SUBOP_COPY_LINEAR = 0; + +const unsigned int SDMA_SUBOP_WRITE_LINEAR = 0; + +/* +** Definitions for SDMA_PKT_COPY_LINEAR packet +*/ + +typedef struct SDMA_PKT_COPY_LINEAR_TAG +{ + + union + { + struct + { + unsigned int op:8; + unsigned int sub_op:8; + unsigned int reserved_0:11; + unsigned int broadcast:1; + unsigned int reserved_1:4; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union + { + struct + { + unsigned int count:22; + unsigned int reserved_0:10; + }; + unsigned int DW_1_DATA; + } COUNT_UNION; + + union + { + struct + { + unsigned int reserved_0:16; + unsigned int dst_sw:2; + unsigned int reserved_1:4; + unsigned int dst_ha:1; + unsigned int reserved_2:1; + unsigned int src_sw:2; + unsigned int reserved_3:4; + unsigned int src_ha:1; + unsigned int reserved_4:1; + }; + unsigned int DW_2_DATA; + } PARAMETER_UNION; + + union + { + struct + { + unsigned int src_addr_31_0:32; + }; + unsigned int DW_3_DATA; + } SRC_ADDR_LO_UNION; + + union + { + struct + { + unsigned int src_addr_63_32:32; + }; + unsigned int DW_4_DATA; + } SRC_ADDR_HI_UNION; + + union + { + struct + { + unsigned int dst_addr_31_0:32; + }; + unsigned int DW_5_DATA; + } DST_ADDR_LO_UNION; + + union + { + struct + { + unsigned int dst_addr_63_32:32; + }; + unsigned int DW_6_DATA; + } DST_ADDR_HI_UNION; +} SDMA_PKT_COPY_LINEAR, *PSDMA_PKT_COPY_LINEAR; + +/* +** Definitions for SDMA_PKT_WRITE_UNTILED packet +*/ + +typedef struct SDMA_PKT_WRITE_UNTILED_TAG +{ + + union + { + struct + { + unsigned int op:8; + unsigned int sub_op:8; + unsigned int reserved_0:16; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union + { + struct + { + unsigned int dst_addr_31_0:32; + }; + unsigned int DW_1_DATA; + } DST_ADDR_LO_UNION; + + union + { + struct + { + unsigned int dst_addr_63_32:32; + }; + unsigned int DW_2_DATA; + } DST_ADDR_HI_UNION; + + union + { + struct + { + unsigned int count:22; + unsigned int reserved_0:2; + unsigned int sw:2; + unsigned int reserved_1:6; + }; + unsigned int DW_3_DATA; + } DW_3_UNION; + + union + { + struct + { + unsigned int data0:32; + }; + unsigned int DW_4_DATA; + } DATA0_UNION; +} SDMA_PKT_WRITE_UNTILED, *PSDMA_PKT_WRITE_UNTILED; + +/* +** Definitions for SDMA_PKT_FENCE packet +*/ + +typedef struct SDMA_PKT_FENCE_TAG +{ + + union + { + struct + { + unsigned int op:8; + unsigned int sub_op:8; + unsigned int reserved_0:16; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union + { + struct + { + unsigned int addr_31_0:32; + }; + unsigned int DW_1_DATA; + } ADDR_LO_UNION; + + union + { + struct + { + unsigned int addr_63_32:32; + }; + unsigned int DW_2_DATA; + } ADDR_HI_UNION; + + union + { + struct + { + unsigned int data:32; + }; + unsigned int DW_3_DATA; + } DATA_UNION; +} SDMA_PKT_FENCE, *PSDMA_PKT_FENCE; + +/* +** Definitions for SDMA_PKT_CONSTANT_FILL packet +*/ + +typedef struct SDMA_PKT_CONSTANT_FILL_TAG +{ + + union + { + struct + { + unsigned int op:8; + unsigned int sub_op:8; + unsigned int sw:2; + unsigned int reserved_0:12; + unsigned int fillsize:2; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union + { + struct + { + unsigned int dst_addr_31_0:32; + }; + unsigned int DW_1_DATA; + } DST_ADDR_LO_UNION; + + union + { + struct + { + unsigned int dst_addr_63_32:32; + }; + unsigned int DW_2_DATA; + } DST_ADDR_HI_UNION; + + union + { + struct + { + unsigned int src_data_31_0:32; + }; + unsigned int DW_3_DATA; + } DATA_UNION; + + union + { + struct + { + unsigned int count:22; + unsigned int reserved_0:10; + }; + unsigned int DW_4_DATA; + } COUNT_UNION; +} SDMA_PKT_CONSTANT_FILL, *PSDMA_PKT_CONSTANT_FILL; + +/* +** Definitions for SDMA_PKT_TRAP packet +*/ + +typedef struct SDMA_PKT_TRAP_TAG +{ + + union + { + struct + { + unsigned int op:8; + unsigned int sub_op:8; + unsigned int reserved_0:16; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union + { + struct + { + unsigned int int_context:28; + unsigned int reserved_0:4; + }; + unsigned int DW_1_DATA; + } INT_CONTEXT_UNION; +} SDMA_PKT_TRAP, *PSDMA_PKT_TRAP; + + +#endif // __SDMA_PKT_STRUCT_H__ diff --git a/tests/kfdtest/kfdtest.exclude b/tests/kfdtest/kfdtest.exclude new file mode 100644 index 0000000000..c2f4772fb5 --- /dev/null +++ b/tests/kfdtest/kfdtest.exclude @@ -0,0 +1,142 @@ +# Permanent exclusions +# These tests are included for debugging, but are not executed in normal execution on any ASIC +# CU Masking Linear/Split are not working correctly due to how the HW distributes work over CUs. +# They are available for testing but are not currently expected to pass on CI/VI/AI +PERMANENT_BLACKLIST_ALL_ASICS=\ +"-KFDEventTest.MeasureInterruptConsumption:"\ +"KFDQMTest.BasicCuMaskingLinear:"\ +"RDMATest.GPUDirect" + +# This is the temporary blacklist for all ASICs. This is to be used when a test is failing consistently +# on every ASIC (Kaveri, Carrizo, Hawaii, Tonga, Fiji, Polaris10, Polaris11 and Vega10 . +# TODO means that a JIRA ticket needs to be created for this issue, as no documentation regarding +# failures can be found +# NOTE: If you update this alphabetical listing, add the corresponding JIRA ticket for reference +# +# KFDGraphicsInterop.RegisterForeignDeviceMem (KFD-366) +# KFDPMTest.SuspendWith[ActiveProcess|IdleQueue|IdleQueueAfterWork] (KFD-326|327|328) +# KFDPNPTest.DisableAndCreateQueue (KFD-329) +# KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) +# KFDQMTest.mGPUShareBO (KFD-334) +TEMPORARY_BLACKLIST_ALL_ASICS=\ +"KFDGraphicsInterop.RegisterForeignDeviceMem:"\ +"KFDPMTest.SuspendWithActiveProcess:"\ +"KFDPMTest.SuspendWithIdleQueue:"\ +"KFDPMTest.SuspendWithIdleQueueAfterWork:"\ +"KFDPNPTest.DisableAndCreateQueue:"\ +"KFDQMTest.GPUDoorbellWrite:"\ +"KFDQMTest.mGPUShareBO" + +BLACKLIST_ALL_ASICS=\ +"$PERMANENT_BLACKLIST_ALL_ASICS:"\ +"$TEMPORARY_BLACKLIST_ALL_ASICS" + +# SDMA-based tests (KFDIPCTest.BasicTest, KFDQM.*Sdma*, KFDMemoryTest.MMBench) are all +# disabled on non-Hawaii due to SDMA instability - SWDEV-101666 +SDMA_BLACKLIST=\ +"KFDIPCTest.*:"\ +"KFDMemoryTest.MMBench:"\ +"KFDQMTest.*Sdma*:"\ +"KFDQMTest.P2PTest" + +# Anything involving CP queue creation is failing on Kaveri. Separate them here for convenience (KFD-336) +KV_QUEUE_BLACKLIST=\ +"KFDExceptionTest.InvalidBadAddress:"\ +"KFDExceptionTest.InvalidWriteAddress:"\ +"KFDExceptionTest.InvalidPPR*:"\ +"KFDLocalMemoryTest.*:"\ +"KFDEventTest.Signal*Event*:"\ +"KFDQMTest.CreateQueueStressSingleThreaded:"\ +"KFDQMTest.*CpQueue*:"\ +"KFDQMTest.*Dispatch*:"\ +"KFDQMTest.Atomics:"\ +"KFDQMTest.GPUDoorbellWrite" + +# KFDQMTest.CpuWriteCoherence fails. 0 dwordsAvailable (KFD-338) +# KFDMemoryTest.MemoryRegister fails on SDMA queue creation (KFD-337) +KV_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ +"$KV_QUEUE_BLACKLIST:"\ +"KFDMemoryTest.MemoryRegister:"\ +"KFDQMTest.CpuWriteCoherence" + +# KFDLocalMemoryTest.BasicTest is failing intermittently (KFD-368) +# KFDMemoryTest.BigBufferStressTest was failing intermittently on 4.9 and hangs when executed twice (KFD-312) +# KFDQMTest.GPUDoorbellWrite fails on Hawaii. Could be HW-related (KFD-342) +HI_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDLocalMemoryTest.BasicTest:"\ +"KFDMemoryTest.BigBufferStressTest:"\ +"KFDQMTest.GPUDoorbellWrite" + +CZ_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ +"KFDExceptionTest.InvalidWriteAddress" + +# KFDPerfCountersTest.*Trace fail (KFD-339) +# KFDMemoryTest.QueryPointerInfo/MemoryRegister* (KFD-341) +# The remaining tests listed here fail on map memory to GPU with a VA conflict (KFD-340) +TONGA_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ +"KFDCWSRTest.BasicTest:"\ +"KFDEventTest.*:"\ +"KFDLocalMemoryTest.*:"\ +"KFDMemoryTest.FlatScratchAccess:"\ +"KFDMemoryTest.MapMemoryToGPU:"\ +"KFDMemoryTest.MemoryRegister*:"\ +"KFDMemoryTest.QueryPointerInfo:"\ +"KFDPerfCountersTest.*:"\ +"KFDDBGTest.BasicAddressWatch:"\ +"KFDQMTest.*" + +# Fragmentation test intermittently failing on 4096 order 3 blocks. Investigating in conjunction with KFD-344 +FIJI_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDLocalMemoryTest.Fragmentation:"\ +"$SDMA_BLACKLIST" + +ELLESMERE_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST" + +# Fragmentation test failing on 4096 order 3 blocks. Investigating in conjunction with KFD-344 +BAFFIN_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDLocalMemoryTest.Fragmentation:"\ +"$SDMA_BLACKLIST" + +# KFDDBG.BasicAddressWatch fails with message "E,INTERR: Setting ASIC VI (was previously set to GFX9), this changes the backend after we have already started parsing the shader, not currently supported by sp3, sorry." (KFD-317) +# KFDExceptionTest.InvalidBadAddress causes some following tests to fail on SRIOV (KFD-343) +# KFDLocalMemoryTest.Fragmentation is still flaking out intermittently (KFD-344) +# KFDQMTest.BasicCuMaskingEven fails intermittently (WIP) +# KFDExceptionTest.InvalidWriteAddress is failing once we enable no-retry (KFD-380) +VEGA10_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDDBGTest.BasicAddressWatch:"\ +"KFDExceptionTest.InvalidBadAddress:"\ +"KFDExceptionTest.InvalidWriteAddress:"\ +"KFDQMTest.CreateQueueStressSingleThreaded:"\ +"KFDQMTest.GPUDoorbellWrite:"\ +"KFDMemoryTest.MemoryRegister:"\ +"KFDLocalMemoryTest.Fragmentation" + +VEGA20_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDDBGTest.BasicAddressWatch:"\ +"KFDExceptionTest.InvalidBadAddress:"\ +"KFDExceptionTest.InvalidWriteAddress:"\ +"KFDQMTest.CreateQueueStressSingleThreaded:"\ +"KFDQMTest.GPUDoorbellWrite:"\ +"KFDMemoryTest.MemoryRegister:"\ +"KFDLocalMemoryTest.Fragmentation" + + +RAVEN_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDDBGTest.BasicAddressWatch:"\ +"KFDQMTest.CreateQueueStressSingleThreaded:"\ +"KFDQMTest.GPUDoorbellWrite:"\ +"KFDMemoryTest.MemoryRegister" diff --git a/tests/kfdtest/sp3/README.txt b/tests/kfdtest/sp3/README.txt new file mode 100644 index 0000000000..7cbe800f50 --- /dev/null +++ b/tests/kfdtest/sp3/README.txt @@ -0,0 +1,6 @@ +Note: This folder is primarily intended for AMD internal developers. + +The folder lib_helper contains the script to generate SP3 library libamdsp3.a +and the associated header files in the current folder for kfdtest to use. +cmake is required for the script to run. Just run ./build_sp3.sh after setting +up the environment variables (source build/envsetup.sh). diff --git a/tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt b/tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt new file mode 100644 index 0000000000..673285ddb4 --- /dev/null +++ b/tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + diff --git a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt new file mode 100644 index 0000000000..bb822e7beb --- /dev/null +++ b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt @@ -0,0 +1,75 @@ +# +# Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# + +cmake_minimum_required(VERSION 2.8 FATAL_ERROR) + +project(amdsp3) + +#set ( CMAKE_VERBOSE_MAKEFILE on ) + +find_package(PkgConfig) + +set ( P4_PATH $ENV{WORK_ROOT}/p4/driver/drivers ) + +set ( SCLIB_SRC ${PROJECT_SOURCE_DIR} ) +#if( DEFINED ENV{SCLIB_SRC} ) +# set ( SCLIB_SRC $ENV{SCLIB_SRC} ) +#else() +# set ( SCLIB_SRC ${P4_PATH}/sc/Chip ) +#endif() + +include_directories(${SCLIB_SRC}/sp3) +include_directories(${SCLIB_SRC}/sp3/release_headers) +include_directories(${SCLIB_SRC}/sp3/gen) + +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-asic.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-dispatch.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-eval.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-gc.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-int.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-lib.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-native.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-cipher.c ) +set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-vm.c ) + +aux_source_directory(${SCLIB_SRC}/sp3/gen SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/si SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/ci SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/gfx8 SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/gfx81 SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/gfx9 SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx81 SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx9 SRC_FILES) + + +message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) +#message( STATUS "SRC_FILES: ") +#foreach(file ${SRC_FILES}) +# message(STATUS "${file}") +#endforeach() + +set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU" ) + +add_library(amdsp3 ${SRC_FILES}) + + diff --git a/tests/kfdtest/sp3/lib_helper/build_sp3.sh b/tests/kfdtest/sp3/lib_helper/build_sp3.sh new file mode 100755 index 0000000000..7cd20ccfb5 --- /dev/null +++ b/tests/kfdtest/sp3/lib_helper/build_sp3.sh @@ -0,0 +1,57 @@ +# +# Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# + +#!/bin/bash + +if [ "$KFDTEST_ROOT" == "" ] || [ "$P4_ROOT" == "" ]; then + echo "Environment variables should be set before running this script" + exit 1 +fi + +cd $KFDTEST_ROOT/sp3/lib_helper + +SP3_PROJECT=$P4_ROOT/driver/drivers/sc/Chip/ +LIB_OUTPUT=$KFDTEST_ROOT/sp3/ + +cp CMakeLists_sp3.txt $SP3_PROJECT/CMakeLists.txt + +mkdir -p build +echo "Building SP3 lib" +pushd build +cmake $SP3_PROJECT/ +make +popd + +rsync --progress -a build/libamdsp3.a $LIB_OUTPUT +# Put the intermediate header files in the current folder for further processing +rsync --progress -a $SP3_PROJECT/sp3/sp3.h . + +# Remove the build folder and CMakeLists.txt put into SP source folder +rm -r build +rm $SP3_PROJECT/CMakeLists.txt + +# Replace the license statement in the header files +{ cat AMD_opensource_license.txt; sed -e '1,/#ifndef/ { /#ifndef/b; d }' sp3.h; } > $LIB_OUTPUT/sp3.h + +# Delete the intermediate header files +rm sp3.h diff --git a/tests/kfdtest/sp3/sp3.h b/tests/kfdtest/sp3/sp3.h new file mode 100644 index 0000000000..d6235be5d8 --- /dev/null +++ b/tests/kfdtest/sp3/sp3.h @@ -0,0 +1,590 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __SP3_H__ +#define __SP3_H__ + +#ifdef __cplusplus +extern "C" { +#endif + + +/// @file sp3.h +/// @brief sp3 API +#include + +// Export tags +#define SP3_EXPORT + + +/// @defgroup sp3main SP3 Main API +/// +/// Main API to assemble and disassemble SP3 shaders. +/// +/// @{ + + +/// Valid shader stages. +enum sp3_shtype { + SP3_SHTYPE_NONE = -1, + SP3_SHTYPE_PS = 0, + SP3_SHTYPE_VS = 1, + SP3_SHTYPE_GS = 2, + SP3_SHTYPE_ES = 3, + SP3_SHTYPE_HS = 4, + SP3_SHTYPE_LS = 5, + SP3_SHTYPE_CS = 6, + SP3_SHTYPE_ACV = 7, +}; + +/// Assorted constants used by sp3 API. +enum sp3_count { + SP3_NUM_MRT = 8, ///< Maximum number of render targets supported. + SP3_NUM_STRM = 4, ///< Maximum number of streams supported. +}; + +/// Disassembly flags. Bitwise-OR flags to set options. +enum sp3_flag { + SP3DIS_NO_STATE = 0x01, ///< Do not include state header at top of shader. + SP3DIS_NO_BINARY = 0x02, ///< Do not include comments with raw binary microcode. + SP3DIS_COMMENTS = 0x04, ///< Do not include comments. + SP3DIS_NO_GPR_COUNT = 0x08, ///< Do not include GPR allocation counts. + SP3DIS_FORCEVALID = 0x10, ///< Force all bytes of microcode to be disassembled. + SP3DIS_NO_ASIC = 0x20, ///< Do not emit the asic header at top of shader. +}; + +/// Shader context. Contains no user-visible fields. +struct sp3_context; + +/// Memory object. Contains no user-visible fields. +struct sp3_vma; + +/// VM addresses are 64-bit and the address unit is 32 bits +typedef uint64_t sp3_vmaddr; + +/// Storage entry for register streams. +struct sp3_reg { + uint32_t index; ///< One of the MM aperture register addresses. + uint32_t value; ///< 32-bit register data. +}; + +/// Bits for a single instruction. +struct sp3_inst_bits { + uint32_t val[5]; ///< Largest single instruction in any backend is 5 dwords. +}; + +/// Wrapped shader metadata. +/// +/// After generation, shaders are encapsulated in sp3_shader structures. +/// +/// Those structures contain the shader binary, its register stream, constants and constant +/// buffers and metadata needed for SC compatibility. +/// +struct sp3_shader { + enum sp3_shtype type; ///< One of the SHTYPE_* constants. + uint32_t asic_int; ///< Internal ASIC index. Do not use. + char asic[0x100]; ///< ASIC name as a string ("RV870" etc). + uint32_t size; ///< Size of the compiled shader, in 32-bit words. + uint32_t nsgprs; ///< Number of scalar GPRs used. + uint32_t nvgprs; ///< Number of vector GPRs used. + uint32_t nsvgprs; ///< Number of shared vector GPRs used. + uint32_t nsgprs_manual_alloc; + uint32_t nvgprs_manual_alloc; + uint32_t nsvgprs_manual_alloc; + uint32_t trap_present; + uint32_t user_sgpr_count; + uint32_t scratch_en; + uint32_t dispatch_draw_en; + uint32_t so_en; + uint32_t so_base0_en; + uint32_t so_base1_en; + uint32_t so_base2_en; + uint32_t so_base3_en; + uint32_t oc_lds_en; + uint32_t tg_size_en; + uint32_t tidig_comp_cnt; ///< Number of components(-1) enabled for thread id in group + uint32_t tgid_x_en; + uint32_t tgid_y_en; + uint32_t tgid_z_en; + uint32_t wave_cnt_en; + uint32_t primgen_en; + uint32_t pc_base_en; + uint32_t sgpr_scratch; + uint32_t sgpr_psvs_state; + uint32_t sgpr_gs2vs_offset; + uint32_t sgpr_so_write_index; + uint32_t sgpr_so_base_offset0; + uint32_t sgpr_so_base_offset1; + uint32_t sgpr_so_base_offset2; + uint32_t sgpr_so_base_offset3; + uint32_t sgpr_offchip_lds; + uint32_t sgpr_is_offchip; + uint32_t sgpr_ring_offset; + uint32_t sgpr_gs_wave_id; + uint32_t sgpr_global_wave_id; + uint32_t sgpr_tg_size; + uint32_t sgpr_tgid_x; + uint32_t sgpr_tgid_y; + uint32_t sgpr_tgid_z; + uint32_t sgpr_tf_base; + uint32_t sgpr_pc_base; + uint32_t sgpr_wave_cnt; + uint32_t wave_size; ///< Number of threads in a wavefront (only certain ASICs; 0 = don't care). + uint32_t pc_exports; ///< Range of parameters exported (if VS). + uint32_t pos_export; ///< Shader executes a position export (if VS). + uint32_t cb_exports; ///< Range of MRTs exported (if PS). + uint32_t mrtz_export_format;///< Export format of the mrtz export. + uint32_t z_export; ///< Shader executes a Z export (if PS). + uint32_t pops_en; ///< Shader is POPS (PS) + uint32_t pops_num_samples; ///< (PS) + uint32_t load_collision_waveid; ///< Shader sets load collision waveid (if PS). + uint32_t load_intrawave_collision; ///< Shader is in intrawave mode (if PS). + uint32_t stencil_test_export; ///< Shader exports stencil (if PS). + uint32_t stencil_op_export; ///< Shader exports stencil (if PS). + uint32_t kill_used; ///< Shader executes ALU KILL operations. + uint32_t cb_masks[SP3_NUM_MRT]; ///< Component masks for each MRT exported (if PS). + uint32_t emit_used; ///< EMIT opcodes used (if GS). + uint32_t covmask_export; ///< Shader exports coverage mask (if PS). + uint32_t mask_export; ///< Shader exports mask (if PS). + uint32_t strm_used[SP3_NUM_STRM]; ///< Streamout operations used (map). + uint32_t scratch_used; ///< Scratch SMX exports used. + uint32_t scratch_itemsize; ///< Scratch ring item size. + uint32_t reduction_used; ///< Reduction SMX exports used. + uint32_t ring_used; ///< ESGS/GSVS ring SMX exports used. + uint32_t ring_itemsize; ///< ESGS/GSVS ring item size (for ES/GS respectively). + uint32_t vertex_size[4]; ///< GSVS ring vertex size (for GS). + uint32_t mem_used; ///< Raw memory SMX exports used. + uint32_t rats_used; ///< Mask of RATs (UAVs) used + uint32_t group_size[3]; ///< Wavefront group size (for ELF files). + uint32_t alloc_lds; ///< Number of LDS bytes allocated for wave group. (translates to lds_size in CS and LS) + uint32_t *data; ///< Shader binary data. + uint32_t nregs; ///< Number of register writes in the stream. + uint64_t crc64; ///< CRC64 of compiled shader, may be used for identification/fingerprinting. + uint32_t crc32; ///< 32-bit CRC of compiled shader (based on crc64), may be used for identification/fingerprinting. + struct sp3_reg *regs; ///< Register writes (index-value pairs). + struct sp3_shader *merged_2nd_shader; ///< Merged es/gs, ls/hs shader, this points to start of the second shader (only certain ASICs). +}; + +/// Comment callback. +typedef const char *(*sp3_comment_cb)(void *, int); + + +/// Get version of the sp3 library. +/// +/// @return String containing the version number. +/// +SP3_EXPORT const char *sp3_version(void); + +/// Create a new sp3 context. +/// +/// @return A new context for use in assembling and disassembling shaders. Free with sp3_close(). +/// +SP3_EXPORT struct sp3_context *sp3_new(void); + +/// Set option for sp3. +/// +/// @param state sp3 context. +/// @param option Option name. Unknown options will raise an error. +/// @param value Option value. NULL is used to represent value-less options. +/// +/// Currently supported options: +/// +/// Werror (boolean) -- indicates whether warnings should be treated as errors. +/// +/// wave_size (integer) -- sets the wave size being used by the draw calls that will be using +/// this shader. Ignored in certain ASICs. You may set this to 32, 64 or the special value 0 +/// to indicate no preference on wave size. The shader will be checked to ensure it is +/// compatible with the size specified here. +/// +/// omit_version (boolean) -- omit generation of the S_VERSION opcode. +/// +/// omit_code_end (boolean) -- omit generation of the S_CODE_END footer. +/// +SP3_EXPORT void sp3_set_option( + struct sp3_context *state, + const char *option, + const char *value); + +/// Parse a file into a context. +/// +/// Use sp3_compile to generate binary microcode after the shader is parsed. +/// +/// @param state Context to use for parsing. +/// @param file File to read. If NULL, parse from stdin. +/// +SP3_EXPORT void sp3_parse_file(struct sp3_context *state, const char *file); + +/// Parse a string into a context. +/// +/// Use sp3_compile to generate binary microcode after the shader is parsed. +/// +/// @param state Context to use for parsing. +/// @param string String to parse. +/// +SP3_EXPORT void sp3_parse_string(struct sp3_context *state, const char *string); + +/// Parse a file from the standard library into a context. +/// +/// Use sp3_compile to generate binary microcode after the shader is parsed. +/// +/// @param state Context to use for parsing. +/// @param name Path to the standard library; files in this directory are parsed. +/// +SP3_EXPORT void sp3_parse_library(struct sp3_context *state, const char *name); + +/// Call a sp3 function. +/// +SP3_EXPORT void sp3_call(struct sp3_context *state, const char *func); + +/// Compile a shader program that has been parsed into the context. +/// +/// @param state sp3 context. +/// @param cffunc Name of clause to call. By convention, this is "main". +/// @return A compiled and linked shader. Free memory with sp3_free_shader(). +/// +SP3_EXPORT struct sp3_shader *sp3_compile( + struct sp3_context *state, + const char *cffunc); + +/// Free a sp3_shader. +/// +/// @param sh Shader object to delete. +/// +SP3_EXPORT void sp3_free_shader(struct sp3_shader *sh); + +/// Get current ASIC name set for a context. +/// +/// @param state Context to query. +/// @return Name of ASIC. +/// +SP3_EXPORT const char *sp3_getasic(struct sp3_context *state); + +/// Set current ASIC name for a context. +/// +/// @param state Context to modify. +/// @param chip Case-insensitive string representing the ASIC to compile or disassemble for. +/// +SP3_EXPORT void sp3_setasic(struct sp3_context *state, const char *chip); + +/// Set global variable in context to an integer. +/// +SP3_EXPORT void sp3_set_param_int( + struct sp3_context *state, + const char *name, + int32_t value); + +/// Set global variable in context to an integer vector. +/// +SP3_EXPORT void sp3_set_param_intvec( + struct sp3_context *state, + const char *name, + uint32_t size, + const int32_t *value); + +/// Set global variable in context to a float. +/// +SP3_EXPORT void sp3_set_param_float( + struct sp3_context *state, + const char *name, + float value); + +/// Set global variable in context to a float vector. +/// +SP3_EXPORT void sp3_set_param_floatvec( + struct sp3_context *state, + const char *name, + uint32_t size, + const float *value); + +/// Set error message header. +/// +/// @param state Context to modify. +/// @param str Text to include in error message header. +/// +SP3_EXPORT void sp3_set_error_header(struct sp3_context *state, const char *str); + +/// Get ASIC metrics for the ASIC in current state. +/// +/// Used by ELF tools to fill in some CAL fields. +/// +/// @param state Context to query. +/// @param name Name of ASIC metric. +/// @return Value of ASIC metric. +/// +SP3_EXPORT int sp3_asicinfo(struct sp3_context *state, const char *name); + +/// Free a context allocated by sp3_new/open/parse. +/// +/// @param state Context to delete. +/// +SP3_EXPORT void sp3_close(struct sp3_context *state); + +/// Disassemble a shader. +/// +/// This call is likely to change to something that will take a filled sp3_shader structure +/// later on. +/// +/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). +/// @param bin Memory map with the opcodes (see sp3-vm.h). +/// @param base Start of the shader in the memory map (in VM entries, i.e. 32-bit words). +/// @param name Same to give the disassembled shader. +/// @param shader_type One of the SHTYPE_* constants. +/// @param include Literal text to include in the CF clause (NULL includes nothing). +/// @param max_len Maximum length of CF clause. Matters if SP3DIS_FORCEVALID is set. +/// @param flags A bitmask of SP3DIS_* flags. +/// +/// @return Shader disassembly as a string. Free memory with sp3_free(). +/// +SP3_EXPORT char *sp3_disasm( + struct sp3_context *state, + struct sp3_vma *bin, + sp3_vmaddr base, + const char *name, + enum sp3_shtype shader_type, + const char *include, + uint32_t max_len, + uint32_t flags); + +/// Disassemble a single shader instruction. +/// +/// This call is likely to change to something that will take a filled sp3_shader structure +/// later on. +/// +/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). +/// @param inst Pointer to dwords containing instruction (exact number of dwords required depends on instruction). +/// @param base Start of the shader in the memory map (in VM entries, i.e. 32-bit words). +/// @param addr Address of the instruction being disassembled (in VM entries, i.e. 32-bit words). +/// @param shader_type One of the SHTYPE_* constants. +/// @param flags A mask of SP3DIS_* flags. +/// +/// @return Shader disassembly as a string. Free memory with sp3_free(). +/// +SP3_EXPORT char *sp3_disasm_inst( + struct sp3_context *state, + const struct sp3_inst_bits *inst, + sp3_vmaddr base, + sp3_vmaddr addr, + enum sp3_shtype shader_type, + uint32_t flags); + +/// Parse a register stream. +/// +/// Can be called before sp3_disasm to preset things like ALU, boolean and loop constants. +/// +/// This call is likely to merge with sp3_disasm later on. +/// +/// @param state sp3 context to fill with state. +/// @param nregs Number of register entries. +/// @param regs Register stream to parse. +/// @param shader_type One of the SHTYPE_* constants. +/// +SP3_EXPORT void sp3_setregs( + struct sp3_context *state, + uint32_t nregs, + const struct sp3_reg *regs, + enum sp3_shtype shader_type); + + +/// Set shader comments +/// +/// @param state sp3 context. +/// @param map Map of comments (0 for no comment, other values will be passed to the callback). +/// @param f_top Callback returning comment to place above the opcode. +/// @param f_right Callback returning comment to place to the right of the opcode. +/// @param ctx Void pointer to pass to comment callbacks. +/// +SP3_EXPORT void sp3_setcomments( + struct sp3_context *state, + struct sp3_vma *map, + sp3_comment_cb f_top, + sp3_comment_cb f_right, + void *ctx); + +/// Set alternate shader entry points +/// +/// Used for disassembly; this marks an additional location in memory +/// (besides the start address) where shader code may be found. Generally +/// required for jump tables and any case where the shader may perform +/// indirect jumps to ensure that disassembly locates all shader +/// instructions. +/// +/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). +/// @param addr Address of the instruction being disassembled (in VM entries, i.e. 32-bit words). +/// +SP3_EXPORT void sp3_setentrypoint( + struct sp3_context *state, + sp3_vmaddr addr); + +/// Clear alternate shader entry points. +/// +/// Clear all entry points previously set with sp3_setentrypoint. +/// +/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). +/// +SP3_EXPORT void sp3_clearentrypoints(struct sp3_context *state); + +/// Free memory allocated by sp3. +/// +/// Windows DLLs that allocate memory have to free it. This function +/// should be used to free the result of sp3_disasm, sp3_compile etc. +/// +SP3_EXPORT void sp3_free(void *ptr); + +/// SP3 API to merge two shaders given file names as input. +/// +SP3_EXPORT struct sp3_shader* sp3_merge_shaders( + struct sp3_context *pointer, + const char *first_file, + const char *second_file); + +/// SP3 API to merge two shaders given shader strings as input. +/// +SP3_EXPORT struct sp3_shader* sp3_merge_shader_strings( + struct sp3_context *pointer, + const char *first_string, + const char *second_string); + + +/// @} + + +/// @defgroup sp3vm SP3 Memory Objects +/// +/// The VM API is used to manage virtual memory maps. Those maps are used for binary storage +/// for disassembly, as they can naturally mirror the GPU's memory map (so no register +/// translation is needed). +/// +/// @{ + +/// Callback function that will fill a VMA on demand +/// +/// The VMA to be filled will be specified through the request address. +/// The callback should fill the VMA using sp3_vm_write calls. +/// +typedef void (* sp3_vmfill)(struct sp3_vma *vm, sp3_vmaddr addr, void *ctx); + +/// Create a new VM that is empty. +/// +/// Free the object with sp3_vm_free(). +/// +/// @return New VM object. +/// +SP3_EXPORT +struct sp3_vma *sp3_vm_new(void); + +/// Create a new VM that has a sp3_vmfill callback. +/// +/// Free the object with sp3_vm_free(). +/// +/// @param fill Function used to populate data in VM. The function will be pass the new VM object, the address and a context. +/// @param ctx User-specified context. Passed to the fill function and not used by sp3 itself. +/// @return New VM object. +/// +SP3_EXPORT +struct sp3_vma *sp3_vm_new_fill(sp3_vmfill fill, void *ctx); + +/// Create a new VM from an array of words. +/// +/// Free the object with sp3_vm_free(). +/// +/// @param base VM address to load array at. +/// @param len Number of 32-bit words in the array. +/// @param data Pointer to the array. +/// @return New VM object. +/// +SP3_EXPORT +struct sp3_vma *sp3_vm_new_ptr(sp3_vmaddr base, sp3_vmaddr len, const uint32_t *data); + +/// Find a VMA, optionally adding it. +/// +/// @param vm VM to search in. +/// @param addr Address to search for. +/// @param add Flag indicating whether a failure should result in adding a new VMA. +/// @return VM object matching the specified address. +/// +SP3_EXPORT +struct sp3_vma *sp3_vm_find(struct sp3_vma *vm, sp3_vmaddr addr, uint32_t add); + +/// Write a word to a VM. +/// +/// @param vm VM to write. +/// @param addr Address to write. +/// @param val 32-bits of data to write. +/// +SP3_EXPORT +void sp3_vm_write(struct sp3_vma *vm, sp3_vmaddr addr, uint32_t val); + +/// Read a word from a VM. +/// +/// @param vm VM to read. +/// @param addr Address to read. +/// @return 32-bits of data at specified address. +/// +SP3_EXPORT +uint32_t sp3_vm_read(struct sp3_vma *vm, sp3_vmaddr addr); + +/// Probe VM for presence. +/// +/// @param vm VM to probe. +/// @param addr Address to search for. +/// @return 1 if the specified address is backed in the VM, 0 otherwise. +/// +SP3_EXPORT +int sp3_vm_present(struct sp3_vma *vm, sp3_vmaddr addr); + +/// Return base address of VM. +/// +/// @param vm VM to query. +/// @return Base address. +/// +SP3_EXPORT +sp3_vmaddr sp3_vm_base(struct sp3_vma *vm); + +/// Return next VM. +/// +/// @param vm VM to query. +/// @return Next VM in list. +/// +SP3_EXPORT +struct sp3_vma *sp3_vm_next(struct sp3_vma *vm); + +/// Free a VM and all its storage. +/// +/// Use this function to free memory allocated by sp3_vm_new, sp3_vm_new_fill and +/// sp3_vm_new_ptr. +/// +/// @param vm VM to free. +/// +SP3_EXPORT +void sp3_vm_free(struct sp3_vma *vm); + + +/// @} + + +#ifdef __cplusplus +} +#endif + + +#endif /* __SP3_H__ */ diff --git a/tests/kfdtest/src/AqlQueue.cpp b/tests/kfdtest/src/AqlQueue.cpp new file mode 100644 index 0000000000..d8e9b43ab6 --- /dev/null +++ b/tests/kfdtest/src/AqlQueue.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "AqlQueue.hpp" +#include "GoogleTestExtension.hpp" + + +AqlQueue::AqlQueue(void) { +} + + +AqlQueue::~AqlQueue(void) { +} + +unsigned int AqlQueue::Wptr() { + return *m_Resources.Queue_write_ptr; +} + +unsigned int AqlQueue::Rptr() { + return *m_Resources.Queue_read_ptr; +} + +unsigned int AqlQueue::RptrWhenConsumed() { + return Wptr(); +} + +void AqlQueue::SubmitPacket() { + // m_pending Wptr is in DWORDs + *m_Resources.Queue_write_ptr = m_pendingWptr; + *(m_Resources.Queue_DoorBell) = Wptr(); +} + diff --git a/tests/kfdtest/src/AqlQueue.hpp b/tests/kfdtest/src/AqlQueue.hpp new file mode 100644 index 0000000000..45395f70eb --- /dev/null +++ b/tests/kfdtest/src/AqlQueue.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_AQL_QUEUE__H__ +#define __KFD_AQL_QUEUE__H__ + +#include "BaseQueue.hpp" + +class AqlQueue : public BaseQueue { + public: + AqlQueue(); + virtual ~AqlQueue(); + + // @brief update queue write pointer and sets the queue doorbell to the queue write pointer + virtual void SubmitPacket(); + + // @ return read pointer are in DWORDs + virtual unsigned int Rptr(); + // @ return write pointer are in DWORDs + virtual unsigned int Wptr(); + // @ return expected m_Resources.Queue_read_ptr when all packets consumed + virtual unsigned int RptrWhenConsumed(); + + protected: + virtual PACKETTYPE PacketTypeSupported() { return PACKETTYPE_AQL; } + + virtual _HSA_QUEUE_TYPE GetQueueType() { return HSA_QUEUE_COMPUTE_AQL; } +}; + +#endif diff --git a/tests/kfdtest/src/BasePacket.cpp b/tests/kfdtest/src/BasePacket.cpp new file mode 100644 index 0000000000..fd714ca756 --- /dev/null +++ b/tests/kfdtest/src/BasePacket.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2017-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "BasePacket.hpp" +#include "KFDTestUtil.hpp" + +void BasePacket::Dump() const { + unsigned int size = SizeInDWords(); + const HSAuint32 *packet = (const HSAuint32 *)GetPacket(); + std::ostream &log = LOG(); + unsigned int i; + + log << "Packet dump:" << std::hex; + for (i = 0; i < size; i++) + log << " " << std::setw(8) << std::setfill('0') << packet[i]; + log << std::endl; +} diff --git a/tests/kfdtest/src/BasePacket.hpp b/tests/kfdtest/src/BasePacket.hpp new file mode 100644 index 0000000000..b2cc6e7da6 --- /dev/null +++ b/tests/kfdtest/src/BasePacket.hpp @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_BASE_PACKET__H__ +#define __KFD_BASE_PACKET__H__ + +/** + * all packets profiles must be defined here + * every type defined here has sub-types + */ +enum PACKETTYPE { + PACKETTYPE_PM4, + PACKETTYPE_SDMA, + PACKETTYPE_AQL +}; + +// @class BasePacket +class BasePacket { + public: + BasePacket(void) {} + virtual ~BasePacket(void) {} + + // @returns the packet type + virtual PACKETTYPE PacketType() const = 0; + // @returns a pointer to the packet + virtual const void *GetPacket() const = 0; + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const = 0; + // @returns the packet size in DWORDS + unsigned int SizeInDWords() const { return SizeInBytes()/sizeof(unsigned int); } + + void Dump() const; + + protected: +}; + +#endif diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp new file mode 100644 index 0000000000..f64a02fc9f --- /dev/null +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "BaseQueue.hpp" +#include "SDMAQueue.hpp" +#include "PM4Queue.hpp" +#include "AqlQueue.hpp" +#include "hsakmt.h" + +BaseQueue::BaseQueue() + :m_Type(HSA_QUEUE_TYPE_SIZE), + m_QueueBuf(NULL), + m_SkipWaitConsumption(true) { +} + +BaseQueue::~BaseQueue(void) { + Destroy(); +} + +HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint64 *pointers) { + HSAKMT_STATUS status; + + if (m_QueueBuf != NULL) { + // queue already exist, one queue per object + Destroy(); + } + + memset(&m_Resources, 0, sizeof(m_Resources)); + + m_Type = GetQueueType(); + m_QueueBuf = new HsaMemoryBuffer(size, NodeId, true/*zero*/, false/*local*/, true/*exec*/); + + if (m_Type == HSA_QUEUE_COMPUTE_AQL) { + m_Resources.Queue_read_ptr_aql = &pointers[0]; + m_Resources.Queue_write_ptr_aql = &pointers[1]; + } + + status = hsaKmtCreateQueue(NodeId, + m_Type, + DEFAULT_QUEUE_PERCENTAGE, + DEFAULT_PRIORITY, + m_QueueBuf->As(), + m_QueueBuf->Size(), + NULL, + &m_Resources); + + if (status != HSAKMT_STATUS_SUCCESS) { + return status; + } + + if (m_Resources.Queue_read_ptr == NULL) { + WARN() << "CreateQueue: read pointer value should be 0" << std::endl; + status = HSAKMT_STATUS_ERROR; + } + + if (m_Resources.Queue_write_ptr == NULL) { + WARN() << "CreateQueue: write pointer value should be 0" << std::endl; + status = HSAKMT_STATUS_ERROR; + } + + // needs to match the queue write ptr + m_pendingWptr = 0; + m_pendingWptr64 = 0; + m_Node = NodeId; + return status; +} + +HSAKMT_STATUS BaseQueue::Update(unsigned int percent, HSA_QUEUE_PRIORITY priority, bool nullifyBuffer) { + void* pNewBuffer = (nullifyBuffer ? NULL : m_QueueBuf->As()); + HSAuint64 newSize = (nullifyBuffer ? 0 : m_QueueBuf->Size()); + + return hsaKmtUpdateQueue(m_Resources.QueueId, percent, priority, pNewBuffer, newSize, NULL); +} + +HSAKMT_STATUS BaseQueue::SetCUMask(unsigned int *mask, unsigned int mask_count) { + return hsaKmtSetQueueCUMask(m_Resources.QueueId, mask_count, mask); +} + +HSAKMT_STATUS BaseQueue::Destroy() { + HSAKMT_STATUS status = HSAKMT_STATUS_SUCCESS; + + if (m_QueueBuf != NULL) { + status = hsaKmtDestroyQueue(m_Resources.QueueId); + + if (status == HSAKMT_STATUS_SUCCESS) { + delete m_QueueBuf; + m_QueueBuf = NULL; + } + } + + return status; +} + +void BaseQueue::PlaceAndSubmitPacket(const BasePacket &packet) { + PlacePacket(packet); + SubmitPacket(); +} + +void BaseQueue::Wait4PacketConsumption() { + ASSERT_TRUE(WaitOnValue(m_Resources.Queue_read_ptr, RptrWhenConsumed())); +} + +bool BaseQueue::AllPacketsSubmitted() { + return Wptr() == Rptr(); +} + +void BaseQueue::PlacePacket(const BasePacket &packet) { + ASSERT_EQ(packet.PacketType(), PacketTypeSupported()) << "Cannot add a packet since packet type doesn't match queue"; + + unsigned int readPtr = Rptr(); + unsigned int writePtr = m_pendingWptr; + HSAuint64 writePtr64 = m_pendingWptr64; + + unsigned int packetSizeInDwords = packet.SizeInDWords(); + unsigned int dwordsRequired = packetSizeInDwords; + unsigned int queueSizeInDWord = m_QueueBuf->Size() / sizeof(uint32_t); + + if (writePtr + packetSizeInDwords > queueSizeInDWord) { + // wraparound expected. We need enough room to also place NOPs to avoid crossing the buffer end. + dwordsRequired += queueSizeInDWord - writePtr; + } + + unsigned int dwordsAvailable = (readPtr - 1 - writePtr + queueSizeInDWord) % queueSizeInDWord; + ASSERT_GE(dwordsAvailable, dwordsRequired) << "Cannot add a packet, buffer overrun"; + + ASSERT_GE(queueSizeInDWord, packetSizeInDwords) << "Cannot add a packet, packet size too large"; + + if (writePtr + packetSizeInDwords >= queueSizeInDWord) { + // wraparound + while (writePtr + packetSizeInDwords > queueSizeInDWord) { + m_QueueBuf->As()[writePtr] = CMD_NOP; + writePtr = (writePtr + 1) % queueSizeInDWord; + writePtr64++; + } + + // not updating Wptr since we might want to do place packet without submission + m_pendingWptr = (writePtr % queueSizeInDWord); + m_pendingWptr64 = writePtr64; + } + + memcpy(m_pendingWptr + m_QueueBuf->As(), packet.GetPacket(), packetSizeInDwords * 4); + + m_pendingWptr = (m_pendingWptr + packetSizeInDwords) % queueSizeInDWord; + m_pendingWptr64 += packetSizeInDwords; +} + +BaseQueue* QueueArray::GetQueue(unsigned int Node) { + // If a queue exists for that node then return, else create one + for (unsigned int i = 0; i < m_QueueList.size(); i++) { + if (Node == m_QueueList.at(i)->GetNodeId()) + return m_QueueList.at(i); + } + + BaseQueue *pQueue = NULL; + + switch (m_QueueType) { + case HSA_QUEUE_COMPUTE: + pQueue = new PM4Queue(); + break; + case HSA_QUEUE_SDMA: + pQueue = new SDMAQueue(); + break; + case HSA_QUEUE_COMPUTE_AQL: + pQueue = new AqlQueue(); + break; + } + + if (pQueue) { + pQueue->Create(Node); + m_QueueList.push_back(pQueue); + } + return pQueue; +} + +void QueueArray::Destroy() { + for (unsigned int i = 0; i < m_QueueList.size(); i++) + delete m_QueueList.at(i); + + m_QueueList.clear(); +} diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp new file mode 100644 index 0000000000..730807f97e --- /dev/null +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_BASE_QUEUE__H__ +#define __KFD_BASE_QUEUE__H__ + +#include +#include "KFDTestUtil.hpp" +#include "BasePacket.hpp" + +// @class BasePacket +class BaseQueue { + public: + static const unsigned int DEFAULT_QUEUE_SIZE = PAGE_SIZE; + static const HSA_QUEUE_PRIORITY DEFAULT_PRIORITY = HSA_QUEUE_PRIORITY_NORMAL; + static const unsigned int DEFAULT_QUEUE_PERCENTAGE = 100; + static const unsigned int ZERO_QUEUE_PERCENTAGE = 0; + static const unsigned int FLUSH_GPU_CACHES_TO = 1000; + + BaseQueue(void); + virtual ~BaseQueue(void); + + /** Create the queue. + * @see hsaKmtCreateQueue + * @param pointers is used only for creating AQL queues. Otherwise it is omitted. + */ + virtual HSAKMT_STATUS Create(unsigned int NodeId, unsigned int size = DEFAULT_QUEUE_SIZE, + HSAuint64 *pointers = NULL); + /** Update the queue. + * @see hsaKmtUpdateQueue + * @param percent the new queue percentage + * @param priority the new queue priority + * @param nullifyBuffer + * if 'true', set the new buffer address to NULL and the size to 0. Otherwise + * don't change the queue buffer address/size. + */ + virtual HSAKMT_STATUS Update(unsigned int percent, HSA_QUEUE_PRIORITY priority, bool nullifyBuffer); + virtual HSAKMT_STATUS SetCUMask(unsigned int *mask, unsigned int mask_count); + /** Destroy the queue. + * @see hsaKmtDestroyQueue + */ + virtual HSAKMT_STATUS Destroy(); + /** Wait for all the packets submitted to the queue to be consumed. (i.e. wait until RPTR=WPTR). + * Note that all packets being consumed is not the same as all packets being processed. + */ + virtual void Wait4PacketConsumption(); + /** @brief place packet and submit it in one go + */ + virtual void PlaceAndSubmitPacket(const BasePacket &packet); + /** @brief copy packet to queue and update write pointer + */ + virtual void PlacePacket(const BasePacket &packet); + /** @brief update queue write pointer and sets the queue doorbell to the queue write pointer + */ + virtual void SubmitPacket() = 0; + /** @brief checkes if all packets in queue already processed + * compares queue read and write pointers + */ + bool AllPacketsSubmitted(); + + void SetSkipWaitConsump(int val) { m_SkipWaitConsumption = val; } + int GetSkipWaitConsump() { return m_SkipWaitConsumption; } + int Size() { return m_QueueBuf->Size(); } + + HsaQueueResource *GetResource() { return &m_Resources; } + unsigned int GetPendingWptr() { return m_pendingWptr; } + HSAuint64 GetPendingWptr64() { return m_pendingWptr64; } + virtual _HSA_QUEUE_TYPE GetQueueType() = 0; + unsigned int GetNodeId() { return m_Node; } + + protected: + static const unsigned int CMD_NOP_TYPE_2 = 0x80000000; + static const unsigned int CMD_NOP_TYPE_3 = 0xFFFF1002; + + unsigned int CMD_NOP; + unsigned int m_pendingWptr; + HSAuint64 m_pendingWptr64; + HSA_QUEUE_TYPE m_Type; + HsaQueueResource m_Resources; + HsaMemoryBuffer *m_QueueBuf; + unsigned int m_Node; + + // @ return write pointer modulo queue size in DWORDs + virtual unsigned int Wptr() = 0; + // @ return read pointer modulo queue size in DWORDs + virtual unsigned int Rptr() = 0; + // @ return expected m_Resources.Queue_read_ptr when all packets consumed + virtual unsigned int RptrWhenConsumed() = 0; + virtual PACKETTYPE PacketTypeSupported() = 0; + + private: + // Some tests(such as exception) may not need wait pm4 packet consumption on CZ. + int m_SkipWaitConsumption; +}; + + +// @class QueueArray +// Managed QueueArray for different GPU Nodes +class QueueArray { + // List of Queues. One for each GPU + std::vector m_QueueList; + _HSA_QUEUE_TYPE m_QueueType; + + public: + QueueArray(_HSA_QUEUE_TYPE type): m_QueueType(type) {} + ~QueueArray() { + Destroy(); + } + + BaseQueue* GetQueue(unsigned int Node); + void Destroy(); +}; + +#endif diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp new file mode 100644 index 0000000000..d3c50f7e60 --- /dev/null +++ b/tests/kfdtest/src/Dispatch.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "Dispatch.hpp" + +#include "PM4Packet.hpp" + +#include "asic_reg/gfx_7_2_d.h" +#include "asic_reg/gfx_7_2_sh_mask.h" + +Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset) + :m_IsaBuf(isaBuf), m_IndirectBuf(PACKETTYPE_PM4, PAGE_SIZE / sizeof(unsigned int), isaBuf.Node()), + m_DimX(1), m_DimY(1), m_DimZ(1), m_pArg1(NULL), m_pArg2(NULL), m_pEop(NULL), m_ScratchEn(false), + m_ComputeTmpringSize(0), m_scratch_base(0ll), m_SpiPriority(0) { + HsaEventDescriptor eventDesc; + eventDesc.EventType = HSA_EVENTTYPE_SIGNAL; + eventDesc.NodeId = isaBuf.Node(); + eventDesc.SyncVar.SyncVar.UserData = NULL; + eventDesc.SyncVar.SyncVarSize = 0; + + hsaKmtCreateEvent(&eventDesc, !eventAutoReset, false, &m_pEop); +} + +Dispatch::~Dispatch() { + if (m_pEop != NULL) + hsaKmtDestroyEvent(m_pEop); +} + +void Dispatch::SetArgs(void* pArg1, void* pArg2) { + m_pArg1 = pArg1; + m_pArg2 = pArg2; +} + +void Dispatch::SetDim(unsigned int x, unsigned int y, unsigned int z) { + m_DimX = x; + m_DimY = y; + m_DimZ = z; +} + +void Dispatch::SetScratch(int numWaves, int waveSize, unsigned long long scratch_base) { + m_ComputeTmpringSize = ((waveSize << 12) | (numWaves)); + m_ScratchEn = true; + m_scratch_base = scratch_base; +} + +void Dispatch::SetSpiPriority(unsigned int priority) { + m_SpiPriority = priority; +} + +void Dispatch::Submit(BaseQueue& queue) { + ASSERT_NE(m_pEop, (void*)0); + + BuildIb(); + + queue.PlaceAndSubmitPacket(PM4IndirectBufPacket(&m_IndirectBuf)); + + // Write data to SyncVar for synchronization purpose + if (m_pEop->EventData.EventData.SyncVar.SyncVar.UserData != NULL) { + queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int*)m_pEop-> + EventData.EventData.SyncVar.SyncVar.UserData, m_pEop->EventId)); + } + + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, m_pEop->EventData.HWData2, m_pEop->EventId)); + + if (!queue.GetSkipWaitConsump()) + queue.Wait4PacketConsumption(); +} + +void Dispatch::Sync(unsigned int timeout) { + ASSERT_SUCCESS(hsaKmtWaitOnEvent(m_pEop, timeout)); +} + +// returning with status in order to allow to take actions before proc termination +int Dispatch::SyncWithStatus(unsigned int timeout) { + int stat; + + return ((stat = hsaKmtWaitOnEvent(m_pEop, timeout)) != HSAKMT_STATUS_SUCCESS); +} + +void Dispatch::BuildIb() { + unsigned long long shiftedIsaAddr = m_IsaBuf.As() >> 8; + unsigned int arg0, arg1, arg2, arg3; + SplitU64(reinterpret_cast(m_pArg1), arg0, arg1); + SplitU64(reinterpret_cast(m_pArg2), arg2, arg3); + + // starts at COMPUTE_START_X + const unsigned int COMPUTE_DISPATCH_DIMS_VALUES[] = { + 0, // START_X + 0, // START_Y + 0, // START_Z + 1, // NUM_THREADS_X - this is actually the number of threads in a thread group + 1, // NUM_THREADS_Y + 1, // NUM_THREADS_Z + 0, // COMPUTE_PIPELINESTAT_ENABLE + 0, // COMPUTE_PERFCOUNT_ENABLE + }; + + unsigned int pgmRsrc2 = 0; + pgmRsrc2 |= (m_ScratchEn << COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT) + & COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK; + pgmRsrc2 |= ((m_scratch_base ? 6 : 4 ) << COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT) + & COMPUTE_PGM_RSRC2__USER_SGPR_MASK; + pgmRsrc2 |= (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT) + & COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK; + pgmRsrc2 |= (1 << COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT) + & COMPUTE_PGM_RSRC2__TGID_X_EN_MASK; + pgmRsrc2 |= (1 << COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT) + & COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK; + pgmRsrc2 |= (0 << COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT) + & COMPUTE_PGM_RSRC2__EXCP_EN_MASK; + pgmRsrc2 |= (1 << COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT) + & COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK; + + const unsigned int COMPUTE_PGM_RSRC[] = { + 0x000c0084 | ((m_SpiPriority & 3) << 10), // PGM_RSRC1 = { VGPRS: 16 SGPRS: 16 PRIORITY: m_SpiPriority FLOAT_MODE: c0 PRIV: 0 DX10_CLAMP: 0 DEBUG_MODE: 0 IEEE_MODE: 0 BULKY: 0 CDBG_USER: 0 } + pgmRsrc2 + }; + + // starts at COMPUTE_PGM_LO + const unsigned int COMPUTE_PGM_VALUES_GFX8[] = { + static_cast(shiftedIsaAddr), // PGM_LO + static_cast(shiftedIsaAddr >> 32) // PGM_HI + | (is_dgpu() ? 0 : (1<<8)) // including PGM_ATC=? + }; + + // starts at COMPUTE_PGM_LO + const unsigned int COMPUTE_PGM_VALUES_GFX9[] = { + static_cast(shiftedIsaAddr), // PGM_LO + static_cast(shiftedIsaAddr >> 32) // PGM_HI + | (is_dgpu() ? 0 : (1<<8)), // including PGM_ATC=? + 0, + 0, + static_cast(m_scratch_base >> 8), // compute_dispatch_scratch_base + static_cast(m_scratch_base >> 40) + }; + + // starts at COMPUTE_RESOURCE_LIMITS + const unsigned int COMPUTE_RESOURCE_LIMITS[] = { + 0, // COMPUTE_RESOURCE_LIMITS + }; + + // starts at COMPUTE_TMPRING_SIZE + const unsigned int COMPUTE_TMPRING_SIZE[] = { + m_ComputeTmpringSize, // COMPUTE_TMPRING_SIZE + }; + + // starts at COMPUTE_RESTART_X + const unsigned int COMPUTE_RESTART_VALUES[] = { + 0, // COMPUTE_RESTART_X + 0, // COMPUTE_RESTART_Y + 0, // COMPUTE_RESTART_Z + 0 // COMPUTE_THREAD_TRACE_ENABLE + }; + + // starts at COMPUTE_USER_DATA_0 + const unsigned int COMPUTE_USER_DATA_VALUES[] = { + // Reg name - use in KFDtest - use in ABI + arg0, // COMPUTE_USER_DATA_0 - arg0 - resource descriptor for the scratch buffer - 1st DWORD + arg1, // COMPUTE_USER_DATA_1 - arg1 - resource descriptor for the scratch buffer - 2nd DWORD + arg2, // COMPUTE_USER_DATA_2 - arg2 - resource descriptor for the scratch buffer - 3rd DWORD + arg3, // COMPUTE_USER_DATA_3 - arg3 - resource descriptor for the scratch buffer - 4th DWORD + static_cast(m_scratch_base), // COMPUTE_USER_DATA_4 - flat_scratch_lo + static_cast(m_scratch_base >> 32), // COMPUTE_USER_DATA_4 - flat_scratch_hi + 0, // COMPUTE_USER_DATA_6 - - AQL queue address, low part + 0, // COMPUTE_USER_DATA_7 - - AQL queue address, high part + 0, // COMPUTE_USER_DATA_8 - - kernel arguments block, low part + 0, // COMPUTE_USER_DATA_9 - - kernel arguments block, high part + 0, // COMPUTE_USER_DATA_10 - - unused + 0, // COMPUTE_USER_DATA_11 - - unused + 0, // COMPUTE_USER_DATA_12 - - unused + 0, // COMPUTE_USER_DATA_13 - - unused + 0, // COMPUTE_USER_DATA_14 - - unused + 0, // COMPUTE_USER_DATA_15 - - unused + }; + + const unsigned int DISPATCH_INIT_VALUE = 0x00000021 | (is_dgpu() ? 0 : 0x1000); + // {COMPUTE_SHADER_EN=1, PARTIAL_TG_EN=0, FORCE_START_AT_000=0, ORDERED_APPEND_ENBL=0, + // ORDERED_APPEND_MODE=0, USE_THREAD_DIMENSIONS=1, ORDER_MODE=0, DISPATCH_CACHE_CNTL=0, + // SCALAR_L1_INV_VOL=0, VECTOR_L1_INV_VOL=0, DATA_ATC=?, RESTORE=0} + + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket()); + + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_START_X, COMPUTE_DISPATCH_DIMS_VALUES, ARRAY_SIZE(COMPUTE_DISPATCH_DIMS_VALUES))); + + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO, + (g_TestGPUFamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8, + (g_TestGPUFamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC, ARRAY_SIZE(COMPUTE_PGM_RSRC))); + + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_RESOURCE_LIMITS, COMPUTE_RESOURCE_LIMITS, ARRAY_SIZE(COMPUTE_RESOURCE_LIMITS))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_TMPRING_SIZE, COMPUTE_TMPRING_SIZE, ARRAY_SIZE(COMPUTE_TMPRING_SIZE))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_RESTART_X, COMPUTE_RESTART_VALUES, ARRAY_SIZE(COMPUTE_RESTART_VALUES))); + + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_USER_DATA_0, COMPUTE_USER_DATA_VALUES, ARRAY_SIZE(COMPUTE_USER_DATA_VALUES))); + + m_IndirectBuf.AddPacket(PM4DispatchDirectPacket(m_DimX, m_DimY, m_DimZ, DISPATCH_INIT_VALUE)); + + m_IndirectBuf.AddPacket(PM4PartialFlushPacket()); + + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket()); +} diff --git a/tests/kfdtest/src/Dispatch.hpp b/tests/kfdtest/src/Dispatch.hpp new file mode 100644 index 0000000000..36983554fb --- /dev/null +++ b/tests/kfdtest/src/Dispatch.hpp @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_DISPATCH__H__ +#define __KFD_DISPATCH__H__ +#include "KFDTestUtil.hpp" +#include "IndirectBuffer.hpp" +#include "BaseQueue.hpp" + +class Dispatch { + public: + Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset = false); + ~Dispatch(); + + void SetArgs(void* pArg1, void* pArg2); + + void SetDim(unsigned int x, unsigned int y, unsigned int z); + + void Submit(BaseQueue& queue); + + void Sync(unsigned int timeout = HSA_EVENTTIMEOUT_INFINITE); + + int SyncWithStatus(unsigned int timeout); + + void SetScratch(int numWaves, int waveSize, unsigned long long scratch_base); + + void SetSpiPriority(unsigned int priority); + + HsaEvent *GetHsaEvent() { return m_pEop; } + + private: + void BuildIb(); + + private: + const HsaMemoryBuffer& m_IsaBuf; + + IndirectBuffer m_IndirectBuf; + + unsigned int m_DimX; + unsigned int m_DimY; + unsigned int m_DimZ; + + void* m_pArg1; + void* m_pArg2; + + HsaEvent* m_pEop; + + bool m_ScratchEn; + unsigned int m_ComputeTmpringSize; + + unsigned long long m_scratch_base; + unsigned int m_SpiPriority; +}; + +#endif // __KFD_DISPATCH__H__ diff --git a/tests/kfdtest/src/GoogleTestExtension.cpp b/tests/kfdtest/src/GoogleTestExtension.cpp new file mode 100644 index 0000000000..9e510e11e2 --- /dev/null +++ b/tests/kfdtest/src/GoogleTestExtension.cpp @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "GoogleTestExtension.hpp" +#include "OSWrapper.hpp" + +bool Ok2Run(unsigned int testProfile) { + bool testMatchProfile = true; + if ((testProfile & g_TestRunProfile) == 0) { + // display msg to notify a test that is not running + WARN() << "Test is skipped beacuse profile does not match current run mode" << std::endl; + testMatchProfile = false; + } + + return testMatchProfile; +} + +// This predication is used when specific HW capabilites must exist for the test to succeed. +bool TestReqEnvCaps(unsigned int envCaps) { + bool testMatchEnv = true; + if ((envCaps & g_TestENVCaps) != envCaps) { + // display msg to notify a test that is not running + WARN() << "Test is skipped due to HW capability issues" << std::endl; + testMatchEnv = false; + } + + return testMatchEnv; +} + +// This predication is used when specific HW capabilites must abscent for the test to succeed. +// e.g testing capabilites not supported by HW scheduling +bool TestReqNoEnvCaps(unsigned int envCaps) { + bool testMatchEnv = true; + if ((envCaps & g_TestENVCaps) != 0) { + // display msg to notify a test that is not running + WARN() << "Test is skipped due to HW capability issues" << std::endl; + testMatchEnv = false; + } + + return testMatchEnv; +} + +std::ostream& operator<< (KFDLog log, LOGTYPE level) { + const char *heading; + + if (level == LOGTYPE_WARNING) { + SetConsoleTextColor(TEXTCOLOR_YELLOW); + heading = "[----------] "; + } else { + SetConsoleTextColor(TEXTCOLOR_GREEN); + heading = "[ ] "; + } + + std::clog << heading; + SetConsoleTextColor(TEXTCOLOR_WHITE); + + return std::clog; +} + diff --git a/tests/kfdtest/src/GoogleTestExtension.hpp b/tests/kfdtest/src/GoogleTestExtension.hpp new file mode 100644 index 0000000000..5633a156bf --- /dev/null +++ b/tests/kfdtest/src/GoogleTestExtension.hpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __GOOGLETEST_EXTENSION__H__ +#define __GOOGLETEST_EXTENSION__H__ + +#include +#include "hsakmt.h" +#include "KFDTestFlags.hpp" + +enum LOGTYPE { + LOGTYPE_INFO, // msg header in green + LOGTYPE_WARNING // msg header in yellow +}; + +class KFDLog{}; +std::ostream& operator << (KFDLog log ,LOGTYPE level); + +// @brief log additional details, to be displayed in the same format as other google test outputs +// currently not supported by google test +// should be used like cout: LOG() << "message" << value << std::endl; +#define LOG() KFDLog() << LOGTYPE_INFO +#define WARN() KFDLog() << LOGTYPE_WARNING + +// all test MUST be in a try catch since google test flag to throw exception on any fatal fail is on +#define TEST_START(testProfile) if (Ok2Run(testProfile)) try { +#define TEST_END } catch (...) {} + +// used to wrape setup and teardown functions, anything that is build-in gtest and is not a test +#define ROUTINE_START try { +#define ROUTINE_END }catch(...) {} + +#define TEST_REQUIRE_ENV_CAPABILITIES(envCaps) if (!TestReqEnvCaps(envCaps)) return; +#define TEST_REQUIRE_NO_ENV_CAPABILITIES(envCaps) if (!TestReqNoEnvCaps(envCaps)) return; + +#define ASSERT_SUCCESS(_val) ASSERT_EQ(HSAKMT_STATUS_SUCCESS, (_val)) +#define EXPECT_SUCCESS(_val) EXPECT_EQ(HSAKMT_STATUS_SUCCESS, (_val)) + +#define ASSERT_NOTNULL(_val) ASSERT_NE((void *)NULL, _val) +#define EXPECT_NOTNULL(_val) EXPECT_NE((void *)NULL, _val) + +// @brief determines if its ok to run a test given input flags +bool Ok2Run(unsigned int testProfile); + +// @brief checks if all HW capabilities needed for a test to run exist +bool TestReqEnvCaps(unsigned int hwCaps); + +// @brief checks if all HW capabilities that prevents a test from running are non existing +bool TestReqNoEnvCaps(unsigned int hwCaps); + +#endif diff --git a/tests/kfdtest/src/IndirectBuffer.cpp b/tests/kfdtest/src/IndirectBuffer.cpp new file mode 100644 index 0000000000..fb31a9203c --- /dev/null +++ b/tests/kfdtest/src/IndirectBuffer.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "IndirectBuffer.hpp" +#include "GoogleTestExtension.hpp" +#include "pm4_pkt_struct_common.h" +#include "PM4Packet.hpp" + + +IndirectBuffer::IndirectBuffer(PACKETTYPE type, unsigned int sizeInDWords, unsigned int NodeId) + :m_NumOfPackets(0), m_MaxSize(sizeInDWords), m_ActualSize(0), m_PacketTypeAllowed(type) { + m_IndirectBuf = new HsaMemoryBuffer(sizeInDWords*sizeof(unsigned int), NodeId, true/*zero*/, false/*local*/, true/*exec*/); +} + +IndirectBuffer::~IndirectBuffer(void) { + delete m_IndirectBuf; +} + +void IndirectBuffer::AddPacket(const BasePacket &packet) { + ASSERT_EQ(packet.PacketType(), m_PacketTypeAllowed) << "Cannot add a packet since packet type doesn't match queue"; + + unsigned int writePtr = m_ActualSize; + + ASSERT_GE(m_MaxSize, packet.SizeInDWords() + writePtr) << "Cannot add a packet, not enough room"; + + memcpy(m_IndirectBuf->As() + writePtr , packet.GetPacket(), packet.SizeInBytes()); + m_ActualSize += packet.SizeInDWords(); + m_NumOfPackets++; +} + + diff --git a/tests/kfdtest/src/IndirectBuffer.hpp b/tests/kfdtest/src/IndirectBuffer.hpp new file mode 100644 index 0000000000..14fa1e75f8 --- /dev/null +++ b/tests/kfdtest/src/IndirectBuffer.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __INDIRECT_BUFFER__H__ +#define __INDIRECT_BUFFER__H__ + +#include "BasePacket.hpp" +#include "KFDTestUtil.hpp" + +/** @class IndirectBuffer + * when working with indirect buffer, create IndirectBuffer, fill it with all the packets you want + * create an indirect packet to point to it, and submit the packet to queue + */ +class IndirectBuffer { + public: + // @param[size] queue max size in DWords + // @param[type] packets type allowed in queue + IndirectBuffer(PACKETTYPE type, unsigned int sizeInDWords, unsigned int NodeId); + ~IndirectBuffer(void); + + // @brief add packet to queue, all validations are done with gtest ASSERT and EXPECT + void AddPacket(const BasePacket &packet); + // @returns the actual size of the indirect queue in DWord, equivalent to write pointer + unsigned int SizeInDWord() { return m_ActualSize; } + // @returns indirect queue address + unsigned int *Addr() { return m_IndirectBuf->As(); } + + protected: + // how many packets in queue + unsigned int m_NumOfPackets; + // max size of queue in DWords + unsigned int m_MaxSize; + // current size of queue in DWords + unsigned int m_ActualSize; + HsaMemoryBuffer *m_IndirectBuf; + // defines what packets are supported in this queue + PACKETTYPE m_PacketTypeAllowed; +}; + +#endif // __INDIRECT_BUFFER__H__ diff --git a/tests/kfdtest/src/IsaGenerator.cpp b/tests/kfdtest/src/IsaGenerator.cpp new file mode 100644 index 0000000000..83f34b948e --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "IsaGenerator.hpp" + +#include +#include + +#include "IsaGenerator_Gfx72.hpp" +#include "IsaGenerator_Gfx8.hpp" +#include "IsaGenerator_Gfx9.hpp" + +#include "GoogleTestExtension.hpp" + +#include "sp3.h" + +const std::string IsaGenerator::ADDRESS_WATCH_SP3( + "var REG_TRAPSTS_EXCP_MASK = 0x000001ff\n" + "var WAVE_COUNT_OFFSET = 12\n" + "var TMA_CYCLE_OFFSET = 16\n" + "\n" + "/*\n" + " * ttmp[0:1] -- The ISA address that triggered this trap handler\n" + " * ttmp[10:11] -- The TMA user provided, used to store the debug info in this shader\n" + " * v[10:14] ttmp[7:8] -- temp use inside this shader\n" + " * s5 -- store the counts that this trap been triggered\n" + " * Each time when the trap is triggered , this shader will write\n" + " * ttmp[0] : ttmp[1] : Trap_Status : [reserved]\n" + " * to TMA + (trap count * TMA_CYCLE_OFFSET)\n" + " * The TMA + WAVE_COUNT_OFFSET(the first [reserved] address)\n" + " * used to store the total triggered trap count.\n" + " */\n" + "shader main\n" + "\n" + " asic(VI)\n" + "\n" + " type(CS)\n" + " v_mov_b32 v10, ttmp10\n" + " v_mov_b32 v11, ttmp11\n" + " s_mov_b32 ttmp7, s5\n" + " s_mulk_i32 ttmp7, TMA_CYCLE_OFFSET\n" + " s_addk_i32 s5, 1\n" + " v_mov_b32 v12, ttmp0\n" + " v_add_u32 v10, vcc, ttmp7, v10\n" + " flat_store_dword v[10,11], v12 slc glc\n" + " v_mov_b32 v12, ttmp1\n" + " v_add_u32 v10, vcc, 4, v10\n" + " flat_store_dword v[10,11], v12 slc glc\n" + " s_getreg_b32 ttmp8, hwreg(HW_REG_TRAPSTS)\n" + " s_and_b32 ttmp8, ttmp8, REG_TRAPSTS_EXCP_MASK\n" + " v_mov_b32 v12, ttmp8\n" + " v_add_u32 v10, vcc, 4, v10\n" + " flat_store_dword v[10,11], v12 glc\n" + " v_mov_b32 v10, ttmp10\n" + " v_add_u32 v10, vcc, WAVE_COUNT_OFFSET, v10\n" + " v_mov_b32 v13, 1\n" + " flat_atomic_add v14, v[10:11], v13 slc glc\n" + " s_and_b32 ttmp1, ttmp1, 0xffff\n" + " s_rfe_b64 [ttmp0,ttmp1]\n" + "end\n" +); + +IsaGenerator* IsaGenerator::Create(unsigned int familyId) { + switch (familyId) { + case FAMILY_CI: + case FAMILY_KV: + return new IsaGenerator_Gfx72; + case FAMILY_VI: + case FAMILY_CZ: + return new IsaGenerator_Gfx8; + case FAMILY_AI: + case FAMILY_RV: + return new IsaGenerator_Gfx9; + default: + LOG() << "Error: Invalid ISA" << std::endl; + return NULL; + } +} + +void IsaGenerator::GetAwTrapHandler(HsaMemoryBuffer& rBuf) { + CompileShader(ADDRESS_WATCH_SP3.c_str(), "main", rBuf); +} + +void IsaGenerator::CompileShader(const char* shaderCode, const char* shaderName, HsaMemoryBuffer& rBuf) { + sp3_context* pSp3 = sp3_new(); + sp3_setasic(pSp3, GetAsicName().c_str()); + sp3_parse_string(pSp3, shaderCode); + sp3_shader* pShader = sp3_compile(pSp3, shaderName); + + std::copy(pShader->data, pShader->data + pShader->size, rBuf.As()); + sp3_free_shader(pShader); + + /** Inside this close function, there is an unknown reason of free memory not used by compiler. + * Comment out this as a workaround. System will do the garbage collection after this + * application is closed. + */ + // sp3_close(pSp3); +} diff --git a/tests/kfdtest/src/IsaGenerator.hpp b/tests/kfdtest/src/IsaGenerator.hpp new file mode 100644 index 0000000000..11f1d0ca5b --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _ISAGENERATOR_H_ +#define _ISAGENERATOR_H_ + +#include "KFDTestUtil.hpp" + +/* isa generation class - interface */ +class IsaGenerator { + +public: + static IsaGenerator* Create(unsigned int familyId); + + virtual ~IsaGenerator() {} + + virtual void GetNoopIsa(HsaMemoryBuffer& rBuf) = 0; + virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf) = 0; + virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) = 0; + virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf) = 0; + virtual void GetCwsrTrapHandler(HsaMemoryBuffer& rBuf) {} + virtual void GetAwTrapHandler(HsaMemoryBuffer& rBuf); + + void CompileShader(const char* shaderCode, const char* shaderName, HsaMemoryBuffer& rBuf); + +protected: + virtual const std::string& GetAsicName() = 0; + +private: + static const std::string ADDRESS_WATCH_SP3; +}; + +#endif //_ISAGENERATOR_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Gfx72.cpp b/tests/kfdtest/src/IsaGenerator_Gfx72.cpp new file mode 100644 index 0000000000..e0d98fb5c8 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx72.cpp @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "IsaGenerator_Gfx72.hpp" + +#include +#include + +const std::string IsaGenerator_Gfx72::ASIC_NAME = "CI"; + +const uint32_t IsaGenerator_Gfx72::NOOP_ISA[] = { + 0xbf810000 // S_ENDPGM +}; + +/* The below arrays are filled with hex values in order not to reference + * proprietary header files, but we still leave the code here for future + * reference. + */ +#if 0 +const uint32_t IsaGenerator_Gfx72::COPY_DWORD_ISA[] = { + (63u << SQ_VOP1__ENCODING__SHIFT) | (0 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v0, s0 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (1 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v1, s1 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (2 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (2 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v2, s2 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (3 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (3 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v3, s3 (VOP1) + + (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_LOAD_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT)/*(3 << 16)*/, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) + (4u << SQ_FLAT_1__VDST__SHIFT) | (0 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V0:V1, VDST = V4 (FLAT_1) + + (383u << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_WAITCNT << SQ_SOPP__OP__SHIFT) | (0 << SQ_SOPP__SIMM16__SHIFT), // s_waitcnt 0 (SOPP) + + (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_STORE_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT), // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) + (4u << SQ_FLAT_1__DATA__SHIFT) | (2 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V2:V3, DATA = V4 (FLAT_1) + + 0xBF810000u // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 +}; + +const uint32_t IsaGenerator_Gfx72::INFINITE_LOOP_ISA[] = { + (0x17F << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_BRANCH << SQ_SOPP__OP__SHIFT) | ( (const uint32_t)-1 << SQ_SOPP__SIMM16__SHIFT), // s_branch -1 (PC <- PC + SIMM*4)+4 + 0xBF810000u // S_ENDPGM +}; + +const uint32_t IsaGenerator_Gfx72::ATOMIC_INC_ISA[] = { + (63u << SQ_VOP1__ENCODING__SHIFT) | (0 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v0, s0 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (1 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v1, s1 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (2 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0xC1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 0xFFFFFFFF, s2 (VOP1) + + (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_ATOMIC_INC << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (0 << SQ_FLAT_0__GLC__SHIFT), // SQ_FLAT_0, flat_atomic_inc, slc = 1, glc = 0 (FLAT_0) + (3u << SQ_FLAT_1__VDST__SHIFT) | (2u << SQ_FLAT_1__DATA__SHIFT) | (0 << SQ_FLAT_1__ADDR__SHIFT), // ADDR/dst = V0:V1, VDST/ret = V3, DATA/src=V2 (FLAT_1) + 0xBF810000u // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 +}; +#endif + +const uint32_t IsaGenerator_Gfx72::COPY_DWORD_ISA[] = { + 0x7e000200, // v_mov_b32 v0, s0 (VOP1) + 0x7e020201, // v_mov_b32 v1, s1 (VOP1) + 0x7e040202, // v_mov_b32 v2, s2 (VOP1) + 0x7e060203, // v_mov_b32 v3, s3 (VOP1) + + 0xdc330000, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) + 0x04000000, // ADDR = V0:V1, VDST = V4 (FLAT_1) + + 0xbf8c0000, // s_waitcnt 0 (SOPP) + + 0xdc730000, // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) + 0x00000402, // ADDR = V2:V3, DATA = V4 (FLAT_1) + + 0xbf810000 // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 +}; + +const uint32_t IsaGenerator_Gfx72::INFINITE_LOOP_ISA[] = { + 0xbf82ffff, // s_branch -1 (PC <- PC + SIMM*4)+4 + 0xbf810000 // S_ENDPGM +}; + +const uint32_t IsaGenerator_Gfx72::ATOMIC_INC_ISA[] = { + 0x7e000200, // v_mov_b32 v0, s0 (VOP1) + 0x7e020201, // v_mov_b32 v1, s1 (VOP1) + 0x7e0402c1, // v_mov_b32 0xFFFFFFFF, s2 (VOP1) + + 0xdcf20000, // SQ_FLAT_0, flat_atomic_inc, slc = 1, glc = 0 (FLAT_0) + 0x03000200, // ADDR/dst = V0:V1, VDST/ret = V3, DATA/src=V2 (FLAT_1) + 0xbf810000 // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 +}; + +void IsaGenerator_Gfx72::GetNoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx72::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { + std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx72::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx72::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { + std::copy(ATOMIC_INC_ISA, ATOMIC_INC_ISA+ARRAY_SIZE(ATOMIC_INC_ISA), rBuf.As()); +} + +const std::string& IsaGenerator_Gfx72::GetAsicName() { + return ASIC_NAME; +} diff --git a/tests/kfdtest/src/IsaGenerator_Gfx72.hpp b/tests/kfdtest/src/IsaGenerator_Gfx72.hpp new file mode 100644 index 0000000000..5c39ffa216 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx72.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _ISAGENERATOR_GFX72_H_ +#define _ISAGENERATOR_GFX72_H_ + +#include +#include "IsaGenerator.hpp" + +class IsaGenerator_Gfx72 : public IsaGenerator { + public: + virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); + virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); + + protected: + virtual const std::string& GetAsicName(); + + private: + static const std::string ASIC_NAME; + + static const uint32_t NOOP_ISA[]; + static const uint32_t COPY_DWORD_ISA[]; + static const uint32_t INFINITE_LOOP_ISA[]; + static const uint32_t ATOMIC_INC_ISA[]; +}; + +#endif // _ISAGENERATOR_GFX72_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Gfx8.cpp b/tests/kfdtest/src/IsaGenerator_Gfx8.cpp new file mode 100644 index 0000000000..65e0df6836 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx8.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "IsaGenerator_Gfx8.hpp" + +#include +#include + +const std::string IsaGenerator_Gfx8::ASIC_NAME = "VI"; + +const uint32_t IsaGenerator_Gfx8::NOOP_ISA[] = { + 0xbf810000 // S_ENDPGM +}; + +/** The below arrays are filled with hex values in order not to reference + * proprietary header files, but we still leave the code here for future + * reference. + */ +#if 0 +const uint32_t IsaGenerator_Gfx8::COPY_DWORD_ISA[] = { + (63u << SQ_VOP1__ENCODING__SHIFT) | (0 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v0, s0 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (1 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v1, s1 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (2 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (2 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v2, s2 (VOP1) + (63u << SQ_VOP1__ENCODING__SHIFT) | (3 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (3 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v3, s3 (VOP1) + + (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_LOAD_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT)/*(3 << 16)*/, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) + (4u << SQ_FLAT_1__VDST__SHIFT) | (0 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V0:V1, VDST = V4 (FLAT_1) + + (383u << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_WAITCNT << SQ_SOPP__OP__SHIFT) | (0 << SQ_SOPP__SIMM16__SHIFT), // s_waitcnt 0 (SOPP) + + (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_STORE_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT), // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) + (4u << SQ_FLAT_1__DATA__SHIFT) | (2 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V2:V3, DATA = V4 (FLAT_1) + + 0xBF810000u // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 +}; + +const uint32_t IsaGenerator_Gfx8::INFINITE_LOOP_ISA[] = { + (0x17F << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_BRANCH << SQ_SOPP__OP__SHIFT) | ( (const uint32_t)-1 << SQ_SOPP__SIMM16__SHIFT), // s_branch -1 (PC <- PC + SIMM*4)+4 + 0xBF810000u // S_ENDPGM +}; +#endif + +const uint32_t IsaGenerator_Gfx8::COPY_DWORD_ISA[] = { + 0x7e000200, // v_mov_b32 v0, s0 (VOP1) + 0x7e020201, // v_mov_b32 v1, s1 (VOP1) + 0x7e040202, // v_mov_b32 v2, s2 (VOP1) + 0x7e060203, // v_mov_b32 v3, s3 (VOP1) + + 0xdc530000, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) + 0x04000000, // ADDR = V0:V1, VDST = V4 (FLAT_1) + + 0xbf8c0000, // s_waitcnt 0 (SOPP) + + 0xdc730000, // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) + 0x00000402, // ADDR = V2:V3, DATA = V4 (FLAT_1) + + 0xbf810000 // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 +}; + +const uint32_t IsaGenerator_Gfx8::INFINITE_LOOP_ISA[] = { + 0xbf82ffff, // s_branch -1 (PC <- PC + SIMM*4)+4 + 0xbf810000 // S_ENDPGM +}; + +/** + * The atomic_add_isa binary is generated from following ISA + * The original atomic_inc is not support by some PCIE, so use atomic_add instead + * + */ +/* +shader atomic_add +asic(VI) +type(CS) + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, 1 + flat_atomic_add v3, v[0:1], v2 slc glc + s_waitcnt 0 + s_endpgm +end +*/ + +const uint32_t IsaGenerator_Gfx8::ATOMIC_ADD_ISA[] = { + 0x7e000200, 0x7e020201, + 0x7e040281, 0xdd0b0000, + 0x03000200, 0xbf8c0000, + 0xbf810000, 0x00000000 +}; + +void IsaGenerator_Gfx8::GetNoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx8::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { + std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx8::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx8::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { + std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); +} + +const std::string& IsaGenerator_Gfx8::GetAsicName() { + return ASIC_NAME; +} diff --git a/tests/kfdtest/src/IsaGenerator_Gfx8.hpp b/tests/kfdtest/src/IsaGenerator_Gfx8.hpp new file mode 100644 index 0000000000..7e5b9e3c89 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx8.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _ISAGENERATOR_GFX8_H_ +#define _ISAGENERATOR_GFX8_H_ + +#include +#include "IsaGenerator.hpp" + +class IsaGenerator_Gfx8 : public IsaGenerator { + public: + virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); + virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); + + protected: + virtual const std::string& GetAsicName(); + + private: + static const std::string ASIC_NAME; + + static const uint32_t NOOP_ISA[]; + static const uint32_t COPY_DWORD_ISA[]; + static const uint32_t INFINITE_LOOP_ISA[]; + static const uint32_t ATOMIC_ADD_ISA[]; +}; + +#endif // _ISAGENERATOR_GFX72_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Gfx9.cpp b/tests/kfdtest/src/IsaGenerator_Gfx9.cpp new file mode 100644 index 0000000000..8eaab32a5e --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx9.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "IsaGenerator_Gfx9.hpp" + +#include +#include + +const std::string IsaGenerator_Gfx9::ASIC_NAME = "GFX9"; + +/* The binaries are generated from following ISA */ +#if 0 +/* flat_atomic_inc will not support by some PCIE, use flat_atomic_add instead */ +shader atomic_add +asic(GFX9) +type(CS) + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, 1 + flat_atomic_add v3, v[0:1], v2 slc glc + s_waitcnt 0 + s_endpgm +end + +shader copy_dword +asic(GFX9) +type(CS) +/* copy the parameters from scalar registers to vector registers */ + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 +/* copy a dword between the passed addresses */ + flat_load_dword v4, v[0:1] slc glc + s_waitcnt 0 + flat_store_dword v[2:3], v4 slc glc + s_endpgm +end + +shader main +asic(GFX9) +type(CS) +loop: + s_branch loop + s_endpgm +end + + +#endif + +const uint32_t IsaGenerator_Gfx9::NOOP_ISA[] = { + 0xbf810000 +}; + +const uint32_t IsaGenerator_Gfx9::COPY_DWORD_ISA[] = { + 0x7e000200, 0x7e020201, + 0x7e040202, 0x7e060203, + 0xdc530000, 0x047f0000, + 0xbf8c0000, 0xdc730000, + 0x007f0402, 0xbf810000 +}; + +const uint32_t IsaGenerator_Gfx9::INFINITE_LOOP_ISA[] = { + 0xbf82ffff, 0xbf810000 +}; + +const uint32_t IsaGenerator_Gfx9::ATOMIC_ADD_ISA[] = { + 0x7e000200, 0x7e020201, + 0x7e040281, 0xdd0b0000, + 0x037f0200, 0xbf8c0000, + 0xbf810000, 0x00000000 +}; + +void IsaGenerator_Gfx9::GetNoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx9::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { + std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx9::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx9::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { + std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); +} + +const std::string& IsaGenerator_Gfx9::GetAsicName() { + return ASIC_NAME; +} + diff --git a/tests/kfdtest/src/IsaGenerator_Gfx9.hpp b/tests/kfdtest/src/IsaGenerator_Gfx9.hpp new file mode 100644 index 0000000000..32103c0a15 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx9.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _ISAGENERATOR_GFX9_H_ +#define _ISAGENERATOR_GFX9_H_ + +#include +#include "IsaGenerator.hpp" + +class IsaGenerator_Gfx9 : public IsaGenerator { + public: + virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); + virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); + + protected: + virtual const std::string& GetAsicName(); + + private: + static const std::string ASIC_NAME; + + static const uint32_t NOOP_ISA[]; + static const uint32_t COPY_DWORD_ISA[]; + static const uint32_t INFINITE_LOOP_ISA[]; + static const uint32_t ATOMIC_ADD_ISA[]; +}; + +#endif // _ISAGENERATOR_GFX9_H_ diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp new file mode 100644 index 0000000000..0a7c1659bd --- /dev/null +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDBaseComponentTest.hpp" +#include "KFDTestUtil.hpp" + +void KFDBaseComponentTest::SetUpTestCase() { +} + +void KFDBaseComponentTest::TearDownTestCase() { +} + +void KFDBaseComponentTest::SetUp() { + ROUTINE_START + + ASSERT_SUCCESS(hsaKmtOpenKFD()); + EXPECT_SUCCESS(hsaKmtGetVersion(&m_VersionInfo)); + memset( &m_SystemProperties, 0, sizeof(m_SystemProperties) ); + + /** in order to be correctly testing the KFD interfaces and ensure + * that the KFD acknowledges relevant node parameters + * for the rest of the tests and used for more specific topology tests + * call to GetSystemProperties for a system snapshot of the topology here + */ + ASSERT_SUCCESS(hsaKmtAcquireSystemProperties(&m_SystemProperties)); + ASSERT_GT(m_SystemProperties.NumNodes, HSAuint32(0)) << "HSA has no nodes."; + + m_NodeInfo.Init(m_SystemProperties.NumNodes); + + // setting memory flags with default values , can be modified according to needs + m_MemoryFlags.ui32.NonPaged = 0; // Paged + m_MemoryFlags.ui32.CachePolicy = HSA_CACHING_NONCACHED; // Non cached + m_MemoryFlags.ui32.ReadOnly = 0; // Read/Write + m_MemoryFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; // 4KB page + m_MemoryFlags.ui32.HostAccess = 1; // host accessible + m_MemoryFlags.ui32.NoSubstitute = 0; // fall back to node 0 if needed + m_MemoryFlags.ui32.GDSMemory = 0; + m_MemoryFlags.ui32.Scratch = 0; + + const HsaNodeProperties *nodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + ASSERT_NOTNULL(nodeProperties) << "failed to get HSA default GPU Node properties"; + + g_TestGPUFamilyId = FamilyIdFromNode(nodeProperties); + + m_FamilyId = g_TestGPUFamilyId; + + ROUTINE_END +} + +void KFDBaseComponentTest::TearDown() { + ROUTINE_START + + ASSERT_SUCCESS(hsaKmtReleaseSystemProperties()); + ASSERT_SUCCESS(hsaKmtCloseKFD()); + + ROUTINE_END +} + +HSAuint64 KFDBaseComponentTest::GetSysMemSize() { + const HsaNodeProperties *nodeProps; + HsaMemoryProperties cpuMemoryProps; + HSAuint64 systemMemSize = 0; + + /* Find System Memory size */ + for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { + nodeProps = m_NodeInfo.GetNodeProperties(node); + if (nodeProps != NULL && nodeProps->NumCPUCores > 0 && nodeProps->NumMemoryBanks > 0) { + /* For NUMA nodes, memory is distributed among differnt nodes. + * Compute total system memory size. KFD driver also computes + * the system memory (si_meminfo) similarly + */ + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(node, 1, &cpuMemoryProps)); + systemMemSize += cpuMemoryProps.SizeInBytes; + } + } + + return systemMemSize; +} + +HSAuint64 KFDBaseComponentTest::GetVramSize(int defaultGPUNode) { + const HsaNodeProperties *nodeProps; + + /* Find framebuffer size */ + nodeProps = m_NodeInfo.GetNodeProperties(defaultGPUNode); + EXPECT_NE((const HsaNodeProperties *)NULL, nodeProps); + HSAuint32 numBanks = nodeProps->NumMemoryBanks; + HsaMemoryProperties memoryProps[numBanks]; + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(defaultGPUNode, numBanks, memoryProps)); + unsigned bank; + for (bank = 0; bank < numBanks; bank++) { + if (memoryProps[bank].HeapType == HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE + || memoryProps[bank].HeapType == HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC) + return memoryProps[bank].SizeInBytes; + } + + return 0; +} diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp new file mode 100644 index 0000000000..18ab7b2a4a --- /dev/null +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "hsakmt.h" +#include "OSWrapper.hpp" +#include "KFDTestUtil.hpp" + +#ifndef __KFD_BASE_COMPONENT_TEST__H__ +#define __KFD_BASE_COMPONENT_TEST__H__ + +// @class KFDBaseComponentTest +class KFDBaseComponentTest : public testing::Test { + public: + KFDBaseComponentTest(void) { m_MemoryFlags.Value = 0; } + ~KFDBaseComponentTest(void) {} + + HSAuint64 GetSysMemSize(); + HSAuint64 GetVramSize(int defaultGPUNode); + + protected: + HsaVersionInfo m_VersionInfo; + HsaSystemProperties m_SystemProperties; + unsigned int m_FamilyId; + HsaMemFlags m_MemoryFlags; + HsaNodeInfo m_NodeInfo; + + // @brief SetUpTestCase function run before the first test that uses KFDOpenCloseKFDTest class fixture, and opens KFD. + static void SetUpTestCase(); + // @brief TearDownTestCase function run after the last test from KFDOpenCloseKFDTest class fixture and calls close KFD. + static void TearDownTestCase(); + // @brief SetUp function run before every test that uses KFDOpenCloseKFDTest class fixture, sets all common settings for the tests. + virtual void SetUp(); + // @brief TearDown function run after every test that uses KFDOpenCloseKFDTest class fixture. + virtual void TearDown(); +}; + +#endif // __KFD_BASE_COMPONENT_TEST__H__ diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp new file mode 100644 index 0000000000..594500c032 --- /dev/null +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2015-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDCWSRTest.hpp" +#include "Dispatch.hpp" + +static const char* iterate_isa_gfx8 = \ +"\ +shader iterate_isa\n\ +asic(VI)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ +/*v0 stores the workitem ID inside the workgroup, use it to caculate the dest*/\n\ +// v_lshlrev_b32 v0, 2, v0\n\ +// v_add_u32 v2, vcc, v0, v2\n\ + v_mov_b32 v0, s0\n\ + flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + v_mov_b32 v5, 0\n\ +LOOP:\n\ + v_add_u32 v5, vcc, 1, v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ + v_cmp_lt_u32 vcc, v5, v4\n\ + s_cbranch_vccnz LOOP\n\ + flat_store_dword v[2,3], v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + s_endpgm\n\ +end\n\ +"; + +static const char* iterate_isa_gfx9 = \ +"\ +shader iterate_isa\n\ +asic(GFX9)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ + flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + v_mov_b32 v5, 0\n\ +LOOP:\n\ + v_add_co_u32 v5, vcc, 1, v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ + v_cmp_lt_u32 vcc, v5, v4\n\ + s_cbranch_vccnz LOOP\n\ + flat_store_dword v[2,3], v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + s_endpgm\n\ +end\n\ +"; + +/* +v[0:1] = target iteration value +v[2:3] = iterate result +*/ + +void KFDCWSRTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + // TODO: Seems in the ISA, I can not get the workitem_id as expected, so I can not set the destination based on workitem_id. + // Set the wave_num to 1 for now as a workarpound. Will set it to 8 or even 256 in the future. + wave_number = 1; + + ROUTINE_END +} + +void KFDCWSRTest::TearDown() { + ROUTINE_START + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +/** + * KFDCWSRTest.BasicTest + * + * This test dispatches the loop_inc_isa shader and lets it run, ensuring its destination pointer gets incremented. + * It then triggers CWSR and ensures the shader stops running. + * It then resumes the shader, ensures that it's running again and terminates it. + */ + +TEST_F(KFDCWSRTest, BasicTest) { + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + if (m_FamilyId >= FAMILY_VI) { + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer iterateBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + HsaMemoryBuffer resultBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + + unsigned int* iter = iterateBuf.As(); + unsigned int* result = resultBuf.As(); + + m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? iterate_isa_gfx9 : iterate_isa_gfx8 , + "iterate_isa", isaBuffer); + + PM4Queue queue1, queue2; + + EXPECT_SUCCESS(queue1.Create(defaultGPUNode)); + + Dispatch *dispatch1, *dispatch2; + + dispatch1 = new Dispatch(isaBuffer); + dispatch2 = new Dispatch(isaBuffer); + + dispatch1->SetArgs(&iter[0], &result[0]); + dispatch1->SetDim(wave_number, 1, 1); + dispatch2->SetArgs(&iter[1], &result[wave_number]); + dispatch2->SetDim(wave_number, 1, 1); + + iter[0] = 40000000; + iter[1] = 20000000; + + // submit the shader, queue1 + dispatch1->Submit(queue1); + // Create queue2 during queue1 still running will trigger the CWSR + EXPECT_SUCCESS(queue2.Create(defaultGPUNode)); + // submit the shader + dispatch2->Submit(queue2); + dispatch1->Sync(); + dispatch2->Sync(); + // ensure all the waves complete as expected + int i; + for (i = 0 ; i < wave_number; ++i) { + if (result[i] != iter[0]) { + LOG() << "Dispatch 1, work item " << i << ' ' << result[i] << std::endl; + break; + } + if (result[i + wave_number] != iter[1]) { + LOG() << "Dispatch 2, work item " << i << ' ' << result[i] << std::endl; + break; + } + } + EXPECT_EQ(i, wave_number); + + EXPECT_SUCCESS(queue1.Destroy()); + EXPECT_SUCCESS(queue2.Destroy()); + + delete dispatch1; + delete dispatch2; + + } else { + LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + } + + TEST_END +} diff --git a/tests/kfdtest/src/KFDCWSRTest.hpp b/tests/kfdtest/src/KFDCWSRTest.hpp new file mode 100644 index 0000000000..40526cc7fb --- /dev/null +++ b/tests/kfdtest/src/KFDCWSRTest.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2015-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_CWSR_TEST__H__ +#define __KFD_CWSR_TEST__H__ + +#include + +#include "PM4Queue.hpp" +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" + +class KFDCWSRTest : public KFDBaseComponentTest { + public: + KFDCWSRTest() :m_pIsaGen(NULL) {} + ~KFDCWSRTest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: // members + unsigned wave_number; + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_CWSR_TEST__H__ diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp new file mode 100644 index 0000000000..abff218160 --- /dev/null +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDDBGTest.hpp" +#include "KFDQMTest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "Dispatch.hpp" + + +static const char* loop_inc_isa = \ +"\ +shader loop_inc_isa\n\ +asic(VI)\n\ +type(CS)\n\ +vgpr_count(16)\n\ +trap_present(1)\n\ +/* TODO Enable here Address Watch Exception: */ \n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ + v_mov_b32 v5, 0 \n\ +/*Iteration 1*/\n\ + v_mov_b32 v4, -1\n\ + flat_atomic_inc v5, v[0:1], v4 glc \n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + flat_load_dword v8, v[2:3] \n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + v_cmp_gt_u32 vcc, 2, v5 \n\ +/*Iteration 2*/\n\ + v_mov_b32 v4, -1\n\ + flat_atomic_inc v5, v[0:1], v4 glc \n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + flat_load_dword v8, v[2:3] \n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + v_cmp_gt_u32 vcc, 2, v5 \n\ +/*Epilogue*/\n\ + v_mov_b32 v5, 0 \n\ + s_endpgm\n\ + \n\ +end\n\ +"; + +void KFDDBGTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDDBGTest::TearDown() { + ROUTINE_START + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + /* reset the user trap handler */ + hsaKmtSetTrapHandler(m_NodeInfo.HsaDefaultGPUNode(), 0, 0, 0, 0); + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDDBGTest, BasicAddressWatch) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_VI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + static const unsigned int TMA_TRAP_COUNT_OFFSET = 3; + static const unsigned int TMA_TRAP_COUNT_VALUE = 3; + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + // Allocate Buffers on System Memory + HsaMemoryBuffer trapHandler(PAGE_SIZE*2, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer dstBuf(PAGE_SIZE*4, defaultGPUNode, true, false, false); + HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); + + // Get Address Watch TrapHandler + m_pIsaGen->GetAwTrapHandler(trapHandler); + + ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, + trapHandler.As(), + 0x1000, + tmaBuf.As(), + 0x1000)); + + m_pIsaGen->CompileShader(loop_inc_isa, "loop_inc_isa", isaBuf); + PM4Queue queue, queue_flush; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + ASSERT_SUCCESS(queue_flush.Create(defaultGPUNode)); + + // Set Address Watch Params + // TODO: Set atchMode[1] to Atomic in case we want to test this mode. + + HSA_DBG_WATCH_MODE WatchMode[2]; + HSAuint64 WatchAddress[2]; + HSAuint64 WatchMask[2]; + HSAKMT_STATUS AddressWatchSuccess; + HSAuint64 AddressMask64 = 0x0; + unsigned char *secDstBuf = (unsigned char *)dstBuf.As() + 8192; + + WatchMode[0] = HSA_DBG_WATCH_ALL; + WatchAddress[0] = (HSAuint64) dstBuf.As() & (~AddressMask64); + WatchMask[0] = ~AddressMask64; + WatchMode[1] = HSA_DBG_WATCH_ALL; + WatchAddress[1] = (HSAuint64)secDstBuf & (~AddressMask64); + WatchMask[1] = ~AddressMask64; + + queue_flush.PlaceAndSubmitPacket(PM4WriteDataPacket(dstBuf.As(), 0x0, 0x0)); + queue_flush.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)secDstBuf, 0x0, 0x0)); + Delay(50); + ASSERT_SUCCESS(hsaKmtDbgRegister(defaultGPUNode)); + + AddressWatchSuccess = hsaKmtDbgAddressWatch( + defaultGPUNode, // IN + 2, // # watch points + &WatchMode[0], // IN + (void **) &WatchAddress[0], // IN + &WatchMask[0], // IN, optional + NULL // IN, optional + ); + + EXPECT_EQ(AddressWatchSuccess, HSAKMT_STATUS_SUCCESS); + + Dispatch dispatch(isaBuf); + dispatch.SetArgs(dstBuf.As(), (void *)secDstBuf); + dispatch.SetDim(1, 1, 1); + + // TODO: use Memory ordering rules w/ atomics + // for host-GPU memory syncs. + // set to: std::memory_order_seq_cst + + dispatch.Submit(queue); + + Delay(50); + dispatch.Sync(g_TestTimeOut); + + // Check that we got trap handler calls due to add watch triggers + ASSERT_GE(*(tmaBuf.As()+ TMA_TRAP_COUNT_OFFSET), TMA_TRAP_COUNT_VALUE); + + ASSERT_SUCCESS(hsaKmtDbgUnregister(defaultGPUNode)); + ASSERT_SUCCESS(queue.Destroy()); + ASSERT_SUCCESS(queue_flush.Destroy()); + } else { + LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + } + TEST_END +} diff --git a/tests/kfdtest/src/KFDDBGTest.hpp b/tests/kfdtest/src/KFDDBGTest.hpp new file mode 100644 index 0000000000..331b9017f8 --- /dev/null +++ b/tests/kfdtest/src/KFDDBGTest.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_DBG_TEST__H__ +#define __KFD_DBG_TEST__H__ + +#include + +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" + +class KFDDBGTest : public KFDBaseComponentTest { + public: + KFDDBGTest() :m_pIsaGen(NULL) {} + ~KFDDBGTest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: // members + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_DBG_TEST__H__ diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp new file mode 100644 index 0000000000..200fe638b4 --- /dev/null +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -0,0 +1,285 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include + +#include "KFDEventTest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" + + +void KFDEventTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + m_pHsaEvent = NULL; + + ROUTINE_END +} + +void KFDEventTest::TearDown() { + ROUTINE_START + + // not all tests create event, destroy only if there is one + if (m_pHsaEvent != NULL) { + // hsaKmtDestroyEvent moved to TearDown to make sure it is being called + EXPECT_SUCCESS(hsaKmtDestroyEvent(m_pHsaEvent)); + } + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDEventTest, CreateDestroyEvent) { + TEST_START(TESTPROFILE_RUNALL); + + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_NodeInfo.HsaDefaultGPUNode(), &m_pHsaEvent)); + EXPECT_NE(0, m_pHsaEvent->EventData.HWData2); + + // destroy event is being called in test TearDown + TEST_END; +} + +TEST_F(KFDEventTest, CreateMaxEvents) { + TEST_START(TESTPROFILE_RUNALL); + + static const unsigned int MAX_EVENT_NUMBER = 256; + + HsaEvent* pHsaEvent[MAX_EVENT_NUMBER]; + + unsigned int i = 0; + + for (i = 0; i < MAX_EVENT_NUMBER; i++) { + pHsaEvent[i] = NULL; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_NodeInfo.HsaDefaultGPUNode(), &pHsaEvent[i])); + } + + for (i = 0; i < MAX_EVENT_NUMBER; i++) { + EXPECT_SUCCESS(hsaKmtDestroyEvent(pHsaEvent[i])); + } + + TEST_END; +} + +TEST_F(KFDEventTest, SignalEvent) { + TEST_START(TESTPROFILE_RUNALL); + + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_NodeInfo.HsaDefaultGPUNode(), &m_pHsaEvent)); + ASSERT_NE(0, m_pHsaEvent->EventData.HWData2); + + PM4Queue queue; + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); + + queue.Wait4PacketConsumption(); + + ASSERT_SUCCESS(hsaKmtWaitOnEvent(m_pHsaEvent, g_TestTimeOut)); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END; +} + +static uint64_t gettime() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((int64_t)ts.tv_sec) * 1000 * 1000 * 1000 + ts.tv_nsec; +} + +static inline double pow2_round_up(int num) { + return pow(2, ceil(log(num)/log(2))); +} + +class QueueAndSignalBenchmark { + private: + static const int HISTORY_SIZE = 100; + + int mNumEvents; + int mHistorySlot; + uint64_t mTimeHistory[HISTORY_SIZE]; + uint64_t mLatHistory[HISTORY_SIZE]; + + public: + QueueAndSignalBenchmark(int events) : mNumEvents(events), mHistorySlot(0) { + memset(mTimeHistory, 0, sizeof(mTimeHistory)); + memset(mLatHistory, 0, sizeof(mLatHistory)); + } + + int queueAndSignalEvents(int node, int eventCount, uint64_t &time, uint64_t &latency) { + int r; + uint64_t startTime; + PM4Queue queue; + + HsaEvent** pHsaEvent = (HsaEvent**) calloc(eventCount, sizeof(HsaEvent*)); + size_t packetSize = PM4ReleaseMemoryPacket(false, 0, 0).SizeInBytes(); + int qSize = fmax(PAGE_SIZE, pow2_round_up(packetSize*eventCount + 1)); + + time = 0; + + r = queue.Create(node, qSize); + if (r != HSAKMT_STATUS_SUCCESS) + goto exit; + + for (int i = 0; i < eventCount; i++) { + r = CreateQueueTypeEvent(false, false, node, &pHsaEvent[i]); + if (r != HSAKMT_STATUS_SUCCESS) + goto exit; + + queue.PlacePacket(PM4ReleaseMemoryPacket(false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); + } + + startTime = gettime(); + queue.SubmitPacket(); + for (int i = 0; i < eventCount; i++) { + r = hsaKmtWaitOnEvent(pHsaEvent[i], g_TestTimeOut); + + if (r != HSAKMT_STATUS_SUCCESS) + goto exit; + + if (i == 0) + latency = gettime() - startTime; + } + time = gettime() - startTime; + +exit: + for (int i = 0; i < eventCount; i++) { + if (pHsaEvent[i]) + hsaKmtDestroyEvent(pHsaEvent[i]); + } + queue.Destroy(); + + return r; + } + + void run(int node) { + int r = 0; + uint64_t time = 0, latency = 0; + uint64_t avgLat = 0, avgTime = 0; + uint64_t minTime = ULONG_MAX, maxTime = 0; + uint64_t minLat = ULONG_MAX, maxLat = 0; + + r = queueAndSignalEvents(node, mNumEvents, time, latency); + ASSERT_EQ(r, HSAKMT_STATUS_SUCCESS); + + mTimeHistory[mHistorySlot%HISTORY_SIZE] = time; + mLatHistory[mHistorySlot%HISTORY_SIZE] = latency; + + for (int i = 0; i < HISTORY_SIZE; i++) { + minTime = mTimeHistory[i] < minTime ? mTimeHistory[i] : minTime; + maxTime = mTimeHistory[i] > maxTime ? mTimeHistory[i] : maxTime; + avgTime += mTimeHistory[i]; + + minLat = mLatHistory[i] < minLat ? mLatHistory[i] : minLat; + maxLat = mLatHistory[i] > maxLat ? mLatHistory[i] : maxLat; + avgLat += mLatHistory[i]; + } + + avgTime /= HISTORY_SIZE; + avgLat /= HISTORY_SIZE; + mHistorySlot++; + + printf("\033[KEvents: %d History: %d/%d\n", mNumEvents, mHistorySlot, HISTORY_SIZE); + printf("\033[KMin Latency: %f ms\n", (float)minLat/1000000); + printf("\033[KMax Latency: %f ms\n", (float)maxLat/1000000); + printf("\033[KAvg Latency: %f ms\n", (float)avgLat/1000000); + printf("\033[K Min Rate: %f IH/ms\n", ((float)mNumEvents)/maxTime*1000000); + printf("\033[K Max Rate: %f IH/ms\n", ((float)mNumEvents)/minTime*1000000); + printf("\033[K Avg Rate: %f IH/ms\n", ((float)mNumEvents)/avgTime*1000000); + } +}; + +TEST_F(KFDEventTest, MeasureInterruptConsumption) { + TEST_START(TESTPROFILE_RUNALL); + QueueAndSignalBenchmark latencyBench(128); + QueueAndSignalBenchmark sustainedBench(4096); + + printf("\033[2J"); + while (true) { + printf("\033[H"); + printf("--------------------------\n"); + latencyBench.run(m_NodeInfo.HsaDefaultGPUNode()); + printf("--------------------------\n"); + sustainedBench.run(m_NodeInfo.HsaDefaultGPUNode()); + printf("--------------------------\n"); + } + + TEST_END; +} + +TEST_F(KFDEventTest, SignalMaxEvents) { + TEST_START(TESTPROFILE_RUNALL); + + static const unsigned int MAX_EVENT_NUMBER = 4096; + uint64_t time, latency; + + QueueAndSignalBenchmark maxEventTest(MAX_EVENT_NUMBER); + maxEventTest.queueAndSignalEvents(m_NodeInfo.HsaDefaultGPUNode(), MAX_EVENT_NUMBER, + time, latency); + + TEST_END; +} + +TEST_F(KFDEventTest, SignalMultipleEventsWaitForAll) { + TEST_START(TESTPROFILE_RUNALL); + + static const unsigned int EVENT_NUMBER = 64; // 64 is the maximum for hsaKmtWaitOnMultipleEvents + static const unsigned int WAIT_BETWEEN_SUBMISSIONS_MS = 50; + + HsaEvent* pHsaEvent[EVENT_NUMBER]; + unsigned int i = 0; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + for (i = 0; i < EVENT_NUMBER; i++) { + pHsaEvent[i] = NULL; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &pHsaEvent[i])); + } + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + unsigned int pktSizeDwords = 0; + for (i = 0; i < EVENT_NUMBER; i++) { + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); + queue.Wait4PacketConsumption(); + + Delay(WAIT_BETWEEN_SUBMISSIONS_MS); + } + + ASSERT_SUCCESS(hsaKmtWaitOnMultipleEvents(pHsaEvent, EVENT_NUMBER, true, g_TestTimeOut)); + + ASSERT_SUCCESS(queue.Destroy()); + + for (i = 0; i < EVENT_NUMBER; i++) + EXPECT_SUCCESS(hsaKmtDestroyEvent(pHsaEvent[i])); + + TEST_END; +} diff --git a/tests/kfdtest/src/KFDEventTest.hpp b/tests/kfdtest/src/KFDEventTest.hpp new file mode 100644 index 0000000000..43b6935f7f --- /dev/null +++ b/tests/kfdtest/src/KFDEventTest.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDBaseComponentTest.hpp" + +#ifndef __KFD_EVENT_TEST__H__ +#define __KFD_EVENT_TEST__H__ + +class KFDEventTest : public KFDBaseComponentTest { + public: + KFDEventTest(void) {} + ~KFDEventTest(void) {} + + // @brief SetUp function runs before every test in KFDEventTest. + virtual void SetUp(); + // @brief TearDown function runs after every test in KFDEventTest. + virtual void TearDown(); + + protected: + static const unsigned int EVENT_TIMEOUT = 5000; // 5 seconds + HsaEvent* m_pHsaEvent; +}; + +#endif // __KFD_EVENT_TEST__H__ diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp new file mode 100644 index 0000000000..156dabbf34 --- /dev/null +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -0,0 +1,454 @@ +/* + * Copyright (C) 2017-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include "KFDEvictTest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "Dispatch.hpp" + +#define N_PROCESSES (8) /* number of processes running in parallel, at least 2 */ +#define ALLOCATE_BUF_SIZE_MB (64) +#define ALLOCATE_RETRY_TIMES (3) + +HSAint32 KFDEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) { + HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT; + HSAuint64 sysMemSize = GetSysMemSize(); + HSAuint64 size, sizeInPages; + HSAuint32 count; + + LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; + + /* use one third of total system memory for eviction buffer to test + * limit max allocate size to duoble of vramSize + * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) + */ + size = sysMemSize/3 + vramSize; + size = size > vramSize<<1 ? vramSize<<1 : size; + sizeInPages = size >> PAGE_SHIFT; + count = sizeInPages / (vramBufSizeInPages * N_PROCESSES); + + return count; +} + +void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, + std::vector &pBuffers) { + HSAuint64 totalMB; + + totalMB = N_PROCESSES*count*(vramBufSize>>20); + if (m_IsParent) { + LOG() << "Testing " << N_PROCESSES << "*" << count << "*" << (vramBufSize>>20) << "(="<< totalMB << ")MB" << std::endl; + } + + HSAKMT_STATUS ret; + HSAuint32 retry = 0; + + m_Flags.Value = 0; + m_Flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + m_Flags.ui32.HostAccess = 0; + m_Flags.ui32.NonPaged = 1; + + for (HSAuint32 i = 0; i < count; ) { + ret = hsaKmtAllocMemory(defaultGPUNode, vramBufSize, m_Flags, &m_pBuf); + if (ret == HSAKMT_STATUS_SUCCESS) { + if (is_dgpu()) { + if (hsaKmtMapMemoryToGPU(m_pBuf, vramBufSize, NULL) == HSAKMT_STATUS_ERROR) { + EXPECT_SUCCESS(hsaKmtFreeMemory(m_pBuf, vramBufSize)); + break; + } + } + pBuffers.push_back(m_pBuf); + + i++; + retry = 0; + } else { + if (retry++ > ALLOCATE_RETRY_TIMES) { + break; + } + + /* wait for 1 second to try allocate again */ + sleep(1); + } + } +} + +void KFDEvictTest::FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize) { + for (HSAuint32 i = 0; i < pBuffers.size(); i++) { + m_pBuf = pBuffers[i]; + if (m_pBuf != NULL) { + if (is_dgpu()) + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(m_pBuf)); + EXPECT_SUCCESS(hsaKmtFreeMemory(m_pBuf, vramBufSize)); + } + } +} + +void KFDEvictTest::ForkChildProcesses(int nprocesses) { + int i; + + for (i = 0; i < nprocesses - 1; ++i) { + pid_t pid = fork(); + ASSERT_GE(pid, 0); + + if (pid == 0) { + /* Child process */ + /* Cleanup file descriptors copied from parent process + * then call SetUp->hsaKmtOpenKFD to create new process + */ + m_psName = "Test process " + std::to_string(i) + " "; + TearDown(); + SetUp(); + m_ChildPids.clear(); + m_IsParent = false; + return; + } + + /* Parent process */ + m_ChildPids.push_back(pid); + } + + m_psName = "Test process " + std::to_string(i) + " "; +} + +void KFDEvictTest::WaitChildProcesses() { + if (m_IsParent) { + /* only run by parent process */ + int childStatus; + int childExitOkNum = 0; + int size = m_ChildPids.size(); + + for (HSAuint32 i = 0; i < size; i++) { + pid_t pid = m_ChildPids.front(); + + waitpid(pid, &childStatus, 0); + if (WIFEXITED(childStatus) == 1 && WEXITSTATUS(childStatus) == 0) + childExitOkNum++; + + m_ChildPids.erase(m_ChildPids.begin()); + } + + ASSERT_EQ(childExitOkNum, size); + } + + /* child process or parent process finished successfullly */ + m_ChildStatus = HSAKMT_STATUS_SUCCESS; +} + +/* Evict and restore procedure basic test + * + * Use N_PROCESSES processes to allocate vram buf size larger than total vram size + * + * ALLOCATE_BUF_SIZE_MB buf allocation size + * + * number of buf is equal to (vramSizeMB / (vramBufSizeMB * N_PROCESSES) ) + 8 + * Total vram all processes allocated: 8GB for 4GB Fiji, and 20GB for 16GB Vega10 + * + * many times of eviction and restore will happen: + * ttm will evict buffers of another process if not enough free vram + * process restore will evict buffers of another process + * + * Sometimes the allocate may fail (maybe that is normal) + * ALLOCATE_RETRY_TIMES max retry times to allocate + * + * This is basic test, no queue so vram are not used by GPU during test + * + * Todo: + * - Synchronization between the processes, so they know for sure when + * they are done allocating memory + */ +TEST_F(KFDEvictTest, BasicTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + + if (!vramSize) { + LOG() << "No VRAM found, skipping the test" << std::endl; + return; + } else { + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; + } + + HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); + if (count == 0) { + LOG() << "Not enough system memory, skipping the test" << std::endl; + return; + } + + /* Fork the child processes */ + ForkChildProcesses(N_PROCESSES); + + std::vector pBuffers; + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + + /* wait for other processes to finish allocation, then free buffer */ + sleep(ALLOCATE_RETRY_TIMES); + + LOG() << m_psName << "free buffer" << std::endl; + FreeBuffers(pBuffers, vramBufSize); + + WaitChildProcesses(); + + TEST_END +} + +/* Shader to read local buffers using multiple wavefronts in parallel + * until address buffer is filled with specific value 0x5678 by host program, + * then each wavefront fills value 0x5678 at corresponding result buffer and quit + * + * initial state: + * s[0:1] - address buffer base address + * s[2:3] - result buffer base address + * s4 - workgroup id + * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 + * registers: + * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X + * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 + * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 + * v[6:7] - local buf address used for read test + */ +static const char* gfx9_ReadMemory = +"\ + shader ReadMemory\n\ + asic(GFX9)\n\ + type(CS)\n\ + \n\ + // compute address of corresponding output buffer\n\ + v_mov_b32 v0, s4 // use workgroup id as index\n\ + v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ + v_mov_b32 v5, s3\n\ + v_add_u32 v5, vcc_lo, v5\n\ + \n\ + // compute input buffer offset used to store corresponding local buffer address\n\ + v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ + v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ + v_mov_b32 v3, s1\n\ + v_add_u32 v3, vcc_lo, v3\n\ + \n\ + // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ + flat_load_dwordx2 v[6:7], v[2:3] slc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + \n\ + v_mov_b32 v8, 0x5678\n\ + s_movk_i32 s8, 0x5678\n\ +L_REPEAT:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + s_cmp_eq_i32 s16, s8\n\ + s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ + // loop read 64M local buffer starting at v[6:7]\n\ + // every 4k page only read once\n\ + v_mov_b32 v9, 0\n\ + v_mov_b32 v10, 0x1000 // 4k page\n\ + v_mov_b32 v11, 0x4000000 // 64M size\n\ + v_mov_b32 v12, v6\n\ + v_mov_b32 v13, v7\n\ +L_LOOP_READ:\n\ + flat_load_dwordx2 v[14:15], v[12:13] slc\n\ + v_add_u32 v9, v9, v10 \n\ + v_add_co_u32 v12, vcc, v12, v10\n\ + v_add_u32 v13, vcc_lo, v13\n\ + v_cmp_lt_u32 vcc, v9, v11\n\ + s_cbranch_vccnz L_LOOP_READ\n\ + s_branch L_REPEAT\n\ +L_QUIT:\n\ + flat_store_dword v[4:5], v8\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ + s_endpgm\n\ + end\n\ +"; + +static const char* gfx8_ReadMemory = +"\ + shader ReadMemory\n\ + asic(VI)\n\ + type(CS)\n\ + \n\ + // compute address of corresponding output buffer\n\ + v_mov_b32 v0, s4 // use workgroup id as index\n\ + v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ + v_mov_b32 v5, s3\n\ + v_addc_u32 v5, vcc, v5, 0, vcc\n\ + \n\ + // compute input buffer offset used to store corresponding local buffer address\n\ + v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ + v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ + v_mov_b32 v3, s1\n\ + v_addc_u32 v3, vcc, v3, 0, vcc\n\ + \n\ + // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ + flat_load_dwordx2 v[6:7], v[2:3] slc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + \n\ + v_mov_b32 v8, 0x5678\n\ + s_movk_i32 s8, 0x5678\n\ +L_REPEAT:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + s_cmp_eq_i32 s16, s8\n\ + s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ + // loop read 64M local buffer starting at v[6:7]\n\ + // every 4k page only read once\n\ + v_mov_b32 v9, 0\n\ + v_mov_b32 v10, 0x1000 // 4k page\n\ + v_mov_b32 v11, 0x4000000 // 64M size\n\ + v_mov_b32 v12, v6\n\ + v_mov_b32 v13, v7\n\ +L_LOOP_READ:\n\ + flat_load_dwordx2 v[14:15], v[12:13] slc\n\ + v_add_u32 v9, vcc, v9, v10 \n\ + v_add_u32 v12, vcc, v12, v10\n\ + v_addc_u32 v13, vcc, v13, 0, vcc\n\ + v_cmp_lt_u32 vcc, v9, v11\n\ + s_cbranch_vccnz L_LOOP_READ\n\ + s_branch L_REPEAT\n\ +L_QUIT:\n\ + flat_store_dword v[4:5], v8\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ + s_endpgm\n\ + end\n\ +"; + +std::string KFDEvictTest::CreateShader() { + if (m_FamilyId >= FAMILY_AI) + return gfx9_ReadMemory; + else + return gfx8_ReadMemory; +} + +/* Evict and restore queue test + * + * N_PROCESSES processes read all local buffers in parallel while buffers are evicted and restored + * If GPU vm page fault happens, then test shader will stop and failed to write specific value + * at dest buffer. Test will report failed. + * + * Steps: + * - fork N_PROCESSES processes, each process does the same below + * - allocate local buffers, each buffer size is 64MB + * - allocate zero initialized host access address buffer and result buffer + * address buffer to pass address of local buffers to shader + * result buffer to store shader output result + * - submit queue to run ReadMemory shader + * - shader start m_DimX wavefronts, each wavefront keep reading one local buffer + * - notify shader to quit + * - check result buffer with specific value to confirm all wavefronts quit normally + */ +TEST_F(KFDEvictTest, QueueTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + + const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + + /* Skip test for chip it doesn't have CWSR, which the test depends on */ + if (m_FamilyId < FAMILY_VI || isTonga(pNodeProperties)) { + LOG() << std::hex << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + return; + } + + HSAuint32 i; + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + + if (!vramSize) { + LOG() << "No VRAM found, skipping the test" << std::endl; + return; + } else { + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; + } + + HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); + if (count == 0) { + LOG() << "Not enough system memory, skipping the test" << std::endl; + return; + } + /* assert all buffer address can be stored within one page + * because only one page host memory srcBuf is allocated + */ + ASSERT_LE(count, PAGE_SIZE/sizeof(unsigned int *)); + + /* Fork the child processes */ + ForkChildProcesses(N_PROCESSES); + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer addrBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer resultBuffer(PAGE_SIZE, defaultGPUNode); + + std::vector pBuffers; + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + + unsigned int wavefront_num = pBuffers.size(); + LOG() << m_psName << "wavefront number " << wavefront_num << std::endl; + + void **localBufAddr = addrBuffer.As(); + unsigned int *result = resultBuffer.As(); + + for (i = 0; i < wavefront_num; i++) + *(localBufAddr + i) = pBuffers[i]; + + m_pIsaGen->CompileShader(CreateShader().c_str(), "ReadMemory", isaBuffer); + + PM4Queue pm4Queue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + + Dispatch dispatch0(isaBuffer); + dispatch0.SetArgs(localBufAddr, result); + dispatch0.SetDim(wavefront_num, 1, 1); + /* submit the packet and start shader */ + dispatch0.Submit(pm4Queue); + + /* doing evict/restore queue test for 5 seconds while queue is running */ + sleep(5); + + /* LOG() << m_psName << "notify shader to quit" << std::endl; */ + /* fill address buffer so shader quits */ + addrBuffer.Fill(0x5678); + + /* wait for shader to finish or timeout if shade has vm page fault */ + dispatch0.SyncWithStatus(120000); + + ASSERT_SUCCESS(pm4Queue.Destroy()); + /* LOG() << m_psName << "free buffer" << std::endl; */ + /* cleanup */ + FreeBuffers(pBuffers, vramBufSize); + + /* check if all wavefronts finish successfully */ + for (i = 0; i < wavefront_num; i++) + ASSERT_EQ(0x5678, *(result + i)); + + WaitChildProcesses(); + + TEST_END +} + diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp new file mode 100644 index 0000000000..3f500b2655 --- /dev/null +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2017-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_EVICT_TEST__H__ +#define __KFD_EVICT_TEST__H__ + +#include +#include +#include "KFDLocalMemoryTest.hpp" +#include "KFDBaseComponentTest.hpp" +#include "IsaGenerator.hpp" + +// @class KFDEvictTest +// Test eviction and restore procedure using two processes +class KFDEvictTest : public KFDLocalMemoryTest { + public: + KFDEvictTest(void): m_ChildStatus(HSAKMT_STATUS_ERROR), m_IsParent(true) {} + + ~KFDEvictTest(void) { + if (!m_IsParent) { + /* child process has to exit + * otherwise gtest will continue other tests + */ + exit(m_ChildStatus); + } + + try { + WaitChildProcesses(); + } catch (...) {} + } + + protected: + std::string CreateShader(); + void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, + std::vector &pBuffers); + void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); + void ForkChildProcesses(int nprocesses); + void WaitChildProcesses(); + HSAint32 GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize); + + protected: // members + std::string m_psName; + std::vector m_ChildPids; + HsaMemFlags m_Flags; + void* m_pBuf; + HSAKMT_STATUS m_ChildStatus; + bool m_IsParent; +}; + +#endif // __KFD_EVICT_TEST__H__ diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp new file mode 100644 index 0000000000..1a4a715d97 --- /dev/null +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -0,0 +1,355 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDExceptionTest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "Dispatch.hpp" + +// All tests are marked by their serial number in the QCM FDD + +void KFDExceptionTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDExceptionTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +/* Test for memory exception. The function expects a Memory Fault to be + * triggered by the GPU when it tries to copy dword from pSrc to pDst. + * Should be called from a Child Process since the Memory Fault causes + * all the queues to be halted. +*/ +void KFDExceptionTest::TestMemoryException(int defaultGPUNode, HSAuint64 pSrc, + HSAuint64 pDst, unsigned int dimX, + unsigned int dimY, unsigned int dimZ) { + PM4Queue queue; + HsaEvent *vmFaultEvent; + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HSAuint64 faultAddress, page_mask = ~((HSAuint64)PAGE_SIZE - 1); + Dispatch dispatch(isaBuffer, false); + + HsaEventDescriptor eventDesc; + eventDesc.EventType = HSA_EVENTTYPE_MEMORY; + eventDesc.NodeId = defaultGPUNode; + eventDesc.SyncVar.SyncVar.UserData = NULL; + eventDesc.SyncVar.SyncVarSize = 0; + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + m_ChildStatus = queue.Create(defaultGPUNode); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "Queue create failed" << std::endl; + return; + } + m_ChildStatus = hsaKmtCreateEvent(&eventDesc, true, false, &vmFaultEvent); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "Event create failed" << std::endl; + goto queuefail; + } + + dispatch.SetDim(dimX, dimY, dimZ); + dispatch.SetArgs((void *)pSrc, (void *)pDst); + dispatch.Submit(queue); + + m_ChildStatus = hsaKmtWaitOnEvent(vmFaultEvent, g_TestTimeOut); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "Wait failed. No Exception triggered" << std::endl; + goto eventfail; + } + + if (vmFaultEvent->EventData.EventType != HSA_EVENTTYPE_MEMORY) { + WARN() << "Unexpected Event Received " << vmFaultEvent->EventData.EventType + << std::endl; + m_ChildStatus = HSAKMT_STATUS_ERROR; + goto eventfail; + } + faultAddress = vmFaultEvent->EventData.EventData.MemoryAccessFault.VirtualAddress; + if (faultAddress != (pSrc & page_mask) && + faultAddress != (pDst & page_mask) ) { + WARN() << "Unexpected Fault Address " << faultAddress + << " expected " << (pSrc & page_mask) << " or " + << (pDst & page_mask) << std::endl; + m_ChildStatus = HSAKMT_STATUS_ERROR; + } + +eventfail: + hsaKmtDestroyEvent(vmFaultEvent); +queuefail: + queue.Destroy(); +} + +/* Test Bad Address access in a child process */ +TEST_F(KFDExceptionTest, InvalidBadAddress) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + return; + } + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + m_ChildStatus = hsaKmtOpenKFD(); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in child process" << std::endl; + return; + } + + HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode, false); + + srcBuffer.Fill(0xAA55AA55); + TestMemoryException(defaultGPUNode, srcBuffer.As(), + 0x12345678ULL); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + if (is_dgpu()) { + ASSERT_EQ(WIFEXITED(childStatus), true); + ASSERT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } else { + ASSERT_EQ(WIFSIGNALED(childStatus), true); + ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + } + + TEST_END +} + +/* TODO: Check why this test doesn't behave as expected. + * Expected behaviour: Since pDst is not mapped in GPU VM, an attempt + * to write to pDst should trigger a Memory Fault. + * However, If pDst > 40bits (MC Size) GPU does not create Memory Fault. + * An EOP interrupt is generated however as expected no data is copied + * into pDst. + */ +TEST_F(KFDExceptionTest, InvalidPPRWriteProtection) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (is_dgpu()) { + LOG() << "Not an APU, no PPR available, skip the test" << std::endl; + return; + } + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + return; + } + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + m_ChildStatus = hsaKmtOpenKFD(); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in child process" << std::endl; + return; + } + + HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode, false); + void* pDst = VirtualAllocMemory(NULL, PAGE_SIZE, MEM_READ); + + LOG() << "PPR dst address is " << pDst << std::endl; + + srcBuffer.Fill(0xAA55AA55); + TestMemoryException(defaultGPUNode, srcBuffer.As(), + (HSAuint64)pDst); + + /* Wait enough time here to ensure this process got killed by kernel + * due to PPR exception. + */ + sleep(5); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + ASSERT_EQ(WIFSIGNALED(childStatus), true); + ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + + TEST_END +} + +/* TODO: Same as previous test InvalidPPRWriteProtection + */ +TEST_F(KFDExceptionTest, InvalidPPRReadProtection) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (is_dgpu()) { + LOG() << "Not an APU, no PPR available, skip the test" << std::endl; + return; + } + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + return; + } + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + m_ChildStatus = hsaKmtOpenKFD(); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in child process" << std::endl; + return; + } + + HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode, false); + void* pSrc = VirtualAllocMemory(NULL, PAGE_SIZE, MEM_NONE); + + LOG() << "PPR src address is " << pSrc << std::endl; + + dstBuffer.Fill(0xAA55AA55); + TestMemoryException(defaultGPUNode, (HSAuint64)pSrc, + dstBuffer.As()); + + /* Wait enough time here to ensure this process got killed by kernel + * due to PPR exception. + */ + sleep(5); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + ASSERT_EQ(WIFSIGNALED(childStatus), true); + ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + + TEST_END +} + +/* Allocate Read Only buffer. Test Memory Exception failure by + * attempting to write to that buffer in the child process. + */ +TEST_F(KFDExceptionTest, InvalidWriteAddress) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + return; + } + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + m_ChildStatus = hsaKmtOpenKFD(); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in child process" << std::endl; + return; + } + + HsaMemoryBuffer readOnlyBuffer(PAGE_SIZE, defaultGPUNode, false /*zero*/, + false /*isLocal*/, true /*isExec*/, + false /*isScratch*/, true /*isReadOnly*/); + HsaMemoryBuffer srcSysBuffer(PAGE_SIZE, defaultGPUNode, false); + + srcSysBuffer.Fill(0xAA55AA55); + + TestMemoryException(defaultGPUNode, srcSysBuffer.As(), + readOnlyBuffer.As()); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + if (is_dgpu()) { + ASSERT_EQ(WIFEXITED(childStatus), true); + ASSERT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } else { + ASSERT_EQ(WIFSIGNALED(childStatus), true); + ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + } + + TEST_END +} + +/* Test VM fault storm handling by copying to/from invalid pointers + * with lots of work items at the same time + */ +TEST_F(KFDExceptionTest, FaultStorm) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + return; + } + + HSAKMT_STATUS status; + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + m_ChildStatus = hsaKmtOpenKFD(); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in child process" << std::endl; + return; + } + + TestMemoryException(defaultGPUNode, 0x12345678, 0x76543210, 1024, 1024, 1); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + if (is_dgpu()) { + ASSERT_EQ(WIFEXITED(childStatus), true); + ASSERT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } else { + ASSERT_EQ(WIFSIGNALED(childStatus), true); + ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + } + + TEST_END +} diff --git a/tests/kfdtest/src/KFDExceptionTest.hpp b/tests/kfdtest/src/KFDExceptionTest.hpp new file mode 100644 index 0000000000..24bf433437 --- /dev/null +++ b/tests/kfdtest/src/KFDExceptionTest.hpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_EXCEPTION_TEST__H__ +#define __KFD_EXCEPTION_TEST__H__ + +#include + +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" + +class KFDExceptionTest : public KFDBaseComponentTest { + public: + KFDExceptionTest() :m_pIsaGen(NULL), m_ChildPid(-1) { + /* Because there could be early return before m_ChildPid is set + * by fork(), we should initialize m_ChildPid to a non-zero value + * to avoid possible exit of the main process. + */ + } + + ~KFDExceptionTest() { + /* exit() is necessary for the child process. Otherwise when the + * child process finishes, gtest assumes the test has finished and + * starts the next test while the parent is still active. + */ + if (m_ChildPid == 0) + exit(m_ChildStatus); + } + + protected: + virtual void SetUp(); + virtual void TearDown(); + + void TestMemoryException(int defaultGPUNode, HSAuint64 pSrc, HSAuint64 pDst, + unsigned int dimX = 1, unsigned int dimY = 1, + unsigned int dimZ = 1); + + protected: // members + pid_t m_ChildPid; + HSAKMT_STATUS m_ChildStatus; + + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_EXCEPTION_TEST__H__ diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp new file mode 100644 index 0000000000..b99c1f3b9c --- /dev/null +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -0,0 +1,317 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDGraphicsInterop.hpp" + +extern "C" { +#include +#include +#include +} +#include +#include +#include +#include +#include +#include "Dispatch.hpp" +#include "PM4Queue.hpp" + +void KFDGraphicsInterop::SetUp() { + ROUTINE_START + + KFDMemoryTest::SetUp(); + + // Try to open and initialize a render node for each device + for (int i = 0; i < MAX_RENDER_NODES; i++) { + m_RenderNodes[i].fd = drmOpenRender(i + 128); + + if (m_RenderNodes[i].fd <= 0) + continue; + + if (amdgpu_device_initialize(m_RenderNodes[i].fd, + &m_RenderNodes[i].major_version, + &m_RenderNodes[i].minor_version, + &m_RenderNodes[i].device_handle) != 0) { + drmClose(m_RenderNodes[i].fd); + m_RenderNodes[i].fd = 0; + } + + // Try to determine the bus-ID from sysfs + char path[PATH_MAX], link[PATH_MAX]; + snprintf(path, PATH_MAX, "/sys/class/drm/renderD%d", i+128); + if (readlink(path, link, PATH_MAX) < 0) { + LOG() << "Failed to read sysfs link " << path + << ", can't determine bus ID." << std::endl; + continue; + } + + char *state = NULL; + char *prev = NULL; + char *tok = strtok_r(link, "/", &state); + while (tok && strcmp(tok, "drm")) { + prev = tok; + tok = strtok_r(NULL, "/", &state); + } + unsigned domain, bus, device, func; + if (!prev || + sscanf(prev, "%04x:%02x:%02x.%1x", &domain, &bus, &device, &func) + != 4) { + LOG() << "Failed to parse sysfs link " << path + << ", can't determine bus ID." << std::endl; + continue; + } + + m_RenderNodes[i].bdf = (bus << 8) | (device << 3) | func; + } + + ROUTINE_END +} + +void KFDGraphicsInterop::TearDown() { + ROUTINE_START + + for (int i = 0; i < MAX_RENDER_NODES; i++) { + if (m_RenderNodes[i].fd <= 0) + continue; + + EXPECT_EQ(0, amdgpu_device_deinitialize(m_RenderNodes[i].device_handle)); + EXPECT_EQ(0, drmClose(m_RenderNodes[i].fd)); + } + + KFDMemoryTest::TearDown(); + + ROUTINE_END +} + +int KFDGraphicsInterop::FindDRMRenderNode(int gpuNode) { + int rn; + const HsaNodeProperties *pNodeProps = + m_NodeInfo.GetNodeProperties(gpuNode); + + for (rn = 0; rn < MAX_RENDER_NODES; rn++) { + if (m_RenderNodes[rn].fd <= 0) + continue; + if (m_RenderNodes[rn].bdf == pNodeProps->LocationId) + return rn; + } + LOG() << "Found no render node corresponding to GPU node " + << gpuNode << std::endl; + LOG() << "Check your device permissions" << std::endl; + return -1; +} + +TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + const HsaNodeProperties *pNodeProps = + m_NodeInfo.GetNodeProperties(defaultGPUNode); + const HSAuint32 familyID = FamilyIdFromNode(pNodeProps); + + if (isTonga(pNodeProps)) { + LOG() << "Skipping test: Tonga workaround in thunk returns incorrect allocation size" << std::endl; + return; + } + + HSAuint32 nodes[1] = {(uint32_t)defaultGPUNode}; + + const char metadata[] = "This data is really meta."; + unsigned metadata_size = strlen(metadata)+1; + int rn = FindDRMRenderNode(defaultGPUNode); + + if (rn < 0) { + LOG() << "Skipping test" << std::endl; + return; + } + + // Create the buffer with metadata and get a dmabuf handle to it + struct amdgpu_bo_alloc_request alloc; + amdgpu_bo_handle handle; + if (familyID == FAMILY_CZ || isTonga(pNodeProps)) + alloc.alloc_size = PAGE_SIZE * 8; + else + alloc.alloc_size = PAGE_SIZE; + alloc.phys_alignment = PAGE_SIZE; + alloc.preferred_heap = AMDGPU_GEM_DOMAIN_VRAM; + alloc.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + ASSERT_EQ(0, amdgpu_bo_alloc(m_RenderNodes[rn].device_handle, &alloc, &handle)); + + void *pCpuMap; + ASSERT_EQ(0, amdgpu_bo_cpu_map(handle, &pCpuMap)); + memset(pCpuMap, 0xaa, PAGE_SIZE); + ASSERT_EQ(0, amdgpu_bo_cpu_unmap(handle)); + + struct amdgpu_bo_metadata meta; + meta.flags = 0; + meta.tiling_info = 0; + meta.size_metadata = metadata_size; + memcpy(meta.umd_metadata, metadata, metadata_size); + ASSERT_EQ(0, amdgpu_bo_set_metadata(handle, &meta)); + + uint32_t dmabufFd; + ASSERT_EQ(0, amdgpu_bo_export(handle, amdgpu_bo_handle_type_dma_buf_fd, &dmabufFd)); + + // Register it with HSA + HsaGraphicsResourceInfo info; + ASSERT_SUCCESS(hsaKmtRegisterGraphicsHandleToNodes(dmabufFd, &info, + 1, nodes)); + + // DMA buffer handle and GEM handle are no longer needed, KFD + // should have taken a reference to the BO + ASSERT_EQ(0, close(dmabufFd)); + ASSERT_EQ(0, amdgpu_bo_free(handle)); + + // Check that buffer size and metadata match + ASSERT_EQ(info.SizeInBytes, alloc.alloc_size); + ASSERT_EQ(info.MetadataSizeInBytes, metadata_size); + ASSERT_EQ(0, strcmp(metadata, (const char *)info.Metadata)); + + // Map the buffer + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(info.MemoryAddress, + info.SizeInBytes, + NULL)); + + // Copy contents to a system memory buffer for comparison + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/); + + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + Dispatch dispatch(isaBuffer); + + dispatch.SetArgs(info.MemoryAddress, dstBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + ASSERT_SUCCESS(queue.Destroy()); + + ASSERT_EQ(dstBuffer.As()[0], 0xaaaaaaaa); + + // Test QueryMem before the cleanup + HsaPointerInfo ptrInfo; + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((const void *)info.MemoryAddress, &ptrInfo)); + EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_GRAPHICS); + EXPECT_EQ(ptrInfo.Node, (HSAuint32)defaultGPUNode); + EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)info.MemoryAddress); + EXPECT_EQ(ptrInfo.SizeInBytes, alloc.alloc_size); + + // Cleanup + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(info.MemoryAddress)); + ASSERT_SUCCESS(hsaKmtDeregisterMemory(info.MemoryAddress)); + + TEST_END +} + +/* Third-party device memory can be registered for GPU access in + * ROCm stack. Test this feature. Third party device is mimicked + * in multi-GPU system using Graphics stack (libdrm). CPU accessible + * device memory is allocated using Graphics stack on gpuNode2 and + * this memory will be registered on gpuNode1 for GPU access. + */ +TEST_F(KFDGraphicsInterop, RegisterForeignDeviceMem) { + TEST_START(TESTPROFILE_RUNALL) + + if (!is_dgpu()) { + LOG() << "Skipping test: Supports only multi-dGPU system" << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: Need at least two GPUs" << std::endl; + return; + } + + /* gpuNode2 must have public memory (large bar) to allocate CPU accessible + * device memory. + */ + HSAint32 gpuNode1 = m_NodeInfo.HsaDefaultGPUNode(), gpuNode2 = 0; + const HsaNodeProperties *pNodeProperties; + + gpuNode2 = m_NodeInfo.FindLargeBarGPUNode(); + if (gpuNode2 < 0) { + LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + return; + } + if (gpuNode1 == gpuNode2) { + for (unsigned i = 0; i < gpuNodes.size(); i++) { + if (gpuNodes.at(i) != gpuNode2) { + gpuNode1 = gpuNodes.at(i); + break; + } + } + } + + const HsaNodeProperties *pNodeProps = + m_NodeInfo.GetNodeProperties(gpuNode2); + const HSAuint32 familyID = FamilyIdFromNode(pNodeProps); + + int rn = FindDRMRenderNode(gpuNode2); + if (rn < 0) { + LOG() << "Skipping test" << std::endl; + return; + } + + // Allocate CPU accessible device memory on gpuNode2 + struct amdgpu_bo_alloc_request alloc; + amdgpu_bo_handle handle; + if (familyID == FAMILY_CZ || isTonga(pNodeProps)) + alloc.alloc_size = PAGE_SIZE * 8; + else + alloc.alloc_size = PAGE_SIZE; + alloc.phys_alignment = PAGE_SIZE; + alloc.preferred_heap = AMDGPU_GEM_DOMAIN_VRAM; + alloc.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + ASSERT_EQ(0, amdgpu_bo_alloc(m_RenderNodes[rn].device_handle, &alloc, &handle)); + + void *pCpuMap; + ASSERT_EQ(0, amdgpu_bo_cpu_map(handle, &pCpuMap)); + memset(pCpuMap, 0xAA, PAGE_SIZE); + + /* Register third-party device memory in KFD. Test GPU access + * by carrying out a simple copy test + */ + HsaMemoryBuffer lockDeviceMemory(pCpuMap, PAGE_SIZE); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, gpuNode1, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer dstBuffer(PAGE_SIZE, gpuNode1, true/*zero*/); + PM4Queue queue; + Dispatch dispatch(isaBuffer); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(queue.Create(gpuNode1)); + + dispatch.SetArgs(lockDeviceMemory.As(), dstBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + ASSERT_SUCCESS(queue.Destroy()); + ASSERT_EQ(dstBuffer.As()[0], 0xAAAAAAAA); + + ASSERT_EQ(0, amdgpu_bo_cpu_unmap(handle)); + ASSERT_EQ(0, amdgpu_bo_free(handle)); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDGraphicsInterop.hpp b/tests/kfdtest/src/KFDGraphicsInterop.hpp new file mode 100644 index 0000000000..9b94062520 --- /dev/null +++ b/tests/kfdtest/src/KFDGraphicsInterop.hpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDMemoryTest.hpp" +extern "C" { +#include +} + +#ifndef __KFD_GRAPHICS_INTEROP_TEST__H__ +#define __KFD_GRAPHICS_INTEROP_TEST__H__ + +// @class KFDGraphicsInteropTest +// Adds access to graphics device for interoperability testing +class KFDGraphicsInterop : public KFDMemoryTest +{ +public: + KFDGraphicsInterop(void) {}; + ~KFDGraphicsInterop(void) {}; +protected: + virtual void SetUp(); + virtual void TearDown(); + +protected: +#define MAX_RENDER_NODES 64 + struct { + int fd; + uint32_t major_version; + uint32_t minor_version; + amdgpu_device_handle device_handle; + uint32_t bdf; + } m_RenderNodes[MAX_RENDER_NODES]; + +// @brief Finds DRM Render node corresponding to gpuNode +// @return DRM Render Node if successful or -1 on failure +int FindDRMRenderNode(int gpuNode); +}; + +#endif diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp new file mode 100644 index 0000000000..4f7e90adac --- /dev/null +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -0,0 +1,731 @@ +/* + * Copyright (C) 2017-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDIPCTest.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAQueue.hpp" +#include "SDMAPacket.hpp" + +void KFDIPCTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + ROUTINE_END +} + +void KFDIPCTest::TearDown() { + ROUTINE_START + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +KFDIPCTest::~KFDIPCTest(void) { + /* exit() is necessary for the child process. Otherwise when the + * child process finishes, gtest assumes the test has finished and + * starts the next test while the parent is still active. + */ + if (m_ChildPid == 0) + exit(0); +} + +/* Imort shared Local Memory from parent process. Check for the pattern + * filled in by the parent process. Then fill a new pattern. + */ +void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { + /* Open KFD device for child process. This needs to called before + * any memory definitions + */ + if (HSAKMT_STATUS_SUCCESS != hsaKmtOpenKFD()) + exit(1); + + SDMAQueue sdmaQueue; + HsaSharedMemoryHandle sharedHandleLM; + HSAuint64 size = PAGE_SIZE, sharedSize; + HsaMemoryBuffer tempSysBuffer(size, defaultGPUNode, false); + HSAuint32 *sharedLocalBuffer = NULL; + + /* Read from Pipe the shared Handle. Import shared Local Memory */ + ASSERT_GE(read(pipefd[0], (void*)&sharedHandleLM, sizeof(sharedHandleLM)), 0); + + ASSERT_SUCCESS(hsaKmtRegisterSharedHandle(&sharedHandleLM, + (void**)&sharedLocalBuffer, &sharedSize)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(sharedLocalBuffer, sharedSize, NULL)); + + /* Check for pattern in the shared Local Memory */ + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + size = size < sharedSize ? size : sharedSize; + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(tempSysBuffer.As(), + sharedLocalBuffer, size)); + sdmaQueue.Wait4PacketConsumption(); + ASSERT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xAAAAAAAA)); + + /* Fill in the Local Memory with different pattern */ + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sharedLocalBuffer, 0xBBBBBBBB)); + sdmaQueue.Wait4PacketConsumption(); + + /* Clean up */ + ASSERT_SUCCESS(sdmaQueue.Destroy()); + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(sharedLocalBuffer)); + ASSERT_SUCCESS(hsaKmtDeregisterMemory(sharedLocalBuffer)); +} + +/* Fill a pattern in to Local Memory and share with the child process. + * Then wait until Child process to exit and check for the new pattern + * fill in by the child process. + */ + +void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pipefd) { + HSAuint64 size = PAGE_SIZE, sharedSize; + int status; + HSAuint64 AlternateVAGPU; + HsaMemoryBuffer toShareLocalBuffer(size, defaultGPUNode, false, true); + HsaMemoryBuffer tempSysBuffer(PAGE_SIZE, defaultGPUNode, false); + SDMAQueue sdmaQueue; + HsaSharedMemoryHandle sharedHandleLM; + + /* Fill a Local Buffer with a pattern */ + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(toShareLocalBuffer.As(), toShareLocalBuffer.Size(), &AlternateVAGPU)); + tempSysBuffer.Fill(0xAAAAAAAA); + + /* Copy pattern in Local Memory before sharing it */ + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(toShareLocalBuffer.As(), + tempSysBuffer.As(), size)); + sdmaQueue.Wait4PacketConsumption(); + + /* Share it with the child process */ + ASSERT_SUCCESS(hsaKmtShareMemory(toShareLocalBuffer.As(), size, &sharedHandleLM)); + + ASSERT_GE(write(pipefd[1], (void*)&sharedHandleLM, sizeof(sharedHandleLM)), 0); + + /* Wait for the child to finish */ + waitpid(cpid, &status, 0); + + ASSERT_EQ(WIFEXITED(status), 1); + ASSERT_EQ(WEXITSTATUS(status), 0); + + /* Check for the new pattern filled in by child process */ + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(tempSysBuffer.As(), + toShareLocalBuffer.As(), size)); + sdmaQueue.Wait4PacketConsumption(); + ASSERT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xBBBBBBBB)); + + /* Clean up */ + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(toShareLocalBuffer.As())); + ASSERT_SUCCESS(sdmaQueue.Destroy()); +} + +/* Test IPC memory. + * 1. Parent Process [Create/Fill] LocalMemory (LM) --share--> Child Process + * 2. Child Process import LM and check for the pattern. + * 3. Child Process fill in a new pattern and quit. + * 4. Parent Process wait for the Child process to finish and then check for + * the new pattern in LM + * + * IPC support is limited to Local Memory. + */ + +TEST_F(KFDIPCTest, BasicTest) { + TEST_START(TESTPROFILE_RUNALL) + + const std::vector& GpuNodes = m_NodeInfo.GetNodesWithGPU(); + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + int pipefd[2]; + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skip the test because of no VRAM found" << std::endl; + return; + } + + /* Test libhsakmt fork() clean up by defining some buffers. These + * buffers gets duplicated in the child process but not are not valid + * as it doesn't have proper mapping in GPU. The clean up code in libhsakmt + * should handle it + */ + volatile HSAuint32 stackData[1]; + HsaMemoryBuffer tmpSysBuffer(PAGE_SIZE, defaultGPUNode, false); + HsaMemoryBuffer tmpUserptrBuffer((void *)&stackData[0], sizeof(HSAuint32)); + + /* Create Pipes for communicating shared handles */ + ASSERT_EQ(pipe(pipefd), 0); + + /* Create a child process and share the above Local Memory with it */ + + m_ChildPid = fork(); + if (m_ChildPid == 0) + BasicTestChildProcess(defaultGPUNode, pipefd); /* Child Process */ + else + BasicTestParentProcess(defaultGPUNode, m_ChildPid, pipefd); /* Parent proces */ + + /* Code path executed by both parent and child with respective fds */ + close(pipefd[1]); + close(pipefd[0]); + + TEST_END +} + +/* Cross Memory Attach Test. Memory Descriptor Array. + * The following 2 2D-arrays describe the source and destination memory arrays used + * by CMA test. The entry is only valid if Size != 0. Each of these buffers will be + * filled intially with "FillPattern". After the test the srcRange is still expected + * to have the same pattern. The dstRange is expected to have srcRange pattern. + * + * For e.g. for TEST_COUNT = 1, + * srcRange has 2 buffers of size 0x1800. Buf1 filled with 0xA5A5A5A5 and Buf2 + * filled with 0xAAAAAAAA + * dstRange has 3 buffers of size 0x1000. All of them filled 0xFFFFFFFF. + * After Copy: dstBuf1[0-0x1000] is expected to be 0xA5A5A5A5 + * dstBuf2[0-0x800] is expected to be 0xA5A5A5A5 + * dstBuf3[0x800-0x1000] is expected to be 0xAAAAAAAA + * and dstBuf4[0x0-0x1000] is expected to be 0xAAAAAAAA + * + * For this CMA test after copy only the first and the last of dstBuf is checked + */ + +static testMemoryDescriptor srcRange[CMA_TEST_COUNT][CMA_MEMORY_TEST_ARRAY_SIZE] = { + { /* Memory Type Size FillPattern FirstItem Last item */ + { CMA_MEM_TYPE_USERPTR, 0x801800, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_USERPTR, 0x1800, 0xAAAAAAAA, 0xAAAAAAAA, 0xAAAAAAAA }, + { CMA_MEM_TYPE_USERPTR, 0x0, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_USERPTR, 0x0, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + }, + { + { CMA_MEM_TYPE_SYSTEM, 0x208000, 0xDEADBEEF, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_SYSTEM, 0x4000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_SYSTEM, 0x6000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_SYSTEM, 0x2000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + }, + { + { CMA_MEM_TYPE_LOCAL_MEM, 0x800000, 0xDEADBEEF, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, + } +}; + +static testMemoryDescriptor dstRange[CMA_TEST_COUNT][CMA_MEMORY_TEST_ARRAY_SIZE] = { + { + /* Memory Type Size FillPattern FirstItem Last item */ + { CMA_MEM_TYPE_USERPTR, 0x801000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_USERPTR, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xAAAAAAAA }, + { CMA_MEM_TYPE_USERPTR, 0x1000, 0xFFFFFFFF, 0xAAAAAAAA, 0xAAAAAAAA }, + { CMA_MEM_TYPE_USERPTR, 0x0, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, + }, + { + { CMA_MEM_TYPE_SYSTEM, 0x202000, 0xFFFFFFFF, 0xDEADBEEF, 0xDEADBEEF }, + { CMA_MEM_TYPE_SYSTEM, 0x4000, 0xFFFFFFFF, 0xDEADBEEF, 0xDEADBEEF }, + { CMA_MEM_TYPE_SYSTEM, 0x8000, 0xFFFFFFFF, 0xDEADBEEF, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_SYSTEM, 0x6000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, + }, + { + { CMA_MEM_TYPE_LOCAL_MEM, 0x800000, 0xFFFFFFFF, 0xDEADBEEF, 0xDEADBEEF }, + { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, + { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, + } +}; + +KFDCMAArray::KFDCMAArray() : m_ValidCount(0), m_QueueArray(HSA_QUEUE_SDMA) { + memset(m_MemArray, 0, sizeof(m_MemArray)); + memset(m_HsaMemoryRange, 0, sizeof(m_HsaMemoryRange)); +} + +CMA_TEST_STATUS KFDCMAArray::Destroy() { + for (int i = 0; i < m_ValidCount; i++) { + if (m_MemArray[i]) { + void *userPtr; + + userPtr = m_MemArray[i]->GetUserPtr(); + delete m_MemArray[i]; + + if (userPtr) + free(userPtr); + } + } + + memset(m_MemArray, 0, sizeof(m_MemArray)); + memset(m_HsaMemoryRange, 0, sizeof(m_HsaMemoryRange)); + m_ValidCount = 0; + + return CMA_TEST_SUCCESS; +} + +/* Initialize KFDCMAArray based on array of testMemoryDescriptor. Usually testMemoryDescriptor[] is + * statically defined array by the user. Only items with non-zero size are considered valid + */ +CMA_TEST_STATUS KFDCMAArray::Init(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE], int node) { + CMA_TEST_STATUS err = CMA_TEST_SUCCESS; + memset(m_MemArray, 0, sizeof(m_MemArray)); + memset(m_HsaMemoryRange, 0, sizeof(m_HsaMemoryRange)); + + m_ValidCount = 0; + for (int i = 0; i < CMA_MEMORY_TEST_ARRAY_SIZE; i++) { + if ((*memDescriptor)[i].m_MemSize == 0) + continue; + + switch ((*memDescriptor)[i].m_MemType) { + case CMA_MEM_TYPE_SYSTEM: + m_MemArray[i] = new HsaMemoryBuffer((*memDescriptor)[i].m_MemSize, node); + break; + + case CMA_MEM_TYPE_USERPTR: + { + void *userPtr = malloc((*memDescriptor)[i].m_MemSize); + m_MemArray[i] = new HsaMemoryBuffer(userPtr, (*memDescriptor)[i].m_MemSize); + break; + } + + case CMA_MEM_TYPE_LOCAL_MEM: + m_MemArray[i] = new HsaMemoryBuffer((*memDescriptor)[i].m_MemSize, node, false, true); + break; + } + + if (m_MemArray[i]) { + m_HsaMemoryRange[i].MemoryAddress = m_MemArray[i]->As(); + m_HsaMemoryRange[i].SizeInBytes = m_MemArray[i]->Size(); + m_ValidCount++; + } else { + err = CMA_TEST_NOMEM; + break; + } + } + + return err; +} + +/* Fill each buffer of KFDCMAArray with the pattern described by testMemoryDescriptor[] */ +void KFDCMAArray::FillPattern(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE]) { + SDMAQueue sdmaQueue; + bool queueCreated = false; + unsigned int queueNode; + + for (int i = 0; i < m_ValidCount; i++) { + if (m_MemArray[i]->isLocal()) + m_MemArray[i]->Fill((*memDescriptor)[i].m_FillPattern, *m_QueueArray.GetQueue(m_MemArray[i]->Node())); + else + m_MemArray[i]->Fill((*memDescriptor)[i].m_FillPattern); + } +} + +/* Check the first and last item of each buffer in KFDCMAArray with the pattern described by + * testMemoryDescriptor[]. Return 0 on success. + */ +CMA_TEST_STATUS KFDCMAArray::checkPattern(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE]) { + HSAuint64 lastItem; + CMA_TEST_STATUS ret = CMA_TEST_SUCCESS; + unsigned int queueNode = 0; + bool queueCreated = false; + HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); + volatile HSAuint32 *tmp = tmpBuffer.As(); + + for (int i = 0; i < m_ValidCount; i++) { + lastItem = m_MemArray[i]->Size(); + lastItem -= sizeof(HSAuint32); + + if (m_MemArray[i]->isLocal()) { + BaseQueue *sdmaQueue = m_QueueArray.GetQueue(m_MemArray[i]->Node()); + + if (!m_MemArray[i]->IsPattern(0, (*memDescriptor)[i].m_CheckFirstWordPattern, *sdmaQueue, tmp) || + !m_MemArray[i]->IsPattern(lastItem, (*memDescriptor)[i].m_CheckLastWordPattern, *sdmaQueue, tmp)) { + ret = CMA_CHECK_PATTERN_ERROR; + break; + } + + } else { + if (!m_MemArray[i]->IsPattern(0, (*memDescriptor)[i].m_CheckFirstWordPattern) || + !m_MemArray[i]->IsPattern(lastItem, (*memDescriptor)[i].m_CheckLastWordPattern)) { + ret = CMA_CHECK_PATTERN_ERROR; + break; + } + } + } + + return ret; +} + + +/* Non-blocking read and write to avoid Test from hanging (block indefinitely) + * if either server or client process exits due to assert failure + */ +static int write_non_block(int fd, const void *buf, int size) { + int total_bytes = 0, cur_bytes = 0; + int retries = 5; + struct timespec tm = { 0, 10000000ULL }; + const char *ptr = (const char *)buf; + + do { + cur_bytes = write(fd, ptr, (size - total_bytes)); + + if (cur_bytes < 0 && errno != EAGAIN) + return cur_bytes; + + if (cur_bytes > 0) { + total_bytes += cur_bytes; + ptr += cur_bytes; + } + + if (total_bytes < size) + nanosleep(&tm, NULL); + } while (total_bytes < size && retries--); + + /* Check for overflow */ + if (total_bytes > size) + return -1; + + return total_bytes; +} + +static int read_non_block(int fd, void *buf, int size) { + int total_bytes = 0, cur_bytes = 0; + int retries = 5; + struct timespec tm = { 0, 100000000ULL }; + char *ptr = (char *)buf; + + do { + cur_bytes = read(fd, ptr, (size - total_bytes)); + + if (cur_bytes < 0 && errno != EAGAIN) + return cur_bytes; + + if (cur_bytes > 0) { + total_bytes += cur_bytes; + ptr += cur_bytes; + } + + if (total_bytes < size) + nanosleep(&tm, NULL); + } while (total_bytes < size && retries--); + + if (total_bytes > size) + return -1; + + return total_bytes; +} + + +/* Send HsaMemoryRange to another process that is connected via writePipe */ +CMA_TEST_STATUS KFDCMAArray::sendCMAArray(int writePipe) { + if (write_non_block(writePipe, (void*)&m_HsaMemoryRange, sizeof(m_HsaMemoryRange)) != + sizeof(m_HsaMemoryRange)) + return CMA_IPC_PIPE_ERROR; + return CMA_TEST_SUCCESS; +} + +/* Send HsaMemoryRange from another process and initialize KFDCMAArray */ +CMA_TEST_STATUS KFDCMAArray::recvCMAArray(int readPipe) { + int i; + + if (read_non_block(readPipe, (void*)&m_HsaMemoryRange, sizeof(m_HsaMemoryRange)) != + sizeof(m_HsaMemoryRange)) + return CMA_IPC_PIPE_ERROR; + + for (i = 0; i < CMA_MEMORY_TEST_ARRAY_SIZE; i++) { + if (m_HsaMemoryRange[i].SizeInBytes) + m_ValidCount++; + } + return CMA_TEST_SUCCESS; +} + + +CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachChildProcess(int defaultGPUNode, int writePipe, + int readPipe, CMA_TEST_TYPE testType) { + KFDCMAArray cmaLocalArray; + char msg[16]; + int testNo; + CMA_TEST_STATUS status; + + /* Initialize and fill Local Buffer Array with a pattern. + * READ_TEST: Send the Array to parent process. Wait for the parent + * to finish reading and checking. Then move to next text case or + * quit if last one. + * WRITE_TEST: Send Local Buffer Array to parent process and and wait + * for parent to write to it. Check for new pattern. Then move to next + * case or quit if last one. + */ + for (testNo = 0; testNo < CMA_TEST_COUNT; testNo++) { + if (testType == CMA_READ_TEST) { + cmaLocalArray.Init(&srcRange[testNo], defaultGPUNode); + cmaLocalArray.FillPattern(&srcRange[testNo]); + } else { + cmaLocalArray.Init(&dstRange[testNo], defaultGPUNode); + cmaLocalArray.FillPattern(&dstRange[testNo]); + } + + if (cmaLocalArray.sendCMAArray(writePipe) < 0) { + status = CMA_IPC_PIPE_ERROR; + break; + } + + /* Wait till the test is over */ + memset(msg, 0, sizeof(msg)); + if (read_non_block(readPipe, msg, 4) < 0) { + status = CMA_IPC_PIPE_ERROR; + break; + } + + if (!strcmp(msg, "CHCK")) + status = cmaLocalArray.checkPattern(&dstRange[testNo]); + else if (!strcmp(msg, "NEXT")) + status = CMA_TEST_SUCCESS; + else if (!strcmp(msg, "EXIT")) + status = CMA_TEST_ABORT; + else + status = CMA_PARENT_FAIL; + + cmaLocalArray.Destroy(); + if (status != CMA_TEST_SUCCESS) + break; + } + + return status; +} + + +CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachParentProcess(int defaultGPUNode, pid_t cid, + int writePipe, int readPipe, + CMA_TEST_TYPE testType) { + KFDCMAArray cmaLocalArray, cmaRemoteArray; + HSAuint64 copied = 0; + int testNo; + CMA_TEST_STATUS status; + + /* Recevie buffer array from child and then initialize and fill in Local Buffer Array. + * READ_TEST: Copy remote buffer array into Local Buffer Array and then check + * for the new pattern. + * WRITE_TEST: Write Local Buffer Array into remote buffer array. Notify child to + * to check for the new pattern. + */ + for (testNo = 0; testNo < CMA_TEST_COUNT; testNo++) { + status = cmaRemoteArray.recvCMAArray(readPipe); + if (status != CMA_TEST_SUCCESS) + break; + + if (testType == CMA_READ_TEST) { + status = cmaLocalArray.Init(&dstRange[testNo], defaultGPUNode); + if (status != CMA_TEST_SUCCESS) + break; + cmaLocalArray.FillPattern(&dstRange[testNo]); + + if (hsaKmtProcessVMRead(cid, cmaLocalArray.getMemoryRange(), + cmaLocalArray.getValidRangeCount(), + cmaRemoteArray.getMemoryRange(), + cmaRemoteArray.getValidRangeCount(), + &copied) != HSAKMT_STATUS_SUCCESS) { + status = CMA_TEST_HSA_READ_FAIL; + break; + } + + status = cmaLocalArray.checkPattern(&dstRange[testNo]); + if (status != CMA_TEST_SUCCESS) + break; + + cmaLocalArray.Destroy(); + cmaRemoteArray.Destroy(); + + if (write_non_block(writePipe, "NEXT", 4) < 0) { + status = CMA_IPC_PIPE_ERROR; + break; + } + } else { + status = cmaLocalArray.Init(&srcRange[testNo], defaultGPUNode); + if (status != CMA_TEST_SUCCESS) + break; + cmaLocalArray.FillPattern(&srcRange[testNo]); + + if (hsaKmtProcessVMWrite(cid, cmaLocalArray.getMemoryRange(), + cmaLocalArray.getValidRangeCount(), + cmaRemoteArray.getMemoryRange(), + cmaRemoteArray.getValidRangeCount(), + &copied) != HSAKMT_STATUS_SUCCESS) { + status = CMA_TEST_HSA_WRITE_FAIL; + break; + } + + cmaLocalArray.Destroy(); + cmaRemoteArray.Destroy(); + if (write_non_block(writePipe, "CHCK", 4) < 0) { + status = CMA_IPC_PIPE_ERROR; + break; + } + } + } /* for loop */ + + return status; +} + +/* Test Cross Memory Attach + * hsaKmtProcessVMRead and hsaKmtProcessVMWrite are GPU address equivalent to + * process_vm_readv and process_vm_writev. These calls transfer data between + * the address space of the calling process ("the local process") and the process + * identified by pid ("the remote process"). + * + * In the tests parent process will be the local process and child will be + * the remote. + */ +TEST_F(KFDIPCTest, CrossMemoryAttachTest) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + int pipeCtoP[2], pipePtoC[2]; + int status; + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skip the test because of no VRAM found" << std::endl; + return; + } + + /* Create Pipes for communicating shared handles */ + ASSERT_EQ(pipe2(pipeCtoP, O_NONBLOCK), 0); + ASSERT_EQ(pipe2(pipePtoC, O_NONBLOCK), 0); + + /* Create a child process and share the above Local Memory with it */ + + m_ChildPid = fork(); + if (m_ChildPid == 0 && hsaKmtOpenKFD() == HSAKMT_STATUS_SUCCESS) { + /* Child Process */ + status = CrossMemoryAttachChildProcess(defaultGPUNode, pipeCtoP[1], + pipePtoC[0], CMA_READ_TEST); + ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Child: Read Test Fail"; + status = CrossMemoryAttachChildProcess(defaultGPUNode, pipeCtoP[1], + pipePtoC[0], CMA_WRITE_TEST); + ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Child: Write Test Fail"; + } else { + int childStatus; + + status = CrossMemoryAttachParentProcess(defaultGPUNode, m_ChildPid, + pipePtoC[1], pipeCtoP[0], CMA_READ_TEST); /* Parent proces */ + ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Read Test Fail"; + status = CrossMemoryAttachParentProcess(defaultGPUNode, m_ChildPid, + pipePtoC[1], pipeCtoP[0], CMA_WRITE_TEST); + ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Write Test Fail"; + + waitpid(m_ChildPid, &childStatus, 0); + ASSERT_EQ(WIFEXITED(childStatus), true); + ASSERT_EQ(WEXITSTATUS(childStatus), 0); + } + + /* Code path executed by both parent and child with respective fds */ + close(pipeCtoP[1]); + close(pipeCtoP[0]); + close(pipePtoC[1]); + close(pipePtoC[0]); + TEST_END +} + +/* Test Cross Memory Attach + * + * hsaKmtProcessVMRead and hsaKmtProcessVMWrite are GPU address equivalent to + * process_vm_readv and process_vm_writev. These calls are used to transfer data + * between the address space of the calling process ("the local process") and the process + * identified by pid ("the remote process"). However, these functions should also work + * with a single process and single BO. + */ +TEST_F(KFDIPCTest, CMABasicTest) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + HSAuint64 size = PAGE_SIZE; + SDMAQueue sdmaQueue; + HsaMemoryRange srcRange, dstRange; + HSAuint64 copied; + const int PATTERN1 = 0xA5A5A5A5, PATTERN2 = 0xFFFFFFFF; + HSAKMT_STATUS status; + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skip the test because of no VRAM found" << std::endl; + return; + } + + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); + volatile HSAuint32 *tmp = tmpBuffer.As(); + + /* Initialize test buffer. Fill first half and second half with + * different pattern + */ + HsaMemoryBuffer testLocalBuffer(size, defaultGPUNode, false, true); + testLocalBuffer.Fill(PATTERN1, sdmaQueue, 0, size/2); + testLocalBuffer.Fill(PATTERN2, sdmaQueue, size/2, size/2); + + /* Test1. Copy (or overwrite) buffer onto itself */ + srcRange.MemoryAddress = testLocalBuffer.As(); + srcRange.SizeInBytes = size; + dstRange.MemoryAddress = testLocalBuffer.As(); + dstRange.SizeInBytes = size; + ASSERT_SUCCESS(hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied)); + ASSERT_EQ(copied, size); + + ASSERT_TRUE(testLocalBuffer.IsPattern(0, PATTERN1, sdmaQueue, tmp)); + ASSERT_TRUE(testLocalBuffer.IsPattern(size - 4, PATTERN2, sdmaQueue, tmp)); + + + /* Test2. Test unaligned byte copy. Write 3 bytes to an unaligned destination address */ + const int unaligned_offset = 1; + const int unaligned_size = 3; + const int unaligned_mask = (((1 << (unaligned_size * 8)) - 1) << (unaligned_offset * 8)); + HSAuint32 expected_pattern; + + srcRange.MemoryAddress = testLocalBuffer.As(); + srcRange.SizeInBytes = size; /* Deliberately set to value > unaligned_size. Only unaligned_size + * should be copied since dstRange.SizeInBytes == unaligned_size + */ + dstRange.MemoryAddress = (void *)(testLocalBuffer.As() + (size / 2) + unaligned_offset); + dstRange.SizeInBytes = unaligned_size; + ASSERT_SUCCESS(hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied)); + ASSERT_EQ(copied, unaligned_size); + + expected_pattern = (PATTERN2 & ~unaligned_mask | (PATTERN1 & unaligned_mask)); + ASSERT_TRUE(testLocalBuffer.IsPattern(size/2, expected_pattern, sdmaQueue, tmp)); + + + /* Test3. Test overflow and expect failure */ + srcRange.MemoryAddress = testLocalBuffer.As(); + srcRange.SizeInBytes = size; + dstRange.MemoryAddress = (void *)(testLocalBuffer.As() + 4); + dstRange.SizeInBytes = size; /* This should overflow since offset is VA + 4 */ + status = hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied); + EXPECT_NE(status, HSAKMT_STATUS_SUCCESS); + EXPECT_LE(copied, (size - 4)); + + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDIPCTest.hpp b/tests/kfdtest/src/KFDIPCTest.hpp new file mode 100644 index 0000000000..2cb208e3da --- /dev/null +++ b/tests/kfdtest/src/KFDIPCTest.hpp @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2017-2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDBaseComponentTest.hpp" +#include "BaseQueue.hpp" +#include "IsaGenerator.hpp" + +#ifndef __KFD_MEMORY_TEST__H__ +#define __KFD_MEMORY_TEST__H__ + +#define CMA_MEMORY_TEST_ARRAY_SIZE 4 +#define CMA_TEST_COUNT 3 + +enum CMA_MEM_TYPE { + CMA_MEM_TYPE_SYSTEM = 0, + CMA_MEM_TYPE_USERPTR, + CMA_MEM_TYPE_LOCAL_MEM, +}; + +enum CMA_TEST_TYPE { + CMA_READ_TEST = 0, + CMA_WRITE_TEST +}; + +enum CMA_TEST_STATUS { + CMA_TEST_SUCCESS = 0, + CMA_IPC_PIPE_ERROR = 1, + CMA_CHECK_PATTERN_ERROR, + CMA_TEST_ABORT, + CMA_TEST_NOMEM, + CMA_PARENT_FAIL, + CMA_TEST_HSA_READ_FAIL, + CMA_TEST_HSA_WRITE_FAIL +}; + +// @struct testMemoryDescriptor +// Describes test buffers for Cross Memory Attach Test. +struct testMemoryDescriptor { + CMA_MEM_TYPE m_MemType; + HSAuint64 m_MemSize; + /* The buffer will be initialized with this pattern */ + HSAuint32 m_FillPattern; + /* After CMA test, this pattern is expected in the first word */ + HSAuint32 m_CheckFirstWordPattern; + /* After CMA test, this pattern is expected in the last word */ + HSAuint32 m_CheckLastWordPattern; + + testMemoryDescriptor(CMA_MEM_TYPE memType, HSAuint64 memSize, + HSAuint32 fillPattern, HSAuint32 firstCheckPattern, + HSAuint32 lastCheckPattern) : + m_MemType(memType), + m_MemSize(memSize), + m_FillPattern(fillPattern), + m_CheckFirstWordPattern(firstCheckPattern), + m_CheckLastWordPattern(lastCheckPattern) {} + ~testMemoryDescriptor(){} +}; + +// @class KFDCMAArray +// Array of buffers that will be passed between the parent and child +// process for Cross memory read and write tests +class KFDCMAArray { + /* Used to store the actual buffer array */ + HsaMemoryBuffer* m_MemArray[CMA_MEMORY_TEST_ARRAY_SIZE]; + /* Used for passing to thunk CMA functions */ + HsaMemoryRange m_HsaMemoryRange[CMA_MEMORY_TEST_ARRAY_SIZE]; + /* Though previous arrays are fixed sizes only m_ValidCount + * ones are valid + */ + HSAuint64 m_ValidCount; + QueueArray m_QueueArray; + + public: + KFDCMAArray(); + ~KFDCMAArray() { + Destroy(); + } + + CMA_TEST_STATUS Init(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE], int node); + CMA_TEST_STATUS Destroy(); + + HsaMemoryRange* getMemoryRange() { return m_HsaMemoryRange; } + HSAuint64 getValidRangeCount() { return m_ValidCount; } + void FillPattern(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE]); + CMA_TEST_STATUS checkPattern(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE]); + CMA_TEST_STATUS sendCMAArray(int writePipe); + CMA_TEST_STATUS recvCMAArray(int readPipe); +}; + + +// @class KFDIPCTest +class KFDIPCTest : public KFDBaseComponentTest { + public: + KFDIPCTest(void) : m_ChildPid(-1) {} + ~KFDIPCTest(void); + protected: + virtual void SetUp(); + virtual void TearDown(); + + /* For IPC testing */ + void BasicTestChildProcess(int defaultGPUNode, int *pipefd); + void BasicTestParentProcess(int defaultGPUNode, pid_t childPid, int *pipefd); + + /* For CMA testing */ + CMA_TEST_STATUS CrossMemoryAttachChildProcess(int defaultGPUNode, int writePipe, + int readPipe, CMA_TEST_TYPE testType); + CMA_TEST_STATUS CrossMemoryAttachParentProcess(int defaultGPUNode, pid_t cid, + int writePipe, int readPipe, CMA_TEST_TYPE testType); + protected: + pid_t m_ChildPid; +}; + +#endif + diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp new file mode 100644 index 0000000000..c6119156bb --- /dev/null +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -0,0 +1,528 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDLocalMemoryTest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "Dispatch.hpp" + +// All tests are marked by their serial number in the QCM FDD + +void KFDLocalMemoryTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDLocalMemoryTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDLocalMemoryTest, BasicTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + HSAuint64 AlternateVAGPU; + unsigned int BufferSize = PAGE_SIZE; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "No VRAM found, skipping the test" << std::endl; + return; + } + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer srcSysBuffer(BufferSize, defaultGPUNode, false); + HsaMemoryBuffer destSysBuffer(BufferSize, defaultGPUNode); + HsaMemoryBuffer srcLocalBuffer(BufferSize, defaultGPUNode, false, true); + HsaMemoryBuffer dstLocalBuffer(BufferSize, defaultGPUNode, false, true); + + srcSysBuffer.Fill(0x01010101); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(srcLocalBuffer.As(), srcLocalBuffer.Size(), &AlternateVAGPU)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dstLocalBuffer.As(), dstLocalBuffer.Size(), &AlternateVAGPU)); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + queue.SetSkipWaitConsump(0); + + Dispatch dispatch(isaBuffer); + + dispatch.SetArgs(srcSysBuffer.As(), srcLocalBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + dispatch.SetArgs(srcLocalBuffer.As(), dstLocalBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + dispatch.SetArgs(dstLocalBuffer.As(), destSysBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + ASSERT_SUCCESS(queue.Destroy()); + + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(srcLocalBuffer.As())); + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(dstLocalBuffer.As())); + ASSERT_EQ(destSysBuffer.As()[0], 0x01010101); + + TEST_END +} + +TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + HSAuint64 AlternateVAGPU; + unsigned int BufferSize = PAGE_SIZE; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "No VRAM found, skipping the test" << std::endl; + return; + } + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer SysBufferA(BufferSize, defaultGPUNode, false); + HsaMemoryBuffer SysBufferB(BufferSize, defaultGPUNode, true); + HsaMemoryBuffer LocalBuffer(BufferSize, defaultGPUNode, true, true); + + SysBufferA.Fill(0x01010101); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + queue.SetSkipWaitConsump(0); + + if (!is_dgpu()) + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU)); + + Dispatch dispatch(isaBuffer); + + dispatch.SetArgs(SysBufferA.As(), LocalBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As())); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU)); + + dispatch.SetArgs(LocalBuffer.As(), SysBufferB.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + ASSERT_SUCCESS(queue.Destroy()); + ASSERT_EQ(SysBufferB.As()[0], 0x01010101); + if (!is_dgpu()) + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As())); + + TEST_END +} + +/* Deliberately fragment GPUVM aperture to fill up address space + * + * General idea: Allocate buffers, but don't map them to GPU. This + * will reserve virtual address space without pinning physical + * memory. It should allow using more address space than physically + * available memory. + * + * Even without pinning memory, TTM will still commit memory at + * allocation time and swap out movable buffers to system memory or + * even the hard drive, if it needs to. So we can't allocate arbitrary + * amounts of virtual memory. + * + * Strategy to maximize the amount of allocated, fragmented address + * space while keeping the amount of committed memory bounded at all + * times: + * + * 1. Allocate N blocks of a given size, initially 1 page + * 2. Free every other block, creating holes in the address space. + * This frees up half the memory + * 3. Allocate N/4 blocks of 2-pages each. This requires as much + * memory as was freed in step 2. The block size is bigger than + * the 1-page holes, so new address space will be used. + * 4. Free half the blocks just allocated, and half of the + * remaining blocks of step 1. This creates 3-page holes between + * the 1-page blocks from step 1, and 2-page holes between the + * 2-page blocks from step 3. It frees up half of the total + * memory. + * 5. Double the block size to 4, devide number of blocks by 2. + * Again, this will require the amount of memory freed in step 4. + * The block size 4 is bigger than the biggest hole (3 pages). + * 6. Free half the memory again, creating 7-page holes between + * 1-page blocks, 6-page holes between 2-page blocks, and 4-page + * holes between 4-page blocks. + * + * Repeat, doubling block size and halving number of blocks in each + * iteration. Each iteration starts and ends with half the total + * memory free. Because the block size is always bigger than the + * biggest hole, each iteration increases the amount of address space + * occupied by half the total memory size. Once the block size reaches + * half of the free memory (1/4 of total memory) the limit is reached. + * + * With 2^n pages available memory, n * 2^(n-1) pages of address space + * can be reserved. At the end of that process, half the memory will + * be free. + * + * Total memory | Fragmented address space + * order | pages | size | pages | size | ratio + * ------+-------+------+-------+-------+------- + * 2 | 4 | 16K | 4 | 16K | 1 + * 3 | 8 | 32K | 12 | 48K | 1.5 + * 4 | 16 | 64K | 32 | 128K | 2 + * 5 | 32 | 128K | 80 | 320K | 2.5 + * 6 | 64 | 256K | 192 | 768K | 3 + * 7 | 128 | 512K | 448 | 1.75M | 3.5 + * 8 | 256 | 1M | 1M | 4M | 4 + * 9 | 512 | 2M | 2.25M | 9M | 4.5 + * 10 | 1K | 4M | 5M | 20M | 5 + * 11 | 2K | 8M | 11M | 44M | 5.5 + * 12 | 4K | 16M | 24M | 96M | 6 + * 13 | 8K | 32M | 52M | 208M | 6.5 + * 14 | 16K | 64M | 112M | 448M | 7 + * 15 | 32K | 128M | 240M | 960M | 7.5 + * 16 | 64K | 256M | 512M | 2G | 8 + * 17 | 128K | 512M | 1088M | 4.25G | 8.5 + * 18 | 256K | 1G | 2.25G | 9G | 9 + * 19 | 512K | 2G | 4.75G | 19G | 9.5 + * 20 | 1M | 4G | 10G | 40G | 10 + */ +TEST_F(KFDLocalMemoryTest, Fragmentation) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HSAuint64 fbSize; + + fbSize = GetVramSize(defaultGPUNode); + + if (!fbSize) { + LOG() << "No VRAM found, skipping test." << std::endl; + return; + } else { + LOG() << "Found VRAM of " << std::dec << (fbSize >> 20) << "MB." << std::endl; + } + + /* Use up to half of available memory. Using more results in + * excessive memory movement in TTM and slows down the test too + * much. maxOrder is the size of the biggest block that will be + * allocated. It's 1/4 of the usable memory, so 1/8 the total FB + * size in pages. + * + * Use 8x bigger page size on dGPU to match Tonga alignment + * workaround. Also nicely matches the 8x bigger GPUVM address + * space on AMDGPU compared to RADEON. + */ + unsigned pageSize = is_dgpu() ? PAGE_SIZE*8 : PAGE_SIZE; + fbSize /= pageSize; + unsigned maxOrder = 0; + // Limit maxOrder up to 14 so this test doesn't run longer than 10 mins + while (((fbSize >> maxOrder) >= 16) && (maxOrder < 14)) + maxOrder++; + + /* Queue and memory used by the shader copy tests */ + HsaMemoryBuffer sysBuffer(PAGE_SIZE, defaultGPUNode, false); + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + /* Allocate and test memory using the strategy explained at the top */ + HSAKMT_STATUS status; + HsaMemFlags memFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 0; + memFlags.ui32.NonPaged = 1; + struct { + void **pointers; + unsigned long nPages; + } pages[maxOrder+1]; + unsigned order, o; + unsigned long p; + HSAuint64 size; + unsigned value = 0; + memset(pages, 0, sizeof(pages)); + for (order = 0; order <= maxOrder; order++) { + // At maxOrder, block sizes is 1/4 of available memory + pages[order].nPages = 1UL << (maxOrder - order + 2); + // At order != 0, 1/2 the memory is already allocated + if (order > 0) + pages[order].nPages >>= 1; + // Allocate page pointers + pages[order].pointers = new void *[pages[order].nPages]; + EXPECT_NE((void **)NULL, pages[order].pointers) + << "Couldn't allocate memory for " << pages[order].nPages + << " pointers at order " << order << std::endl; + if (!pages[order].pointers) { + pages[order].nPages = 0; + break; + } + /* Allocate buffers and access the start and end of every one: + * 1. Copy from sysBuffer[0] to start of block + * 2. Copy from start of block to end of block + * 3. Copy from end of block to sysBuffer[1] + * 4. Compare results */ + size = (HSAuint64)(1 << order) * pageSize; + LOG() << std::dec << "Trying to allocate " << pages[order].nPages + << " order " << order << " blocks " << std::endl; + for (p = 0; p < pages[order].nPages; p++) { + status = hsaKmtAllocMemory(defaultGPUNode, size, + memFlags, &pages[order].pointers[p]); + if (status != HSAKMT_STATUS_SUCCESS) { + EXPECT_EQ(HSAKMT_STATUS_NO_MEMORY, status); + pages[order].nPages = p; + break; + } + + void *bufferEnd = (void *)((unsigned long)pages[order].pointers[p] + + size - sizeof(unsigned)); + sysBuffer.As()[0] = ++value; + + status = hsaKmtMapMemoryToGPU(pages[order].pointers[p], + size, NULL); + if (status != HSAKMT_STATUS_SUCCESS) { + ASSERT_SUCCESS(hsaKmtFreeMemory(pages[order].pointers[p], + size)); + pages[order].nPages = p; + break; + } + Dispatch dispatch1(isaBuffer); + dispatch1.SetArgs(sysBuffer.As(), pages[order].pointers[p]); + dispatch1.Submit(queue); + // no sync needed for multiple GPU dispatches to the same queue + + Dispatch dispatch2(isaBuffer); + dispatch2.SetArgs(pages[order].pointers[p], bufferEnd); + dispatch2.Submit(queue); + // no sync needed for multiple GPU dispatches to the same queue + + Dispatch dispatch3(isaBuffer); + dispatch3.SetArgs(bufferEnd, + (void *)&(sysBuffer.As()[1])); + dispatch3.Submit(queue); + dispatch3.Sync(g_TestTimeOut); + EXPECT_EQ(value, sysBuffer.As()[1]); + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pages[order].pointers[p])); + } + LOG() << " Got " << pages[order].nPages + << ", end of last block addr: " + << (void *)((unsigned long)pages[order].pointers[p-1] + size - 1) + << std::endl; + + // Now free half the memory + for (o = 0; o <= order; o++) { + unsigned long step = 1UL << (order - o + 1); + unsigned long offset = (step >> 1) - 1; + size = (HSAuint64)(1 << o) * pageSize; + LOG() << " Freeing every " << step << "th order " + << o << " block starting with " << offset << std::endl; + for (p = offset; p < pages[o].nPages; p += step) { + ASSERT_NE((void **)NULL, pages[o].pointers[p]); + EXPECT_SUCCESS(hsaKmtFreeMemory(pages[o].pointers[p], size)); + pages[o].pointers[p] = NULL; + } + } + } + + /* Clean up */ + for (order = 0; order <= maxOrder; order++) { + if (pages[order].pointers == NULL) + continue; + + size = (HSAuint64)(1 << order) * pageSize; + for (p = 0; p < pages[order].nPages; p++) + if (pages[order].pointers[p] != NULL) + EXPECT_SUCCESS(hsaKmtFreeMemory(pages[order].pointers[p], size)); + + delete[] pages[order].pointers; + } + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDLocalMemoryTest, CheckZeroInitializationVram) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + /* Testing VRAM */ + HSAuint64 vramSizeMB = GetVramSize(defaultGPUNode) >> 20; + + if (!vramSizeMB) { + LOG() << "No VRAM found, skipping the test" << std::endl; + return; + } + + HSAuint64 vramBufSizeMB = vramSizeMB >> 2; + /* limit the buffer size in order not to overflow the SDMA queue buffer. */ + if (vramBufSizeMB > 1024) { + vramBufSizeMB = 1024; + } + HSAuint64 vramBufSize = vramBufSizeMB * 1024 * 1024; + + /* Make sure the entire VRAM is used at least once */ + int count = (vramSizeMB + vramBufSizeMB - 1) / vramBufSizeMB + 1; + + LOG() << "Using " << std::dec << vramBufSizeMB + << "MB VRAM buffer to test " << std::dec << count + << " times"<< std::endl; + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode, 8 * PAGE_SIZE)); + + HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); + volatile HSAuint32 *tmp = tmpBuffer.As(); + + unsigned int offset = 2060; // a constant offset, should be 4 aligned. + + while (count--) { + HsaMemoryBuffer localBuffer(vramBufSize, defaultGPUNode, false, true); + + EXPECT_TRUE(localBuffer.IsPattern(0, 0, sdmaQueue, tmp)); + + for (HSAuint64 i = offset; i < vramBufSize;) { + EXPECT_TRUE(localBuffer.IsPattern(i, 0, sdmaQueue, tmp)); + i += 4096; + } + + /* Checking last 4 bytes */ + EXPECT_TRUE(localBuffer.IsPattern(vramBufSize - 4, 0, sdmaQueue, tmp)); + + localBuffer.Fill(0xABCDEFFF, sdmaQueue); + } + + TEST_END +} + +TEST_F(KFDLocalMemoryTest, MapVramToGPUNodesTest) { + TEST_START(TESTPROFILE_RUNALL); + + HSAint32 src_node; + HSAint32 dst_node; + HsaPointerInfo info; + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: Need at least two GPUs" << std::endl; + return; + } + + if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { + src_node = g_TestNodeId; + dst_node = g_TestDstNodeId; + } else { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + dst_node = m_NodeInfo.FindLargeBarGPUNode(); + if (dst_node < 0) { + LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + return; + } + + if (dst_node != defaultGPUNode) { + /* at least one node should be defaultGPUNode */ + src_node = defaultGPUNode; + } else { + for (auto node : gpuNodes) { + if (node != dst_node) { + src_node = node; + break; + } + } + } + } + + LOG() << "Testing from GPU " << src_node << " to GPU " << dst_node << std::endl; + + void *shared_addr; + HSAuint32 nodes[] = { (HSAuint32)src_node, (HSAuint32)dst_node }; + HsaMemFlags memFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 1; + memFlags.ui32.NonPaged = 1; + memFlags.ui32.ExecuteAccess = 1; + + HsaMemMapFlags mapFlags = {0}; + + EXPECT_SUCCESS(hsaKmtAllocMemory(nodes[1], PAGE_SIZE, memFlags, &shared_addr)); + EXPECT_SUCCESS(hsaKmtRegisterMemoryToNodes(shared_addr, PAGE_SIZE, 2, nodes)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(shared_addr, PAGE_SIZE, NULL, mapFlags, 2, nodes)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(shared_addr, &info)); + EXPECT_EQ(info.NRegisteredNodes, 2); + EXPECT_EQ(info.NMappedNodes, 2); + + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(shared_addr, PAGE_SIZE, NULL, mapFlags, 1, &nodes[0])); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(shared_addr, &info)); + EXPECT_EQ(info.NRegisteredNodes, 2); + EXPECT_EQ(info.NMappedNodes, 1); + EXPECT_EQ(info.MappedNodes[0], nodes[0]); + + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(shared_addr, PAGE_SIZE, NULL, mapFlags, 1, &nodes[1])); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(shared_addr, &info)); + EXPECT_EQ(info.NRegisteredNodes, 2); + EXPECT_EQ(info.NMappedNodes, 1); + EXPECT_EQ(info.MappedNodes[0], nodes[1]); + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(shared_addr)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(shared_addr, &info)); + EXPECT_EQ(info.NRegisteredNodes, 2); + EXPECT_EQ(info.NMappedNodes, 0); + + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(shared_addr, PAGE_SIZE, NULL, mapFlags, 1, &nodes[0])); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(shared_addr, &info)); + EXPECT_EQ(info.NRegisteredNodes, 2); + EXPECT_EQ(info.NMappedNodes, 1); + EXPECT_EQ(info.MappedNodes[0], nodes[0]); + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(shared_addr)); + EXPECT_SUCCESS(hsaKmtFreeMemory(shared_addr, PAGE_SIZE)); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.hpp b/tests/kfdtest/src/KFDLocalMemoryTest.hpp new file mode 100644 index 0000000000..368e92b21f --- /dev/null +++ b/tests/kfdtest/src/KFDLocalMemoryTest.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_LOCALMEMORY_TEST__H__ +#define __KFD_LOCALMEMORY_TEST__H__ + +#include + +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" + +class KFDLocalMemoryTest : public KFDBaseComponentTest { + public: + KFDLocalMemoryTest() :m_pIsaGen(NULL) {} + ~KFDLocalMemoryTest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: // members + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_LOCALMEMORY_TEST__H__ diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp new file mode 100644 index 0000000000..6f12caed65 --- /dev/null +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -0,0 +1,1628 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDMemoryTest.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Dispatch.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAQueue.hpp" +#include "SDMAPacket.hpp" + +const char* gfx8_ScratchCopyDword = +"\ +shader ScratchCopyDword\n\ +asic(VI)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ +/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ + s_mov_b32 flat_scratch_lo, 8/*2 dwords of scratch per thread*/\n\ + s_mov_b32 flat_scratch_hi, 0/*offset in units of 256bytes*/\n\ +/*copy a dword between the passed addresses*/\n\ + flat_load_dword v4, v[0:1] slc\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + flat_store_dword v[2:3], v4 slc\n\ + \n\ + s_endpgm\n\ + \n\ +end\n\ +"; + +const char* gfx9_ScratchCopyDword = +"\ +shader ScratchCopyDword\n\ +asic(GFX9)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ +/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ + s_mov_b32 flat_scratch_lo, s4\n\ + s_mov_b32 flat_scratch_hi, s5\n\ +/*copy a dword between the passed addresses*/\n\ + flat_load_dword v4, v[0:1] slc\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + flat_store_dword v[2:3], v4 slc\n\ + \n\ + s_endpgm\n\ + \n\ +end\n\ +"; + +/* Continuously poll src buffer and check buffer value + * After src buffer is filled with specific value (0x5678, + * by host program), fill dst buffer with specific + * value(0x5678) and quit + */ +const char* gfx9_PollMemory = +"\ +shader ReadMemory\n\ +asic(GFX9)\n\ +type(CS)\n\ +/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ + s_movk_i32 s18, 0x5678\n\ + LOOP:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 LOOP\n\ + s_store_dword s18, s[2:3], 0x0 glc\n\ + s_endpgm\n\ + end\n\ +"; + +void KFDMemoryTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDMemoryTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +#include +#define GB(x) ((x)<<30) + +/* + * try to map as much as possible system memory to gpu. + * lets see if kfd support 1TB memory correctly or not. + * And after this test case, we can observe if there is any sideeffect. + * NOTICE: there are memory usage limit checks in hsa/kfd according to the total + * physical system memory. + */ +TEST_F(KFDMemoryTest, MMapLarge) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + if (!is_dgpu()) { + LOG() << "Skip the test on APU" << std::endl; + return; + } + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + const unsigned long nObjects = 1<<14; + HSAuint64 *AlternateVAGPU = new HSAuint64[nObjects]; + ASSERT_NE((unsigned long)AlternateVAGPU, 0); + HsaMemMapFlags mapFlags = {0}; + unsigned long s; + char *addr; + unsigned long flags = MAP_ANONYMOUS | MAP_PRIVATE; + + /* Test up to 1TB memory*/ + s = GB(1024ULL) / nObjects; + addr = (char*)mmap(0, s, PROT_READ | PROT_WRITE, flags, -1, 0); + ASSERT_NE(addr, MAP_FAILED); + memset(addr, 0, s); + + int i = 0; + /* Allocate 1024GB, aka 1TB*/ + for (; i < nObjects; i++) { + if (hsaKmtRegisterMemory(addr + i, s - i)) + break; + if (hsaKmtMapMemoryToGPUNodes(addr + i, s - i, + &AlternateVAGPU[i], mapFlags, 1, (HSAuint32 *)&defaultGPUNode)) { + hsaKmtDeregisterMemory((void*)AlternateVAGPU[i]); + break; + } + } + + LOG() << "Successfully registered and mapped " << (i * s >> 30) + << "GB system memory to gpu" << std::endl; + + while (i--) { + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU((void*)AlternateVAGPU[i])); + ASSERT_SUCCESS(hsaKmtDeregisterMemory((void*)AlternateVAGPU[i])); + } + + munmap(addr, s); + delete []AlternateVAGPU; + + TEST_END +} + +/* keep memory mapped to default node + * Keep mapping/unmapping memory to/from non-default node + * A shader running on default node consistantly access + * memory - make sure memory is always accessible on default, + * i.e., there is no gpu vm fault. + * Synchronization b/t host program and shader: + * 1. host initialize src and dst buffer to 0 + * 2. shader keep reading src buffer and check value + * 3. host write src buffer to 0x5678 to indicate quit, polling dst until it becomes 0x5678 + * 4. shader write dst buffer to 0x5678 after src changed to 0x5678, quit + * 5. host program quit after dst becomes 0x5678 + * Need at least two gpu nodes to run the test. The defaut node has to be a gfx9 node. + * Otherwise, test is skipped. Use kfdtest --node=$$ to specify the defaut node + * This test case is introduced as a side-result of investigation of SWDEV-134798, which + * is a gpu vm fault while running rocr conformance test. Here we try to simulate the + * same test behaviour. + */ +TEST_F(KFDMemoryTest, MapUnmapToNodes) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId != FAMILY_AI) { + LOG() << "Skipping test: Test uses gfx9-based shader, skip on other ASICs" << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: Need at least two GPUs" << std::endl; + return; + } + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + LOG() << "default GPU node" << defaultGPUNode << std::endl; + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HSAuint32 nondefaultNode; + for (unsigned i = 0; i < gpuNodes.size(); i++) { + if (gpuNodes.at(i) != defaultGPUNode) { + nondefaultNode = gpuNodes.at(i); + break; + } + } + HSAuint32 mapNodes[2] = {defaultGPUNode, nondefaultNode}; + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); + + m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + + PM4Queue pm4Queue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + + Dispatch dispatch0(isaBuffer); + dispatch0.SetArgs(srcBuffer.As(), dstBuffer.As()); + dispatch0.Submit(pm4Queue); + + HsaMemMapFlags memFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 1; + + for (unsigned i = 0; i < 1<<14; i ++) { + hsaKmtMapMemoryToGPUNodes(srcBuffer.As(), PAGE_SIZE, NULL, memFlags, (i>>5)&1+1, mapNodes); + } + + /* fill src buffer so shader quits */ + srcBuffer.Fill(0x5678); + WaitOnValue(dstBuffer.As(), 0x5678); + ASSERT_EQ(*dstBuffer.As(), 0x5678); + ASSERT_SUCCESS(pm4Queue.Destroy()); + TEST_END +} + +// basic test of hsaKmtMapMemoryToGPU and hsaKmtUnmapMemoryToGPU +TEST_F(KFDMemoryTest , MapMemoryToGPU) { + TEST_START(TESTPROFILE_RUNALL) + + unsigned int *nullPtr = NULL; + unsigned int* pDb = NULL; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode /* system */, PAGE_SIZE, m_MemoryFlags, (void**)&pDb)); + // verify that pDb is not null before it's being used + ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(pDb, PAGE_SIZE, NULL)); + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + // Release the buffers + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, PAGE_SIZE)); + + TEST_END +} + +// following tests are for hsaKmtAllocMemory with invalid params +TEST_F(KFDMemoryTest, InvalidMemoryPointerAlloc) { + TEST_START(TESTPROFILE_RUNALL) + + EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtAllocMemory(0 /* system */, PAGE_SIZE, m_MemoryFlags, NULL)); + + TEST_END +} + +TEST_F(KFDMemoryTest, ZeroMemorySizeAlloc) { + TEST_START(TESTPROFILE_RUNALL) + + unsigned int* pDb = NULL; + EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtAllocMemory(0 /* system */, 0, m_MemoryFlags, (void**)&pDb)); + + TEST_END +} + +// basic test for hsaKmtAllocMemory +TEST_F(KFDMemoryTest, MemoryAlloc) { + TEST_START(TESTPROFILE_RUNALL) + + unsigned int* pDb = NULL; + EXPECT_SUCCESS(hsaKmtAllocMemory(0 /* system */, PAGE_SIZE, m_MemoryFlags, (void**)&pDb)); + + TEST_END +} + +TEST_F(KFDMemoryTest, AccessPPRMem) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (is_dgpu()) { + LOG() << "Not an APU, no PPR available, skip the test" << std::endl; + return; + } + + unsigned int *destBuf = (unsigned int *)VirtualAllocMemory(NULL, PAGE_SIZE, + MEM_READ | MEM_WRITE); + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf, + 0xABCDEF09, 0x12345678)); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf, 0xABCDEF09); + WaitOnValue(destBuf + 1, 0x12345678); + + ASSERT_SUCCESS(queue.Destroy()); + + /* This sleep hides the dmesg PPR message storm on Raven, which happens + * when the CPU buffer is freed before the excessive PPRs are all + * consumed by IOMMU HW. Because of that, a kernel driver workaround + * is put in place to address that, so we don't need to wait here. + */ + //sleep(5); + + VirtualFreeMemory(destBuf, PAGE_SIZE); + + TEST_END +} + +// Linux OS-specific Test for registering OS allocated memory +TEST_F(KFDMemoryTest, MemoryRegister) { + const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + if (isTonga(pNodeProperties)) { + LOG() << "Skipping test: Workaround in thunk for Tonga causes failure:" << std::endl; + return; + } + + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + /* Different unaligned memory locations to be mapped for GPU + * access: + * + * - initialized data segment (file backed) + * - stack (anonymous memory) + * + * Separate them enough so they are in different cache lines + * (64-byte = 16-dword). + */ + static volatile HSAuint32 globalData = 0xdeadbeef; + volatile HSAuint32 stackData[17] = {0}; + const unsigned dstOffset = 0; + const unsigned sdmaOffset = 16; + + HsaMemoryBuffer srcBuffer((void *)&globalData, sizeof(HSAuint32)); + HsaMemoryBuffer dstBuffer((void *)&stackData[dstOffset], sizeof(HSAuint32)); + HsaMemoryBuffer sdmaBuffer((void *)&stackData[sdmaOffset], sizeof(HSAuint32)); + + /* Create PM4 and SDMA queues before fork+COW to test queue + * eviction and restore */ + PM4Queue pm4Queue; + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + /* First submit just so the queues are not empty, and to get the + * TLB populated (in case we need to flush TLBs somewhere after + * updating the page tables) */ + Dispatch dispatch0(isaBuffer); + dispatch0.SetArgs(srcBuffer.As(), dstBuffer.As()); + dispatch0.Submit(pm4Queue); + dispatch0.Sync(g_TestTimeOut); + + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaBuffer.As(), 0x12345678)); + sdmaQueue.Wait4PacketConsumption(); + ASSERT_TRUE(WaitOnValue(&stackData[sdmaOffset], 0x12345678)); + + /* Fork a child process to mark pages as COW */ + pid_t pid = fork(); + ASSERT_GE(pid, 0); + if (pid == 0) { + /* Child process waits for a SIGTERM from the parent. It can't + * make any write access to the stack because we want the + * parent to make the first write access and get a new copy. A + * busy loop is the safest way to do that, since any function + * call (e.g. sleep) would write to the stack. */ + while (1) + {} + WARN() << "Shouldn't get here!" << std::endl; + exit(0); + } + + /* Parent process writes to COW page(s) and gets a new copy. MMU + * notifier needs to update the GPU mapping(s) for the test to + * pass. */ + globalData = 0xD00BED00; + stackData[dstOffset] = 0xdeadbeef; + stackData[sdmaOffset] = 0xdeadbeef; + + /* Terminate the child process before a possible test failure that + * would leave it spinning in the background indefinitely. */ + int status; + EXPECT_EQ(0, kill(pid, SIGTERM)); + EXPECT_EQ(pid, waitpid(pid, &status, 0)); + EXPECT_NE(0, WIFSIGNALED(status)); + EXPECT_EQ(SIGTERM, WTERMSIG(status)); + + /* Now check that the GPU is accessing the correct page */ + Dispatch dispatch1(isaBuffer); + dispatch1.SetArgs(srcBuffer.As(), dstBuffer.As()); + dispatch1.Submit(pm4Queue); + dispatch1.Sync(g_TestTimeOut); + + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaBuffer.As(), 0xD0BED0BE)); + sdmaQueue.Wait4PacketConsumption(); + + ASSERT_SUCCESS(pm4Queue.Destroy()); + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + ASSERT_EQ(0xD00BED00, globalData); + ASSERT_EQ(0xD00BED00, stackData[dstOffset]); + ASSERT_EQ(0xD0BED0BE, stackData[sdmaOffset]); + + TEST_END +} + +TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { + if (!is_dgpu()) { + LOG() << "Skipping test: Will run on APU once APU+dGPU supported:" << std::endl; + return; + } + + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + HSAuint64 nGPU = gpuNodes.size(); // number of gpu nodes + static volatile HSAuint32 mem[4]; + HSAuint64 gpuva1, gpuva2; + + /* Same address, different size */ + EXPECT_SUCCESS(hsaKmtRegisterMemory((void *)&mem[0], sizeof(HSAuint32)*2)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU((void *)&mem[0], sizeof(HSAuint32)*2, + &gpuva1)); + EXPECT_SUCCESS(hsaKmtRegisterMemory((void *)&mem[0], sizeof(HSAuint32))); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU((void *)&mem[0], sizeof(HSAuint32), + &gpuva2)); + EXPECT_TRUE(gpuva1 != gpuva2); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva1)); + EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva1)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva2)); + EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva2)); + + /* Same address, same size */ + HsaMemMapFlags memFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 1; + + HSAuint32 nodes[nGPU]; + for (unsigned int i = 0; i < nGPU; i++) + nodes[i] = gpuNodes.at(i); + EXPECT_SUCCESS(hsaKmtRegisterMemoryToNodes((void *)&mem[2], + sizeof(HSAuint32)*2, nGPU, nodes)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes((void *)&mem[2], + sizeof(HSAuint32) * 2, + &gpuva1, memFlags, nGPU, nodes)); + EXPECT_SUCCESS(hsaKmtRegisterMemoryToNodes((void *)&mem[2], + sizeof(HSAuint32) * 2, nGPU, nodes)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes((void *)&mem[2], + sizeof(HSAuint32) * 2, + &gpuva2, memFlags, nGPU, nodes)); + EXPECT_EQ(gpuva1, gpuva2); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva1)); + EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva1)); + /* Confirm that we still have access to the memory, mem[2] */ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + mem[2] = 0x0; + queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)gpuva2, + 0xdeadbeef)); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(true, 0, 0)); + queue.Wait4PacketConsumption(); + EXPECT_EQ(true, WaitOnValue((unsigned int *)&mem[2], 0xdeadbeef)); + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva2)); + EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva2)); + + TEST_END +} + +// FlatScratchAccess +// Since HsaMemoryBuffer has to be associated with a specific GPU node, this function in the current form +// will not work for multiple GPU nodes. For now test only one default GPU node. +// TODO: Generalize it to support multiple nodes + +#define SCRATCH_SLICE_SIZE 0x10000 +#define SCRATCH_SLICE_NUM 3 +#define SCRATCH_SIZE (SCRATCH_SLICE_NUM * SCRATCH_SLICE_SIZE) +#define SCRATCH_SLICE_OFFSET(i) ((i) * SCRATCH_SLICE_SIZE) + +TEST_F(KFDMemoryTest, FlatScratchAccess) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId == FAMILY_CI || m_FamilyId == FAMILY_KV) { + LOG() << "Skipping test: Test uses VI-based shader, fails on CI" << std::endl; + return; + } + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer scratchBuffer(SCRATCH_SIZE, defaultGPUNode, false/*zero*/, false/*local*/, false/*exec*/, true /*scratch*/); + + // Unmap scratch for sub-allocation mapping tests + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As())); + + // Map and unmap a few slices in different order: 2-0-1, 0-2-1 + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(2), + SCRATCH_SLICE_SIZE, NULL)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(0), + SCRATCH_SLICE_SIZE, NULL)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(1), + SCRATCH_SLICE_SIZE, NULL)); + + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(1))); + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(2))); + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(0))); + + // Map everything for test below + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(scratchBuffer.As(), SCRATCH_SIZE, NULL)); + + // source & destination memory buffers + HsaMemoryBuffer srcMemBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer dstMemBuffer(PAGE_SIZE, defaultGPUNode); + + + // Initialize the srcBuffer to some fixed value + srcMemBuffer.Fill(0x01010101); + + // Initialize a buffer with a DWORD copy ISA + m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? gfx9_ScratchCopyDword : gfx8_ScratchCopyDword, + "ScratchCopyDword", isaBuffer); + + const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); + + // TODO: Add support to all GPU Nodes. + // The loop over the system nodes is removed as the test can be executed only on GPU nodes. This + // also requires changes to be made to all the HsaMemoryBuffer variables defined above, as + // HsaMemoryBuffer is now associated with a Node. + if (pNodeProperties != NULL) { + // Get the aperture of the scratch buffer + HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(defaultGPUNode, pNodeProperties->NumMemoryBanks, memoryProperties)); + + for (unsigned int bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) { + if (memoryProperties[bank].HeapType == HSA_HEAPTYPE_GPU_SCRATCH) { + int numWaves = 4; // WAVES must be >= # SE + int waveSize = 1; // amount of space used by each wave in units of 256 dwords... + + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + HSAuint64 scratchApertureAddr = memoryProperties[bank].VirtualBaseAddress; + + // Create a dispatch packet to copy + Dispatch dispatchSrcToScratch(isaBuffer); + + // setup the dispatch packet + // Copying from the source Memory Buffer to the scratch buffer + dispatchSrcToScratch.SetArgs(srcMemBuffer.As(), reinterpret_cast(scratchApertureAddr)); + dispatchSrcToScratch.SetDim(1, 1, 1); + dispatchSrcToScratch.SetScratch(numWaves, waveSize, scratchBuffer.As()); + // submit the packet + dispatchSrcToScratch.Submit(queue); + dispatchSrcToScratch.Sync(); + + // Create another dispatch packet to copy scratch buffer contents to destination buffer. + Dispatch dispatchScratchToDst(isaBuffer); + + // set the arguments to copy from the scratch buffer + // to the destination buffer + dispatchScratchToDst.SetArgs(reinterpret_cast(scratchApertureAddr), dstMemBuffer.As()); + dispatchScratchToDst.SetDim(1, 1, 1); + dispatchScratchToDst.SetScratch(numWaves, waveSize, scratchBuffer.As()); + + // submit the packet + dispatchScratchToDst.Submit(queue); + dispatchScratchToDst.Sync(); + + // Check that the scratch buffer contents were correctly copied over to the system memory buffer + ASSERT_EQ(dstMemBuffer.As()[0], 0x01010101); + } + } + + delete [] memoryProperties; + } + + TEST_END +} + +TEST_F(KFDMemoryTest, GetTileConfigTest) { + TEST_START(TESTPROFILE_RUNALL) + + HSAuint32 tile_config[32] = {0}; + HSAuint32 macro_tile_config[16] = {0}; + unsigned int i; + HsaGpuTileConfig config = {0}; + + config.TileConfig = tile_config; + config.MacroTileConfig = macro_tile_config; + config.NumTileConfigs = 32; + config.NumMacroTileConfigs = 16; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + ASSERT_SUCCESS(hsaKmtGetTileConfig(defaultGPUNode, &config)); + + LOG() << "tile_config:" << std::endl; + for (i = 0; i < config.NumTileConfigs; i++) + LOG() << "\t" << std::dec << i << ": 0x" << std::hex + << tile_config[i] << std::endl; + + LOG() << "macro_tile_config:" << std::endl; + for (i = 0; i < config.NumMacroTileConfigs; i++) + LOG() << "\t" << std::dec << i << ": 0x" << std::hex + << macro_tile_config[i] << std::endl; + + LOG() << "gb_addr_config: 0x" << std::hex << config.GbAddrConfig + << std::endl; + LOG() << "num_banks: 0x" << std::hex << config.NumBanks << std::endl; + LOG() << "num_ranks: 0x" << std::hex << config.NumRanks << std::endl; + + TEST_END +} + +void KFDMemoryTest::BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granularityMB, + HSAuint64 *lastSize) { + HSAuint64 sysMemSizeMB; + HsaMemMapFlags mapFlags = {0}; + HSAuint64 AlternateVAGPU; + int ret; + + sysMemSizeMB = GetSysMemSize() >> 20; + + LOG() << "Found System Memory of " << std::dec << sysMemSizeMB + << "MB" << std::endl; + + /* Testing big buffers in system memory */ + unsigned int * pDb = NULL; + HSAuint64 lowMB = 0; + HSAuint64 highMB = (sysMemSizeMB + granularityMB - 1) & ~(granularityMB - 1); + + HSAuint64 sizeMB; + HSAuint64 size = 0; + HSAuint64 lastTestedSize = 0; + + while (highMB - lowMB > granularityMB) { + sizeMB = (lowMB + highMB) / 2; + size = sizeMB * 1024 * 1024; + ret = hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, + (void**)&pDb); + if (ret) { + highMB = sizeMB; + continue; + } + + ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, + mapFlags, 1, (HSAuint32 *)&defaultGPUNode); + if (ret) { + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + highMB = sizeMB; + continue; + } + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + + lowMB = sizeMB; + lastTestedSize = sizeMB; + } + + /* Save the biggest allocated system buffer forsignal handling test */ + LOG() << "The biggest allocated system buffer is " << std::dec + << lastTestedSize << "MB" << std::endl; + if (lastSize) + *lastSize = lastTestedSize * 1024 *1024; +} + +void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, + HSAuint64 *lastSize) { + HSAuint64 AlternateVAGPU; + int ret; + HSAuint64 vramSizeMB; + HsaMemFlags memFlags; + HsaMemMapFlags mapFlags = {0}; + + vramSizeMB = GetVramSize(defaultGPUNode) >> 20; + + LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; + + /* Testing big buffers in VRAM */ + unsigned int * pDb = NULL; + HSAuint64 lowMB = 0; + HSAuint64 highMB = (vramSizeMB + granularityMB - 1) & ~(granularityMB - 1); + + HSAuint64 sizeMB; + HSAuint64 size = 0; + HSAuint64 lastTestedSize = 0; + + memset(&memFlags, 0, sizeof(memFlags)); + memFlags.ui32.HostAccess = 0; + memFlags.ui32.NonPaged = 1; + + while (highMB - lowMB > granularityMB) { + sizeMB = (lowMB + highMB) / 2; + size = sizeMB * 1024 * 1024; + ret = hsaKmtAllocMemory(defaultGPUNode, size, memFlags, + (void**)&pDb); + if (ret) { + highMB = sizeMB; + continue; + } + + ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, + mapFlags, 1, (HSAuint32 *)&defaultGPUNode); + if (ret) { + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + highMB = sizeMB; + continue; + } + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + + lowMB = sizeMB; + lastTestedSize = sizeMB; + } + + LOG() << "The biggest allocated VRAM buffer is " << std::dec + << lastTestedSize << "MB" << std::endl; + if (lastSize) + *lastSize = lastTestedSize * 1024 * 1024; +} + +/* BigBufferStressTest allocs, maps/unmaps, and frees the biggest possible system + * buffers. Its size is found using binary search in the range (0, RAM SIZE) with + * a granularity of 128M. Repeat the similar logic on local buffers (VRAM). + * Finally, it allocs and maps 128M system buffers in a loop until it + * fails, then unmaps and frees them afterwards. + * Please note we limit the biggest possible system buffer to be smaller than + * the RAM size. The reason is that the system buffer can make use of virtual + * memory so that a system buffer could be very large even though the RAM size + * is small. For example, on a typical Carrizo platform, the biggest allocated + * system buffer could be more than 14G even though it only has 4G memory. + * In that situation, it will take too much time to finish the test, because of + * the onerous memory swap operation. So we limit the buffer size that way.*/ +TEST_F(KFDMemoryTest, BigBufferStressTest) { + if (!is_dgpu()) { + LOG() << "Skipping test: Running on APU fails and locks the system" << std::endl; + return; + } + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint64 AlternateVAGPU; + HsaMemMapFlags mapFlags = {0}; + int ret; + + HSAuint64 granularityMB = 128; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + BigBufferSystemMemory(defaultGPUNode, granularityMB, NULL); + + BigBufferVRAM(defaultGPUNode, granularityMB, NULL); + + /* Repeatedly allocate and map big buffers in system memory until it fails, + * then unmap and free them. */ +#define ARRAY_ENTRIES 2048 + + int i = 0; + unsigned int* pDb_array[ARRAY_ENTRIES]; + HSAuint64 block_size_mb = 128; + HSAuint64 block_size = block_size_mb * 1024 * 1024; + + do { + ret = hsaKmtAllocMemory(0 /* system */, block_size, m_MemoryFlags, + (void**)&pDb_array[i]); + if (ret) { + break; + } + + ret = hsaKmtMapMemoryToGPUNodes(pDb_array[i], block_size, + &AlternateVAGPU, mapFlags, 1, (HSAuint32 *)&defaultGPUNode); + if (ret) { + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); + break; + } + } while (++i < ARRAY_ENTRIES); + + LOG() << "Allocated system buffers: " << std::dec << i << "x" + << block_size_mb << "MB" << std::endl; + + while (i--) { + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[i])); + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); + } + + TEST_END +} + +TEST_F(KFDMemoryTest, MMBench) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + const unsigned nBufs = 1000; /* measure us, report ns */ + unsigned testIndex, sizeIndex, memType, nMemTypes; + const char *memTypeStrings[2] = {"SysMem", "VRAM"}; + const unsigned nSizes = 4; + const unsigned bufSizes[nSizes] = {PAGE_SIZE, PAGE_SIZE*4, PAGE_SIZE*16, PAGE_SIZE*64}; + const unsigned nTests = nSizes << 2; +#define TEST_BUFSIZE(index) (bufSizes[(index) & (nSizes-1)]) +#define TEST_MEMTYPE(index) ((index / nSizes) & 0x1) +#define TEST_SDMA(index) (((index / nSizes) >> 1) & 0x1) + + void *bufs[nBufs]; + unsigned long long start, end; + unsigned i; + HSAKMT_STATUS ret; + HsaMemFlags memFlags = {0}; + HsaMemMapFlags mapFlags = {0}; + HSAuint64 altVa; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HSAuint64 vramSizeMB = GetVramSize(defaultGPUNode) >> 20; + + LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; + + if (vramSizeMB == 0) + nMemTypes = 1; + else + nMemTypes = 2; + + /* Two SDMA queues to interleave user mode SDMA with memory + * management on either SDMA engine. Make the queues long enough + * to buffer at least nBufs x WriteData packets (7 dwords per + * packet). */ + SDMAQueue sdmaQueue[2]; + ASSERT_SUCCESS(sdmaQueue[0].Create(defaultGPUNode, PAGE_SIZE*8)); + ASSERT_SUCCESS(sdmaQueue[1].Create(defaultGPUNode, PAGE_SIZE*8)); + HsaMemoryBuffer sdmaBuffer(PAGE_SIZE, 0); /* system memory */ +#define INTERLEAVE_SDMA() do { \ + if (interleaveSDMA) { \ + sdmaQueue[0].PlaceAndSubmitPacket( \ + SDMAWriteDataPacket(sdmaBuffer.As(), \ + 0x12345678)); \ + sdmaQueue[1].PlaceAndSubmitPacket( \ + SDMAWriteDataPacket(sdmaBuffer.As()+16, \ + 0x12345678)); \ + } \ + } while (0) +#define IDLE_SDMA() do { \ + if (interleaveSDMA) { \ + sdmaQueue[0].Wait4PacketConsumption(); \ + sdmaQueue[1].Wait4PacketConsumption(); \ + } \ + } while (0) + + LOG() << "Test (avg. ns)\t alloc mapOne umapOne mapAll umapAll free" << std::endl; + for (testIndex = 0; testIndex < nTests; testIndex++) { + unsigned bufSize = TEST_BUFSIZE(testIndex); + unsigned memType = TEST_MEMTYPE(testIndex); + bool interleaveSDMA = TEST_SDMA(testIndex); + unsigned long long allocTime, map1Time, unmap1Time, mapAllTime, unmapAllTime, freeTime; + HSAuint32 allocNode; + + if ((testIndex & (nSizes-1)) == 0) + LOG() << "--------------------------------------------------------------------" << std::endl; + + if (memType >= nMemTypes) + continue; // skip unsupported mem types + + if (memType == 0) { + allocNode = 0; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 1; + memFlags.ui32.NonPaged = 0; + } else { + allocNode = defaultGPUNode; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 0; + memFlags.ui32.NonPaged = 1; + } + + /* Allocation */ + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + ASSERT_SUCCESS(hsaKmtAllocMemory(allocNode, bufSize, memFlags, + &bufs[i])); + INTERLEAVE_SDMA(); + } + allocTime = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); + + /* Map to one GPU */ + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(bufs[i], bufSize, + &altVa, mapFlags, 1, + &defaultGPUNode)); + INTERLEAVE_SDMA(); + } + map1Time = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); + + /* Unmap from GPU */ + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); + INTERLEAVE_SDMA(); + } + unmap1Time = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); + + /* Map to all GPUs */ + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(bufs[i], bufSize, &altVa)); + INTERLEAVE_SDMA(); + } + mapAllTime = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); + + /* Unmap from all GPUs */ + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); + INTERLEAVE_SDMA(); + } + unmapAllTime = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); + + /* Free */ + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + ASSERT_SUCCESS(hsaKmtFreeMemory(bufs[i], bufSize)); + INTERLEAVE_SDMA(); + } + freeTime = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); + + LOG() << std::dec << std::setiosflags(std::ios::right) + << std::setw(3) << (bufSize >> 10) << "K-" + << memTypeStrings[memType] << "-" + << (interleaveSDMA ? "SDMA\t" : "noSDMA\t") + << std::setw(8) << allocTime + << std::setw(8) << map1Time + << std::setw(8) << unmap1Time + << std::setw(8) << mapAllTime + << std::setw(8) << unmapAllTime + << std::setw(8) << freeTime << std::endl; + } + + TEST_END +} + +TEST_F(KFDMemoryTest, QueryPointerInfo) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + unsigned int bufSize = PAGE_SIZE * 8; // CZ and Tonga need 8 pages + HsaPointerInfo ptrInfo; + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + HSAuint64 nGPU = gpuNodes.size(); // number of gpu nodes + + /* GraphicHandle is tested at KFDGraphicsInterop.RegisterGraphicsHandle */ + + /*** Memory allocated on CPU node ***/ + HsaMemoryBuffer hostBuffer(bufSize, 0/*node*/, false, false/*local*/); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(hostBuffer.As(), &ptrInfo)); + EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED); + EXPECT_EQ(ptrInfo.Node, 0); + EXPECT_EQ(ptrInfo.MemFlags.Value, hostBuffer.Flags().Value); + EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As()); + EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)hostBuffer.As()); + EXPECT_EQ(ptrInfo.SizeInBytes, (HSAuint64)hostBuffer.Size()); + if (is_dgpu()) { + EXPECT_EQ((HSAuint64)ptrInfo.NMappedNodes, nGPU); + // Check NMappedNodes again after unmapping the memory + hsaKmtUnmapMemoryToGPU(hostBuffer.As()); + hsaKmtQueryPointerInfo(hostBuffer.As(), &ptrInfo); + } + EXPECT_EQ((HSAuint64)ptrInfo.NMappedNodes, 0); + + /* Skip testing local memory if the platform does not have it */ + if (GetVramSize(defaultGPUNode)) { + HsaMemoryBuffer localBuffer(bufSize, defaultGPUNode, false, true); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(localBuffer.As(), &ptrInfo)); + EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED); + EXPECT_EQ(ptrInfo.Node, defaultGPUNode); + EXPECT_EQ(ptrInfo.MemFlags.Value, localBuffer.Flags().Value); + EXPECT_EQ(ptrInfo.CPUAddress, localBuffer.As()); + EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)localBuffer.As()); + EXPECT_EQ(ptrInfo.SizeInBytes, (HSAuint64)localBuffer.Size()); + + HSAuint32 *addr = localBuffer.As() + 4; + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)addr, &ptrInfo)); + EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)localBuffer.As()); + } + + /** Registered memory: user pointer */ + static volatile HSAuint32 mem[4]; // 8 bytes for register only and + // 8 bytes for register to nodes + HsaMemoryBuffer hsaBuffer((void *)&mem[0], sizeof(HSAuint32)*2); + if (is_dgpu()) { // APU doesn't use userptr + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)&mem[0], &ptrInfo)); + EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); + EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); + EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)hsaBuffer.As()); + EXPECT_EQ(ptrInfo.SizeInBytes, sizeof(HSAuint32)*2); + EXPECT_EQ(ptrInfo.NRegisteredNodes, 0); + EXPECT_EQ(ptrInfo.NMappedNodes, nGPU); + // Register to nodes + HSAuint32 nodes[nGPU]; + for (unsigned int i = 0; i < nGPU; i++) + nodes[i] = gpuNodes.at(i); + EXPECT_SUCCESS(hsaKmtRegisterMemoryToNodes((void *)&mem[2], + sizeof(HSAuint32)*2, nGPU, nodes)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)&mem[2], &ptrInfo)); + EXPECT_EQ(ptrInfo.NRegisteredNodes, nGPU); + EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)&mem[2])); + } + + /* Not a starting address, but an address inside the memory range + * should also get the memory information + */ + HSAuint32 *address = hostBuffer.As() + 1; + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)address, &ptrInfo)); + EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED); + EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As()); + if (is_dgpu()) { + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)&mem[1], &ptrInfo)); + EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); + EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); + } + + /*** Set user data ***/ + char userData[16] = "This is a test."; + EXPECT_SUCCESS(hsaKmtSetMemoryUserData(hostBuffer.As(), (void *)userData)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(hostBuffer.As(), &ptrInfo)); + EXPECT_EQ(ptrInfo.UserData, (void *)userData); + + TEST_END +} + +/* Linux OS-specific test for a debugger accessing HSA memory in a + * debugged process. + * + * Allocates a system memory and a visible local memory buffer (if + * possible). Forks a child process that PTRACE_ATTACHes to the parent + * to access its memory like a debugger would. Child copies data in + * the parent process using PTRACE_PEEKDATA and PTRACE_POKEDATA. After + * the child terminates, the parent checks that the copy was + * successful. */ +TEST_F(KFDMemoryTest, PtraceAccess) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemFlags memFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 1; + + void *mem[2]; + unsigned i; + + // Offset in the VRAM buffer to test crossing non-contiguous + // buffer boundaries. The second access starting from offset + // sizeof(long)+1 will cross a node boundary in a single access, + // for node sizes of 4MB or smaller. + const HSAuint64 VRAM_OFFSET = (4 << 20) - 2 * sizeof(long); + + // alloc system memory from node 0 and initialize it + memFlags.ui32.NonPaged = 0; + ASSERT_SUCCESS(hsaKmtAllocMemory(0, PAGE_SIZE*2, memFlags, &mem[0])); + for (i = 0; i < 4*sizeof(long) + 4; i++) { + ((HSAuint8 *)mem[0])[i] = i; // source + ((HSAuint8 *)mem[0])[PAGE_SIZE+i] = 0; // destination + } + + // try to alloc local memory from GPU node + memFlags.ui32.NonPaged = 1; + if (m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode)) { + EXPECT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE*2 + (4 << 20), + memFlags, &mem[1])); + mem[1] = (void *)((HSAuint8 *)mem[1] + VRAM_OFFSET); + for (i = 0; i < 4*sizeof(long) + 4; i++) { + ((HSAuint8 *)mem[1])[i] = i; + ((HSAuint8 *)mem[1])[PAGE_SIZE+i] = 0; + } + } else { + LOG() << "Not testing local memory, it's invisible" << std::endl; + mem[1] = NULL; + } + + // Allow any process to trace this one. If kernel is built without + // Yama, this is not needed, and this call will fail. +#ifdef PR_SET_PTRACER + prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif + + // Find out my pid so the child can trace it + pid_t tracePid = getpid(); + + // Fork the child + pid_t childPid = fork(); + ASSERT_GE(childPid, 0); + if (childPid == 0) { + int traceStatus; + int err = 0, r; + + // Child process: don't use ASSERTs after attaching to parent + // process because terminating without detaching from the + // traced process leaves it stopped. Unfortunately, main() + // sets throw_on_failure to true, which seems to affect EXPECT + // as well. So we catch any exceptions and detach before + // terminating. + r = ptrace(PTRACE_ATTACH, tracePid, NULL, NULL); + if (r) { + WARN() << "PTRACE_ATTACH failed: " << r << std::endl; + exit(1); + } + try { + do { + waitpid(tracePid, &traceStatus, 0); + } while (!WIFSTOPPED(traceStatus)); + + for (i = 0; i < 4; i++) { + // Test 4 different (mis-)alignments, leaving 1-byte + // gaps between longs + HSAuint8 *addr = (HSAuint8 *)((long *)mem[0] + i) + i; + errno = 0; + long data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, + (void *)data)); + + if (mem[1] == NULL) + continue; + + addr = (HSAuint8 *)((long *)mem[1] + i) + i; + errno = 0; + data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, + (void *)data)); + } + } catch (...) { + err = 1; + } + r = ptrace(PTRACE_DETACH, tracePid, NULL, NULL); + if (r) { + WARN() << "PTRACE_DETACH failed: " << r << std::endl; + exit(1); + } + exit(err); + } else { + int childStatus; + + // Parent process, just wait for the child to finish + EXPECT_EQ(childPid, waitpid(childPid, &childStatus, 0)); + EXPECT_NE(0, WIFEXITED(childStatus)); + EXPECT_EQ(0, WEXITSTATUS(childStatus)); + } + + // Clear gaps in the source that should not have been copied + ((uint8_t*)mem[0])[ sizeof(long) ] = 0; + ((uint8_t*)mem[0])[2*sizeof(long) + 1] = 0; + ((uint8_t*)mem[0])[3*sizeof(long) + 2] = 0; + ((uint8_t*)mem[0])[4*sizeof(long) + 3] = 0; + // Check results + EXPECT_EQ(0, memcmp(mem[0], (HSAuint8 *)mem[0] + PAGE_SIZE, + sizeof(long)*4 + 4)); + // Free memory + EXPECT_SUCCESS(hsaKmtFreeMemory(mem[0], PAGE_SIZE*2)); + + if (mem[1]) { + ((uint8_t*)mem[1])[ sizeof(long) ] = 0; + ((uint8_t*)mem[1])[2*sizeof(long) + 1] = 0; + ((uint8_t*)mem[1])[3*sizeof(long) + 2] = 0; + ((uint8_t*)mem[1])[4*sizeof(long) + 3] = 0; + EXPECT_EQ(0, memcmp(mem[1], (HSAuint8 *)mem[1] + PAGE_SIZE, + sizeof(long)*4 + 4)); + mem[1] = (void *)((HSAuint8 *)mem[1] - VRAM_OFFSET); + EXPECT_SUCCESS(hsaKmtFreeMemory(mem[1], PAGE_SIZE*2)); + } + + TEST_END +} + +TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) +{ + char *hsaDebug = getenv("HSA_DEBUG"); + + if (!is_dgpu()) { + LOG() << "Skipping test: No VRAM on APU" << std::endl; + return; + } + + if (!hsaDebug || !strcmp(hsaDebug, "0")) { + LOG() << "Skipping test: HSA_DEBUG environment variable not set" << std::endl; + return; + } + + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemFlags memFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + /* Allocate host not accessible vram */ + memFlags.ui32.HostAccess = 0; + memFlags.ui32.NonPaged = 1; + + void *mem, *mem0, *mem1; + unsigned size = PAGE_SIZE*2 + (4 << 20); + HSAuint64 data[2] = {0xdeadbeefdeadbeef, 0xcafebabecafebabe}; + unsigned int data0[2] = {0xdeadbeef, 0xdeadbeef}; + unsigned int data1[2] = {0xcafebabe, 0xcafebabe}; + + const HSAuint64 VRAM_OFFSET = (4 << 20) - sizeof(HSAuint64); + + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, size, memFlags, &mem)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(mem, size, NULL)); + /* set the word before 4M boundary to 0xdeadbeefdeadbeef + * and the word after 4M boundary to 0xcafebabecafebabe + */ + mem0 = (void *)((HSAuint8 *)mem + VRAM_OFFSET); + mem1 = (void *)((HSAuint8 *)mem + VRAM_OFFSET + sizeof(HSAuint64)); + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)mem0, + data0[0], data0[1])); + queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)mem1, + data1[0], data1[1])); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(true, 0, 0)); + queue.Wait4PacketConsumption(); + + /* Allow any process to trace this one. If kernel is built without + * Yama, this is not needed, and this call will fail. + */ +#ifdef PR_SET_PTRACER + prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif + + // Find out my pid so the child can trace it + pid_t tracePid = getpid(); + + // Fork the child + pid_t childPid = fork(); + ASSERT_GE(childPid, 0); + if (childPid == 0) { + int traceStatus; + int err = 0, r; + + /* Child process: don't use ASSERTs after attaching to parent + * process because terminating without detaching from the + * traced process leaves it stopped. Unfortunately, main() + * sets throw_on_failure to true, which seems to affect EXPECT + * as well. So we catch any exceptions and detach before + * terminating. + */ + r = ptrace(PTRACE_ATTACH, tracePid, NULL, NULL); + if (r) { + WARN() << "PTRACE_ATTACH failed: " << r << std::endl; + exit(1); + } + try { + do { + waitpid(tracePid, &traceStatus, 0); + } while (!WIFSTOPPED(traceStatus)); + + /* peek the memory */ + errno = 0; + long data0 = ptrace(PTRACE_PEEKDATA, tracePid, mem0, NULL); + EXPECT_EQ(0, errno); + EXPECT_EQ(data[0], data0); + long data1 = ptrace(PTRACE_PEEKDATA, tracePid, mem1, NULL); + EXPECT_EQ(0, errno); + EXPECT_EQ(data[1], data1); + + /* swap mem0 and mem1 by poking */ + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, (void *)data[1])); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, (void *)data[0])); + EXPECT_EQ(0, errno); + } catch (...) { + err = 1; + } + r = ptrace(PTRACE_DETACH, tracePid, NULL, NULL); + if (r) { + WARN() << "PTRACE_DETACH failed: " << r << std::endl; + exit(1); + } + exit(err); + } else { + int childStatus; + + // Parent process, just wait for the child to finish + EXPECT_EQ(childPid, waitpid(childPid, &childStatus, 0)); + EXPECT_NE(0, WIFEXITED(childStatus)); + EXPECT_EQ(0, WEXITSTATUS(childStatus)); + } + + /* Use shader to read back data to check poke results */ + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + //dstBuffer is cpu accessible gtt memory + HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); + m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? gfx9_ScratchCopyDword : gfx8_ScratchCopyDword, + "ScratchCopyDword",isaBuffer); + Dispatch dispatch0(isaBuffer); + dispatch0.SetArgs(mem0, dstBuffer.As()); + dispatch0.Submit(queue); + dispatch0.Sync(); + ASSERT_EQ(data1[0], dstBuffer.As()[0]); + + Dispatch dispatch1(isaBuffer); + dispatch1.SetArgs(mem1, dstBuffer.As()); + dispatch1.Submit(queue); + dispatch1.Sync(); + WaitOnValue(dstBuffer.As(), data0[0]); + ASSERT_EQ(data0[0], dstBuffer.As()[0]); + + // Clean up + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(mem)); + ASSERT_SUCCESS(hsaKmtFreeMemory(mem, size)); + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +void CatchSignal(int IntrSignal) { + LOG() << "Interrupt Signal " << std::dec << IntrSignal + << " Received" << std::endl; +} + +TEST_F(KFDMemoryTest, SignalHandling) { + TEST_START(TESTPROFILE_RUNALL) + + if (!is_dgpu()) { + LOG() << "Skip the test on APU" << std::endl; + return; + } + + unsigned int *nullPtr = NULL; + unsigned int* pDb = NULL; + struct sigaction sa; + SDMAQueue queue; + HSAuint64 size, sysMemSize; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + sa.sa_handler = CatchSignal; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + pid_t ParentPid = getpid(); + EXPECT_EQ(0, sigaction(SIGUSR1, &sa, NULL)) << "An error occurred while setting a signal handler"; + + sysMemSize = GetSysMemSize(); + + /* System (kernel) memory are limited to 3/8th System RAM + * Try to allocate 1/4th System RAM + */ + size = (sysMemSize >> 2) & ~(HSAuint64)(PAGE_SIZE - 1); + + ASSERT_SUCCESS(hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, (void**)&pDb)); + // verify that pDb is not null before it's being used + ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; + + pid_t childPid = fork(); + ASSERT_GE(childPid, 0); + if (childPid == 0) { + ASSERT_EQ(0, kill(ParentPid, SIGUSR1)); + exit(0); + } else { + LOG() << "Start Memory Mapping..." << std::endl; + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(pDb, size, NULL)); + LOG() << "Mapping finished" << std::endl; + int childStatus; + + // Parent process, just wait for the child to finish + ASSERT_EQ(childPid, waitpid(childPid, &childStatus, 0)); + ASSERT_NE(0, WIFEXITED(childStatus)); + ASSERT_EQ(0, WEXITSTATUS(childStatus)); + } + + pDb[0] = 0x02020202; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(pDb, 0x01010101) ); + queue.Wait4PacketConsumption(); + ASSERT_TRUE(WaitOnValue(pDb, 0x01010101)); + ASSERT_SUCCESS(queue.Destroy()); + + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + // Release the buffers + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + + TEST_END +} + +TEST_F(KFDMemoryTest, CheckZeroInitializationSysMem) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + int ret; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HSAuint64 sysMemSizeMB = GetSysMemSize() >> 20; + + /* Testing system memory */ + HSAuint64 * pDb = NULL; + + HSAuint64 sysBufSizeMB = sysMemSizeMB >> 2; + HSAuint64 sysBufSize = sysBufSizeMB * 1024 * 1024; + + int count = 5; + + LOG() << "Using " << std::dec << sysBufSizeMB + << "MB system buffer to test " << std::dec << count + << " times" << std::endl; + + unsigned int offset = 257; // a constant offset, should be smaller than 512. + unsigned int size = sysBufSize / sizeof(*pDb); + + while (count--) { + ret = hsaKmtAllocMemory(0 /* system */, sysBufSize, m_MemoryFlags, + (void**)&pDb); + if (ret) { + LOG() << "Failed to allocate system buffer of" << std::dec << sysBufSizeMB + << "MB" << std::endl; + return; + } + + /* check the first 64 bit */ + EXPECT_EQ(0, pDb[0]); + pDb[0] = 1; + + for (HSAuint64 i = offset; i < size;) { + EXPECT_EQ(0, pDb[i]); + pDb[i] = i + 1; // set it to non zero + + i += 4096 / sizeof(*pDb); + } + + /* check the last 64 bit */ + EXPECT_EQ(0, pDb[size-1]); + pDb[size-1] = size; + + ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, sysBufSize)); + } + + TEST_END +} + +static inline void access(volatile void *sd, int size, int rw) +{ + /* Most like sit in cache*/ + static struct DUMMY { + char dummy[1024]; + } dummy; + + while ((size -= sizeof(dummy)) >= 0) { + if (rw == 0) + dummy = *(struct DUMMY *)((char*)sd + size); + else + *(struct DUMMY *)((char*)sd + size) = dummy; + } +} + +/* + * on large-ber system, test the visible vram access speed. + * kfd is not allowd to alloc visible vram on non-largebar system. + */ +TEST_F(KFDMemoryTest, MMBandWidth) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + const unsigned nBufs = 1000; /* measure us, report ns */ + unsigned testIndex, sizeIndex, memType; + const unsigned nMemTypes = 2; + const char *memTypeStrings[nMemTypes] = {"SysMem", "VRAM "}; + const unsigned nSizes = 4; + const unsigned bufSizes[nSizes] = {PAGE_SIZE, PAGE_SIZE*4, PAGE_SIZE*16, PAGE_SIZE*64}; + const unsigned nTests = nSizes * nMemTypes; + const unsigned tmpBufferSize = PAGE_SIZE*64; +#define _TEST_BUFSIZE(index) (bufSizes[index % nSizes]) +#define _TEST_MEMTYPE(index) ((index / nSizes) % nMemTypes) + + void *bufs[nBufs]; + unsigned long long start; + unsigned i; + HSAKMT_STATUS ret; + HsaMemFlags memFlags = {0}; + HsaMemMapFlags mapFlags = {0}; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HSAuint64 vramSizeMB = GetVramSize(defaultGPUNode) >> 20; + + LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; + + if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode) || !vramSizeMB) { + LOG() << "not a largebar system, skip!"<> 10) << "K-" + << memTypeStrings[memType] << "\t" + << std::setw(12) << mcpRTime + << std::setw(12) << mcpWTime + << std::setw(12) << accessRTime + << std::setw(12) << accessWTime + << std::endl; + } + + munmap(tmp, tmpBufferSize); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp new file mode 100644 index 0000000000..8982fcb2df --- /dev/null +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDBaseComponentTest.hpp" +#include "IsaGenerator.hpp" + +#ifndef __KFD_MEMORY_TEST__H__ +#define __KFD_MEMORY_TEST__H__ + +// @class KFDTopologyTest +// this class has no additional features to KFDBaseComponentTest +// the separation was made so we are able to goup all memeory tests together +class KFDMemoryTest : public KFDBaseComponentTest { + public: + KFDMemoryTest(void) :m_pIsaGen(NULL) {} + ~KFDMemoryTest(void) {} + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: + IsaGenerator* m_pIsaGen; + + void BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); + void BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); +}; + +#endif // __KFD_MEMORY_TEST__H__ diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp b/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp new file mode 100644 index 0000000000..68351c847b --- /dev/null +++ b/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDOpenCloseKFDTest.hpp" +#include "KFDTestUtil.hpp" + +// before every test from this class fixture - open KFD +void KFDOpenCloseKFDTest::SetUp() { + ROUTINE_START + + ASSERT_SUCCESS(hsaKmtOpenKFD() ); + + ROUTINE_END +} + +// after every test from this class fixture - close KFD +void KFDOpenCloseKFDTest::TearDown() { + ROUTINE_START + + EXPECT_SUCCESS(hsaKmtCloseKFD() ); + + ROUTINE_END +} + +// this test does not use class KFDOpenCloseKFDTest but is placed here +// since it's testing same topic as other test +// verify that calling hsaKmtCloseKFD on a closed KFD will return right status +TEST(KFDCloseKFDTest, CloseAClosedKfd ) { + TEST_START(TESTPROFILE_RUNALL) + + ASSERT_EQ(HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED, hsaKmtCloseKFD()); + + TEST_END +} + +// verify that calling hsaKmtCloseKFD on an already opened KFD will return +// right status +TEST_F(KFDOpenCloseKFDTest, OpenAlreadyOpenedKFD ) { + TEST_START(TESTPROFILE_RUNALL) + + EXPECT_EQ(HSAKMT_STATUS_KERNEL_ALREADY_OPENED, hsaKmtOpenKFD()); + + EXPECT_SUCCESS(hsaKmtCloseKFD()); + + TEST_END +} + +// testing the normal scenario: open followed by close (done in the setup and +// teardown functions) +TEST_F(KFDOpenCloseKFDTest, OpenCloseKFD ) { +} + +TEST_F(KFDOpenCloseKFDTest, InvalidKFDHandleTest ) { + TEST_START(TESTPROFILE_RUNALL) + + HsaVersionInfo m_VersionInfo; + pid_t m_ChildPid = fork(); + if (m_ChildPid == 0) { + EXPECT_EQ(HSAKMT_STATUS_ERROR, hsaKmtGetVersion(&m_VersionInfo)); + exit(0); + } else { + int childStatus; + EXPECT_EQ(m_ChildPid, waitpid(m_ChildPid, &childStatus, 0)); + EXPECT_NE(0, WIFEXITED(childStatus)); + EXPECT_EQ(0, WEXITSTATUS(childStatus)); + } + TEST_END +} diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp b/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp new file mode 100644 index 0000000000..6a2d26646f --- /dev/null +++ b/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "hsakmt.h" + +#ifndef __KFD_OPEN_CLOSE_KFD_TEST__H__ +#define __KFD_OPEN_CLOSE_KFD_TEST__H__ + +// @class KFDOpenCloseKFDTest +class KFDOpenCloseKFDTest : public testing::Test { + public: + KFDOpenCloseKFDTest(void) {} + ~KFDOpenCloseKFDTest(void) {} + + protected: + // @brief SetUp function run before every test that uses KFDOpenCloseKFDTest class fixture, sets all common settings for the tests. + virtual void SetUp(); + // @brief TearDown function run after every test that uses KFDOpenCloseKFDTest class fixture. + virtual void TearDown(); +}; + +#endif // __KFD_OPEN_CLOSE_KFD_TEST__H__ diff --git a/tests/kfdtest/src/KFDPMTest.cpp b/tests/kfdtest/src/KFDPMTest.cpp new file mode 100644 index 0000000000..6e63f7ef9e --- /dev/null +++ b/tests/kfdtest/src/KFDPMTest.cpp @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDPMTest.hpp" +#include "KFDTestUtil.hpp" +#include "PM4Packet.hpp" +#include "PM4Queue.hpp" +#include "hsakmt.h" + +bool KFDPMTest::m_SetupSuccess = false; + +void KFDPMTest::SetUpTestCase() { + ROUTINE_START + + m_SetupSuccess = false; + AcquirePrivilege(OS_SUSPEND); + + // if AcquirePrivilege fails, it will throw and we will not reach here. + m_SetupSuccess = true; + + ROUTINE_END +} + +void KFDPMTest::TearDownTestCase() { +} + + +void KFDPMTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + ROUTINE_END +} + +void KFDPMTest::TearDown() { + ROUTINE_START + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDPMTest, SuspendWithActiveProcess) { + TEST_START(TESTPROFILE_RUNALL) + + ASSERT_EQ(true, SuspendAndWakeUp()); + + TEST_END +} + +TEST_F(KFDPMTest, SuspendWithIdleQueue) { + TEST_START(TESTPROFILE_RUNALL) + + PM4Queue queue; + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + ASSERT_EQ(true, SuspendAndWakeUp()); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDPMTest, SuspendWithIdleQueueAfterWork) { + TEST_START(TESTPROFILE_RUNALL) + + PM4Queue queue; + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuffer(PAGE_SIZE, defaultGPUNode); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuffer.As(), 0x1, 0x2)); + queue.Wait4PacketConsumption(); + WaitOnValue(&(destBuffer.As()[0]), 0x1); + WaitOnValue(&(destBuffer.As()[1]), 0x2); + + destBuffer.Fill(0); + + ASSERT_EQ(true, SuspendAndWakeUp()); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(&(destBuffer.As()[2]), 0x3, 0x4)); + queue.Wait4PacketConsumption(); + + ASSERT_EQ(destBuffer.As()[0], 0); + ASSERT_EQ(destBuffer.As()[1], 0); + + WaitOnValue(&(destBuffer.As()[2]), 0x3); + WaitOnValue(&(destBuffer.As()[3]), 0x4); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +// TODO suspend while workload is being executed by a queue diff --git a/tests/kfdtest/src/KFDPMTest.hpp b/tests/kfdtest/src/KFDPMTest.hpp new file mode 100644 index 0000000000..7cc7c85ad6 --- /dev/null +++ b/tests/kfdtest/src/KFDPMTest.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFDPMTEST_HPP__ +#define __KFDPMTEST_HPP__ + +#include +#include "KFDBaseComponentTest.hpp" + +class KFDPMTest : public KFDBaseComponentTest { + public: + KFDPMTest() {} + ~KFDPMTest() {} + + protected: + static void SetUpTestCase(); + static void TearDownTestCase(); + + virtual void SetUp(); + virtual void TearDown(); + + protected: + static bool m_SetupSuccess; +}; + +#endif // __KFDPMTEST_HPP__ diff --git a/tests/kfdtest/src/KFDPNPTest.cpp b/tests/kfdtest/src/KFDPNPTest.cpp new file mode 100644 index 0000000000..7a1f77ee9d --- /dev/null +++ b/tests/kfdtest/src/KFDPNPTest.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDPNPTest.hpp" +#include "KFDTestUtil.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "hsakmt.h" + +bool KFDPNPTest::m_SetupSuccess = false; + +void KFDPNPTest::SetUpTestCase() { + ROUTINE_START + + AcquirePrivilege(OS_DRIVER_OPERATIONS); + + // if AcquirePrivilege fails, it will throw and we will not reach here. + m_SetupSuccess = true; + + ROUTINE_END +} + +void KFDPNPTest::TearDownTestCase() { +} + + +void KFDPNPTest::SetUp() { + ROUTINE_START + + ASSERT_TRUE(m_SetupSuccess); + + KFDBaseComponentTest::SetUp(); + + ROUTINE_END +} + +void KFDPNPTest::TearDown() { + ROUTINE_START + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDPNPTest, DisableWithActiveProcess) { + TEST_START(TESTPROFILE_RUNALL); + + DisableKfd(); + EnableKfd(); + + TEST_END +} + +TEST_F(KFDPNPTest, DisableAndCreateQueue) { + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuffer(PAGE_SIZE, defaultGPUNode); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuffer.As(), 0x1, 0x2)); + queue.Wait4PacketConsumption(); + + WaitOnValue(&(destBuffer.As()[0]), 0x1); + WaitOnValue(&(destBuffer.As()[1]), 0x2); + + ASSERT_SUCCESS(queue.Destroy()); + + DisableKfd(); + EnableKfd(); + + ASSERT_NE(HSAKMT_STATUS_SUCCESS, queue.Create(defaultGPUNode)) << "Queue creation should fail after a topology change."; + + TEST_END +} diff --git a/tests/kfdtest/src/KFDPNPTest.hpp b/tests/kfdtest/src/KFDPNPTest.hpp new file mode 100644 index 0000000000..e9ce4d619a --- /dev/null +++ b/tests/kfdtest/src/KFDPNPTest.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_PNP_TEST__H__ +#define __KFD_PNP_TEST__H__ + +#include +#include "KFDBaseComponentTest.hpp" + + +class KFDPNPTest : public KFDBaseComponentTest { + public: + KFDPNPTest() {} + ~KFDPNPTest() {} + + protected: + static void SetUpTestCase(); + static void TearDownTestCase(); + + virtual void SetUp(); + virtual void TearDown(); + + protected: + static bool m_SetupSuccess; +}; + +#endif // __KFD_PNP_TEST__H__ diff --git a/tests/kfdtest/src/KFDPerfCounters.cpp b/tests/kfdtest/src/KFDPerfCounters.cpp new file mode 100644 index 0000000000..6735b5115a --- /dev/null +++ b/tests/kfdtest/src/KFDPerfCounters.cpp @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDPerfCounters.hpp" + +void KFDPerfCountersTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + ROUTINE_END +} + +void KFDPerfCountersTest::TearDown() { + ROUTINE_START + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +static struct block_name_table { + char name[32]; + HSA_UUID uuid; +} block_lookup_table[] = { + {"CB ", {0x9ba429c6, 0xaf2d, 0x4b38, 0xb3, 0x49, 0x15, 0x72, 0x71, 0xbe, 0xac, 0x6a}}, + {"CPF ", {0x2b0ad2b5, 0x1c43, 0x4f46, 0xa7, 0xbc, 0xe1, 0x19, 0x41, 0x1e, 0xa6, 0xc9}}, + {"CPG ", {0x590ec94d, 0x20f0, 0x448f, 0x8d, 0xff, 0x31, 0x6c, 0x67, 0x9d, 0xe7, 0xff}}, + {"DB ", {0x3d1a47fc, 0x0013, 0x4ed4, 0x83, 0x06, 0x82, 0x2c, 0xa0, 0xb7, 0xa6, 0xc2}}, + {"GDS ", {0xf59276ec, 0x2526, 0x4bf8, 0x8e, 0xc0, 0x11, 0x8f, 0x77, 0x70, 0x0d, 0xc9}}, + {"GRBM ", {0x8f00933c, 0xc33d, 0x4801, 0x97, 0xb7, 0x70, 0x07, 0xf7, 0x85, 0x73, 0xad}}, + {"GRBMSE ", {0x34ebd8d7, 0x7c8b, 0x4d15, 0x88, 0xfa, 0x0e, 0x4e, 0x4a, 0xf5, 0x9a, 0xc1}}, + {"IA ", {0x34276944, 0x4264, 0x4fcd, 0x9d, 0x6e, 0xae, 0x26, 0x45, 0x82, 0xec, 0x51}}, + {"MC ", {0x13900b57, 0x4956, 0x4d98, 0x81, 0xd0, 0x68, 0x52, 0x19, 0x37, 0xf5, 0x9c}}, + {"PASC ", {0xb0e7fb5d, 0x0efc, 0x4744, 0xb5, 0x16, 0x5d, 0x23, 0xdc, 0x1f, 0xd5, 0x6c}}, + {"PASU ", {0x9a152b6a, 0x1fad, 0x45f2, 0xa5, 0xbf, 0xf1, 0x63, 0x82, 0x6b, 0xd0, 0xcd}}, + {"SPI ", {0xeda81044, 0xd62c, 0x47eb, 0xaf, 0x89, 0x4f, 0x6f, 0xbf, 0x3b, 0x38, 0xe0}}, + {"SRBM ", {0x9f8040e0, 0x6830, 0x4019, 0xac, 0xc8, 0x46, 0x3c, 0x9e, 0x44, 0x5b, 0x89}}, + {"SQ ", {0xb5c396b6, 0xd310, 0x47e4, 0x86, 0xfc, 0x5c, 0xc3, 0x4, 0x3a, 0xf5, 0x8}}, + {"SX ", {0xbdb8d737, 0x43cc, 0x4162, 0xbe, 0x52, 0x51, 0xcf, 0xb8, 0x47, 0xbe, 0xaf}}, + {"TA ", {0xc01ee43d, 0xad92, 0x44b1, 0x8a, 0xb9, 0xbe, 0x5e, 0x69, 0x6c, 0xee, 0xa7}}, + {"TCA ", {0x333e393f, 0xe147, 0x4f49, 0xa6, 0xd1, 0x60, 0x91, 0x4c, 0x70, 0x86, 0xb0}}, + {"TCC ", {0x848ce855, 0xd805, 0x4566, 0xa8, 0xab, 0x73, 0xe8, 0x84, 0xcc, 0x6b, 0xff}}, + {"TCP ", {0xe10a013b, 0x17d4, 0x4bf5, 0xb0, 0x89, 0x42, 0x95, 0x91, 0x05, 0x9b, 0x60}}, + {"TCS ", {0x4126245c, 0x4d96, 0x4d1a, 0x8a, 0xed, 0xa9, 0x39, 0xd4, 0xcc, 0x8e, 0xc9}}, + {"TD ", {0x7d7c0fe4, 0xfe41, 0x4fea, 0x92, 0xc9, 0x45, 0x44, 0xd7, 0x70, 0x6d, 0xc6}}, + {"VGT ", {0x0b6a8cb7, 0x7a01, 0x409f, 0xa2, 0x2c, 0x30, 0x14, 0x85, 0x4f, 0x13, 0x59}}, + {"WD ", {0x0e176789, 0x46ed, 0x4b02, 0x97, 0x2a, 0x91, 0x6d, 0x2f, 0xac, 0x24, 0x4a}}, + {"IOMMUV2", {0x80969879, 0xb0f6, 0x4be6, 0x97, 0xf6, 0x6a, 0x63, 0x0, 0xf5, 0x10, 0x1d}}, + {"DRIVER ", {0xea9b5ae1, 0x6c3f, 0x44b3, 0x89, 0x54, 0xda, 0xf0, 0x75, 0x65, 0xa9, 0xa}} +}; + +void KFDPerfCountersTest::GetBlockName(HSA_UUID uuid, char *name, uint32_t name_len, char *uuid_str, uint32_t uuid_str_len) { + uint32_t i, table_size; + + table_size = sizeof(block_lookup_table) / sizeof(struct block_name_table); + + snprintf(name, name_len, "unknown"); + for (i = 0; i < table_size; i++) { + if (!memcmp(&block_lookup_table[i].uuid, &uuid, sizeof(HSA_UUID))) { + if (name) + snprintf(name, name_len, "%s", block_lookup_table[i].name); + break; + } + } + + if (uuid_str) + snprintf(uuid_str, uuid_str_len, + "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", + uuid.Data1, uuid.Data2, uuid.Data3, + uuid.Data4[0], uuid.Data4[1], uuid.Data4[2], + uuid.Data4[3], uuid.Data4[4], uuid.Data4[5], + uuid.Data4[6], uuid.Data4[7]); +} + +TEST_F(KFDPerfCountersTest, GetCounterProperties) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaCounterProperties* pProps = NULL; + ASSERT_SUCCESS(hsaKmtPmcGetCounterProperties(defaultGPUNode, &pProps)); + /* Verifying that there is at least one block */ + ASSERT_NE(0, pProps->NumBlocks) << "No performance counters blocks"; + + LOG() << std::dec << pProps->NumBlocks << " blocks found." << std::endl; + + HsaCounterBlockProperties *block; + block = &pProps->Blocks[0]; + for (HSAuint32 i = 0; i < pProps->NumBlocks; i++) { + char uuid_string[37] = ""; + char name[32] = ""; + GetBlockName(block->BlockId, name, 32, uuid_string, 37); + + char type[32]; + switch (block->Counters[0].Type) { + case HSA_PROFILE_TYPE_PRIVILEGED_IMMEDIATE: + snprintf(type, sizeof(type), "Priv Immediate"); + break; + case HSA_PROFILE_TYPE_PRIVILEGED_STREAMING: + snprintf(type, sizeof(type), "Priv Streaming"); + break; + case HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE: + snprintf(type, sizeof(type), "Non-priv Immediate"); + break; + case HSA_PROFILE_TYPE_NONPRIV_STREAMING: + snprintf(type, sizeof(type), "Non-priv Immediate"); + break; + default: + snprintf(type, sizeof(type), "Unknown"); + break; + } + + LOG() << name << " (" << uuid_string << "): " << type << ", " << + block->NumCounters << " counter IDs" << std::endl; + block = (HsaCounterBlockProperties *)&block->Counters[block->NumCounters]; + } + + TEST_END +} + +TEST_F(KFDPerfCountersTest, RegisterTrace) { + TEST_START(TESTPROFILE_RUNALL) + HsaCounterProperties* pProps; + + HsaPmcTraceRoot root; + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + pProps = NULL; + ASSERT_SUCCESS(hsaKmtPmcGetCounterProperties(defaultGPUNode, &pProps)); + + /* Verifying that there is at least one block */ + ASSERT_NE(0, pProps->NumBlocks) << "No performance counters blocks"; + + HsaCounterBlockProperties *block = &pProps->Blocks[0]; + bool priv_block_found = false; + for (HSAuint32 i = 0; i < pProps->NumBlocks; i++) { + if (block->Counters[0].Type <= HSA_PROFILE_TYPE_PRIVILEGED_STREAMING) { + priv_block_found = true; + break; + } + block = (HsaCounterBlockProperties *)&block->Counters[block->NumCounters]; + } + + if (!priv_block_found) { + LOG() << "No privileged block is found. Skipping RegisterTrace test." + << std::endl; + return; + } + + /* Registering trace */ + ASSERT_SUCCESS(hsaKmtPmcRegisterTrace(defaultGPUNode, + block->NumConcurrent, + block->Counters, + &root)); + ASSERT_SUCCESS(hsaKmtPmcUnregisterTrace(defaultGPUNode, root.TraceId)); + + TEST_END +} + +TEST_F(KFDPerfCountersTest, StartStopQueryTrace) { + TEST_START(TESTPROFILE_RUNALL) + + HsaPmcTraceRoot root; + HsaCounterProperties* pProps; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + pProps = NULL; + ASSERT_SUCCESS(hsaKmtPmcGetCounterProperties(defaultGPUNode, &pProps)); + + /* Verifying that there is at least one block */ + ASSERT_NE(0, pProps->NumBlocks) << "No performance counters blocks"; + + HsaCounterBlockProperties *block = &pProps->Blocks[0]; + bool priv_block_found = false; + for (HSAuint32 i = 0; i < pProps->NumBlocks; i++) { + if (block->Counters[0].Type <= HSA_PROFILE_TYPE_PRIVILEGED_STREAMING) { + priv_block_found = true; + break; + } + block = (HsaCounterBlockProperties *)&block->Counters[block->NumCounters]; + } + + if (!priv_block_found) { + LOG() << "No privileged block is found. Skipping StartStopQuery test." + << std::endl; + return; + } + + if (getuid()) { /* Non-root */ + LOG() << "Privileged counters requires the user as root." << std::endl + << "Skipping StartStopQuery test." << std::endl; + return; + } + + /* Registering trace */ + ASSERT_SUCCESS(hsaKmtPmcRegisterTrace(defaultGPUNode, + block->NumConcurrent, + block->Counters, + &root)); + + /* Acquiring access for the trace */ + ASSERT_SUCCESS(hsaKmtPmcAcquireTraceAccess(defaultGPUNode, root.TraceId)); + + /* Allocating memory buffer for the trace */ + HsaMemoryBuffer membuf(PAGE_SIZE, defaultGPUNode); + + /* Starting the trace */ + ASSERT_SUCCESS(hsaKmtPmcStartTrace(root.TraceId, + membuf.As(), + membuf.Size())); + + /* Delay between START and STOP tracing */ + Delay(START_STOP_DELAY); + + /* Stopping th trace */ + ASSERT_SUCCESS(hsaKmtPmcStopTrace(root.TraceId)); + + /* Querying the trace */ + ASSERT_SUCCESS(hsaKmtPmcQueryTrace(root.TraceId)); + uint64_t *buf = membuf.As(); + for (uint32_t i = 0; i < block->NumConcurrent; i++, buf++) + LOG() << "Counter " << std::dec << i << ": " << *buf << std::endl; + + /* Releasing the trace */ + ASSERT_SUCCESS(hsaKmtPmcReleaseTraceAccess(0, root.TraceId)); + + ASSERT_SUCCESS(hsaKmtPmcUnregisterTrace(defaultGPUNode, root.TraceId)); + + TEST_END +} + +TEST_F(KFDPerfCountersTest, ClockCountersBasicTest) { + TEST_START(TESTPROFILE_RUNALL) + + HsaClockCounters counters1; + HsaClockCounters counters2; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + EXPECT_SUCCESS(hsaKmtGetClockCounters(defaultGPUNode, &counters1)); + + Delay(100); + + EXPECT_SUCCESS(hsaKmtGetClockCounters(defaultGPUNode, &counters2)); + + EXPECT_NE(0, counters1.GPUClockCounter); + EXPECT_NE(0, counters2.GPUClockCounter); + EXPECT_NE(0, counters1.SystemClockCounter); + EXPECT_NE(0, counters2.SystemClockCounter); + + EXPECT_GT(counters2.GPUClockCounter, counters1.GPUClockCounter); + EXPECT_GT(counters2.SystemClockCounter, counters1.SystemClockCounter); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDPerfCounters.hpp b/tests/kfdtest/src/KFDPerfCounters.hpp new file mode 100644 index 0000000000..ce65eaf963 --- /dev/null +++ b/tests/kfdtest/src/KFDPerfCounters.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFDPERFCOUNTERS_HPP__ +#define __KFDPERFCOUNTERS_HPP__ + +#include +#include "KFDBaseComponentTest.hpp" + +class KFDPerfCountersTest : public KFDBaseComponentTest { + public: + KFDPerfCountersTest() {} + ~KFDPerfCountersTest() {} + + static const unsigned int START_STOP_DELAY = 10000; // 10 sec tracing + + protected: + virtual void SetUp(); + virtual void TearDown(); + void GetBlockName(HSA_UUID uuid, char *name, uint32_t name_len, + char *uuid_str, uint32_t uuid_str_len); +}; + +#endif // __KFDPERFCOUNTERS_HPP__ diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp new file mode 100644 index 0000000000..c43457fcfe --- /dev/null +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -0,0 +1,1566 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include "KFDQMTest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "AqlQueue.hpp" + +#include "Dispatch.hpp" + +// All tests are marked by their serial number in the QCM FDD + +void KFDQMTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDQMTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDQMTest, CreateCpQueue) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFF); + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0, 0)); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf.As(), 0); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, CreateSdmaQueue) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFF); + + SDMAQueue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0x02020202)); + + queue.Wait4PacketConsumption(); + + ASSERT_TRUE(WaitOnValue(destBuf.As(), 0x02020202)); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + int bufSize = PAGE_SIZE; + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + unsigned int MAX_SDMA_QUEUES; + + if (m_FamilyId == FAMILY_RV) + MAX_SDMA_QUEUES = 2; + else + MAX_SDMA_QUEUES = 4; + + HsaMemoryBuffer destBuf(bufSize << 1 , defaultGPUNode, false); + HsaMemoryBuffer srcBuf(bufSize, defaultGPUNode, false); + destBuf.Fill(0xFF); + + SDMAQueue queues[MAX_SDMA_QUEUES]; + + for (unsigned int qidx = 0; qidx < MAX_SDMA_QUEUES; ++qidx) + ASSERT_SUCCESS(queues[qidx].Create(defaultGPUNode)); + + for (unsigned int qidx = 0; qidx < MAX_SDMA_QUEUES; ++qidx) { + destBuf.Fill(0x0); + srcBuf.Fill(qidx + 0xa0); + queues[qidx].PlaceAndSubmitPacket( + SDMACopyDataPacket(destBuf.As(), srcBuf.As(), bufSize)); + queues[qidx].PlaceAndSubmitPacket( + SDMAWriteDataPacket(destBuf.As() + bufSize/4, 0x02020202)); + + queues[qidx].Wait4PacketConsumption(); + + ASSERT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); + + ASSERT_SUCCESS(memcmp( + destBuf.As(), srcBuf.As(), bufSize)); + } + + for (unsigned int qidx = 0; qidx < MAX_SDMA_QUEUES; ++qidx) + ASSERT_SUCCESS(queues[qidx].Destroy()); + + TEST_END +} + +/* The following test is designed to reproduce an intermittent hang on + * Fiji and other VI/Polaris GPUs. This test typically hangs in a few + * seconds. According to analysis done by HW engineers, the culprit + * seems to be PCIe speed switching. The problem can be worked around + * by disabling the lowest DPM level on Fiji. */ +TEST_F(KFDQMTest, SdmaConcurrentCopies) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + +#define BUFFER_SIZE (64*1024) +#define NPACKETS 1 +#define COPY_SIZE (BUFFER_SIZE / NPACKETS) + HsaMemoryBuffer srcBuf(BUFFER_SIZE, 0, true); + HsaMemoryBuffer dstBuf(BUFFER_SIZE, defaultGPUNode, false, is_dgpu() ? true : false); + + SDMAQueue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + std::ostream &log = LOG(); + char progress[] = "-\b"; + log << "Running ... "; + + for (unsigned i = 0; i < 100000; i++) { + if (i % 1000 == 0) { + const char progressSteps[4] = {'-', '\\', '|', '/'}; + progress[0] = progressSteps[(i/1000) % 4]; + log << progress; + } + + for (unsigned j = 0; j < NPACKETS; j++) + queue.PlacePacket( + SDMACopyDataPacket(dstBuf.As()+COPY_SIZE*j, + srcBuf.As()+COPY_SIZE*j, COPY_SIZE)); + queue.SubmitPacket(); + + // Waste a variable amount of time. Submission timing + // while SDMA runs concurrently seems to be critical for + // reproducing the hang + for (int k = 0; k < (i & 0xfff); k++) + memcpy(srcBuf.As()+PAGE_SIZE, srcBuf.As(), 1024); + + // Wait for idle every 8 packets to allow the SDMA engine to + // run concurrently for a bit without getting too far ahead + if ((i & 0x7) == 0) + queue.Wait4PacketConsumption(); + } + log << "Done." << std::endl; + + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(srcBuf.As(), 0x02020202)); + queue.Wait4PacketConsumption(); + ASSERT_TRUE(WaitOnValue(srcBuf.As(), 0x02020202)); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, CreateMultipleCpQueues) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + static const unsigned int MAX_CP_QUEUES = 16; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFF); + + PM4Queue queues[MAX_CP_QUEUES]; + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + ASSERT_SUCCESS(queues[qidx].Create(defaultGPUNode)) << " QueueId=" << qidx; + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) { + queues[qidx].PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As()+qidx*2, qidx, qidx)); + + queues[qidx].Wait4PacketConsumption(); + + WaitOnValue(destBuf.As()+qidx*2, qidx); + } + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + ASSERT_SUCCESS(queues[qidx].Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFFFFFFFF); + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0, 0)); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf.As(), 0); + + destBuf.Fill(0xFFFFFFFF); + + ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true)); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 1, 1)); + + // don't sync since we don't expect rptr to change when the queue is disabled. + Delay(2000); + + ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; + + ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf.As(), 1); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, DisableSdmaQueueByUpdateWithNullAddress) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFFFFFFFF); + + SDMAQueue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0)); + + WaitOnValue(destBuf.As(), 0); + + destBuf.Fill(0xFFFFFFFF); + + ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true)); + + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0)); + + // don't sync since we don't expect rptr to change when the queue is disabled. + Delay(2000); + + ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; + + ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf.As(), 0); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFFFFFFFF); + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + PM4WriteDataPacket packet1, packet2; + packet1.InitPacket(destBuf.As(), 0, 0); + packet2.InitPacket(destBuf.As(), 1, 1); + + queue.PlaceAndSubmitPacket(packet1); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf.As(), 0); + + destBuf.Fill(0xFFFFFFFF); + + ASSERT_SUCCESS(queue.Update(0/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false)); + + queue.PlaceAndSubmitPacket(packet2); + + // don't sync since we don't expect rptr to change when the queue is disabled. + Delay(2000); + + ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; + + ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf.As(), 1); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, CreateQueueStressSingleThreaded) { + TEST_START(TESTPROFILE_RUNALL) + + static const unsigned long long TEST_TIME_SEC = 15; + + unsigned long long initialTime = GetSystemTickCountInMicroSec(); + + unsigned int numIter = 0; + + unsigned long long timePassed = 0; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + do { + // the following means we'll get the order 0,0 => 0,1 => 1,0 => 1,1 so we cover all options. + unsigned int firstToCreate = (numIter % 2 != 0) ? 1 : 0; + unsigned int firstToDestroy = (numIter % 4 > 1) ? 1 : 0; + + unsigned int secondToCreate = (firstToCreate + 1)%2; + unsigned int secondToDestroy = (firstToDestroy + 1)%2; + + BaseQueue *queues[2] = {new PM4Queue(), new SDMAQueue()}; + + ASSERT_SUCCESS(queues[firstToCreate]->Create(defaultGPUNode)); + ASSERT_SUCCESS(queues[secondToCreate]->Create(defaultGPUNode)); + + ASSERT_SUCCESS(queues[firstToDestroy]->Destroy()); + ASSERT_SUCCESS(queues[secondToDestroy]->Destroy()); + + delete queues[0]; + delete queues[1]; + ++numIter; + + unsigned long long curTime = GetSystemTickCountInMicroSec(); + timePassed = (curTime - initialTime) / 1000000; + } while (timePassed < TEST_TIME_SEC); + + TEST_END +} + +TEST_F(KFDQMTest, OverSubscribeCpQueues) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId == FAMILY_CI || m_FamilyId == FAMILY_KV) { + LOG() << "Skipping test: CI doesn't have HW scheduling" << std::endl; + return; + } + + static const unsigned int MAX_CP_QUEUES = 65; + static const unsigned int MAX_PACKETS = 100; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFF); + + PM4Queue queues[MAX_CP_QUEUES]; + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + ASSERT_SUCCESS(queues[qidx].Create(defaultGPUNode)) << " QueueId=" << qidx; + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) { + unsigned int pktSizeDw = 0; + for (unsigned int i = 0; i < MAX_PACKETS; i++) { + PM4WriteDataPacket packet; + packet.InitPacket(destBuf.As()+qidx*2, qidx+i, qidx+i); // two DWORDs per packet + queues[qidx].PlacePacket(packet); + } + } + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + queues[qidx].SubmitPacket(); + + // Delaying for 5 seconds in order to get all the results + Delay(5000); + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + ASSERT_TRUE(queues[qidx].AllPacketsSubmitted())<< "QueueId=" << qidx;; + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + ASSERT_SUCCESS(queues[qidx].Destroy()); + + TEST_END +} + +/* A simple isa loop program with dense mathematic operations + * s1 controls the number iterations of the loop + */ +static const char *loop_isa = \ +"\ +shader loop_isa\n\ +type(CS)\n\ + s_movk_i32 s0, 0x0008\n\ + s_movk_i32 s1, 0x00ff\n\ + v_mov_b32 v0, 0\n\ + v_mov_b32 v1, 0\n\ + v_mov_b32 v2, 0\n\ + v_mov_b32 v3, 0\n\ + v_mov_b32 v4, 0\n\ + v_mov_b32 v5, 0\n\ + v_mov_b32 v6, 0\n\ + v_mov_b32 v7, 0\n\ + v_mov_b32 v8, 0\n\ + v_mov_b32 v9, 0\n\ + v_mov_b32 v10, 0\n\ + v_mov_b32 v11, 0\n\ + v_mov_b32 v12, 0\n\ + v_mov_b32 v13, 0\n\ + v_mov_b32 v14, 0\n\ + v_mov_b32 v15, 0\n\ + v_mov_b32 v16, 0\n\ + LOOP:\n\ + s_mov_b32 s8, s4\n\ + s_mov_b32 s9, s1\n\ + s_mov_b32 s10, s6\n\ + s_mov_b32 s11, s7\n\ + s_cmp_le_i32 s1, s0\n\ + s_cbranch_scc1 END_OF_PGM\n\ + s_buffer_load_dwordx8 s[8:15], s[8:11], 0x10\n\ + v_add_f32 v0, 2.0, v0\n\ + v_cvt_f32_i32 v17, s1\n\ +s_waitcnt lgkmcnt(0)\n\ + v_add_f32 v18, s8, v17\n\ + v_add_f32 v19, s9, v17\n\ + v_add_f32 v20, s10, v17\n\ + v_add_f32 v21, s11, v17\n\ + v_add_f32 v22, s12, v17\n\ + v_add_f32 v23, s13, v17\n\ + v_add_f32 v24, s14, v17\n\ + v_add_f32 v17, s15, v17\n\ + v_log_f32 v25, v18\n\ + v_mul_legacy_f32 v25, v22, v25\n\ + v_exp_f32 v25, v25\n\ + v_log_f32 v26, v19\n\ + v_mul_legacy_f32 v26, v23, v26\n\ + v_exp_f32 v26, v26\n\ + v_log_f32 v27, v20\n\ + v_mul_legacy_f32 v27, v24, v27\n\ + v_exp_f32 v27, v27\n\ + v_log_f32 v28, v21\n\ + v_mul_legacy_f32 v28, v17, v28\n\ + v_exp_f32 v28, v28\n\ + v_add_f32 v5, v5, v25\n\ + v_add_f32 v6, v6, v26\n\ + v_add_f32 v7, v7, v27\n\ + v_add_f32 v8, v8, v28\n\ + v_mul_legacy_f32 v18, 0x3fb8aa3b, v18\n\ + v_exp_f32 v18, v18\n\ + v_mul_legacy_f32 v19, 0x3fb8aa3b, v19\n\ + v_exp_f32 v19, v19\n\ + v_mul_legacy_f32 v20, 0x3fb8aa3b, v20\n\ + v_exp_f32 v20, v20\n\ + v_mul_legacy_f32 v21, 0x3fb8aa3b, v21\n\ + v_exp_f32 v21, v21\n\ + v_add_f32 v9, v9, v18\n\ + v_add_f32 v10, v10, v19\n\ + v_add_f32 v11, v11, v20\n\ + v_add_f32 v12, v12, v21\n\ + v_sqrt_f32 v18, v22\n\ + v_sqrt_f32 v19, v23\n\ + v_sqrt_f32 v20, v24\n\ + v_sqrt_f32 v21, v17\n\ + v_add_f32 v13, v13, v18\n\ + v_add_f32 v14, v14, v19\n\ + v_add_f32 v15, v15, v20\n\ + v_add_f32 v16, v16, v21\n\ + v_rsq_f32 v18, v22\n\ + v_rsq_f32 v19, v23\n\ + v_rsq_f32 v20, v24\n\ + v_rsq_f32 v17, v17\n\ + v_add_f32 v1, v1, v18\n\ + v_add_f32 v2, v2, v19\n\ + v_add_f32 v3, v3, v20\n\ + v_add_f32 v4, v4, v17\n\ + s_add_u32 s0, s0, 1\n\ + s_branch LOOP\n\ + END_OF_PGM:\n\ + s_endpgm\n\ + end\n\ +"; + +long long KFDQMTest::TimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count) { + HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer dstBuffer(PAGE_SIZE, node, true, false, false); + HsaMemoryBuffer ctlBuffer(PAGE_SIZE, node, true, false, false); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + m_pIsaGen->CompileShader(loop_isa, "loop_isa", isaBuffer); + + Dispatch dispatch(isaBuffer); + dispatch.SetDim(1024, 16, 16); + + PM4Queue queue; + EXPECT_SUCCESS(queue.Create(node)); + EXPECT_SUCCESS(queue.SetCUMask(mask, mask_count)); + queue.SetSkipWaitConsump(true); + + struct timeval start, now; + gettimeofday(&start, NULL); + dispatch.Submit(queue); + dispatch.Sync(); + gettimeofday(&now, NULL); + + EXPECT_SUCCESS(queue.Destroy()); + return (now.tv_sec *1000LL + now.tv_usec/1000LL - start.tv_sec * 1000LL - start.tv_usec/1000LL); +} + +/* To cover for outliers, allow us to get the Average time based on a specified number of iterations */ +long long KFDQMTest::GetAverageTimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count, int iterations) { + long long timeArray[iterations]; + long long timeTotal = 0; + if (iterations < 1) { + LOG() << "ERROR: At least 1 iteration must be performed" << std::endl; + return 0; + } + + for (int x = 0; x < iterations; x++) { + timeArray[x] = TimeConsumedwithCUMask(node, mask, mask_count); + timeTotal += timeArray[x]; + } + + if (timeTotal == 0) { + LOG() << "ERROR: Total time reported as 0. Exiting" << std::endl; + return 0; + } + + for (int x = 0; x < iterations; x++) { + long long variance = timeArray[x] / (timeTotal / iterations); + if (variance < CuNegVariance || variance > CuPosVariance) + LOG() << "WARNING: Measurement #" << x << "/" << iterations << " (" << timeArray[x] << ") is at least " << CuVariance*100 << "% away from the mean (" << timeTotal/iterations << ")" << std::endl; + } + + return timeTotal / iterations; +} + +/** + * Apply CU masking in a linear fashion, adding 1 CU per iteration + * until all Shader Engines are full ... + */ +TEST_F(KFDQMTest, BasicCuMaskingLinear) { + TEST_START(TESTPROFILE_RUNALL); + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + if (m_FamilyId >= FAMILY_VI) { + const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); + uint32_t ActiveCU = (pNodeProperties->NumFComputeCores / pNodeProperties->NumSIMDPerCU); + uint32_t numSEs = pNodeProperties->NumShaderBanks; + LOG() << std::hex << "# Compute cores: 0x" << pNodeProperties->NumFComputeCores << std::endl; + LOG() << std::hex << "# SIMDs per CPU: 0x" << pNodeProperties->NumSIMDPerCU << std::endl; + LOG() << std::hex << "# Shader engines: 0x" << numSEs << std::endl; + LOG() << std::hex << "# Active CUs: 0x" << ActiveCU << std::endl; + long long TimewithCU1, TimewithCU; + uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */ + uint32_t maskNumBits = maskNumDwords * 32; + uint32_t mask[maskNumDwords]; + double ratio; + + mask[0] = 0x1; + for (int i = 1; i < maskNumDwords; i++) + mask[i] = 0x0; + + /* Execute once to get any HW optimizations out of the way */ + TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); + + LOG() << "Getting baseline performance numbers (CU Mask: 0x1)" << std::endl; + TimewithCU1 = GetAverageTimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits, 3); + + for (int nCUs = 2; nCUs <= ActiveCU; nCUs++) { + int maskIndex = (nCUs - 1) / 32; + mask[maskIndex] |= 1 << ((nCUs - 1) % 32); + + TimewithCU = TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); + ratio = (double)TimewithCU1 / ((double)TimewithCU * nCUs); + + LOG() << "Expected performance of " << nCUs << " CUs vs 1 CU:" << std::endl; + LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; + + ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); + } + } else { + LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + } + + TEST_END +} + +/** + * Apply CU masking where the number of CUs is equal across all Shader Engines + * This will work due to the HW splitting the workload unevenly across the Shader + * Engines when ((#ofCUs)/(#ofShaderEngines)) is not a whole number. The tests above + * will not yield viable results when an uneven distribution of CUs is used over multiple + * shader engines (e.g. 0x1000100030003), until the HW changes how it schedules work. + */ +TEST_F(KFDQMTest, BasicCuMaskingEven) { + TEST_START(TESTPROFILE_RUNALL); + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId >= FAMILY_VI) { + const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); + uint32_t ActiveCU = (pNodeProperties->NumFComputeCores / pNodeProperties->NumSIMDPerCU); + uint32_t numShaderEngines = pNodeProperties->NumShaderBanks; + if (numShaderEngines == 1) { + LOG() << "Only 1 Shader Engine. Skipping test." << std::endl; + return; + } + + LOG() << std::hex << "# Compute cores: 0x" << pNodeProperties->NumFComputeCores << std::endl; + LOG() << std::hex << "# SIMDs per CPU: 0x" << pNodeProperties->NumSIMDPerCU << std::endl; + LOG() << std::hex << "# Shader engines: 0x" << numShaderEngines << std::endl; + LOG() << std::hex << "# Active CUs: 0x" << ActiveCU << std::endl; + long long TimewithCU1, TimewithCU; + uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */ + uint32_t maskNumBits = maskNumDwords * 32; + uint32_t mask[maskNumDwords]; + int numCuPerShader = ActiveCU / numShaderEngines; + double ratio; + + /* Set Mask to 1 for a single CU */ + mask[0] = 0x1; + for (int i = 1; i < maskNumDwords; i++) + mask[i] = 0x0; + + /* Execute once to get any HW optimizations out of the way */ + TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); + + LOG() << "Getting baseline performance numbers (1 CU)" << std::endl; + TimewithCU1 = GetAverageTimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits, 3); + + /* Each loop will add 1 more CU per SE. We use the mod and divide to handle + * when SEs aren't distributed in multiples of 32 (e.g. Tonga) + * OR the new bit in for simplicity instead of re-creating the mask each iteration + */ + for (int x = 0; x < numCuPerShader; x++) { + for (int se = 0; se < numShaderEngines; se++) { + int offset = x * numShaderEngines + se; + int maskIndex = (offset / 32) % maskNumDwords; + mask[maskIndex] |= 1 << (offset % 32); + } + int nCUs = numShaderEngines * (x + 1); + + TimewithCU = TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); + ratio = (double)TimewithCU1 / ((double)TimewithCU * nCUs); + + LOG() << "Expected performance of " << nCUs << " CUs vs 1 CU:" << std::endl; + LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; + + ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); + } + } else { + LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + } + + TEST_END +} + +TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { + TEST_START(TESTPROFILE_RUNALL); + + if (m_FamilyId < FAMILY_VI) { + LOG() << "Skipping test: Shader won't run on CI" << std::endl; + return; + } + + int node = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(node, 0) << "failed to get default GPU Node"; + HsaMemoryBuffer syncBuf(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); + HSAint32 *syncBuffer = syncBuf.As(); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); + + m_pIsaGen->CompileShader(loop_isa, "loop_isa", isaBuffer); + + Dispatch dispatch[2] = { + Dispatch(isaBuffer, true), + Dispatch(isaBuffer, true) + }; + + int activeTaskBitmap = 0x3; + struct timeval start, end[2]; + HsaEvent *pHsaEvent[2]; + int numEvent = 2; + PM4Queue queue[2]; + HSA_QUEUE_PRIORITY priority[2] = { + HSA_QUEUE_PRIORITY_LOW, + HSA_QUEUE_PRIORITY_HIGH + }; + int i; + + for (i = 0; i < 2; i++) { + syncBuffer[i] = -1; + EXPECT_SUCCESS(queue[i].Create(node)); + queue[i].Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, priority[i], false); + pHsaEvent[i] = dispatch[i].GetHsaEvent(); + pHsaEvent[i]->EventData.EventData.SyncVar.SyncVar.UserData = &syncBuffer[i]; + dispatch[i].SetDim(1024, 16, 16); + } + + gettimeofday(&start, NULL); + for (i = 0; i < 2; i++) + dispatch[i].Submit(queue[i]); + + while (activeTaskBitmap > 0) { + hsaKmtWaitOnMultipleEvents(pHsaEvent, numEvent, false, g_TestTimeOut); + for (i = 0; i < 2; i++) { + if ((activeTaskBitmap & (1 << i)) && (syncBuffer[i] == pHsaEvent[i]->EventId)) { + gettimeofday(&end[i], NULL); + activeTaskBitmap &= ~(1 << i); + } + } + } + + for (i = 0; i < 2; i++) { + EXPECT_SUCCESS(queue[i].Destroy()); + int ms = end[i].tv_sec *1000LL + end[i].tv_usec/1000LL - start.tv_sec * 1000LL - start.tv_usec/1000LL; + LOG() << "Task priority: " << std::dec << priority[i] << "\t"; + LOG() << "Task duration: " << std::dec << ms << "ms" << std::endl; + } + + TEST_END +} + +TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { + TEST_START(TESTPROFILE_RUNALL); + + if (m_FamilyId < FAMILY_VI) { + LOG() << "Skipping test: Shader won't run on CI" << std::endl; + return; + } + + int node = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(node, 0) << "failed to get default GPU Node"; + HsaMemoryBuffer syncBuf(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); + HSAint32 *syncBuffer = syncBuf.As(); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); + + m_pIsaGen->CompileShader(loop_isa, "loop_isa", isaBuffer); + + Dispatch dispatch[2] = { + Dispatch(isaBuffer, true), + Dispatch(isaBuffer, true) + }; + + int activeTaskBitmap = 0x3; + struct timeval start, end[2]; + HsaEvent *pHsaEvent[2]; + int numEvent = 2; + PM4Queue queue[13]; + HSA_QUEUE_PRIORITY priority[2] = { + HSA_QUEUE_PRIORITY_LOW, + HSA_QUEUE_PRIORITY_HIGH + }; + int i; + + /* queue[2..12] are dummy queues. Create queue in this sequence to + * render queue[0] and queue[1] on same pipe with no assumptions + * about the number of pipes used by KFD. Queue #12 is a multiple + * of 1, 2, 3 and 4, so it falls on pipe 0 for any number of pipes + */ + EXPECT_SUCCESS(queue[0].Create(node)); // Queue 0 is on Pipe 0 + for (i = 2; i <= 12; i++) + EXPECT_SUCCESS(queue[i].Create(node)); + EXPECT_SUCCESS(queue[1].Create(node)); // Queue 12 is on Pipe 0 + + for (i = 0; i < 2; i++) { + syncBuffer[i] = -1; + queue[i].Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, priority[i], false); + pHsaEvent[i] = dispatch[i].GetHsaEvent(); + pHsaEvent[i]->EventData.EventData.SyncVar.SyncVar.UserData = &syncBuffer[i]; + dispatch[i].SetDim(1024, 16, 16); + } + + gettimeofday(&start, NULL); + for (i = 0; i < 2; i++) + dispatch[i].Submit(queue[i]); + + while (activeTaskBitmap > 0) { + hsaKmtWaitOnMultipleEvents(pHsaEvent, numEvent, false, g_TestTimeOut); + for (i = 0; i < 2; i++) { + if ((activeTaskBitmap & (1 << i)) && (syncBuffer[i] == pHsaEvent[i]->EventId)) { + gettimeofday(&end[i], NULL); + activeTaskBitmap &= ~(1 << i); + } + } + } + + for (i = 0; i < 2; i++) { + int ms = end[i].tv_sec *1000LL + end[i].tv_usec/1000LL - start.tv_sec * 1000LL - start.tv_usec/1000LL; + LOG() << "Task priority: " << std::dec << priority[i] << "\t"; + LOG() << "Task duration: " << std::dec << ms << "ms" << std::endl; + } + + for (i = 0; i <= 12; i++) { + EXPECT_SUCCESS(queue[i].Destroy()); + } + + TEST_END +} + + +void KFDQMTest::SyncDispatch(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf, int node) { + PM4Queue queue; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + if (node != -1) + defaultGPUNode = node; + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(pSrcBuf, pDstBuf); + dispatch.SetDim(1, 1, 1); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + dispatch.Submit(queue); + dispatch.Sync(); + + ASSERT_SUCCESS(queue.Destroy()); +} + +TEST_F(KFDQMTest, EmptyDispatch) { + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + + m_pIsaGen->GetNoopIsa(isaBuffer); + + SyncDispatch(isaBuffer, NULL, NULL); + + TEST_END +} + +TEST_F(KFDQMTest, SimpleWriteDispatch) { + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode, false); + HsaMemoryBuffer destBuffer(PAGE_SIZE, defaultGPUNode); + + srcBuffer.Fill(0x01010101); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + SyncDispatch(isaBuffer, srcBuffer.As(), destBuffer.As()); + + ASSERT_EQ(destBuffer.As()[0], 0x01010101); + + TEST_END +} + +TEST_F(KFDQMTest, MultipleCpQueuesStressDispatch) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + static const unsigned int MAX_CP_QUEUES = 16; + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode, false); + HsaMemoryBuffer destBuffer(PAGE_SIZE, defaultGPUNode); + + unsigned int* src = srcBuffer.As(); + unsigned int* dst = destBuffer.As(); + + static const unsigned long long TEST_TIME_SEC = 15; + unsigned long long initialTime, curTime; + unsigned int numIter = 0; + unsigned long long timePassed = 0; + + unsigned int i; + PM4Queue queues[MAX_CP_QUEUES]; + Dispatch* dispatch[MAX_CP_QUEUES]; + + destBuffer.Fill(0xFF); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + for (i = 0; i < MAX_CP_QUEUES; ++i) + ASSERT_SUCCESS(queues[i].Create(defaultGPUNode)) << " QueueId=" << i; + + initialTime = GetSystemTickCountInMicroSec(); + + do { + for (i = 0; i < MAX_CP_QUEUES; ++i) { + dispatch[i] = new Dispatch(isaBuffer); + src[i] = numIter; + dst[i] = 0xff; + dispatch[i]->SetArgs(&src[i], &dst[i]); + dispatch[i]->SetDim(1, 1, 1); + dispatch[i]->Submit(queues[i]); + } + for (i = 0; i < MAX_CP_QUEUES; ++i) { + dispatch[i]->Sync(); + ASSERT_EQ(dst[i], src[i]); + delete dispatch[i]; + } + ++numIter; + curTime = GetSystemTickCountInMicroSec(); + timePassed = (curTime - initialTime) / 1000000; + } while (timePassed < TEST_TIME_SEC); + + LOG() << "Total iterated : " << std::dec << numIter << std::endl; + + for (i = 0; i < MAX_CP_QUEUES; ++i) + ASSERT_SUCCESS(queues[i].Destroy()); + + TEST_END +} + + + +TEST_F(KFDQMTest, CpuWriteCoherence) { + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + /* The queue might be full and we fail to submit. There is always one word space unused in queue. + * So let rptr one step ahead then we continually submit packet. + */ + queue.PlaceAndSubmitPacket(PM4NopPacket()); + queue.Wait4PacketConsumption(); + EXPECT_EQ(1, queue.Rptr()); + + do { + queue.PlaceAndSubmitPacket(PM4NopPacket()); + } while (queue.Wptr() != 0); + + queue.Wait4PacketConsumption(); + + EXPECT_EQ(0, queue.Rptr()); + + // now that the GPU has cached the PQ contents, we modify them in CPU cache and ensure that the GPU sees the updated value: + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0x42, 0x42)); + + queue.Wait4PacketConsumption(); + + WaitOnValue(destBuf.As(), 0x42); + + TEST_END +} + +TEST_F(KFDQMTest, CreateAqlCpQueue) { + TEST_START(TESTPROFILE_RUNALL) + + AqlQueue queue; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer pointers(PAGE_SIZE, defaultGPUNode); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode, PAGE_SIZE, pointers.As())); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +#define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) + +#include + +TEST_F(KFDQMTest, QueueLatency) { + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + const int queueSize = PAGE_SIZE * 2; + const int packetSize = PM4ReleaseMemoryPacket(0,0,0,0,0).SizeInBytes(); + /* We always leave one NOP(dword) empty after packet which is required by ring itself. + * We also place NOPs when queue wraparound to avoid crossing buffer end. See PlacePacket(). + * So the worst case is that we need two packetSize space to place one packet. + * Like below, N=NOP,E=Empty,P=Packet. + * |E|E|E|E|E|E|E|rptr...wptr|E|E|E|E|E| ---> |P|P|P|P|P|P|E|rptr...wptr|N|N|N|N|N| + * So to respect that, we reserve packetSize space for these additional NOPs. + * Also we reserve the remainder of the division by packetSize explicitly. + */ + const int reservedSpace = packetSize + queueSize % packetSize; + const int slots = (queueSize - reservedSpace) / packetSize; + long queue_latency_avg = 0, queue_latency_min, queue_latency_max, queue_latency_med; + long overhead, workload; + long *queue_latency_arr = (long*)calloc(slots, sizeof(long)); + const int skip = 2; + const char *fs[skip] = {"1st", "2nd"}; + HsaClockCounters *ts; + HSAuint64 *qts; + int i = 0; + + ASSERT_NE((unsigned long)queue_latency_arr, 0); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode, queueSize)); + + LOG() << "Queue Submit Clock Counter (" << slots << " Packets)" << std::endl; + + HsaMemoryBuffer buf(ALIGN_UP(slots * sizeof(HsaClockCounters), PAGE_SIZE), 0); + ts = buf.As(); + + HsaMemoryBuffer qbuf(ALIGN_UP(slots * sizeof(HSAuint64), PAGE_SIZE), 0); + qts = qbuf.As(); + + /* GpuCounter overhead*/ + do { + hsaKmtGetClockCounters(defaultGPUNode, &ts[i]); + } while (++i < slots); + overhead = ts[slots-1].GPUClockCounter - ts[0].GPUClockCounter; + overhead /= 2 * (slots - 1); + + /* Submit packets serially*/ + i = 0; + do { + queue.PlacePacket(PM4ReleaseMemoryPacket(true, + (unsigned long)&qts[i], + 0, + true, + 1)); + hsaKmtGetClockCounters(defaultGPUNode, &ts[i]); + queue.SubmitPacket(); + queue.Wait4PacketConsumption(); + } while (++i < slots); + + /* Calculate timing which includes workload and overhead*/ + i = 0; + do { + long queue_latency = qts[i] - ts[i].GPUClockCounter; + + ASSERT_GE(queue_latency, 0); + + queue_latency_arr[i] = queue_latency; + if (i >= skip) + queue_latency_avg += queue_latency; + } while (++i < slots); + /*Calculate avg from packet[skip, slots-1]*/ + queue_latency_avg /= (slots - skip); + + /*workload of queue packet itself*/ + i = 0; + do { + queue.PlacePacket(PM4ReleaseMemoryPacket(true, + (unsigned long)&qts[i], + 0, + true, + 1)); + } while (++i < slots); + queue.SubmitPacket(); + queue.Wait4PacketConsumption(); + + /* qts[i] records the timestamp of the end of packet[i] which is + * approximate that of the beginging of packet[i+1]. + * The workload total is [0, skip], [skip+1, slots-1]. + * And We ignore [0, skip], that means we ignore (skip+1) packets. + */ + workload = qts[slots - 1] - qts[skip]; + workload /= (slots - 1 - skip); + + ASSERT_GE(workload, 0); + + i = 0; + do { + /* The queue_latency is not that correct as the workload and overhead are average*/ + queue_latency_arr[i] -= workload + overhead; + /* The First submit takes a long time*/ + if (i < skip) + LOG() << "Queue Latency " << fs[i] << ": \t" << queue_latency_arr[i] << std::endl; + } while (++i < slots); + + std::sort(queue_latency_arr + skip, queue_latency_arr + slots); + + queue_latency_min = queue_latency_arr[skip]; + queue_latency_med = queue_latency_arr[(slots+skip)/2]; + queue_latency_max = queue_latency_arr[slots-1]; + + LOG() << "Queue Latency Avg: \t" << queue_latency_avg << std::endl; + LOG() << "Queue Latency Min: \t" << queue_latency_min << std::endl; + LOG() << "Queue Latency Median: \t" << queue_latency_med << std::endl; + LOG() << "Queue Latency Max: \t" << queue_latency_max << std::endl; + LOG() << "Queue Packet Workload: \t" << workload << std::endl; + LOG() << "Get GpuCounter Overhead: \t" << overhead << std::endl; + + TEST_END +} + + +TEST_F(KFDQMTest, CpQueueWraparound) { + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + for (unsigned int pktIdx = 0; pktIdx <= PAGE_SIZE/sizeof(PM4WRITE_DATA_CI); ++pktIdx) { + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), pktIdx, pktIdx)); + queue.Wait4PacketConsumption(); + WaitOnValue(destBuf.As(), pktIdx); + } + + for (unsigned int pktIdx = 0; pktIdx <= PAGE_SIZE/sizeof(PM4WRITE_DATA_CI); ++pktIdx) { + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), pktIdx, pktIdx)); + queue.Wait4PacketConsumption(); + WaitOnValue(destBuf.As(), pktIdx); + } + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, SdmaQueueWraparound) { + TEST_START(TESTPROFILE_RUNALL); + int bufSize = PAGE_SIZE; + + SDMAQueue queue; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(bufSize << 1, defaultGPUNode, false); + HsaMemoryBuffer srcBuf(bufSize, defaultGPUNode, false); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + for (unsigned int pktIdx = 0; pktIdx <= queue.Size()/sizeof(SDMA_PKT_COPY_LINEAR); ++pktIdx) { + destBuf.Fill(0x0); + srcBuf.Fill(pktIdx); + queue.PlaceAndSubmitPacket( + SDMACopyDataPacket(destBuf.As(), srcBuf.As(), bufSize)); + queue.PlaceAndSubmitPacket( + SDMAWriteDataPacket(destBuf.As() + bufSize/4, 0x02020202)); + queue.Wait4PacketConsumption(); + + ASSERT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); + + ASSERT_SUCCESS(memcmp( + destBuf.As(), srcBuf.As(), bufSize)); + } + + for (unsigned int pktIdx = 0; pktIdx <= queue.Size()/sizeof(SDMA_PKT_WRITE_UNTILED); ++pktIdx) { + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), pktIdx)); + queue.Wait4PacketConsumption(); + WaitOnValue(destBuf.As(), pktIdx); + } + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +struct AtomicIncThreadParams { + long* pDest; + volatile unsigned int count; + volatile bool stop; +}; + +unsigned int AtomicIncThread(void* pCtx) { + AtomicIncThreadParams* pArgs = (AtomicIncThreadParams*)pCtx; + + while (pArgs->stop) + {} + + while (!pArgs->stop) { + AtomicInc(pArgs->pDest); + ++pArgs->count; + } + + return 0; +} + +TEST_F(KFDQMTest, Atomics) { + TEST_START(TESTPROFILE_RUNALL); + /* CI doesn't support Atomics. KV does, but gets its own FAMILY_KV def */ + if (m_FamilyId == FAMILY_CI) { + LOG() << "Skipping test: CI doesn't support Atomics" << std::endl; + return; + } + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode); + + PM4Queue queue; + + m_pIsaGen->GetAtomicIncIsa(isaBuf); + + Dispatch dispatch(isaBuf); + dispatch.SetArgs(destBuf.As(), NULL); + dispatch.SetDim(1024, 1, 1); + + hsaKmtSetMemoryPolicy(defaultGPUNode, HSA_CACHING_CACHED, HSA_CACHING_CACHED, NULL, 0); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + AtomicIncThreadParams params; + params.pDest = destBuf.As(); + params.stop = true; + params.count = 0; + + uint64_t threadId; + + ASSERT_EQ(true, StartThread(&AtomicIncThread, ¶ms, threadId)); + + params.stop = false; + + while (params.count == 0) + {} + + dispatch.Submit(queue); + dispatch.Sync(); + + params.stop = true; + + WaitForThread(threadId); + + EXPECT_EQ(destBuf.As()[0], 1024 + params.count); + + LOG() << "GPU increments: 1024, CPU increments: " << params.count << std::endl; + + queue.Destroy(); + + TEST_END +} + +TEST_F(KFDQMTest, mGPUShareBO) { + TEST_START(TESTPROFILE_RUNALL); + + unsigned int src_node = 2; + unsigned int dst_node = 1; + + if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { + src_node = g_TestNodeId; + dst_node = g_TestDstNodeId; + } + + HsaMemoryBuffer shared_addr(PAGE_SIZE, dst_node, true, false, false, false); + + HsaMemoryBuffer srcNodeMem(PAGE_SIZE, src_node); + HsaMemoryBuffer dstNodeMem(PAGE_SIZE, dst_node); + + /* Handle ISA to write to local memory BO */ + HsaMemoryBuffer isaBufferSrc(PAGE_SIZE, src_node, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer isaBufferDst(PAGE_SIZE, dst_node, true/*zero*/, false/*local*/, true/*exec*/); + + srcNodeMem.Fill(0x05050505); + + m_pIsaGen->GetCopyDwordIsa(isaBufferSrc); + SyncDispatch(isaBufferSrc, srcNodeMem.As(), shared_addr.As(), src_node); + + m_pIsaGen->GetCopyDwordIsa(isaBufferDst); + SyncDispatch(isaBufferDst, shared_addr.As(), dstNodeMem.As(), dst_node); + + ASSERT_EQ(dstNodeMem.As()[0], 0x05050505); + + EXPECT_SUCCESS(shared_addr.UnmapMemToNodes(&dst_node, 1)); + + TEST_END +} + +TEST_F(KFDQMTest, P2PTest) { + TEST_START(TESTPROFILE_RUNALL); + if (!is_dgpu()) { + LOG() << "Skipping test: Can't have 2 APUs on the same system" << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: Need at least two GPUs" << std::endl; + return; + } + HSAint32 gpuNode1 = m_NodeInfo.HsaDefaultGPUNode(); + HSAint32 gpuNode2 = 0; + + /* This test simulates RT team's P2P part in IPCtest: + * + * +--------------------------------------------+ + * | gpu1 gpu2 | + * |gpu1 mem ----> gpu2 mem ----> system buffer | + * +--------------------------------------------+ + * + * Copy data from GPU-1 memory to GPU-2 memory using GPU-1, then GPU-2 + * memory to system buffer using GPU-2. Verify the system buffer + * (initialized with 0) has the same content as gpu1 memory (0x5). + */ + + /* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */ + if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { + gpuNode1 = g_TestNodeId; + gpuNode2 = g_TestDstNodeId; + } + + /* GPU-2 must have public memory(large bar) to do GPU-to-GPU copy. If + * not specified in the command line, find one. + */ + gpuNode2 = m_NodeInfo.FindLargeBarGPUNode(); + if (gpuNode2 < 0) { + LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + return; + } + if (gpuNode1 == gpuNode2) { + for (unsigned i = 0; i < gpuNodes.size(); i++) { + if (gpuNodes.at(i) != gpuNode2) { + gpuNode1 = gpuNodes.at(i); + break; + } + } + } + + HSAuint32 *sysBuf; + HSAuint32 size = 0xc00000; // bigger than 4MB to test non-contiguous memory + HsaMemFlags memFlags = {0}; + HsaMemMapFlags mapFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 1; + + /* 1. Allocate a system buffer and allow the access to GPU-2 */ + EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, + (void **)&sysBuf)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL, + mapFlags, 1, (HSAuint32*)&gpuNode2)); + + /* 2.- Allocate local memory on GPU-1 + * - Allocate local memory on GPU-2 and allow access to both GPUs + */ + HsaMemoryBuffer gpu1Mem(size, gpuNode1, false, true /*isLocal*/); + HsaMemoryBuffer gpu2Mem(size, gpuNode2, false, true); + + SDMAQueue sdmaQueue1, sdmaQueue2; + ASSERT_SUCCESS(sdmaQueue1.Create(gpuNode1)); + ASSERT_SUCCESS(sdmaQueue2.Create(gpuNode2)); + + /* initialize sysBuf as 0 and fill up gpu1 mem with 0x5 */ + memset(sysBuf, 0, size); + sdmaQueue1.PlaceAndSubmitPacket(SDMAFillDataPacket(gpu1Mem.As(), + 0x5, size)); + sdmaQueue1.Wait4PacketConsumption(); + + /* 3. Copy data from gpuNode1 to gpuNode2 */ + sdmaQueue1.PlaceAndSubmitPacket(SDMACopyDataPacket(gpu2Mem.As(), + gpu1Mem.As(), size)); + sdmaQueue1.Wait4PacketConsumption(); + + /* 4. Copy data from gpuNode2 to system buffer */ + sdmaQueue2.PlaceAndSubmitPacket(SDMACopyDataPacket(sysBuf, + gpu2Mem.As(), size)); + sdmaQueue2.Wait4PacketConsumption(); + + /* 5. Verify the data */ + ASSERT_EQ(sysBuf[0], 0x5); + unsigned int end = size / sizeof(HSAuint32) - 1; + ASSERT_EQ(sysBuf[end], 0x5); + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(sysBuf)); + EXPECT_SUCCESS(hsaKmtFreeMemory(sysBuf, size)); + EXPECT_SUCCESS(sdmaQueue1.Destroy()); + EXPECT_SUCCESS(sdmaQueue2.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, SdmaEventInterrupt) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaEvent* event = NULL; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_NodeInfo.HsaDefaultGPUNode(), &event)); + + SDMAQueue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(SDMAFencePacket((void*)event->EventData.HWData2, event->EventId)); + + queue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); + + queue.Wait4PacketConsumption(); + + ASSERT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + + hsaKmtDestroyEvent(event); + + ASSERT_SUCCESS(queue.Destroy()); + + TEST_END +} + +#define DOORBELL_WRITE_USE_SDMA +TEST_F(KFDQMTest, GPUDoorbellWrite) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer destBuf(PAGE_SIZE, 0, true); + PM4Queue pm4Queue; +#ifdef DOORBELL_WRITE_USE_SDMA + SDMAQueue otherQueue; +#else + PM4Queue otherQueue; +#endif + + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + ASSERT_SUCCESS(otherQueue.Create(defaultGPUNode)); + + /* Place PM4 packet in the queue, but don't submit it */ + pm4Queue.PlacePacket(PM4WriteDataPacket(destBuf.As(), 0x12345678, 0x87654321)); + + HsaQueueResource *qRes = pm4Queue.GetResource(); + + if (m_FamilyId < FAMILY_AI) { + unsigned int pendingWptr = pm4Queue.GetPendingWptr(); + +#ifdef DOORBELL_WRITE_USE_SDMA + /* Write the wptr and doorbell update using the GPU's SDMA + * engine. This should submit the PM4 packet on the first + * queue. */ + otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_write_ptr, + pendingWptr)); + otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_DoorBell, + pendingWptr)); +#else + /* Write the wptr and doorbell update using WRITE_DATA packets + * on a second PM4 queue. This should submit the PM4 packet on + * the first queue. */ + otherQueue.PlacePacket( + PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_write_ptr, + pendingWptr, false)); + otherQueue.PlacePacket( + PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_DoorBell, + pendingWptr, false)); +#endif + + otherQueue.SubmitPacket(); + } else { + HSAuint64 pendingWptr64 = pm4Queue.GetPendingWptr64(); + +#ifdef DOORBELL_WRITE_USE_SDMA + /* Write the wptr and doorbell update using the GPU's SDMA + * engine. This should submit the PM4 packet on the first + * queue. */ + otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_write_ptr, + 2, &pendingWptr64)); + otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_DoorBell, + 2, &pendingWptr64)); +#else + /* Write the 64-bit wptr and doorbell update using RELEASE_MEM + * packets without IRQs on a second PM4 queue. RELEASE_MEM + * should perform one atomic 64-bit access. This should submit + * the PM4 packet on the first queue. */ + otherQueue.PlacePacket( + PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_write_ptr, + pendingWptr64, true)); + otherQueue.PlacePacket( + PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_DoorBell, + pendingWptr64, true)); +#endif + + otherQueue.SubmitPacket(); + } + + /* Check that the PM4 packet has been executed */ + ASSERT_TRUE(WaitOnValue(destBuf.As(), 0x12345678)); + ASSERT_TRUE(WaitOnValue(destBuf.As()+1, 0x87654321)); + + ASSERT_SUCCESS(pm4Queue.Destroy()); + ASSERT_SUCCESS(otherQueue.Destroy()); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDQMTest.hpp b/tests/kfdtest/src/KFDQMTest.hpp new file mode 100644 index 0000000000..1834d4b506 --- /dev/null +++ b/tests/kfdtest/src/KFDQMTest.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_QCM_TEST__H__ +#define __KFD_QCM_TEST__H__ + +#include + +#include "PM4Queue.hpp" +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" +#include "Dispatch.hpp" + +class KFDQMTest : public KFDBaseComponentTest { + public: + KFDQMTest():m_pIsaGen(NULL) {} + + ~KFDQMTest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); + + void SyncDispatch(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf, int node = -1); +// void SyncDispatchWithSleep(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf); + long long TimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count); + long long GetAverageTimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count, int iterations); + protected: // members + /* Acceptable performance for CU Masking should be within 5% of linearly-predicted performance */ + const double CuVariance = 0.15; + const double CuNegVariance = 1.0 - CuVariance; + const double CuPosVariance = 1.0 + CuVariance; + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_QCM_TEST__H__ diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp new file mode 100644 index 0000000000..11321f9b88 --- /dev/null +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_TEST_FLAGS__H__ +#define __KFD_TEST_FLAGS__H__ + +extern unsigned int g_TestRunProfile; +extern unsigned int g_TestENVCaps; +extern unsigned int g_TestTimeOut; +extern int g_TestNodeId; +extern int g_TestDstNodeId; +extern bool g_IsChildProcess; +extern unsigned int g_TestGPUFamilyId; + +// each test should call TEST_START with the test custome profile and HW scheduling + +enum TESTPROFILE{ + TESTPROFILE_DEV = 0x1, + TESTPROFILE_PROMO = 0x2, + // 0x4 - 0x8000 - unsed flags + // can add any flag that will mark only part of the tests to run + TESTPROFILE_RUNALL = 0xFFFF +}; + +enum ENVCAPS{ + ENVCAPS_NOADDEDCAPS = 0x0, + ENVCAPS_HWSCHEDULING = 0x1, + ENVCAPS_16BITPASID = 0x2, + ENVCAPS_32BITLINUX = 0x4, + ENVCAPS_64BITLINUX = 0x8 + // 0x8 - 0x8000 - unsed flags + // can add any flag that will mark specific hw limitation \ capability +}; + +enum KfdFamilyId { + FAMILY_UNKNOWN = 0, + FAMILY_CI, // Sea Islands: Hawaii (P), Maui (P), Bonaire (M) + FAMILY_KV, // Fusion Kaveri: Spectre, Spooky; Fusion Kabini: Kalindi + FAMILY_VI, // Volcanic Islands: Iceland (V), Tonga (M) + FAMILY_CZ, // Carrizo, Nolan, Amur + FAMILY_AI, // Arctic Islands + FAMILY_RV, // Raven +}; + +#endif // __KFD_TEST_FLAGS__H__ diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp new file mode 100644 index 0000000000..7dfd24674e --- /dev/null +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "gtest/gtest.h" +#include "KFDTestFlags.hpp" +#include "KFDTestUtil.hpp" +#include "GoogleTestExtension.hpp" +#include "OSWrapper.hpp" + +#define KFD_TEST_DEFAULT_TIMEOUT 2000 + +std::ostream& operator << (std::ostream& out, TESTPROFILE profile) { + switch (profile) { + case TESTPROFILE_DEV: + out << "Developer Test"; + break; + case TESTPROFILE_PROMO: + out << "Promotion Test"; + break; + case TESTPROFILE_RUNALL: + out << "Full Test"; + break; + default: + out << "INVALID"; + }; + + return out; +} + +unsigned int g_TestRunProfile; +unsigned int g_TestENVCaps; +unsigned int g_TestTimeOut; +int g_TestNodeId; +int g_TestDstNodeId; +bool g_IsChildProcess; +unsigned int g_TestGPUFamilyId; + +GTEST_API_ int main(int argc, char **argv) { + // default values for run parameters + g_TestRunProfile = TESTPROFILE_RUNALL; + g_TestENVCaps = ENVCAPS_NOADDEDCAPS | ENVCAPS_64BITLINUX; + g_TestTimeOut = KFD_TEST_DEFAULT_TIMEOUT; + + // every fatal fail ( = assert that failed ) will throw an exption + testing::GTEST_FLAG(throw_on_failure) = true; + testing::InitGoogleTest(&argc, argv); + + CommandLineArguments args; + memset(&args, 0, sizeof(args)); + + bool success = GetCommandLineArguments(argc, argv, args); + + if (success) { + if ((GetHwCapabilityHWS() || args.HwsEnabled == HWCAP__FORCE_ENABLED) && (args.HwsEnabled != HWCAP__FORCE_DISABLED)) + g_TestENVCaps |= ENVCAPS_HWSCHEDULING; + + g_TestRunProfile = args.TestProfile; + g_IsChildProcess = args.ChildProcess; + + if ( args.TimeOut > 0 ) + g_TestTimeOut = args.TimeOut; + + // If --node is not specified, then args.NodeId == -1 + g_TestNodeId = args.NodeId; + g_TestDstNodeId = args.DstNodeId; + + LOG() << "Profile: " << (TESTPROFILE)g_TestRunProfile << std::endl; + LOG() << "HW capabilities: 0x" << std::hex << g_TestENVCaps << std::endl; + + return RUN_ALL_TESTS(); + } +} diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp new file mode 100644 index 0000000000..f3e3c3fefa --- /dev/null +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -0,0 +1,536 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDTestUtil.hpp" + +#include +#include +#include +#include "BaseQueue.hpp" +#include "Dispatch.hpp" +#include "SDMAPacket.hpp" + +bool WaitOnValue(const volatile unsigned int *buf, unsigned int value) { + unsigned int timeOut = g_TestTimeOut; + + while (timeOut > 0 && *buf != value) { + Delay(1); + + if (timeOut != HSA_EVENTTIMEOUT_INFINITE) + timeOut--; + } + + return *buf == value; +} + +void SplitU64(const unsigned long long value, unsigned int& rLoPart, unsigned int& rHiPart) { + rLoPart = static_cast(value); + rHiPart = static_cast(value >> 32); +} + +bool GetHwCapabilityHWS() { + unsigned int value = 0; + bool valExists = ReadDriverConfigValue(CONFIG_HWS, value); + + /* HWS is enabled by default, so... */ + return ( (!valExists) || ( value > 0)); +} + +HSAKMT_STATUS CreateQueueTypeEvent( + bool ManualReset, // IN + bool IsSignaled, // IN + unsigned int NodeId, // IN + HsaEvent** Event // OUT + ) { + HsaEventDescriptor Descriptor; + +// TODO Create per-OS header with this sort of definitions +#ifdef _WIN32 + Descriptor.EventType = HSA_EVENTTYPE_QUEUE_EVENT; +#else + Descriptor.EventType = HSA_EVENTTYPE_SIGNAL; +#endif + Descriptor.SyncVar.SyncVar.UserData = (void*)0xABCDABCD; + Descriptor.NodeId = NodeId; + + return hsaKmtCreateEvent(&Descriptor, ManualReset, IsSignaled, Event); +} + +static bool is_dgpu_dev = false; + +bool is_dgpu() { + return is_dgpu_dev; +} + +unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { + unsigned int familyId = FAMILY_UNKNOWN; + + switch (props->EngineId.ui32.Major) { + case 7: + if (props->EngineId.ui32.Minor == 0) { + if (props->EngineId.ui32.Stepping == 0) + familyId = FAMILY_KV; + else + familyId = FAMILY_CI; + } + break; + case 8: + familyId = FAMILY_VI; + if (props->EngineId.ui32.Stepping == 1) + familyId = FAMILY_CZ; + break; + case 9: + familyId = FAMILY_AI; + if (props->EngineId.ui32.Stepping == 2) + familyId = FAMILY_RV; + break; + } + + if (familyId == FAMILY_KV || familyId == FAMILY_CZ || familyId == FAMILY_RV) + is_dgpu_dev = false; + else + is_dgpu_dev = true; + + return familyId; +} + +bool isTonga(const HsaNodeProperties *props) { + /* Tonga has some workarounds in the thunk that cause certain failures */ + if (props->EngineId.ui32.Major == 8 && props->EngineId.ui32.Stepping == 2) { + return true; + } + + return false; +} + +const HsaMemoryBuffer HsaMemoryBuffer::Null; + +HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, bool isScratch, bool isReadOnly) + :m_Size(size), + m_pUser(NULL), + m_pBuf(NULL), + m_Local(isLocal), + m_Node(node) { + m_Flags.Value = 0; + + if (isScratch) { + m_Flags.ui32.Scratch = 1; + m_Flags.ui32.HostAccess = 1; + } else { + m_Flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + + if (isLocal) { + m_Flags.ui32.HostAccess = 0; + m_Flags.ui32.NonPaged = 1; + } else { + m_Flags.ui32.HostAccess = 1; + m_Flags.ui32.NonPaged = 0; + } + + if (isExec) + m_Flags.ui32.ExecuteAccess = 1; + } + if (isReadOnly) + m_Flags.ui32.ReadOnly = 1; + + EXPECT_SUCCESS(hsaKmtAllocMemory( m_Node, m_Size, m_Flags, &m_pBuf)); + if (is_dgpu()) { + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(m_pBuf, m_Size, NULL)); + m_MappedNodes = 1 << m_Node; + } + + if (zero && !isLocal) + Fill(0); +} + +HsaMemoryBuffer::HsaMemoryBuffer(void *addr, HSAuint64 size): + m_Size(size), + m_pUser(addr), + m_pBuf(NULL), + m_Local(false), + m_Node(0) { + HSAuint64 gpuva = 0; + EXPECT_SUCCESS(hsaKmtRegisterMemory(m_pUser, m_Size)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(m_pUser, m_Size, &gpuva)); + m_pBuf = gpuva ? (void *)gpuva : m_pUser; +} + +HsaMemoryBuffer::HsaMemoryBuffer() + :m_Size(0), + m_pBuf(NULL) { +} + +void HsaMemoryBuffer::Fill(unsigned char value, HSAuint64 offset, HSAuint64 size) { + HSAuint32 uiValue; + + EXPECT_EQ(m_Local, 0) << "Local Memory. Call Fill(HSAuint32 value, BaseQueue& baseQueue)"; + + size = size ? size : m_Size; + ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; + + if (m_pUser != NULL) + memset((char *)m_pUser + offset, value, size); + else if (m_pBuf != NULL) + memset((char *)m_pBuf + offset, value, size); + else + ASSERT_TRUE(0) << "Invalid HsaMemoryBuffer"; +} + +/* Fill CPU accessible buffer with the value. */ +void HsaMemoryBuffer::Fill(HSAuint32 value, HSAuint64 offset, HSAuint64 size) { + HSAuint64 i; + HSAuint32 *ptr = NULL; + + EXPECT_EQ(m_Local, 0) << "Local Memory. Call Fill(HSAuint32 value, BaseQueue& baseQueue)"; + size = size ? size : m_Size; + EXPECT_EQ((size & (sizeof(HSAuint32) - 1)), 0) << "Not word aligned. Call Fill(unsigned char)"; + ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; + + if (m_pUser != NULL) + ptr = (HSAuint32 *)((char *)m_pUser + offset); + else if (m_pBuf != NULL) + ptr = (HSAuint32 *)((char *)m_pBuf + offset); + + ASSERT_NOTNULL(ptr); + + for (i = 0; i < size / sizeof(HSAuint32); i++) + ptr[i] = value; +} + +/* Fill GPU only accessible Local memory with @value using SDMA Constant Fill Command */ +void HsaMemoryBuffer::Fill(HSAuint32 value, BaseQueue& baseQueue, HSAuint64 offset, HSAuint64 size) { + HsaEvent* event = NULL; + + EXPECT_NE(m_Local, 0) << "Not Local Memory. Call Fill(HSAuint32 value)"; + + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_Node, &event)); + ASSERT_EQ(baseQueue.GetQueueType(), HSA_QUEUE_SDMA) << "Only SDMA queues supported"; + + size = size ? size : m_Size; + ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; + + baseQueue.PlacePacket(SDMAFillDataPacket((void *)(this->As() + offset), value, size)); + baseQueue.PlacePacket(SDMAFencePacket((void*)event->EventData.HWData2, event->EventId)); + baseQueue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); + ASSERT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + + hsaKmtDestroyEvent(event); +} + +/* Check if HsaMemoryBuffer[location] has the pattern specified. + * Return TRUE if correct pattern else return FALSE + * HsaMemoryBuffer has to be CPU accessible + */ +bool HsaMemoryBuffer::IsPattern(HSAuint64 location, HSAuint32 pattern) { + HSAuint32 *ptr = NULL; + + EXPECT_EQ(m_Local, 0) << "Local Memory. Call IsPattern(..baseQueue& baseQueue)"; + + if (location >= m_Size) /* Out of bounds */ + return false; + + if (m_pUser != NULL) + ptr = (HSAuint32 *)m_pUser; + else if (m_pBuf != NULL) + ptr = (HSAuint32 *)m_pBuf; + else + return false; + + if (ptr) + return (ptr[location/sizeof(HSAuint32)] == pattern); + + return false; +} + +/* Check if HsaMemoryBuffer[location] has the pattern specified. + * Return TRUE if correct pattern else return FALSE + * HsaMemoryBuffer is supposed to be only GPU accessible + * Use @baseQueue to copy the HsaMemoryBuffer[location] to stack and check the value + */ + +bool HsaMemoryBuffer::IsPattern(HSAuint64 location, HSAuint32 pattern, BaseQueue& baseQueue, volatile HSAuint32 *tmp) { + HsaEvent* event = NULL; + int ret; + + EXPECT_NE(m_Local, 0) << "Not Local Memory. Call IsPattern(HSAuint64 location, HSAuint32 pattern)"; + EXPECT_EQ(baseQueue.GetQueueType(), HSA_QUEUE_SDMA) << "Only SDMA queues supported"; + + if (location >= m_Size) /* Out of bounds */ + return false; + + ret = CreateQueueTypeEvent(false, false, m_Node, &event); + if (ret) + return false; + + *tmp = ~pattern; + baseQueue.PlacePacket(SDMACopyDataPacket((void *)tmp, + (void *)(this->As() + location), + sizeof(HSAuint32))); + baseQueue.PlacePacket(SDMAFencePacket((void*)event->EventData.HWData2, + event->EventId)); + baseQueue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); + + ret = hsaKmtWaitOnEvent(event, g_TestTimeOut); + hsaKmtDestroyEvent(event); + if (ret) + return false; + + return WaitOnValue(tmp, pattern); +} + +unsigned int HsaMemoryBuffer::Size() { + return m_Size; +} + +HsaMemFlags HsaMemoryBuffer::Flags() { + return m_Flags; +} + +unsigned int HsaMemoryBuffer::Node() const { + return m_Node; +} + +int HsaMemoryBuffer::MapMemToNodes(unsigned int *nodes, unsigned int nodes_num) { + int ret, bit; + + ret = hsaKmtRegisterMemoryToNodes(m_pBuf, m_Size, nodes_num, nodes); + if (ret != 0) + return ret; + ret = hsaKmtMapMemoryToGPU(m_pBuf, m_Size, NULL); + if (ret != 0) { + hsaKmtDeregisterMemory(m_pBuf); + return ret; + } + + for (unsigned int i = 0; i < nodes_num; i++) { + bit = 1 << nodes[i]; + m_MappedNodes |= bit; + } + + return 0; +} + +int HsaMemoryBuffer::UnmapMemToNodes(unsigned int *nodes, unsigned int nodes_num) { + int ret, bit; + + ret = hsaKmtUnmapMemoryToGPU(m_pBuf); + if (ret) + return ret; + + hsaKmtDeregisterMemory(m_pBuf); + for (unsigned int i = 0; i < nodes_num; i++) { + bit = 1 << nodes[i]; + m_MappedNodes &= ~bit; + } + + return 0; +} + +void HsaMemoryBuffer::UnmapAllNodes() { + unsigned int *Arr, size, i, j; + int bit; + + size = 0; + for (i = 0; i < 8; i++) { + bit = 1 << i; + if (m_MappedNodes & bit) + size++; + } + + Arr = (unsigned int *)malloc(sizeof(unsigned int) * size); + if (!Arr) + return; + + for (i = 0, j =0; i < 8; i++) { + bit = 1 << i; + if (m_MappedNodes & bit) + Arr[j++] = i; + } + + /* + * TODO: when thunk will be updated use hsaKmtRegisterToNodes. and then nodes will be used + */ + hsaKmtUnmapMemoryToGPU(m_pBuf); + hsaKmtDeregisterMemory(m_pBuf); + + m_MappedNodes = 0; + + free(Arr); +} + +HsaMemoryBuffer::~HsaMemoryBuffer() { + if (m_pUser != NULL) { + hsaKmtUnmapMemoryToGPU(m_pUser); + hsaKmtDeregisterMemory(m_pUser); + } else if (m_pBuf != NULL) { + if (is_dgpu()) { + if (m_MappedNodes) { + hsaKmtUnmapMemoryToGPU(m_pBuf); + hsaKmtDeregisterMemory(m_pBuf); + } + } + hsaKmtFreeMemory(m_pBuf, m_Size); + } + m_pBuf = NULL; +} + +HsaInteropMemoryBuffer::HsaInteropMemoryBuffer(unsigned long long device_handle, unsigned long long buffer_handle, unsigned long long size, unsigned int node) + :m_Size(0), + m_pBuf(NULL), + m_graphic_handle(0), + m_Node(node) { + HSAuint64 flat_address; + EXPECT_SUCCESS(hsaKmtMapGraphicHandle(m_Node, device_handle, buffer_handle, 0, size, &flat_address)); + m_pBuf = (void*)flat_address; +} + +HsaInteropMemoryBuffer::~HsaInteropMemoryBuffer() { + hsaKmtUnmapGraphicHandle(m_Node, (HSAuint64)m_pBuf, m_Size); +} + + +HsaNodeInfo::HsaNodeInfo() { +} + +// Init - Get and store information about all the HSA nodes from the Thunk Library. +// @NumOfNodes - Number to system nodes returned by hsaKmtAcquireSystemProperties +// @Return - false: if no node information is available +// +bool HsaNodeInfo::Init(int NumOfNodes) { + HsaNodeProperties *nodeProperties; + _HSAKMT_STATUS status; + bool ret = false; + + for (int i = 0; i < NumOfNodes; i++) { + nodeProperties = new HsaNodeProperties(); + + status = hsaKmtGetNodeProperties(i, nodeProperties); + /* this is not a fatal test (not using assert), since even when it fails for one node + * we want to get information regarding others. */ + EXPECT_SUCCESS(status) << "Node index: " << i << "hsaKmtGetNodeProperties returned status " << status; + + if (status == HSAKMT_STATUS_SUCCESS) { + m_HsaNodeProps.push_back(nodeProperties); + ret = true; // Return true if atleast one information is available + + if (nodeProperties->NumFComputeCores) + m_NodesWithGPU.push_back(i); + else + m_NodesWithoutGPU.push_back(i); + } else { + delete nodeProperties; + } + } + + return ret; +} + +HsaNodeInfo::~HsaNodeInfo() { + const HsaNodeProperties *nodeProperties; + + for (unsigned int i = 0; i < m_HsaNodeProps.size(); i++) + delete m_HsaNodeProps.at(i); + + m_HsaNodeProps.clear(); +} + +const std::vector& HsaNodeInfo::GetNodesWithGPU() const { + return m_NodesWithGPU; +} + +const HsaNodeProperties* HsaNodeInfo::GetNodeProperties(int NodeNum) const { + return m_HsaNodeProps.at(NodeNum); +} + +const HsaNodeProperties* HsaNodeInfo::HsaDefaultGPUNodeProperties() const { + int NodeNum = HsaDefaultGPUNode(); + if (NodeNum < 0) + return NULL; + return GetNodeProperties(NodeNum); +} + +const int HsaNodeInfo::HsaDefaultGPUNode() const { + if (m_NodesWithGPU.size() == 0) + return -1; + + if (g_TestNodeId >= 0) { + // Check if this is a valid Id, if so use this else use first + // available + for (unsigned int i = 0; i < m_NodesWithGPU.size(); i++) { + if (g_TestNodeId == m_NodesWithGPU.at(i)) + return g_TestNodeId; + } + } + + return m_NodesWithGPU.at(0); +} + +void HsaNodeInfo::PrintNodeInfo() const { + const HsaNodeProperties *nodeProperties; + + for (unsigned int i = 0; i < m_HsaNodeProps.size(); i++) { + nodeProperties = m_HsaNodeProps.at(i); + + LOG() << "***********************************" << std::endl; + LOG() << "Node " << i << std::endl; + LOG() << "NumCPUCores=\t" << nodeProperties->NumCPUCores << std::endl; + LOG() << "NumFComputeCores=\t" << nodeProperties->NumFComputeCores << std::endl; + LOG() << "NumMemoryBanks=\t" << nodeProperties->NumMemoryBanks << std::endl; + LOG() << "VendorId=\t" << nodeProperties->VendorId << std::endl; + LOG() << "DeviceId=\t" << nodeProperties->DeviceId << std::endl; + LOG() << "***********************************" << std::endl; + } + + LOG() << "Default GPU NODE " << HsaDefaultGPUNode() << std::endl; +} + +const bool HsaNodeInfo::IsGPUNodeLargeBar(int node) const { + const HsaNodeProperties *pNodeProperties; + + pNodeProperties = GetNodeProperties(node); + if (pNodeProperties) { + HsaMemoryProperties *memoryProperties = + new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(node, + pNodeProperties->NumMemoryBanks, memoryProperties)); + for (unsigned bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) + if (memoryProperties[bank].HeapType == + HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC) { + delete [] memoryProperties; + return true; + } + delete [] memoryProperties; + } + + return false; +} + +const int HsaNodeInfo::FindLargeBarGPUNode() const { + const std::vector gpuNodes = GetNodesWithGPU(); + + for (unsigned i = 0; i < gpuNodes.size(); i++) + if (IsGPUNodeLargeBar(gpuNodes.at(i))) + return gpuNodes.at(i); + + return -1; +} diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp new file mode 100644 index 0000000000..8f71b7ecb6 --- /dev/null +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD__TEST__UTIL__H__ +#define __KFD__TEST__UTIL__H__ + +#include +#include +#include "OSWrapper.hpp" +#include "GoogleTestExtension.hpp" +#include "hsakmt.h" + +class BaseQueue; +#define ARRAY_SIZE(_x) (sizeof(_x)/sizeof(_x[0])) + +// @brief: waits until the value is written to the buffer or until time out if received through args +bool WaitOnValue(const volatile unsigned int *buf, unsigned int value); + +void SplitU64(const unsigned long long value, unsigned int& rLoPart, unsigned int& rHiPart); + +bool GetHwCapabilityHWS(); + +HSAKMT_STATUS CreateQueueTypeEvent(bool ManualReset, bool IsSignaled, unsigned int NodeId, HsaEvent** Event); + +bool is_dgpu(); +bool isTonga(const HsaNodeProperties *props); +unsigned int FamilyIdFromNode(const HsaNodeProperties * props); + + +class HsaMemoryBuffer { + public: + static const HsaMemoryBuffer Null; + + public: + HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero = true, bool isLocal = false, + bool isExec = false, bool isScratch = false, bool isReadOnly = false); + HsaMemoryBuffer(void *addr, HSAuint64 size); + template + RetType As() { + return reinterpret_cast(m_pBuf); + } + + template + const RetType As() const { + return reinterpret_cast(m_pBuf); + } + + /* Fill @size bytes of buffer with @value starting from @offset + * If @size is 0, the whole buffer is filled with @value + */ + void Fill(unsigned char value, HSAuint64 offset = 0, HSAuint64 size = 0); + void Fill(HSAuint32 value, HSAuint64 offset = 0, HSAuint64 size = 0); + void Fill(int value, HSAuint64 offset = 0, HSAuint64 size = 0) { + Fill((HSAuint32)value, offset, size); + } + void Fill(HSAuint32 value, BaseQueue& baseQueue, + HSAuint64 offset = 0, HSAuint64 size = 0); + + bool IsPattern(HSAuint64 location, HSAuint32 pattern); + bool IsPattern(HSAuint64 location, HSAuint32 pattern, + BaseQueue& baseQueue, volatile HSAuint32 *tmp); + + unsigned int Size(); + HsaMemFlags Flags(); + unsigned int Node() const; + + int MapMemToNodes(unsigned int *nodes, unsigned int nodes_num); + int UnmapMemToNodes(unsigned int *nodes, unsigned int nodes_num); + + void *GetUserPtr() { return m_pUser; } + bool isLocal() { return m_Local; } + ~HsaMemoryBuffer(); + + private: + // disable copy + HsaMemoryBuffer(const HsaMemoryBuffer&); + const HsaMemoryBuffer& operator=(const HsaMemoryBuffer&); + + void UnmapAllNodes(); + HsaMemoryBuffer(); + + private: + HsaMemFlags m_Flags; + HSAuint64 m_Size; + void* m_pUser; + void* m_pBuf; + bool m_Local; + unsigned int m_Node; + unsigned short m_MappedNodes; +}; + + + +class HsaInteropMemoryBuffer { + public: + HsaInteropMemoryBuffer(unsigned long long device_handle, unsigned long long buffer_handle, unsigned long long size, unsigned int node); + + template + RetType As() { + return reinterpret_cast(m_pBuf); + } + + template + const RetType As() const { + return reinterpret_cast(m_pBuf); + } + + unsigned int Size(); + + ~HsaInteropMemoryBuffer(); + + private: + // disable copy + HsaInteropMemoryBuffer(const HsaInteropMemoryBuffer&); + const HsaInteropMemoryBuffer& operator=(const HsaInteropMemoryBuffer&); + + private: + unsigned long long m_Size; + void* m_pBuf; + unsigned long long m_graphic_handle; + unsigned int m_Node; +}; + +// Class HsaNodeInfo - Gather and store all HSA node information from Thunk. +// +class HsaNodeInfo { + // List containing HsaNodeProperties of all Nodes available + std::vector m_HsaNodeProps; + + // List of HSA Nodes that contain a GPU. This includes both APU and dGPU + std::vector m_NodesWithGPU; + + // List of HSA Nodes with CPU only + std::vector m_NodesWithoutGPU; + + public: + HsaNodeInfo(); + ~HsaNodeInfo(); + + bool Init(int NumOfNodes); + + // This function should be soon depricated. This for transistion purpose only + // Currently, KfdTest is designed to test only ONE node. This function acts + // as transistion. + const HsaNodeProperties* HsaDefaultGPUNodeProperties() const; + const int HsaDefaultGPUNode() const; + + // Future use the following two functions to support multi-GPU. + // const std::vector& GpuNodes = GetNodesWithGPU() + // for (..GpuNodes.size()..) GetNodeProperties(GpuNodes.at(i)) + // + const std::vector& GetNodesWithGPU() const; + + // @param node index of the node we are looking at + // @param nodeProperties HsaNodeProperties returned + const HsaNodeProperties* GetNodeProperties(int NodeNum) const; + + void PrintNodeInfo() const; + const bool IsGPUNodeLargeBar(int node) const; + // @brief Find the first available Large-BAR GPU node + // @return: Node ID if successful or -1 + const int FindLargeBarGPUNode() const; +}; + +#endif // __KFD__TEST__UTIL__H__ diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp new file mode 100644 index 0000000000..877899dcaa --- /dev/null +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -0,0 +1,220 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDTopologyTest.hpp" +#include +#include + +// @todo complete topology test according to whats in: hsathk\source\windows\kmt_topology.cpp + +const unsigned long long KFDTopologyTest::c_4Gigabyte = (1ull << 32) - 1; +const unsigned long long KFDTopologyTest::c_40BitAddressSpace = (1ull << 40); + +TEST_F(KFDTopologyTest , BasicTest) { + TEST_START(TESTPROFILE_RUNALL) + + const HsaNodeProperties *pNodeProperties; + + // goes over all nodes in the sytem properties and check the basic info received + for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { + pNodeProperties = m_NodeInfo.GetNodeProperties(node); + if (pNodeProperties != NULL) { + // checking for cpu core only if it's a cpu only node or if its KAVERY apu. + if (pNodeProperties->DeviceId == 0 || FamilyIdFromNode(pNodeProperties) == FAMILY_KV) { + EXPECT_GT(pNodeProperties->NumCPUCores, HSAuint32(0)) << "Node index: " << node << " No CPUs core are connected for node index"; + } + + // if it's not a cpu only node, look for a gpu core + if (pNodeProperties->DeviceId != 0) { + EXPECT_GT(pNodeProperties->NumFComputeCores, HSAuint32(0)) << "Node index: " << node << "No GPUs core are connected."; + // EngineId only applies to GPU, not CPU-only nodes + EXPECT_GT(pNodeProperties->EngineId.ui32.uCode, 0) << "uCode version is 0"; + EXPECT_GE(pNodeProperties->EngineId.ui32.Major, 7) << "Major Version is less than 7"; + EXPECT_LT(pNodeProperties->EngineId.ui32.Minor, 10) << "Minor Version is greater than 9"; + EXPECT_LT(pNodeProperties->EngineId.ui32.Stepping, 10) << "Stepping is greater than 9"; + EXPECT_GT(pNodeProperties->uCodeEngineVersions.uCodeSDMA, 0) << "sDMA firmware version is 0"; + } + EXPECT_GT(pNodeProperties->NumMemoryBanks, HSAuint32(0)) << "Node index: " << node << "No MemoryBanks."; + EXPECT_GT(pNodeProperties->NumCaches, HSAuint32(0)) << "Node index: " << node << "No Caches."; + } + } + + TEST_END +} + +// this test verify failure status on hsaKmtGetNodeProperties with invalid params +TEST_F(KFDTopologyTest, GetNodePropertiesInvalidParams) { + TEST_START(TESTPROFILE_RUNALL) + + EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtGetNodeProperties(0, NULL)); + + TEST_END +} + +// this test verify failure status on hsaKmtGetNodeProperties with invalid params +TEST_F(KFDTopologyTest, GetNodePropertiesInvalidNodeNum) { + TEST_START(TESTPROFILE_RUNALL) + + HsaNodeProperties nodeProperties; + memset(&nodeProperties, 0, sizeof(nodeProperties)); + EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtGetNodeProperties(m_SystemProperties.NumNodes, &nodeProperties)); + + TEST_END +} + +// test that we can get memory property successfully per node +// @todo check validity of values returned +TEST_F(KFDTopologyTest, GetNodeMemoryProperties) { + TEST_START(TESTPROFILE_RUNALL) + const HsaNodeProperties *pNodeProperties; + + for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { + pNodeProperties = m_NodeInfo.GetNodeProperties(node); + + if (pNodeProperties != NULL) { + HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(node, pNodeProperties->NumMemoryBanks, memoryProperties)); + delete [] memoryProperties; + } + } + + TEST_END +} + + +// test the GPU local memory aperture is valid. +TEST_F(KFDTopologyTest, GpuvmApertureValidate) { + TEST_REQUIRE_NO_ENV_CAPABILITIES(ENVCAPS_32BITLINUX); + + TEST_START(TESTPROFILE_RUNALL) + const HsaNodeProperties *pNodeProperties; + const std::vector GpuNodes = m_NodeInfo.GetNodesWithGPU(); + + for (unsigned i = 0; i < GpuNodes.size(); i++) { + pNodeProperties = m_NodeInfo.GetNodeProperties(GpuNodes.at(i)); + if (pNodeProperties != NULL) { + if (!is_dgpu() && !(FamilyIdFromNode(pNodeProperties) == FAMILY_KV)) { + LOG() << "Skipping test: GPUVM framebuffer heap not exposed on APU except Kaveri" << std::endl; + return; + } + HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(GpuNodes.at(i), pNodeProperties->NumMemoryBanks, memoryProperties)); + bool GpuVMHeapFound = false; + for (unsigned int bank = 0 ; bank < pNodeProperties->NumMemoryBanks ; bank++) { + // Check for either private (small-bar/APU) or public (large-bar) + if ((HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE == memoryProperties[bank].HeapType) || + (HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC == memoryProperties[bank].HeapType)) + GpuVMHeapFound = true; + } + EXPECT_TRUE(GpuVMHeapFound); + delete [] memoryProperties; + } + } + + TEST_END +} + +// test that we can get cache property successfully per node +// @todo check validity of values returned +TEST_F(KFDTopologyTest, GetNodeCacheProperties) { + TEST_START(TESTPROFILE_RUNALL) + + const HsaNodeProperties *pNodeProperties; + + for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { + pNodeProperties = m_NodeInfo.GetNodeProperties(node); + if (pNodeProperties != NULL) { + HsaCacheProperties *cacheProperties = new HsaCacheProperties[pNodeProperties->NumCaches]; + EXPECT_SUCCESS(hsaKmtGetNodeCacheProperties(node, pNodeProperties->CComputeIdLo, pNodeProperties->NumCaches, cacheProperties)); + if (pNodeProperties->NumCPUCores > 0) { // this is a CPU node + LOG() << "CPU Node " << std::dec << node << ": " << pNodeProperties->NumCaches << " caches" << std::endl; + for (unsigned n = 0; n < pNodeProperties->NumCaches; n++) { + LOG()<< n << " - Level " << cacheProperties[n].CacheLevel << + " Type " << cacheProperties[n].CacheType.Value << + " Size " << (cacheProperties[n].CacheSize >> 10) << "K " << + " Associativity " << cacheProperties[n].CacheAssociativity << + " LineSize " << cacheProperties[n].CacheLineSize << + " LinesPerTag " << cacheProperties[n].CacheLinesPerTag << std::endl; + char string[1024] = ""; + char sibling[5] = ""; + for (unsigned i = 0; i < 256; i++) { + if (cacheProperties[n].SiblingMap[i]) { + sprintf(sibling, "%d,", i); + strcat(string, sibling); + } + } + LOG() << " ProcIdLow " << cacheProperties[n].ProcessorIdLow << + " SiblingMap " << string << std::endl; + } + } + delete [] cacheProperties; + } + } + + TEST_END +} + +// test that we can get NodeIoLink property successfully per node +// @todo check validity of values returned +// GetNodeIoLinkProperties is disabled for now, test fails due to bug in BIOS +TEST_F(KFDTopologyTest, GetNodeIoLinkProperties) { + TEST_START(TESTPROFILE_RUNALL) + const HsaNodeProperties *pNodeProperties; + int linkId; + char c; + + LOG() << "Topology. [FromNode]--(Weight)-->[ToNode]" << std::endl; + + for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { + pNodeProperties = m_NodeInfo.GetNodeProperties(node); + + if (pNodeProperties != NULL) { + HsaIoLinkProperties *IolinkProperties = new HsaIoLinkProperties[pNodeProperties->NumIOLinks]; + EXPECT_SUCCESS(hsaKmtGetNodeIoLinkProperties(node, pNodeProperties->NumIOLinks, IolinkProperties)); + if (pNodeProperties->NumIOLinks == 0) { + // No io_links. Just Print the node + LOG() << "[" << node << "]" << std::endl; + continue; + } + + for (linkId = 0; linkId < pNodeProperties->NumIOLinks; linkId++) { + if (linkId == 0) { + // First io_link. Print Parent Node and io_link Node + LOG() << "[" << node << "]--(" << IolinkProperties[linkId].Weight << ")-->" << + "[" << IolinkProperties[linkId].NodeTo << "]" << std::endl; + continue; + } + if (linkId == (pNodeProperties->NumIOLinks - 1)) + c = '`'; // last node + else + c = '|'; + LOG() << " " << c << "--(" << IolinkProperties[linkId].Weight << ")-->" << + "[" << IolinkProperties[linkId].NodeTo << "]" << std::endl; + } + LOG() << std::endl; + delete [] IolinkProperties; + } + } + + TEST_END +} diff --git a/tests/kfdtest/src/KFDTopologyTest.hpp b/tests/kfdtest/src/KFDTopologyTest.hpp new file mode 100644 index 0000000000..2ab631f8a4 --- /dev/null +++ b/tests/kfdtest/src/KFDTopologyTest.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDBaseComponentTest.hpp" + +#ifndef __KFD_TOPOLOGY_TEST__H__ +#define __KFD_TOPOLOGY_TEST__H__ + +// @class KFDTopologyTest +// this class has no additional features to KFDBaseComponentTest +// the separation was made so we are able to goup all topology tests together +class KFDTopologyTest : public KFDBaseComponentTest { + public: + KFDTopologyTest(void) {} + ~KFDTopologyTest(void) {} + static const unsigned long long c_4Gigabyte; + static const unsigned long long c_40BitAddressSpace; +}; + +#endif // __KFD_TOPOLOGY_TEST__H__ diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/tests/kfdtest/src/LinuxOSWrapper.cpp new file mode 100644 index 0000000000..90ea87b88f --- /dev/null +++ b/tests/kfdtest/src/LinuxOSWrapper.cpp @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef WIN32 + +#include "OSWrapper.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int protection_flags[8] = {int(PROT_NONE), int(PROT_READ), int(PROT_WRITE), int(PROT_READ | PROT_WRITE), int(PROT_EXEC), int(PROT_EXEC | PROT_READ), int(PROT_EXEC | PROT_WRITE), int(PROT_EXEC | PROT_WRITE | PROT_READ)}; + +void SetConsoleTextColor(TEXTCOLOR color) { + // TODO complete +} + +void Delay(int delayCount) { + // usleeps accept time in microseconds + usleep(delayCount * 1000); +} + +void *VirtualAllocMemory(void *address, unsigned int size, int memProtection ) { + void *ptr; + + ptr = mmap(address, size, protection_flags[memProtection], MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + + if (ptr == MAP_FAILED) + ptr = NULL; + return ptr; +} + +bool VirtualFreeMemory(void *address, unsigned int size) { + if (munmap(address, size) == 0) + return true; + else + return false; +} + +unsigned long GetLastErrorNo() { + return errno; +} + +bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProcess) { + // TODO IMPLEMENT + return false; +} + +unsigned long long GetSystemTickCountInMicroSec() { + struct timeval t; + gettimeofday(&t, 0); + return t.tv_sec * 1000000ULL + t.tv_usec; +} + +bool SuspendAndWakeUp() { + // TODO IMPLEMENT + return false; +} + +void AcquirePrivilege(OS_PRIVILEGE priv) { + // TODO IMPLEMENT +} + +void DisableKfd() { + // TODO IMPLEMENT +} + +void EnableKfd() { + // TODO IMPLEMENT +} + +bool ReadDriverConfigValue(CONFIG_VALUE config, unsigned int& rValue) { + return false; +} + +void ComandLineArgumentsUsage() { + printf("Invalid option value\n"); + printf("\t--hws arg\t - Force HW capability\n"); + printf("\t--profile arg\t - Test profile\n"); + printf("\t--child arg\t - Child Process\n"); + printf("\t--timeout arg\t - Time Out\n"); + printf("\t--dst_node\t - For testing multiple nodes"); +} + +bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs) { + int option_index = 0; + + /* Make getop silent */ + opterr = 0; + static struct option long_options[] = { + { "hws", required_argument, 0, 0 }, + { "profile", required_argument, 0, 0}, + { "child", required_argument, 0, 0}, + { "timeout", required_argument, 0, 0}, + { "node", required_argument, 0, 0 }, + { "dst_node", required_argument, 0, 0 }, + { 0, 0, 0, 0 } + }; + + rArgs.HwsEnabled = HWCAP__DEFAULT; + rArgs.TestProfile = TESTPROFILE_RUNALL; + rArgs.ChildProcess = false; + rArgs.TimeOut = 0; + rArgs.NodeId = -1; + rArgs.DstNodeId = -1; + + while (true) { + int c = getopt_long(argc, argv, "", long_options, &option_index); + + /* Detect the end of the options. */ + if (c != 0) + break; + + /* If this option set a flag, do nothing else now. */ + if (long_options[option_index].flag != 0) + continue; + + if (optarg == NULL) { + ComandLineArgumentsUsage(); + return false; + } + + switch (option_index) { + /* HWS case */ + case 0: + if (!strcmp(optarg, "disable")) { + rArgs.HwsEnabled = HWCAP__FORCE_DISABLED; + } else if (!strcmp(optarg, "enable")) { + rArgs.HwsEnabled = HWCAP__FORCE_ENABLED; + } else { + ComandLineArgumentsUsage(); + return false; + } + break; + /* TEST PROFILE */ + case 1: + if (!strcmp(optarg, "dev")) { + rArgs.TestProfile = TESTPROFILE_DEV; + } else if (!strcmp(optarg, "promo")) { + rArgs.TestProfile = TESTPROFILE_PROMO; + } else if (!strcmp(optarg, "all")) { + rArgs.TestProfile = TESTPROFILE_RUNALL; + } else { + ComandLineArgumentsUsage(); + return false; + } + break; + + case 2: + rArgs.ChildProcess = true; + break; + + case 3: + { + int timeOut = atoi(optarg); + if (timeOut > 0) + rArgs.TimeOut = timeOut; + } + break; + case 4: + { + int nodeId = atoi(optarg); + if (nodeId >= 0) + rArgs.NodeId = nodeId; + } + break; + case 5: + { + int dstNodeId = atoi(optarg); + if (dstNodeId >= 0) + rArgs.DstNodeId = dstNodeId; + } + break; + } + } + + return true; +} + +void HWMemoryBarrier() { + __sync_synchronize(); +} + +bool StartThread(unsigned int (*thread_func)(void*), void* param, uint64_t& thread_id) { + pthread_t id; + bool ret = false; + typedef void* (*pthread_func_t)(void*); + + if (!pthread_create(&id, NULL, (pthread_func_t)thread_func, param)) { + thread_id = (pthread_t)id; + ret = true; + } + return ret; +} + +bool WaitForThread(uint64_t threadId) { + return 0 == pthread_join((pthread_t)threadId, NULL); +} + +long AtomicInc(volatile long* pValue) { + return __sync_add_and_fetch(pValue, 1); +} + +void MemoryBarrier() { + __sync_synchronize(); +} + +#endif // !WIN32 diff --git a/tests/kfdtest/src/OSWrapper.hpp b/tests/kfdtest/src/OSWrapper.hpp new file mode 100644 index 0000000000..bcba47bb63 --- /dev/null +++ b/tests/kfdtest/src/OSWrapper.hpp @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include + +#include "KFDTestFlags.hpp" + +#ifndef __OS__WRAPPER__H__ +#define __OS__WRAPPER__H__ + +#ifndef PAGE_SIZE +#define PAGE_SIZE (1<<12) +#define PAGE_SHIFT (12) +#endif + +enum TEXTCOLOR { + TEXTCOLOR_WHITE, + TEXTCOLOR_GREEN, + TEXTCOLOR_YELLOW +}; + +enum OS_PRIVILEGE { + OS_DRIVER_OPERATIONS, + OS_SUSPEND +}; + +enum CONFIG_VALUE { + CONFIG_HWS +}; + +enum HwCapabilityStatus { + HWCAP__FORCE_DISABLED, + HWCAP__DEFAULT, + HWCAP__FORCE_ENABLED +}; + +struct CommandLineArguments { + HwCapabilityStatus HwsEnabled; + TESTPROFILE TestProfile; + bool ChildProcess; + unsigned int TimeOut; + int NodeId; + int DstNodeId; +}; + +// It is either MEM_NONE or the bitwise OR of one or more of the following flags +#define MEM_NONE 0x00 +#define MEM_READ 0x01 +#define MEM_WRITE 0x02 +#define MEM_EXECUTE 0x4 + + + +// @brief change console text color +void SetConsoleTextColor(TEXTCOLOR color); +// @params delayCount : delay time in milliseconds +void Delay(int delayCount); +// @brief replacement for windows VirtualAlloc func +void *VirtualAllocMemory(void *address, unsigned int size, int memProtection = MEM_READ | MEM_WRITE); +// @brief replacement for windows FreeVirtual func +bool VirtualFreeMemory(void *address, unsigned int size); +// @brief retrieve the last error number +unsigned long GetLastErrorNo(); + +long AtomicInc(volatile long* pValue); + +void MemoryBarrier(); + +// @brief: runs the selected test case number of times required, each in a separate process +// @params testToRun : can be a specific test testcase like TestCase.TestName or if you want to run all tests in a test case: TestCase.* and so on +// @params numOfProcesses : how many processes to run in parallel +// @params runsPerProcess : how many iteration a test should do per process, must be a positive number +bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProcess = 1); + +unsigned long long GetSystemTickCountInMicroSec(); + +/**Put the system to S3/S4 power state and bring it back to S0. +@return 'true' on success, 'false' on failure. +*/ +bool SuspendAndWakeUp(); + +void AcquirePrivilege(OS_PRIVILEGE priv); + +void DisableKfd(); +void EnableKfd(); + +bool ReadDriverConfigValue(CONFIG_VALUE config, unsigned int& rValue); + +bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs); + +void HWMemoryBarrier(); +bool StartThread(unsigned int (*)(void*), void* pParam, uint64_t& threadId); +bool WaitForThread(uint64_t threadId); + +#endif // __OS__WRAPPER__H__ diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp new file mode 100644 index 0000000000..c0c48337a5 --- /dev/null +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -0,0 +1,302 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "PM4Packet.hpp" +#include "hsakmttypes.h" +#include +#include +#include + +#include "asic_reg/gfx_7_2_enum.h" + +unsigned int PM4Packet::CalcCountValue() const { + return (SizeInDWords() - (sizeof(PM4_TYPE_3_HEADER) / sizeof(uint32_t)) - 1); +} + +void PM4Packet::InitPM4Header(PM4_TYPE_3_HEADER &header, it_opcode_type opCode) { + header.count = CalcCountValue(); + header.opcode = opCode; + header.type = PM4_TYPE_3; + header.shaderType = 1; // compute + header.predicate = 0; + header.reserved1 = 0; +} + +PM4WriteDataPacket::~PM4WriteDataPacket(void) { + if (m_pPacketData) + free(m_pPacketData); +} + +unsigned int PM4WriteDataPacket::SizeInBytes() const { + return (offsetof(PM4WRITE_DATA_CI, data) + m_ndw*sizeof(uint32_t)); +} + +void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) { + m_pPacketData = (PM4WRITE_DATA_CI *)calloc(1, SizeInBytes()); + // verify that the memory is allocated successfully, cannot use assert here + EXPECT_NOTNULL(m_pPacketData); + + InitPM4Header(m_pPacketData->header, IT_WRITE_DATA); + + m_pPacketData->bitfields2.dst_sel = dst_sel_mec_write_data_MEMORY_5; // memory-async + m_pPacketData->bitfields2.addr_incr = addr_incr_mec_write_data_INCREMENT_ADDR_0; // increment addr + m_pPacketData->bitfields2.wr_confirm = wr_confirm_mec_write_data_WAIT_FOR_CONFIRMATION_1; + m_pPacketData->bitfields2.atc = is_dgpu() ? + atc_write_data_NOT_USE_ATC_0 : atc_write_data_USE_ATC_1; + m_pPacketData->bitfields2.cache_policy = cache_policy_mec_write_data_BYPASS_2; + + m_pPacketData->dst_addr_lo = static_cast( + reinterpret_cast(destBuf)); // byte addr + m_pPacketData->dst_address_hi = static_cast( + reinterpret_cast(destBuf) >> 32); + + memcpy(m_pPacketData->data, data, m_ndw * sizeof(uint32_t)); +} + +PM4ReleaseMemoryPacket::~PM4ReleaseMemoryPacket(void) { + if (m_pPacketData) + free(m_pPacketData); +} + +void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, + uint64_t data, bool is64bit, bool isTimeStamp) { + if (g_TestGPUFamilyId < FAMILY_AI) { + PM4_RELEASE_MEM_CI *pkt; + + m_packetSize = sizeof(PM4_RELEASE_MEM_CI); + pkt = (PM4_RELEASE_MEM_CI *)calloc(1, m_packetSize); + m_pPacketData = pkt; + EXPECT_NOTNULL(m_pPacketData); + + InitPM4Header(pkt->header, IT_RELEASE_MEM); + + pkt->bitfields2.event_type = 0x14; + pkt->bitfields2.event_index = event_index_mec_release_mem_EVENT_WRITE_EOP_5; + // Possible values: + // 0101(5): EVENT_WRITE_EOP event types + // 0110(6): Reserved for EVENT_WRITE_EOS packet. + // 0111(7): Reserved (previously) for EVENT_WRITE packet. + pkt->bitfields2.l2_wb = 1; + pkt->bitfields2.l2_inv = 1; + pkt->bitfields2.cache_policy = cache_policy_mec_release_mem_BYPASS_2; + pkt->bitfields2.atc = is_dgpu() ? + atc_mec_release_mem_ci_NOT_USE_ATC_0 : + atc_mec_release_mem_ci_USE_ATC_1; // ATC setting for fences and timestamps to the MC or TCL2. + pkt->bitfields3.dst_sel = dst_sel_mec_release_mem_MEMORY_CONTROLLER_0; + // Possible values: + // 0 - memory_controller. + // 1 - tc_l2. + if (address) { + pkt->bitfields3.int_sel = (isPolling ? + int_sel_mec_release_mem_SEND_DATA_AFTER_WRITE_CONFIRM_3 : + int_sel_mec_release_mem_SEND_INTERRUPT_AFTER_WRITE_CONFIRM_2); + // Possible values: + // 0 - None (Do not send an interrupt). + // 1 - Send Interrupt Only. Program DATA_SEL 0". + // 2 - Send Interrupt when Write Confirm (WC) is received from the MC. + // 3 - Wait for WC, but dont send interrupt (applicable to 7.3+) [g73_1] + // 4 - Reserved for INTERRUPT packet + if (isTimeStamp && is64bit) + pkt->bitfields3.data_sel = data_sel_mec_release_mem_SEND_GPU_CLOCK_COUNTER_3; + else + pkt->bitfields3.data_sel = is64bit ? + data_sel_mec_release_mem_SEND_64_BIT_DATA_2 : + data_sel_mec_release_mem_SEND_32_BIT_LOW_1; + // Possible values: + // 0 - None, i.e., Discard Data. + // 1 - Send 32-bit Data Low (Discard Data High). + // 2 - Send 64-bit Data. + // 3 - Send current value of the 64 bit global GPU clock counter. + // 4 - Send current value of the 64 bit system clock counter. + // 5 - Store GDS Data to memory. + // 6 - Reserved for use by the CP for Signal Semaphore. + // 7 - Reserved for use by the CP for Wait Semaphore. + } else { + pkt->bitfields3.int_sel = (isPolling ? + int_sel_mec_release_mem_NONE_0 : + int_sel_mec_release_mem_SEND_INTERRUPT_ONLY_1); + pkt->bitfields3.data_sel = data_sel_mec_release_mem_NONE_0; + } + + pkt->bitfields4a.address_lo_dword_aligned = static_cast((address&0xffffffff) >> 2); + pkt->addr_hi = static_cast(address>>32); + + pkt->data_lo = static_cast(data); + pkt->data_hi = static_cast(data >> 32); + } else { + PM4MEC_RELEASE_MEM_AI *pkt; + + m_packetSize = sizeof(PM4MEC_RELEASE_MEM_AI); + pkt = (PM4MEC_RELEASE_MEM_AI *)calloc(1, m_packetSize); + m_pPacketData = pkt; + EXPECT_NOTNULL(m_pPacketData); + + InitPM4Header(pkt->header, IT_RELEASE_MEM); + + pkt->bitfields2.event_type = 0x14; + pkt->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe; + pkt->bitfields2.tc_wb_action_ena = 1; + pkt->bitfields2.tc_action_ena = 1; + pkt->bitfields2.cache_policy = cache_policy__mec_release_mem__lru; + + pkt->bitfields3.dst_sel = dst_sel__mec_release_mem__memory_controller; + + if (address) { + pkt->bitfields3.int_sel = (isPolling ? + int_sel__mec_release_mem__send_data_after_write_confirm: + int_sel__mec_release_mem__send_interrupt_after_write_confirm); + + if (isTimeStamp && is64bit) + pkt->bitfields3.data_sel = data_sel__mec_release_mem__send_gpu_clock_counter; + else + pkt->bitfields3.data_sel = is64bit ? + data_sel__mec_release_mem__send_64_bit_data : + data_sel__mec_release_mem__send_32_bit_low; + } else { + pkt->bitfields3.int_sel = (isPolling ? + int_sel__mec_release_mem__none: + int_sel__mec_release_mem__send_interrupt_only); + pkt->bitfields3.data_sel = data_sel__mec_release_mem__none; + } + + pkt->bitfields4a.address_lo_32b = static_cast((address&0xffffffff) >> 2); + pkt->address_hi = static_cast(address>>32); + + pkt->data_lo = static_cast(data); + pkt->data_hi = static_cast(data >> 32); + + pkt->int_ctxid = static_cast(data); + } +} + +PM4IndirectBufPacket::PM4IndirectBufPacket(IndirectBuffer *pIb) { + InitPacket(pIb); +} + +unsigned int PM4IndirectBufPacket::SizeInBytes() const { + return sizeof(PM4MEC_INDIRECT_BUFFER); +} + +void PM4IndirectBufPacket::InitPacket(IndirectBuffer *pIb) { + memset(&m_packetData, 0, SizeInBytes()); + InitPM4Header(m_packetData.header, IT_INDIRECT_BUFFER); + + m_packetData.bitfields2.ib_base_lo = static_cast((reinterpret_cast(pIb->Addr()))) >> 2; + m_packetData.bitfields3.ib_base_hi = reinterpret_cast(pIb->Addr()) >> 32; + m_packetData.bitfields4.ib_size = pIb->SizeInDWord(); + m_packetData.bitfields4.chain = 0; + m_packetData.bitfields4.offload_polling = 0; + m_packetData.bitfields4.volatile_setting = 0; + m_packetData.bitfields4.valid = 1; + m_packetData.bitfields4.vmid = 0; // in iommutest: vmid = queueParams.VMID; + m_packetData.bitfields4.cache_policy = cache_policy_indirect_buffer_BYPASS_2; +} + +PM4AcquireMemoryPacket::PM4AcquireMemoryPacket(void) { + memset(&m_packetData, 0, SizeInBytes()); + InitPM4Header(m_packetData.header, IT_ACQUIRE_MEM); + + m_packetData.bitfields2.coher_cntl = 0x28c00000; // copied from the way the HSART does this. + m_packetData.bitfields2.engine = engine_acquire_mem_PFP_0; + m_packetData.coher_size = 0xFFFFFFFF; + m_packetData.bitfields3.coher_size_hi = 0; + m_packetData.coher_base_lo = 0; + m_packetData.bitfields4.coher_base_hi = 0; + m_packetData.bitfields5.poll_interval = 4; // copied from the way the HSART does this. +} + +unsigned int PM4AcquireMemoryPacket::SizeInBytes() const { + return sizeof(PM4ACQUIRE_MEM); +} + +PM4SetShaderRegPacket::PM4SetShaderRegPacket(void) + : m_packetDataAllocated(false) { +} + +PM4SetShaderRegPacket::PM4SetShaderRegPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs) + : m_packetDataAllocated(false) { + InitPacket(baseOffset, regValues, numRegs); +} + +PM4SetShaderRegPacket::~PM4SetShaderRegPacket(void) { + if (m_packetDataAllocated) + free(m_pPacketData); +} + +void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs) { + m_packetSize = sizeof(PM4SET_SH_REG) + (numRegs-1)*sizeof(uint32_t); // 1st register is a part of the packet struct. + + // allocating the size of the packet, since the packet is assembled from a struct followed by an additional DWORD data + m_pPacketData = (PM4SET_SH_REG *)malloc(m_packetSize); + + ASSERT_NOTNULL(m_pPacketData); + + m_packetDataAllocated = true; + + memset(m_pPacketData, 0, m_packetSize); + + InitPM4Header(m_pPacketData->header, IT_SET_SH_REG); + + m_pPacketData->bitfields2.reg_offset = baseOffset - PERSISTENT_SPACE_START; + + memcpy(m_pPacketData->reg_data, regValues, numRegs*sizeof(uint32_t)); +} + +PM4DispatchDirectPacket::PM4DispatchDirectPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, unsigned int dispatchInit) { + InitPacket(dimX, dimY, dimZ, dispatchInit); +} + +void PM4DispatchDirectPacket::InitPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, unsigned int dispatchInit) { + memset(&m_packetData, 0, SizeInBytes()); + InitPM4Header(m_packetData.header, IT_DISPATCH_DIRECT); + + m_packetData.dim_x = dimX; + m_packetData.dim_y = dimY; + m_packetData.dim_z = dimZ; + m_packetData.dispatch_initiator = dispatchInit; +} + +unsigned int PM4DispatchDirectPacket::SizeInBytes() const { + return sizeof(PM4DISPATCH_DIRECT); +} + +PM4PartialFlushPacket::PM4PartialFlushPacket(void) { + memset(&m_packetData, 0, SizeInBytes()); + InitPM4Header(m_packetData.header, IT_EVENT_WRITE); + + m_packetData.bitfields2.event_index = event_index_event_write_CS_VS_PS_PARTIAL_FLUSH_4; + m_packetData.bitfields2.event_type = CS_PARTIAL_FLUSH; +} + +unsigned int PM4PartialFlushPacket::SizeInBytes() const { + // for PARTIAL_FLUSH_CS packets, the last 2 DWORDS don't exist. + return sizeof(PM4EVENT_WRITE) - sizeof(uint32_t)*2; +} + +PM4NopPacket::PM4NopPacket(void) { + InitPM4Header(m_packetData, IT_NOP); +} + +unsigned int PM4NopPacket::SizeInBytes() const { + return sizeof(m_packetData); +} diff --git a/tests/kfdtest/src/PM4Packet.hpp b/tests/kfdtest/src/PM4Packet.hpp new file mode 100644 index 0000000000..cc09d43b68 --- /dev/null +++ b/tests/kfdtest/src/PM4Packet.hpp @@ -0,0 +1,221 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_PM4_PACKET__H__ +#define __KFD_PM4_PACKET__H__ + +#include "BasePacket.hpp" +#include "kfd_pm4_opcodes.h" +#include "pm4_pkt_struct_common.h" +#include "pm4_pkt_struct_ci.h" +#include "pm4_pkt_struct_ai.h" +#include "IndirectBuffer.hpp" + +// @class PM4Packet: marks a group of all PM4 packets +class PM4Packet : public BasePacket { + public: + PM4Packet(void) {} + virtual ~PM4Packet(void) {} + + virtual PACKETTYPE PacketType() const { return PACKETTYPE_PM4; } + unsigned int CalcCountValue() const; + + protected: + void InitPM4Header(PM4_TYPE_3_HEADER &header, it_opcode_type opCode); +}; + +// @class PM4WriteDataPacket +class PM4WriteDataPacket : public PM4Packet { + public: + // empty constructor, befor using the packet call the init func + PM4WriteDataPacket(void): m_ndw(0), m_pPacketData(NULL) {} + // this contructor will also init the packet, no need for additional calls + PM4WriteDataPacket(unsigned int *destBuf, unsigned int data1): + m_ndw(1), m_pPacketData(NULL) {InitPacket(destBuf, &data1);} + PM4WriteDataPacket(unsigned int *destBuf, unsigned int data1, unsigned int data2): + m_ndw(2), m_pPacketData(NULL) { + unsigned int data[2] = {data1, data2}; + InitPacket(destBuf, data); + } + + virtual ~PM4WriteDataPacket(void); + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const; + // @returns a pointer to the packet + virtual const void *GetPacket() const { return m_pPacketData; } + // @brief initialise the packet + void InitPacket(unsigned int *destBuf, unsigned int data1) { + m_ndw = 1; + InitPacket(destBuf, &data1); + } + void InitPacket(unsigned int *destBuf, unsigned int data1, unsigned int data2) { + unsigned int data[2] = {data1, data2}; + m_ndw = 2; + InitPacket(destBuf, data); + } + void InitPacket(unsigned int *destBuf, void *data); + + protected: + unsigned int m_ndw; + // PM4WRITE_DATA_CI struct contains all the packets data + PM4WRITE_DATA_CI *m_pPacketData; +}; + +// @class PM4ReleaseMemoryPacket +class PM4ReleaseMemoryPacket : public PM4Packet { + public: + // empty constructor, befor using the packet call the init func + PM4ReleaseMemoryPacket(void): m_pPacketData(NULL) {} + // this contructor will also init the packet, no need for adittional calls + PM4ReleaseMemoryPacket(bool isPolling, uint64_t address, uint64_t data, + bool is64bit = false, bool isTimeStamp = false): m_pPacketData(NULL) { + InitPacket(isPolling, address, data, is64bit, isTimeStamp); + } + + virtual ~PM4ReleaseMemoryPacket(void); + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const { return m_packetSize; } + // @returns a pointer to the packet + virtual const void *GetPacket() const { return m_pPacketData; } + // @brief initialise the packet + void InitPacket(bool isPolling, uint64_t address, uint64_t data, + bool is64bit = false, bool isTimeStamp = false); + + private: + void *m_pPacketData; + unsigned int m_packetSize; +}; + +// @class PM4IndirectBufPacket +class PM4IndirectBufPacket : public PM4Packet { + public: + // empty constructor, befor using the packet call the init func + PM4IndirectBufPacket(void) {} + // this contructor will also init the packet, no need for adittional calls + PM4IndirectBufPacket(IndirectBuffer *pIb); + + virtual ~PM4IndirectBufPacket(void) {} + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const; + // @returns a pointer to the packet + virtual const void *GetPacket() const { return &m_packetData; } + // @breif initialise the packet + void InitPacket(IndirectBuffer *pIb); + + private: + // PM4MEC_INDIRECT_BUFFER struct contains all the packets data + PM4MEC_INDIRECT_BUFFER m_packetData; +}; + +// @class PM4AcquireMemoryPacket +class PM4AcquireMemoryPacket : public PM4Packet { + public: + PM4AcquireMemoryPacket(void); + virtual ~PM4AcquireMemoryPacket(void) {} + + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const; + // @returns a pointer to the packet + virtual const void *GetPacket() const { return &m_packetData; } + + private: + // PM4ACQUIRE_MEM struct contains all the packets data + PM4ACQUIRE_MEM m_packetData; +}; + +// @class PM4SetShaderRegPacket packet that writes to consecutive registers starting at baseOffset. +class PM4SetShaderRegPacket : public PM4Packet { + public: + PM4SetShaderRegPacket(void); + + PM4SetShaderRegPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs); + + virtual ~PM4SetShaderRegPacket(void); + + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const { return m_packetSize; } + // @returns a pointer to the packet + virtual const void *GetPacket() const { return m_pPacketData; } + + void InitPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs); + + private: + unsigned int m_packetSize; + bool m_packetDataAllocated; + // PM4SET_SH_REG struct contains all the packets data + PM4SET_SH_REG *m_pPacketData; +}; + +// @class PM4DispatchDirectPacket +class PM4DispatchDirectPacket : public PM4Packet { + public: + PM4DispatchDirectPacket(void) {} + + PM4DispatchDirectPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, unsigned int dispatchInit); + + virtual ~PM4DispatchDirectPacket(void) {} + + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const; + // @returns a pointer to the packet + virtual const void *GetPacket() const { return &m_packetData; } + + void InitPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, unsigned int dispatchInit); + + private: + // PM4DISPATCH_DIRECT struct contains all the packets data + PM4DISPATCH_DIRECT m_packetData; +}; + +// @class PM4PartialFlushPacket +class PM4PartialFlushPacket : public PM4Packet { + public: + PM4PartialFlushPacket(void); + virtual ~PM4PartialFlushPacket(void) {} + + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const; + // @returns a pointer to the packet + virtual const void *GetPacket() const { return &m_packetData; } + + private: + // PM4EVENT_WRITE struct contains all the packets data + PM4EVENT_WRITE m_packetData; +}; + +// @class PM4NopPacket +class PM4NopPacket : public PM4Packet { + public: + PM4NopPacket(void); + virtual ~PM4NopPacket(void) {} + + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const; + // @returns a pointer to the packet + virtual const void *GetPacket() const { return &m_packetData; } + + private: + PM4_TYPE_3_HEADER m_packetData; +}; + +#endif // __KFD_PM4_PACKET__H__ diff --git a/tests/kfdtest/src/PM4Queue.cpp b/tests/kfdtest/src/PM4Queue.cpp new file mode 100644 index 0000000000..f569c84ef1 --- /dev/null +++ b/tests/kfdtest/src/PM4Queue.cpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "PM4Queue.hpp" +#include "pm4_pkt_struct_common.h" +#include "GoogleTestExtension.hpp" +#include "kfd_pm4_opcodes.h" + + +PM4Queue::PM4Queue(void) { + CMD_NOP = CMD_NOP_TYPE_3; +} + +PM4Queue::~PM4Queue(void) { +} + +unsigned int PM4Queue::Wptr() { + /* Write pointer in dwords. Simulate 32-bit wptr that wraps at + * queue size even on Vega10 and later chips with 64-bit wptr. */ + return *m_Resources.Queue_write_ptr % (m_QueueBuf->Size() / 4); +} + +unsigned int PM4Queue::Rptr() { + /* CP read pointer in dwords. It's still 32-bit even on Vega10. */ + return *m_Resources.Queue_read_ptr; +} + +unsigned int PM4Queue::RptrWhenConsumed() { + /* On PM4 queues Rptr is always 32-bit in dword units and wraps at + * queue size. The expected value when all packets are consumed is + * exactly the value returned by Wptr(). */ + return Wptr(); +} + +void PM4Queue::SubmitPacket() { + // m_pending Wptr is in DWORDs + if (g_TestGPUFamilyId < FAMILY_AI) { + // Pre-Vega10 uses 32-bit wptr and doorbell + MemoryBarrier(); + *m_Resources.Queue_write_ptr = m_pendingWptr; + MemoryBarrier(); + *(m_Resources.Queue_DoorBell) = m_pendingWptr; + } else { + // Vega10 and later uses 64-bit wptr and doorbell + MemoryBarrier(); + *m_Resources.Queue_write_ptr_aql = m_pendingWptr64; + MemoryBarrier(); + *(m_Resources.Queue_DoorBell_aql) = m_pendingWptr64; + } +} + diff --git a/tests/kfdtest/src/PM4Queue.hpp b/tests/kfdtest/src/PM4Queue.hpp new file mode 100644 index 0000000000..2acd767622 --- /dev/null +++ b/tests/kfdtest/src/PM4Queue.hpp @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_PM4_QUEUE__H__ +#define __KFD_PM4_QUEUE__H__ + +#include "BaseQueue.hpp" +#include "PM4Packet.hpp" + +class PM4Queue : public BaseQueue { + public: + PM4Queue(void); + virtual ~PM4Queue(void); + + // @brief update queue write pointer and sets the queue doorbell to the queue write pointer + virtual void SubmitPacket(); + + // @ return read pointer modulo queue size in DWORDs + virtual unsigned int Rptr(); + // @ return write pointer modulo queue size in DWORDs + virtual unsigned int Wptr(); + // @ return expected m_Resources.Queue_read_ptr when all packets consumed + virtual unsigned int RptrWhenConsumed(); + + protected: + virtual PACKETTYPE PacketTypeSupported() { return PACKETTYPE_PM4; } + + virtual _HSA_QUEUE_TYPE GetQueueType() { return HSA_QUEUE_COMPUTE; } +}; + +#endif // __KFD_PM4_QUEUE__H__ diff --git a/tests/kfdtest/src/RDMATest.cpp b/tests/kfdtest/src/RDMATest.cpp new file mode 100644 index 0000000000..59a25f08f6 --- /dev/null +++ b/tests/kfdtest/src/RDMATest.cpp @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "RDMATest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "Dispatch.hpp" +#include "RDMAUtil.hpp" + +void RDMATest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void RDMATest::TearDown() { + ROUTINE_START + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(RDMATest, GPUDirect) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + HSAuint64 AlternateVAGPU; + + PM4Queue queue; + unsigned int BufferSize = PAGE_SIZE; + int ret; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer srcSysBuffer(BufferSize, defaultGPUNode, false); + HsaMemoryBuffer srcLocalBuffer(BufferSize, defaultGPUNode, false, true); + + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(srcSysBuffer.As(), + srcSysBuffer.Size(), + &AlternateVAGPU)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(srcLocalBuffer.As(), + srcLocalBuffer.Size(), + &AlternateVAGPU)); + + /* Fill up srcSysBuffer */ + srcSysBuffer.Fill(0xfe); + + /* Put 'copy dword' command to ISA buffer */ + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + Dispatch dispatch(isaBuffer); + + /* Submit the command to GPU so GPU will copy from system memory + * (srcSysBuffer) to local memory(srcLocalBuffer) + */ + dispatch.SetArgs(srcSysBuffer.As(), srcLocalBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); // GPU executed the command + + ASSERT_SUCCESS(queue.Destroy()); + + LocalMemoryAccess Rdma; + + Rdma.Open(); + ASSERT_GE(Rdma.fd, 0) << "Failed to open RDMA"; + + /* GetPages asks the test driver to convert GPU virtual memory to DMA/ + * Physical memory and save it in the list. rdma_mmap maps the memory to + * user space memory. + */ + ret = Rdma.GetPages((uint64_t)srcLocalBuffer.As(), PAGE_SIZE); + ASSERT_EQ(ret, 0) << "Failed to get pages"; + + void *gpuAddr = Rdma.MMap((uint64_t)srcLocalBuffer.As(), PAGE_SIZE); + ASSERT_GE((uint64_t)gpuAddr, 0) << "Failed to map RDMA address."; + + /* Read the memory to confirm that application can read the local memory + * correctly from the mapped address. + */ + ASSERT_EQ(memcmp(gpuAddr, srcSysBuffer.As(), 4), 0); + + Rdma.UnMap(gpuAddr, PAGE_SIZE); + Rdma.Close(); + + TEST_END +} diff --git a/tests/kfdtest/src/RDMATest.hpp b/tests/kfdtest/src/RDMATest.hpp new file mode 100644 index 0000000000..e15ea2395b --- /dev/null +++ b/tests/kfdtest/src/RDMATest.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __RDMA_TEST__H__ +#define __RDMA_TEST__H__ + +#include + +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" + +class RDMATest : public KFDBaseComponentTest { + public: + RDMATest():m_pIsaGen(NULL) {} + ~RDMATest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: // members + IsaGenerator* m_pIsaGen; +}; + +#endif // __RDMA_TEST__H__ diff --git a/tests/kfdtest/src/RDMAUtil.cpp b/tests/kfdtest/src/RDMAUtil.cpp new file mode 100644 index 0000000000..20246f9003 --- /dev/null +++ b/tests/kfdtest/src/RDMAUtil.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "amdp2ptest.h" +#include "RDMAUtil.hpp" + +void LocalMemoryAccess::Open() { + fd = open(AMDP2PTEST_DEVICE_PATH, O_RDWR); +} + +void LocalMemoryAccess::Close() { + close(fd); + fd = -1; +} + +int LocalMemoryAccess::GetPages(uint64_t gpu_va_addr, uint64_t size) { + struct AMDRDMA_IOCTL_GET_PAGES_PARAM param = {0}; + + if (fd <= 0) + return -1; + + param.addr = gpu_va_addr; + param.length = size; + + return ioctl(fd, AMD2P2PTEST_IOCTL_GET_PAGES, ¶m); +} + +void *LocalMemoryAccess::MMap(uint64_t offset, size_t size) { + void *gpuAddr; + + if (fd <= 0) + return NULL; + + gpuAddr = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, offset); + return gpuAddr; +} + +void LocalMemoryAccess::UnMap(void *offset, size_t size) { + munmap(offset, size); +} diff --git a/tests/kfdtest/src/RDMAUtil.hpp b/tests/kfdtest/src/RDMAUtil.hpp new file mode 100644 index 0000000000..335a07c404 --- /dev/null +++ b/tests/kfdtest/src/RDMAUtil.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2016-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __RDMA__UTIL__H__ +#define __RDMA__UTIL__H__ + +class LocalMemoryAccess { + public: + int fd; + void Open(void); + void Close(void); + int GetPages(uint64_t, uint64_t); + void *MMap(uint64_t, size_t); + void UnMap(void *, size_t); +}; + +#endif // __RDMA__UTIL__H__ diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp new file mode 100644 index 0000000000..e7d1020546 --- /dev/null +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include +#include "SDMAPacket.hpp" +#include "KFDTestUtil.hpp" + +/* Byte/dword cound in many SDMA packets is 1-based in AI, meaning a + * count of 1 is encoded as 0. */ +#define SDMA_COUNT(c) (g_TestGPUFamilyId < FAMILY_AI ? (c) : (c)-1) + +SDMAWriteDataPacket::SDMAWriteDataPacket(void): + packetData(NULL) { +} + +SDMAWriteDataPacket::SDMAWriteDataPacket(void* destAddr, unsigned int data): + packetData(NULL) { + InitPacket(destAddr, data); +} + +SDMAWriteDataPacket::SDMAWriteDataPacket(void* destAddr, unsigned int ndw, + void *data): + packetData(NULL) { + InitPacket(destAddr, ndw, data); +} + +SDMAWriteDataPacket::~SDMAWriteDataPacket(void) { + if (packetData) + free(packetData); +} + +void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int data) { + InitPacket(destAddr, 1, &data); +} + +void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int ndw, + void *data) { + packetSize = sizeof(SDMA_PKT_WRITE_UNTILED) + + (ndw - 1) * sizeof(unsigned int); + packetData = (SDMA_PKT_WRITE_UNTILED *)calloc(1, packetSize); + + packetData->HEADER_UNION.op = SDMA_OP_WRITE; + packetData->HEADER_UNION.sub_op = SDMA_SUBOP_WRITE_LINEAR; + + SplitU64(reinterpret_cast(destAddr), + packetData->DST_ADDR_LO_UNION.DW_1_DATA, // dst_addr_31_0 + packetData->DST_ADDR_HI_UNION.DW_2_DATA); // dst_addr_63_32 + + packetData->DW_3_UNION.count = SDMA_COUNT(ndw); + memcpy(&packetData->DATA0_UNION.DW_4_DATA, data, ndw*sizeof(unsigned int)); +} + +#define TWO_MEG (1 << 21) +SDMACopyDataPacket::~SDMACopyDataPacket(void) { + free(packetData); +} + +SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsize) { + int32_t size = 0; + packetSize = ((surfsize + TWO_MEG - 1) >> 21) * sizeof(SDMA_PKT_COPY_LINEAR); + + SDMA_PKT_COPY_LINEAR *pSDMA = (SDMA_PKT_COPY_LINEAR *)malloc(packetSize); + packetData = pSDMA; + + while (surfsize > 0) { + /* sdma support maximum 0x3fffe0 byte in one copy, take 2M here */ + if (surfsize > TWO_MEG) + size = TWO_MEG; + else + size = surfsize; + + memset(pSDMA, 0, sizeof(SDMA_PKT_COPY_LINEAR)); + pSDMA->HEADER_UNION.op = SDMA_OP_COPY; + pSDMA->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR; + pSDMA->COUNT_UNION.count = SDMA_COUNT(size); + SplitU64(reinterpret_cast(src), + pSDMA->SRC_ADDR_LO_UNION.DW_3_DATA, // src_addr_31_0 + pSDMA->SRC_ADDR_HI_UNION.DW_4_DATA); // src_addr_63_32 + SplitU64(reinterpret_cast(dst), + pSDMA->DST_ADDR_LO_UNION.DW_5_DATA, // dst_addr_31_0 + pSDMA->DST_ADDR_HI_UNION.DW_6_DATA); // dst_addr_63_32 + + pSDMA++; + src = (char *)src + size; + dst = (char *)dst + size; + surfsize -= size; + } +} + +SDMAFillDataPacket::~SDMAFillDataPacket() { + free(m_PacketData); +} + +SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned int size) { + unsigned int copy_size; + SDMA_PKT_CONSTANT_FILL *pSDMA; + + /* SDMA support maximum 0x3fffe0 byte in one copy. Use 2M copy_size */ + m_PacketSize = ((size + TWO_MEG - 1) >> 21) * sizeof(SDMA_PKT_CONSTANT_FILL); + pSDMA = (SDMA_PKT_CONSTANT_FILL *)calloc(1, m_PacketSize); + m_PacketData = pSDMA; + + while (size > 0) { + if (size > TWO_MEG) + copy_size = TWO_MEG; + else + copy_size = size; + + pSDMA->HEADER_UNION.op = SDMA_OP_CONST_FILL; + pSDMA->HEADER_UNION.sub_op = 0; + + /* If Both size and address are DW aligned, then use DW fill */ + if (!(copy_size & 0x3) && !((HSAuint64)dst & 0x3)) + pSDMA->HEADER_UNION.fillsize = 2; /* DW Fill */ + else + pSDMA->HEADER_UNION.fillsize = 0; /* Byte Fill */ + + pSDMA->COUNT_UNION.count = SDMA_COUNT(copy_size); + + SplitU64(reinterpret_cast(dst), + pSDMA->DST_ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ + pSDMA->DST_ADDR_HI_UNION.DW_2_DATA); /*dst_addr_63_32*/ + + pSDMA->DATA_UNION.DW_3_DATA = data; + pSDMA++; + + dst = (char *)dst + copy_size; + size -= copy_size; + } +} + +SDMAFencePacket::SDMAFencePacket(void) { +} + +SDMAFencePacket::SDMAFencePacket(void* destAddr, unsigned int data) { + InitPacket(destAddr, data); +} + +SDMAFencePacket::~SDMAFencePacket(void) { +} + +void SDMAFencePacket::InitPacket(void* destAddr, unsigned int data) { + memset(&packetData, 0, SizeInBytes()); + + packetData.HEADER_UNION.op = SDMA_OP_FENCE; + + SplitU64(reinterpret_cast(destAddr), + packetData.ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ + packetData.ADDR_HI_UNION.DW_2_DATA); /*dst_addr_63_32*/ + + packetData.DATA_UNION.data = data; +} + + +SDMATrapPacket::SDMATrapPacket(unsigned int eventID) { + InitPacket(eventID); +} + +SDMATrapPacket::~SDMATrapPacket(void) { +} + +void SDMATrapPacket::InitPacket(unsigned int eventID) { + memset(&packetData, 0, SizeInBytes()); + + packetData.HEADER_UNION.op = SDMA_OP_TRAP; + packetData.INT_CONTEXT_UNION.int_context = eventID; +} diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp new file mode 100644 index 0000000000..89ad4cac07 --- /dev/null +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_SDMA_PACKET__H__ +#define __KFD_SDMA_PACKET__H__ + +#include "BasePacket.hpp" +#include "sdma_pkt_struct.h" + +// @class SDMSPacket: marks a group of all SDMA packets +class SDMAPacket : public BasePacket { + public: + SDMAPacket(void) {} + virtual ~SDMAPacket(void) {} + + virtual PACKETTYPE PacketType() const { return PACKETTYPE_SDMA; } +}; + +class SDMAWriteDataPacket : public SDMAPacket { + public: + // empty constructor, befor using the packet call the init func + SDMAWriteDataPacket(void); + // this contructor will also init the packet, no need for adittional calls + SDMAWriteDataPacket(void* destAddr, unsigned int data); + SDMAWriteDataPacket(void* destAddr, unsigned int ndw, void *data); + + virtual ~SDMAWriteDataPacket(void); + + // @returns a pointer to the packet + virtual const void *GetPacket() const { return packetData; } + // @breif initialise the packet + void InitPacket(void* destAddr, unsigned int data); + void InitPacket(void* destAddr, unsigned int ndw, void *data); + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const { return packetSize; } + + protected: + // SDMA_PKT_WRITE_UNTILED struct contains all the packets data + SDMA_PKT_WRITE_UNTILED *packetData; + unsigned int packetSize; +}; + +class SDMACopyDataPacket : public SDMAPacket { + public: + // this contructor will also init the packet, no need for adittional calls + SDMACopyDataPacket(void *dest, void *src, unsigned int size); + + virtual ~SDMACopyDataPacket(void); + + // @returns a pointer to the packet + virtual const void *GetPacket() const { return packetData; } + + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const { return packetSize; } + + protected: + // SDMA_PKT_COPY_LINEAR struct contains all the packets data + SDMA_PKT_COPY_LINEAR *packetData; + + unsigned int packetSize; +}; + +class SDMAFillDataPacket : public SDMAPacket { + public: + // this contructor will also init the packet, no need for adittional calls + SDMAFillDataPacket(void *dest, unsigned int data, unsigned int size); + + virtual ~SDMAFillDataPacket(void); + + // @returns a pointer to the packet + virtual const void *GetPacket() const { return m_PacketData; } + + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const { return m_PacketSize; } + + protected: + // SDMA_PKT_CONSTANT_FILL struct contains all the packets data + SDMA_PKT_CONSTANT_FILL *m_PacketData; + + unsigned int m_PacketSize; +}; + +class SDMAFencePacket : public SDMAPacket { + public: + // empty constructor, befor using the packet call the init func + SDMAFencePacket(void); + // this contructor will also init the packet, no need for adittional calls + SDMAFencePacket(void* destAddr, unsigned int data); + + virtual ~SDMAFencePacket(void); + + // @returns a pointer to the packet + virtual const void *GetPacket() const { return &packetData; } + // @brief initialise the packet + void InitPacket(void* destAddr, unsigned int data); + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const { return sizeof(SDMA_PKT_FENCE ); } + + protected: + // SDMA_PKT_FENCE struct contains all the packets data + SDMA_PKT_FENCE packetData; +}; + +class SDMATrapPacket : public SDMAPacket { + public: + // empty constructor, befor using the packet call the init func + SDMATrapPacket(unsigned int eventID = 0); + + virtual ~SDMATrapPacket(void); + + // @returns a pointer to the packet + virtual const void *GetPacket() const { return &packetData; } + // @brief initialise the packet + void InitPacket(unsigned int eventID); + // @returns the packet size in bytes + virtual unsigned int SizeInBytes() const { return sizeof(SDMA_PKT_TRAP); } + + protected: + // SDMA_PKT_TRAP struct contains all the packets data + SDMA_PKT_TRAP packetData; +}; + +#endif // __KFD_SDMA_PACKET__H__ diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/tests/kfdtest/src/SDMAQueue.cpp new file mode 100644 index 0000000000..982d919556 --- /dev/null +++ b/tests/kfdtest/src/SDMAQueue.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "SDMAQueue.hpp" + +SDMAQueue::SDMAQueue(void) { + CMD_NOP = 0; +} + +SDMAQueue::~SDMAQueue(void) { +} + +unsigned int SDMAQueue::Wptr() { + /* In SDMA queues write pointers are saved in bytes, convert the + * wptr value to DWORD to fit the way BaseQueue works. On Vega10 + * the write ptr is 64-bit. We only read the low 32 bit (assuming + * the queue buffer is smaller than 4GB) and modulo divide by the + * queue size to simulate a 32-bit read pointer. */ + return (*m_Resources.Queue_write_ptr % m_QueueBuf->Size()) / + sizeof(unsigned int); +} + +unsigned int SDMAQueue::Rptr() { + /* In SDMA queues read pointers are saved in bytes, convert the + * read value to DWORD to fit the way BaseQueue works. On Vega10 + * the read ptr is 64-bit. We only read the low 32 bit (assuming + * the queue buffer is smaller than 4GB) and modulo divide by the + * queue size to simulate a 32-bit read pointer. */ + return (*m_Resources.Queue_read_ptr % m_QueueBuf->Size()) / + sizeof(unsigned int); +} + +unsigned int SDMAQueue::RptrWhenConsumed() { + /* Rptr is same size and byte units as Wptr. Here we only care + * about the low 32-bits. When all packets are consumed, read and + * write pointers should have the same value. */ + return *m_Resources.Queue_write_ptr; +} + +void SDMAQueue::SubmitPacket() { + // m_pending Wptr is in DWORDs + if (g_TestGPUFamilyId < FAMILY_AI) { + // Pre-Vega10 uses 32-bit wptr and doorbell + unsigned int wPtrInBytes = m_pendingWptr * sizeof(unsigned int); + MemoryBarrier(); + *m_Resources.Queue_write_ptr = wPtrInBytes; + MemoryBarrier(); + *(m_Resources.Queue_DoorBell) = wPtrInBytes; + } else { + // Vega10 and later uses 64-bit wptr and doorbell + HSAuint64 wPtrInBytes = m_pendingWptr64 * sizeof(unsigned int); + MemoryBarrier(); + *m_Resources.Queue_write_ptr_aql = wPtrInBytes; + MemoryBarrier(); + *(m_Resources.Queue_DoorBell_aql) = wPtrInBytes; + } +} + diff --git a/tests/kfdtest/src/SDMAQueue.hpp b/tests/kfdtest/src/SDMAQueue.hpp new file mode 100644 index 0000000000..7cf931368a --- /dev/null +++ b/tests/kfdtest/src/SDMAQueue.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_SDMA_QUEUE__H__ +#define __KFD_SDMA_QUEUE__H__ + +#include "BaseQueue.hpp" + +class SDMAQueue : public BaseQueue { + public: + SDMAQueue(void); + virtual ~SDMAQueue(void); + + // @brief update queue write pointer and sets the queue doorbell to the queue write pointer + virtual void SubmitPacket(); + + protected: + // @ return write pointer modulo queue size in DWORDs + virtual unsigned int Wptr(); + // @ return read pointer modulo queue size in DWORDs + virtual unsigned int Rptr(); + // @ return expected m_Resources.Queue_read_ptr when all packets consumed + virtual unsigned int RptrWhenConsumed(); + + virtual PACKETTYPE PacketTypeSupported() { return PACKETTYPE_SDMA; } + + virtual _HSA_QUEUE_TYPE GetQueueType() { return HSA_QUEUE_SDMA; } +}; + +#endif // __KFD_SDMA_QUEUE__H__ From f8472a055c3818fc6b5efc9a1fb0a5c6182e9add Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 30 Jul 2018 16:08:33 -0400 Subject: [PATCH 0361/1247] kfdtest: Use libhsakmt to replace all the occurrences of thunk Thunk is an internal name and we'd better reference it using the library name. Change-Id: I20042bda546e5249530311d3de30c71d99379033 Signed-off-by: Yong Zhao --- tests/kfdtest/CMakeLists.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index fe7fed8132..5afc3f3619 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -21,7 +21,7 @@ # # -# If environment variable DRM_DIR or THUNK_LIB_OUT is set, the script +# If environment variable DRM_DIR or LIBHSAKMT_PATH is set, the script # will pick up the corresponding libraries from those pathes. cmake_minimum_required(VERSION 2.8 FATAL_ERROR) @@ -48,24 +48,24 @@ else() endif() -if( DEFINED ENV{THUNK_LIB_OUT} ) - set ( THUNK_LIB_OUT $ENV{THUNK_LIB_OUT} ) - message ( " THUNK_LIB_OUT environment variable is set" ) +if( DEFINED ENV{LIBHSAKMT_PATH} ) + set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} ) + message ( "LIBHSAKMT_PATH environment variable is set" ) else() set ( ENV{PKG_CONFIG_PATH} /opt/rocm/libhsakmt/ ) pkg_check_modules(HSAKMT libhsakmt) if( NOT HSAKMT_FOUND ) - set ( THUNK_LIB_OUT $ENV{OUT_DIR} ) + set ( LIBHSAKMT_PATH $ENV{OUT_DIR} ) endif() endif() -if( DEFINED THUNK_LIB_OUT ) - set ( HSAKMT_INCLUDE_DIRS ${THUNK_LIB_OUT}/include ) - set ( HSAKMT_LIBRARY_DIRS ${THUNK_LIB_OUT}/lib ) +if( DEFINED LIBHSAKMT_PATH ) + set ( HSAKMT_INCLUDE_DIRS ${LIBHSAKMT_PATH}/include ) + set ( HSAKMT_LIBRARY_DIRS ${LIBHSAKMT_PATH}/lib ) set ( HSAKMT_LIBRARIES hsakmt ) endif() -message ( "Find thunk lib at ${HSAKMT_LIBRARY_DIRS}" ) +message ( "Find libhsakmt at ${HSAKMT_LIBRARY_DIRS}" ) set ( SP3_DIR ${PROJECT_SOURCE_DIR}/sp3 ) From 1d43938ac7789f2e2d0b89e4f9fe6cd2569a6982 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 23 Jul 2018 21:32:55 -0400 Subject: [PATCH 0362/1247] kfdtest: Add run utility files for kfdtest A README.txt file is added to help the opensource community to use kfdtest effectively. After building, run_kfdtest.sh in the building output folder can be used to run the test. Change-Id: I9612d9d5a63bd4cdc3a328efd9961d3cc92a6ba5 Signed-off-by: Yong Zhao --- tests/kfdtest/CMakeLists.txt | 2 + tests/kfdtest/README.txt | 20 ++ tests/kfdtest/{ => scripts}/kfdtest.exclude | 0 tests/kfdtest/scripts/run_kfdtest.sh | 217 ++++++++++++++++++++ 4 files changed, 239 insertions(+) create mode 100644 tests/kfdtest/README.txt rename tests/kfdtest/{ => scripts}/kfdtest.exclude (100%) create mode 100755 tests/kfdtest/scripts/run_kfdtest.sh diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 5afc3f3619..b79f880a74 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -106,3 +106,5 @@ add_executable(kfdtest ${SRC_FILES}) target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LIBRARIES} ${DRM_AMDGPU_LIBRARIES} pthread m stdc++ rt amdsp3) +configure_file ( scripts/kfdtest.exclude kfdtest.exclude COPYONLY ) +configure_file ( scripts/run_kfdtest.sh run_kfdtest.sh COPYONLY ) diff --git a/tests/kfdtest/README.txt b/tests/kfdtest/README.txt new file mode 100644 index 0000000000..b892ba5df1 --- /dev/null +++ b/tests/kfdtest/README.txt @@ -0,0 +1,20 @@ +1. Note on building kfdtest + +To build this kfdtest application, the following libraries should be already +installed on the building machine: +libdrm libdrm_amdgpu libhsakmt + +If libhsakmt is not installed, but the headers and libraries are present +locally, you can specify its directory by +export LIBHSAKMT_PATH=/*your local libhsakmt folder*/ +With that, the headers and libraries are searched under +LIBHSAKMT_PATH/include and LIBHSAKMT_PATH/lib respectively. + + +2. How to run kfdtest + +Just run "./run_kfdtest.sh" under the building output folder. You may need +to specify library path through: +export LD_LIBRARY_PATH=/*your library path containing libhsakmt*/ + +Note: you can use "run_kfdtest.sh -h" to see more options. diff --git a/tests/kfdtest/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude similarity index 100% rename from tests/kfdtest/kfdtest.exclude rename to tests/kfdtest/scripts/kfdtest.exclude diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh new file mode 100755 index 0000000000..a36ebc957b --- /dev/null +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -0,0 +1,217 @@ +#!/bin/bash +# +# Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# + + +if [ "$BIN_DIR" == "" ]; then + BIN_DIR="$(pwd)/$(dirname $0)" +fi + +PLATFORM="" +GDB="" +NODE="" +MULTI_GPU="" +FORCE_HIGH="" + +printUsage() { + echo + echo "Usage: $(basename $0) [options ...] [gtest arguments]" + echo + echo "Options:" + echo " -p , --platform Only run tests that"\ + "pass on the specified platform" + echo " -g , --gdb Run in debugger" + echo " -n , --node NodeId to test. If"\ + "not specified test will be run on all nodes" + echo " -l , --list List available nodes" + echo " --high Force clocks to high for test execution" + echo " -h , --help Prints this help" + echo + echo "Gtest arguments will be forwarded to the app" + echo + echo "Valid platform options: cz, kv, tg, fj, hi, pl/el, plb/bf, vg10, all" + echo "'all' option runs all tests" + + return 0 +} +# Print gtest_filter for the given Platform +# If MULTI_GPU flag is set, then Multi-GPU Tests are selected. Once all tests +# pass in Multi GPU environment, this flag can be removed +# param - Platform. +getFilter() { + local platform=$1; + case "$platform" in + cz ) FILTER="--gtest_filter=$CZ_TESTS_BLACKLIST" ;; + hi ) FILTER="--gtest_filter=$HI_TESTS_BLACKLIST" ;; + kv ) FILTER="--gtest_filter=$KV_TESTS_BLACKLIST" ;; + tg ) FILTER="--gtest_filter=$TONGA_TESTS_BLACKLIST" ;; + fj ) FILTER="--gtest_filter=$FIJI_TESTS_BLACKLIST" ;; + pl | el ) FILTER="--gtest_filter=$ELLESMERE_TESTS_BLACKLIST" ;; + plb | bf ) FILTER="--gtest_filter=$BAFFIN_TESTS_BLACKLIST" ;; + vg10 ) FILTER="--gtest_filter=$VEGA10_TESTS_BLACKLIST" ;; + vg20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; + rv ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; + all ) FILTER="" ;; + *) die "Unsupported platform $PLATFORM or node $NODE. Exiting" ;; + esac + echo "$FILTER" +} + +TOPOLOGY_SYSFS_DIR=/sys/devices/virtual/kfd/kfd/topology/nodes + +# Prints list of HSA Nodes. HSA Nodes are identified from sysfs KFD topology. The nodes +# should have valid SIMD count +getHsaNodes() { + for i in $(find $TOPOLOGY_SYSFS_DIR -maxdepth 1 -mindepth 1 -type d); do + simdcount=$(cat $i/properties | grep simd_count | awk '{print $2}') + if [ $simdcount != 0 ]; then + hsaNodeList+="$(basename $i) " + fi + done + echo "$hsaNodeList" +} + + +# Prints GPU name for a given Device ID +# param - Device ID. +deviceIdToGpuName() { + local deviceId=$1; shift; + case $deviceId in + 1304 | 1305 | 1306 | 1307 | 1309 | 130a | 130b | 130c | 130d | 130e | 130f | \ + 1310 | 1311 | 1312 | 1313 | 1315 | 1316 | 1317 | 1318 | 131b | 131c | 131d ) + platformName="kv" ;; + 67a0 | 67a1 | 67a2 | 67a8 | 67a9 | 67aa | 67b0 | 67b1 | 67b8 | 67b9 | 67ba | 67be ) + platformName="hi" ;; + 9870 | 9874 | 9875 | 9876 | 9877 ) + platformName="cz" ;; + 6920 | 6921 | 6928 | 6929 | 692b | 692f | 6930 | 6938 | 6939 ) + platformName="tg" ;; + 7300 | 730f) + platformName="fj" ;; + 67c0 | 67c1 | 67c2 | 67c4 | 67c7 | 67c8 | 67c9 | 67ca | 67cc | 67cf | 67d0 | 67df ) + platformName="pl" ;; + 67e0 | 67e1 | 67e3 | 67e7 | 67e8 | 67e9 | 67eb | 67ef | 67ff ) + platformName="plb" ;; + 6860 | 6861 | 6862 | 6863 | 6864 | 6867 | 6868 | 6869 | 686a | 686b | 686c | 687f) + platformName="vg10" ;; + 66a0 | 66a1 | 66a2 | 66a3 |66a7 | 66af) + platformName="vg20" ;; + 15dd ) + platformName="rv" ;; + * ) + return ;; + esac + echo "$platformName" +} + + +# Prints GPU Name for the given Node ID +# param - Node ID +getNodeName() { + local nodeId=$1; shift; + local gpuIdInDec=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep device_id | awk '{print $2}') + printf -v gpuIdInHex "%x" "$gpuIdInDec" + local gpuName=$(deviceIdToGpuName $gpuIdInHex) + echo "$gpuName" +} + +# Run KfdTest independently. Two global variables set by command-line +# will influence the tests as indicated below +# PLATFORM - If set all tests will run with this platform filter +# NODE - If set tests will be run only on this NODE, else it will be +# run on all available HSA Nodes +runKfdTest() { + if [ "$NODE" == "" ]; then + hsaNodes=$(getHsaNodes) + else + hsaNodes=$NODE + fi + + source $BIN_DIR/kfdtest.exclude + + for hsaNode in $hsaNodes; do + nodeName=$(getNodeName $hsaNode) + if [ "$PLATFORM" != "" ] && [ "$PLATFORM" != "$nodeName" ]; then + echo "WARNING: Actual ASIC $nodeName treated as $PLATFORM" + nodeName="$PLATFORM" + fi + + gtestFilter=$(getFilter $nodeName) + + echo "" + echo "++++ Starting testing node $hsaNode ($nodeName) ++++" + $GDB $KFDTEST "--node=$hsaNode" $gtestFilter $GTEST_ARGS + echo "---- Finished testing node $hsaNode ($nodeName) ----" + done + +} + +# Prints number of GPUs present in the system +getGPUCount() { + gNodes=$(getHsaNodes) + gNodes=( $gNodes ) + gpuCount=${#gNodes[@]} + echo "$gpuCount" +} + +# set $MULTI_GPU flag if the system has more than 1 GPU +setMultiGPUFlag() { + gpuCount=$(getGPUCount) + if [ $gpuCount -gt 1 ]; then + MULTI_GPU=1 + else + MULTI_GPU=0 + fi +} + +while [ "$1" != "" ]; do + case "$1" in + -p | --platform ) + shift 1; PLATFORM=$1 ;; + -g | --gdb ) + GDB="gdb --args" ;; + -l | --list ) + printGpuNodelist; exit 0 ;; + -n | --node ) + shift 1; NODE=$1 ;; + --high) + FORCE_HIGH="true" ;; + -h | --help ) + printUsage; exit 0 ;; + *) + GTEST_ARGS=$@; break;; + esac + shift 1 +done + +KFDTEST="$BIN_DIR/kfdtest" + +if [ "$FORCE_HIGH" == "true" ]; then + pushGpuDpmState high + pushTrap "popGpuDpmState" EXIT +fi + +setMultiGPUFlag +# Set HSA_DEBUG env to run KFDMemoryTest.PtraceAccessInvisibleVram +export HSA_DEBUG=1 +runKfdTest From 86552aba4ba502fec58d138e0d166ad0922d3b95 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Wed, 1 Aug 2018 10:06:33 +0800 Subject: [PATCH 0363/1247] kfdtest: make the output of QueueLatency test more readable Change-Id: Ib33ac25509b23f2e5869bde126e3f11ef60f017e Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDQMTest.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index c43457fcfe..d88e33c7c1 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1047,6 +1047,7 @@ TEST_F(KFDQMTest, CreateAqlCpQueue) { } #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define CounterToNanoSec(x) ((x) * 1000 / (is_dgpu() ? 27 : 100)) #include @@ -1082,7 +1083,7 @@ TEST_F(KFDQMTest, QueueLatency) { ASSERT_SUCCESS(queue.Create(defaultGPUNode, queueSize)); - LOG() << "Queue Submit Clock Counter (" << slots << " Packets)" << std::endl; + LOG() << std::dec << "Queue Submit NanoSeconds (" << slots << " Packets)" << std::endl; HsaMemoryBuffer buf(ALIGN_UP(slots * sizeof(HsaClockCounters), PAGE_SIZE), 0); ts = buf.As(); @@ -1152,7 +1153,7 @@ TEST_F(KFDQMTest, QueueLatency) { queue_latency_arr[i] -= workload + overhead; /* The First submit takes a long time*/ if (i < skip) - LOG() << "Queue Latency " << fs[i] << ": \t" << queue_latency_arr[i] << std::endl; + LOG() << "Queue Latency " << fs[i] << ": \t" << CounterToNanoSec(queue_latency_arr[i]) << std::endl; } while (++i < slots); std::sort(queue_latency_arr + skip, queue_latency_arr + slots); @@ -1161,12 +1162,12 @@ TEST_F(KFDQMTest, QueueLatency) { queue_latency_med = queue_latency_arr[(slots+skip)/2]; queue_latency_max = queue_latency_arr[slots-1]; - LOG() << "Queue Latency Avg: \t" << queue_latency_avg << std::endl; - LOG() << "Queue Latency Min: \t" << queue_latency_min << std::endl; - LOG() << "Queue Latency Median: \t" << queue_latency_med << std::endl; - LOG() << "Queue Latency Max: \t" << queue_latency_max << std::endl; - LOG() << "Queue Packet Workload: \t" << workload << std::endl; - LOG() << "Get GpuCounter Overhead: \t" << overhead << std::endl; + LOG() << "Queue Latency Avg: \t" << CounterToNanoSec(queue_latency_avg) << std::endl; + LOG() << "Queue Latency Min: \t" << CounterToNanoSec(queue_latency_min) << std::endl; + LOG() << "Queue Latency Median: \t" << CounterToNanoSec(queue_latency_med) << std::endl; + LOG() << "Queue Latency Max: \t" << CounterToNanoSec(queue_latency_max) << std::endl; + LOG() << "Queue Packet Workload: \t" << CounterToNanoSec(workload) << std::endl; + LOG() << "Get GpuCounter Overhead: \t" << CounterToNanoSec(overhead) << std::endl; TEST_END } From f3e787078487f00d0be751b4cee45d776267e035 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 1 Aug 2018 18:33:22 -0400 Subject: [PATCH 0364/1247] kfdtest: Evaluate whether a node is APU based on spec This will facilitate the user cases that some APU asics is used as dGPU. Change-Id: Ib3a79ae31a03e7a618c7785166f56282a7617127 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDTestUtil.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index f3e3c3fefa..f8b36c01f0 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -106,7 +106,7 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { break; } - if (familyId == FAMILY_KV || familyId == FAMILY_CZ || familyId == FAMILY_RV) + if (props->NumCPUCores && props->NumFComputeCores) is_dgpu_dev = false; else is_dgpu_dev = true; From b629dd18884469699b65825a99c829fcc6fc6a0a Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 2 Aug 2018 06:49:48 -0400 Subject: [PATCH 0365/1247] spec: Only remove ldconf file if uninstalling $1 is passed in during install/upgrade/uninstall. If this is an uninstall, remove the ldconf file, otherwise leave it. In yum-based systems, the uninstall from the old package is executed after the post-install of the newer package, which will undo the ldconf work from the newer package. Change-Id: I1681cb3bc65f0bd802bce5703187ae5f4d3f3530 --- RPM/rpm_postun | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RPM/rpm_postun b/RPM/rpm_postun index 0b130e27e8..fb6028fd85 100644 --- a/RPM/rpm_postun +++ b/RPM/rpm_postun @@ -1 +1,3 @@ -rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig +if [ $1 -eq 0 ]; then + rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig +fi From 08b6685dd52ff9d92ab4cae88fc167092b0e2afe Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 1 Aug 2018 18:59:32 -0400 Subject: [PATCH 0366/1247] Change the confusing type and name in topology node is used repeatedly and excessively, which caused unnecessary confusion. Change-Id: I4ae4171887df5e5b85209a5af8a636e6d72e5e82 Signed-off-by: Yong Zhao --- src/topology.c | 235 +++++++++++++++++++++++-------------------------- 1 file changed, 111 insertions(+), 124 deletions(-) diff --git a/src/topology.c b/src/topology.c index 86b812e021..fab47c44fd 100644 --- a/src/topology.c +++ b/src/topology.c @@ -53,10 +53,10 @@ typedef struct { HsaMemoryProperties *mem; /* node->NumBanks elements */ HsaCacheProperties *cache; HsaIoLinkProperties *link; -} node_t; +} node_props_t; static HsaSystemProperties *_system = NULL; -static node_t *node = NULL; +static node_props_t *g_props; static int is_valgrind; static int processor_vendor; @@ -234,30 +234,17 @@ union _cpuid_leaf_ebx { /* Register EBX */ uint32_t full; }; -static void -free_node(node_t *n) +static void free_properties(node_props_t *props, int size) { - assert(n); + if (props) { + int i; + for (i = 0; i < size; i++) { + free(props[i].mem); + free(props[i].cache); + free(props[i].link); + } - if (!n) - return; - - if ((n)->mem) - free((n)->mem); - if ((n)->cache) - free((n)->cache); - if ((n)->link) - free((n)->link); -} - -static void free_nodes(node_t *temp_nodes, int size) -{ - int i; - - if (temp_nodes) { - for (i = 0; i < size; i++) - free_node(&temp_nodes[i]); - free(temp_nodes); + free(props); } } @@ -1082,7 +1069,7 @@ exit: * @cpu_ci_list - the cpu cache information list to look up cache info * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure */ -static HSAKMT_STATUS topology_get_cpu_cache_props(node_t *tbl, +static HSAKMT_STATUS topology_get_cpu_cache_props(node_props_t *tbl, cpu_cacheinfo_t *cpu_ci_list) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -1154,7 +1141,7 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list return HSAKMT_STATUS_SUCCESS; } -static HSAKMT_STATUS topology_get_cpu_cache_props(node_t *tbl, +static HSAKMT_STATUS topology_get_cpu_cache_props(node_props_t *tbl, cpu_cacheinfo_t *cpu_ci_list) { return HSAKMT_STATUS_SUCCESS; @@ -1311,7 +1298,7 @@ err1: */ static HsaIoLinkProperties *topology_get_free_io_link_slot_for_node(uint32_t node_id, const HsaSystemProperties *sys_props, - node_t *nodes) + node_props_t *node_props) { HsaIoLinkProperties *props; @@ -1320,18 +1307,18 @@ static HsaIoLinkProperties *topology_get_free_io_link_slot_for_node(uint32_t nod return NULL; } - props = nodes[node_id].link; + props = node_props[node_id].link; if (!props) { pr_err("No io_link reported for Node [%d]\n", node_id); return NULL; } - if (nodes[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { + if (node_props[node_id].node.NumIOLinks >= sys_props->NumNodes - 1) { pr_err("No more space for io_link for Node [%d]\n", node_id); return NULL; } - return &props[nodes[node_id].node.NumIOLinks]; + return &props[node_props[node_id].node.NumIOLinks]; } /* topology_add_io_link_for_node - If a free slot is available, @@ -1341,7 +1328,7 @@ static HsaIoLinkProperties *topology_get_free_io_link_slot_for_node(uint32_t nod */ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, const HsaSystemProperties *sys_props, - node_t *nodes, + node_props_t *node_props, HSA_IOLINKTYPE IoLinkType, uint32_t NodeTo, uint32_t Weight, bool bi_dir) @@ -1353,7 +1340,7 @@ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, for (i = 0; i < num_links; i++) { props = topology_get_free_io_link_slot_for_node(node_from, - sys_props, nodes); + sys_props, node_props); if (!props) return HSAKMT_STATUS_NO_MEMORY; @@ -1361,7 +1348,7 @@ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, props->NodeFrom = node_from; props->NodeTo = node_to; props->Weight = Weight; - nodes[node_from].node.NumIOLinks++; + node_props[node_from].node.NumIOLinks++; /* switch direction on the 2nd link when num_links=2 */ node_from = NodeTo; node_to = node_id; @@ -1371,16 +1358,16 @@ static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, } /* Find the CPU that this GPU (gpu_node) directly connects to */ -static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, node_t *nodes) +static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, node_props_t *node_props) { - HsaIoLinkProperties *props = nodes[gpu_node].link; + HsaIoLinkProperties *props = node_props[gpu_node].link; uint32_t i; - if (!nodes[gpu_node].gpu_id || !props || - nodes[gpu_node].node.NumIOLinks == 0) + if (!node_props[gpu_node].gpu_id || !props || + node_props[gpu_node].node.NumIOLinks == 0) return -1; - for (i = 0; i < nodes[gpu_node].node.NumIOLinks; i++) + for (i = 0; i < node_props[gpu_node].node.NumIOLinks; i++) if (props[i].IoLinkType == HSA_IOLINKTYPE_PCIEXPRESS && props[i].Weight <= 20) /* >20 is GPU->CPU->GPU */ return props[i].NodeTo; @@ -1392,16 +1379,16 @@ static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, node_t *nodes) * been created in the kernel. */ static HSAKMT_STATUS get_direct_iolink_info(uint32_t node1, uint32_t node2, - node_t *nodes, HSAuint32 *weight, + node_props_t *node_props, HSAuint32 *weight, HSA_IOLINKTYPE *type) { - HsaIoLinkProperties *props = nodes[node1].link; + HsaIoLinkProperties *props = node_props[node1].link; uint32_t i; if (!props) return HSAKMT_STATUS_INVALID_NODE_UNIT; - for (i = 0; i < nodes[node1].node.NumIOLinks; i++) + for (i = 0; i < node_props[node1].node.NumIOLinks; i++) if (props[i].NodeTo == node2) { if (weight) *weight = props[i].Weight; @@ -1414,7 +1401,7 @@ static HSAKMT_STATUS get_direct_iolink_info(uint32_t node1, uint32_t node2, } static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, - node_t *nodes, HSAuint32 *weight, + node_props_t *node_props, HSAuint32 *weight, HSA_IOLINKTYPE *type) { int32_t dir_cpu1 = -1, dir_cpu2 = -1; @@ -1428,13 +1415,13 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, return HSAKMT_STATUS_INVALID_PARAMETER; /* CPU->CPU is not an indirect link */ - if (!nodes[node1].gpu_id && !nodes[node2].gpu_id) + if (!node_props[node1].gpu_id && !node_props[node2].gpu_id) return HSAKMT_STATUS_INVALID_NODE_UNIT; - if (nodes[node1].gpu_id) - dir_cpu1 = gpu_get_direct_link_cpu(node1, nodes); - if (nodes[node2].gpu_id) - dir_cpu2 = gpu_get_direct_link_cpu(node2, nodes); + if (node_props[node1].gpu_id) + dir_cpu1 = gpu_get_direct_link_cpu(node1, node_props); + if (node_props[node2].gpu_id) + dir_cpu2 = gpu_get_direct_link_cpu(node2, node_props); if (dir_cpu1 < 0 && dir_cpu2 < 0) return HSAKMT_STATUS_ERROR; @@ -1449,18 +1436,18 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, if (dir_cpu2 >= 0) { if (dir_cpu1 == dir_cpu2) /* GPU->CPU->GPU*/ { ret = get_direct_iolink_info(node1, dir_cpu1, - nodes, &weight1, NULL); + node_props, &weight1, NULL); if (ret != HSAKMT_STATUS_SUCCESS) return ret; ret = get_direct_iolink_info(dir_cpu1, node2, - nodes, &weight2, type); + node_props, &weight2, type); } else /* GPU->CPU->CPU->GPU*/ { ret = get_direct_iolink_info(node1, dir_cpu1, - nodes, &weight1, NULL); + node_props, &weight1, NULL); if (ret != HSAKMT_STATUS_SUCCESS) return ret; ret = get_direct_iolink_info(dir_cpu1, dir_cpu2, - nodes, &weight2, type); + node_props, &weight2, type); if (ret != HSAKMT_STATUS_SUCCESS) return ret; /* On QPI interconnection, GPUs can't access @@ -1472,22 +1459,22 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, && weight2 > 20) return HSAKMT_STATUS_NOT_SUPPORTED; ret = get_direct_iolink_info(dir_cpu2, node2, - nodes, &weight3, NULL); + node_props, &weight3, NULL); } } else /* GPU->CPU->CPU */ { - ret = get_direct_iolink_info(node1, dir_cpu1, nodes, + ret = get_direct_iolink_info(node1, dir_cpu1, node_props, &weight1, NULL); if (ret != HSAKMT_STATUS_SUCCESS) return ret; - ret = get_direct_iolink_info(dir_cpu1, node2, nodes, + ret = get_direct_iolink_info(dir_cpu1, node2, node_props, &weight2, type); } } else { /* CPU->CPU->GPU */ - ret = get_direct_iolink_info(node1, dir_cpu2, nodes, &weight2, + ret = get_direct_iolink_info(node1, dir_cpu2, node_props, &weight2, type); if (ret != HSAKMT_STATUS_SUCCESS) return ret; - ret = get_direct_iolink_info(dir_cpu2, node2, nodes, &weight3, + ret = get_direct_iolink_info(dir_cpu2, node2, node_props, &weight3, NULL); } @@ -1499,7 +1486,7 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, } static void topology_create_indirect_gpu_links(const HsaSystemProperties *sys_props, - node_t *nodes) + node_props_t *node_props) { uint32_t i, j; @@ -1508,10 +1495,10 @@ static void topology_create_indirect_gpu_links(const HsaSystemProperties *sys_pr for (i = 0; i < sys_props->NumNodes - 1; i++) { for (j = i + 1; j < sys_props->NumNodes; j++) { - get_indirect_iolink_info(i, j, nodes, &weight, &type); + get_indirect_iolink_info(i, j, node_props, &weight, &type); if (!weight) continue; - if (topology_add_io_link_for_node(i, sys_props, nodes, + if (topology_add_io_link_for_node(i, sys_props, node_props, type, j, weight, true) != HSAKMT_STATUS_SUCCESS) pr_err("Fail to add IO link %d->%d\n", i, j); } @@ -1522,7 +1509,7 @@ HSAKMT_STATUS topology_take_snapshot(void) { uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; HsaSystemProperties sys_props; - node_t *temp_nodes = 0; + node_props_t *temp_props = 0; void *cpu_ci_list = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; struct pci_access *pacc; @@ -1544,54 +1531,54 @@ retry: return ret; if (sys_props.NumNodes > 0) { topology_create_temp_cpu_cache_list(&cpu_ci_list); - temp_nodes = calloc(sys_props.NumNodes * sizeof(node_t), 1); - if (!temp_nodes) + temp_props = calloc(sys_props.NumNodes * sizeof(node_props_t), 1); + if (!temp_props) return HSAKMT_STATUS_NO_MEMORY; pacc = pci_alloc(); pci_init(pacc); for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, - &temp_nodes[i].node, - &temp_nodes[i].gpu_id, pacc); + &temp_props[i].node, + &temp_props[i].gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) { - free_nodes(temp_nodes, i); + free_properties(temp_props, i); goto err; } - if (temp_nodes[i].node.NumMemoryBanks) { - temp_nodes[i].mem = calloc(temp_nodes[i].node.NumMemoryBanks * sizeof(HsaMemoryProperties), 1); - if (!temp_nodes[i].mem) { + if (temp_props[i].node.NumMemoryBanks) { + temp_props[i].mem = calloc(temp_props[i].node.NumMemoryBanks * sizeof(HsaMemoryProperties), 1); + if (!temp_props[i].mem) { ret = HSAKMT_STATUS_NO_MEMORY; - free_nodes(temp_nodes, i + 1); + free_properties(temp_props, i + 1); goto err; } - for (mem_id = 0; mem_id < temp_nodes[i].node.NumMemoryBanks; mem_id++) { - ret = topology_sysfs_get_mem_props(i, mem_id, &temp_nodes[i].mem[mem_id]); + for (mem_id = 0; mem_id < temp_props[i].node.NumMemoryBanks; mem_id++) { + ret = topology_sysfs_get_mem_props(i, mem_id, &temp_props[i].mem[mem_id]); if (ret != HSAKMT_STATUS_SUCCESS) { - free_nodes(temp_nodes, i + 1); + free_properties(temp_props, i + 1); goto err; } } } - if (temp_nodes[i].node.NumCaches) { - temp_nodes[i].cache = calloc(temp_nodes[i].node.NumCaches * sizeof(HsaCacheProperties), 1); - if (!temp_nodes[i].cache) { + if (temp_props[i].node.NumCaches) { + temp_props[i].cache = calloc(temp_props[i].node.NumCaches * sizeof(HsaCacheProperties), 1); + if (!temp_props[i].cache) { ret = HSAKMT_STATUS_NO_MEMORY; - free_nodes(temp_nodes, i + 1); + free_properties(temp_props, i + 1); goto err; } - for (cache_id = 0; cache_id < temp_nodes[i].node.NumCaches; cache_id++) { - ret = topology_sysfs_get_cache_props(i, cache_id, &temp_nodes[i].cache[cache_id]); + for (cache_id = 0; cache_id < temp_props[i].node.NumCaches; cache_id++) { + ret = topology_sysfs_get_cache_props(i, cache_id, &temp_props[i].cache[cache_id]); if (ret != HSAKMT_STATUS_SUCCESS) { - free_nodes(temp_nodes, i + 1); + free_properties(temp_props, i + 1); goto err; } } - } else if (!temp_nodes[i].gpu_id) { /* a CPU node */ + } else if (!temp_props[i].gpu_id) { /* a CPU node */ ret = topology_get_cpu_cache_props( - &temp_nodes[i], cpu_ci_list); + &temp_props[i], cpu_ci_list); if (ret != HSAKMT_STATUS_SUCCESS) { - free_nodes(temp_nodes, i + 1); + free_properties(temp_props, i + 1); goto err; } } @@ -1599,18 +1586,18 @@ retry: /* To simplify, allocate maximum needed memory for io_links for each node. This * removes the need for realloc when indirect and QPI links are added later */ - temp_nodes[i].link = calloc(sys_props.NumNodes - 1, sizeof(HsaIoLinkProperties)); - if (!temp_nodes[i].link) { + temp_props[i].link = calloc(sys_props.NumNodes - 1, sizeof(HsaIoLinkProperties)); + if (!temp_props[i].link) { ret = HSAKMT_STATUS_NO_MEMORY; - free_nodes(temp_nodes, i + 1); + free_properties(temp_props, i + 1); goto err; } - if (temp_nodes[i].node.NumIOLinks) { - for (link_id = 0; link_id < temp_nodes[i].node.NumIOLinks; link_id++) { - ret = topology_sysfs_get_iolink_props(i, link_id, &temp_nodes[i].link[link_id]); + if (temp_props[i].node.NumIOLinks) { + for (link_id = 0; link_id < temp_props[i].node.NumIOLinks; link_id++) { + ret = topology_sysfs_get_iolink_props(i, link_id, &temp_props[i].link[link_id]); if (ret != HSAKMT_STATUS_SUCCESS) { - free_nodes(temp_nodes, i+1); + free_properties(temp_props, i+1); goto err; } } @@ -1622,32 +1609,32 @@ retry: /* All direct IO links are created in the kernel. Here we need to * connect GPU<->GPU or GPU<->CPU indirect IO links. */ - topology_create_indirect_gpu_links(&sys_props, temp_nodes); + topology_create_indirect_gpu_links(&sys_props, temp_props); ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { - free_nodes(temp_nodes, sys_props.NumNodes); + free_properties(temp_props, sys_props.NumNodes); goto err; } if (gen_start != gen_end) { - free_nodes(temp_nodes, sys_props.NumNodes); - temp_nodes = 0; + free_properties(temp_props, sys_props.NumNodes); + temp_props = 0; goto retry; } if (!_system) { _system = malloc(sizeof(HsaSystemProperties)); if (!_system) { - free_nodes(temp_nodes, sys_props.NumNodes); + free_properties(temp_props, sys_props.NumNodes); return HSAKMT_STATUS_NO_MEMORY; } } *_system = sys_props; - if (node) - free(node); - node = temp_nodes; + if (g_props) + free(g_props); + g_props = temp_props; err: topology_destroy_temp_cpu_cache_list(cpu_ci_list); return ret; @@ -1658,16 +1645,16 @@ HSAKMT_STATUS topology_drop_snapshot(void) { HSAKMT_STATUS err; - if (!!_system != !!node) { + if (!!_system != !!g_props) { pr_warn("Probably inconsistency?\n"); err = HSAKMT_STATUS_SUCCESS; goto out; } - if (node) { + if (g_props) { /* Remove state */ - free_nodes(node, _system->NumNodes); - node = NULL; + free_properties(g_props, _system->NumNodes); + g_props = NULL; } free(_system); @@ -1680,10 +1667,10 @@ out: HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) { - if (!node || !_system || _system->NumNodes <= nodeid) + if (!g_props || !_system || _system->NumNodes <= nodeid) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (gpu_id) - *gpu_id = node[nodeid].gpu_id; + *gpu_id = g_props[nodeid].gpu_id; return HSAKMT_STATUS_SUCCESS; } @@ -1693,7 +1680,7 @@ HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id) uint64_t node_idx; for (node_idx = 0; node_idx < _system->NumNodes; node_idx++) { - if (node[node_idx].gpu_id == gpu_id) { + if (g_props[node_idx].gpu_id == gpu_id) { *node_id = node_idx; return HSAKMT_STATUS_SUCCESS; } @@ -1771,7 +1758,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, if (err != HSAKMT_STATUS_SUCCESS) return err; - *NodeProperties = node[NodeId].node; + *NodeProperties = g_props[NodeId].node; /* For CPU only node don't add any additional GPU memory banks. */ if (gpu_id) { if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) @@ -1820,9 +1807,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, memset(MemoryProperties, 0, NumBanks * sizeof(HsaMemoryProperties)); - for (i = 0; i < MIN(node[NodeId].node.NumMemoryBanks, NumBanks); i++) { - assert(node[NodeId].mem); - MemoryProperties[i] = node[NodeId].mem[i]; + for (i = 0; i < MIN(g_props[NodeId].node.NumMemoryBanks, NumBanks); i++) { + assert(g_props[NodeId].mem); + MemoryProperties[i] = g_props[NodeId].mem[i]; } /* The following memory banks does not apply to CPU only node */ @@ -1836,7 +1823,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_GPU_LDS; - MemoryProperties[i].SizeInBytes = node[NodeId].node.LDSSizeInKB * 1024; + MemoryProperties[i].SizeInBytes = g_props[NodeId].node.LDSSizeInKB * 1024; i++; } @@ -1844,11 +1831,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, * For dGPU the topology node contains Local Memory and it is added by * the for loop above */ - if (!nodeIsDGPU && i < NumBanks && node[NodeId].node.LocalMemSize > 0 && + if (!nodeIsDGPU && i < NumBanks && g_props[NodeId].node.LocalMemSize > 0 && fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; - MemoryProperties[i].SizeInBytes = node[NodeId].node.LocalMemSize; + MemoryProperties[i].SizeInBytes = g_props[NodeId].node.LocalMemSize; i++; } @@ -1897,14 +1884,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeCacheProperties(HSAuint32 NodeId, goto out; } - if (NodeId >= _system->NumNodes || NumCaches > node[NodeId].node.NumCaches) { + if (NodeId >= _system->NumNodes || NumCaches > g_props[NodeId].node.NumCaches) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } - for (i = 0; i < MIN(node[NodeId].node.NumCaches, NumCaches); i++) { - assert(node[NodeId].cache); - CacheProperties[i] = node[NodeId].cache[i]; + for (i = 0; i < MIN(g_props[NodeId].node.NumCaches, NumCaches); i++) { + assert(g_props[NodeId].cache); + CacheProperties[i] = g_props[NodeId].cache[i]; } err = HSAKMT_STATUS_SUCCESS; @@ -1935,14 +1922,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, goto out; } - if (NodeId >= _system->NumNodes || NumIoLinks > node[NodeId].node.NumIOLinks) { + if (NodeId >= _system->NumNodes || NumIoLinks > g_props[NodeId].node.NumIOLinks) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } - for (i = 0; i < MIN(node[NodeId].node.NumIOLinks, NumIoLinks); i++) { - assert(node[NodeId].link); - IoLinkProperties[i] = node[NodeId].link[i]; + for (i = 0; i < MIN(g_props[NodeId].node.NumIOLinks, NumIoLinks); i++) { + assert(g_props[NodeId].link); + IoLinkProperties[i] = g_props[NodeId].link[i]; } err = HSAKMT_STATUS_SUCCESS; @@ -1954,22 +1941,22 @@ out: uint16_t get_device_id_by_node(HSAuint32 node_id) { - if (!node || !_system || _system->NumNodes <= node_id) + if (!g_props || !_system || _system->NumNodes <= node_id) return 0; - return node[node_id].node.DeviceId; + return g_props[node_id].node.DeviceId; } uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; - if (!node || !_system) + if (!g_props || !_system) return 0; for (i = 0; i < _system->NumNodes; i++) { - if (node[i].gpu_id == gpu_id) - return node[i].node.DeviceId; + if (g_props[i].gpu_id == gpu_id) + return g_props[i].node.DeviceId; } return 0; From f8d19104aa0870c33b298fd7ba549291bd8c4875 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 1 Aug 2018 14:18:40 -0400 Subject: [PATCH 0367/1247] Kfdtest: Change and move drm device function into KFDBaseComponentTest It is for other test to reuse this function. Change-Id: Ib0dbc1a267a5bbcd8078ab3265677b53531f86f3 Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 58 +++++++++++++ tests/kfdtest/src/KFDBaseComponentTest.hpp | 24 +++++- tests/kfdtest/src/KFDGraphicsInterop.cpp | 94 ---------------------- tests/kfdtest/src/KFDGraphicsInterop.hpp | 20 ----- 4 files changed, 79 insertions(+), 117 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 0a7c1659bd..ff7ce063c2 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -36,6 +36,7 @@ void KFDBaseComponentTest::SetUp() { ASSERT_SUCCESS(hsaKmtOpenKFD()); EXPECT_SUCCESS(hsaKmtGetVersion(&m_VersionInfo)); memset( &m_SystemProperties, 0, sizeof(m_SystemProperties) ); + memset(m_RenderNodes, 0, sizeof(m_RenderNodes)); /** in order to be correctly testing the KFD interfaces and ensure * that the KFD acknowledges relevant node parameters @@ -70,6 +71,14 @@ void KFDBaseComponentTest::SetUp() { void KFDBaseComponentTest::TearDown() { ROUTINE_START + for (int i = 0; i < MAX_RENDER_NODES; i++) { + if (m_RenderNodes[i].fd <= 0) + continue; + + amdgpu_device_deinitialize(m_RenderNodes[i].device_handle); + drmClose(m_RenderNodes[i].fd); + } + ASSERT_SUCCESS(hsaKmtReleaseSystemProperties()); ASSERT_SUCCESS(hsaKmtCloseKFD()); @@ -115,3 +124,52 @@ HSAuint64 KFDBaseComponentTest::GetVramSize(int defaultGPUNode) { return 0; } + +int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) { + char path[PATH_MAX], buf[PAGE_SIZE]; + + snprintf(path, PATH_MAX, "/sys/class/kfd/kfd/topology/nodes/%d/properties", gpuNode); + + int fd = open(path, O_RDONLY); + + if (fd < 0) { + LOG() << "Failed to open " << path << std::endl; + return -EINVAL; + } + + read(fd, buf, PAGE_SIZE); + + close(fd); + + char *s = strstr(buf, "drm_render_minor"); + + int minor = atoi(s + 17); + + if (minor < 128) { + LOG() << "Failed to get minor number " << minor << std::endl; + return -EINVAL; + } + + int index = minor - 128; + + if (m_RenderNodes[index].fd == 0) { + m_RenderNodes[index].fd = drmOpenRender(minor); + + if (m_RenderNodes[index].fd < 0) { + LOG() << "Failed to open render node" << std::endl; + return -EINVAL; + } + + if (amdgpu_device_initialize(m_RenderNodes[index].fd, + &m_RenderNodes[index].major_version, + &m_RenderNodes[index].minor_version, + &m_RenderNodes[index].device_handle) != 0) { + drmClose(m_RenderNodes[index].fd); + m_RenderNodes[index].fd = 0; + LOG() << "Failed to initialize amdgpu device" << std::endl; + return -EINVAL; + } + } + + return index; +} diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 18ab7b2a4a..9a12e545ce 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -20,14 +20,20 @@ * OTHER DEALINGS IN THE SOFTWARE. * */ +#ifndef __KFD_BASE_COMPONENT_TEST__H__ +#define __KFD_BASE_COMPONENT_TEST__H__ #include #include "hsakmt.h" #include "OSWrapper.hpp" #include "KFDTestUtil.hpp" - -#ifndef __KFD_BASE_COMPONENT_TEST__H__ -#define __KFD_BASE_COMPONENT_TEST__H__ +#include +#include +#include +#include +#include +#include +#include // @class KFDBaseComponentTest class KFDBaseComponentTest : public testing::Test { @@ -37,6 +43,18 @@ class KFDBaseComponentTest : public testing::Test { HSAuint64 GetSysMemSize(); HSAuint64 GetVramSize(int defaultGPUNode); +#define MAX_RENDER_NODES 64 + struct { + int fd; + uint32_t major_version; + uint32_t minor_version; + amdgpu_device_handle device_handle; + uint32_t bdf; + } m_RenderNodes[MAX_RENDER_NODES]; + +// @brief Finds DRM Render node corresponding to gpuNode +// @return DRM Render Node if successful or -1 on failure + int FindDRMRenderNode(int gpuNode); protected: HsaVersionInfo m_VersionInfo; diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp index b99c1f3b9c..f44130aea8 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.cpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -23,103 +23,9 @@ #include "KFDGraphicsInterop.hpp" -extern "C" { -#include -#include -#include -} -#include -#include -#include -#include -#include #include "Dispatch.hpp" #include "PM4Queue.hpp" -void KFDGraphicsInterop::SetUp() { - ROUTINE_START - - KFDMemoryTest::SetUp(); - - // Try to open and initialize a render node for each device - for (int i = 0; i < MAX_RENDER_NODES; i++) { - m_RenderNodes[i].fd = drmOpenRender(i + 128); - - if (m_RenderNodes[i].fd <= 0) - continue; - - if (amdgpu_device_initialize(m_RenderNodes[i].fd, - &m_RenderNodes[i].major_version, - &m_RenderNodes[i].minor_version, - &m_RenderNodes[i].device_handle) != 0) { - drmClose(m_RenderNodes[i].fd); - m_RenderNodes[i].fd = 0; - } - - // Try to determine the bus-ID from sysfs - char path[PATH_MAX], link[PATH_MAX]; - snprintf(path, PATH_MAX, "/sys/class/drm/renderD%d", i+128); - if (readlink(path, link, PATH_MAX) < 0) { - LOG() << "Failed to read sysfs link " << path - << ", can't determine bus ID." << std::endl; - continue; - } - - char *state = NULL; - char *prev = NULL; - char *tok = strtok_r(link, "/", &state); - while (tok && strcmp(tok, "drm")) { - prev = tok; - tok = strtok_r(NULL, "/", &state); - } - unsigned domain, bus, device, func; - if (!prev || - sscanf(prev, "%04x:%02x:%02x.%1x", &domain, &bus, &device, &func) - != 4) { - LOG() << "Failed to parse sysfs link " << path - << ", can't determine bus ID." << std::endl; - continue; - } - - m_RenderNodes[i].bdf = (bus << 8) | (device << 3) | func; - } - - ROUTINE_END -} - -void KFDGraphicsInterop::TearDown() { - ROUTINE_START - - for (int i = 0; i < MAX_RENDER_NODES; i++) { - if (m_RenderNodes[i].fd <= 0) - continue; - - EXPECT_EQ(0, amdgpu_device_deinitialize(m_RenderNodes[i].device_handle)); - EXPECT_EQ(0, drmClose(m_RenderNodes[i].fd)); - } - - KFDMemoryTest::TearDown(); - - ROUTINE_END -} - -int KFDGraphicsInterop::FindDRMRenderNode(int gpuNode) { - int rn; - const HsaNodeProperties *pNodeProps = - m_NodeInfo.GetNodeProperties(gpuNode); - - for (rn = 0; rn < MAX_RENDER_NODES; rn++) { - if (m_RenderNodes[rn].fd <= 0) - continue; - if (m_RenderNodes[rn].bdf == pNodeProps->LocationId) - return rn; - } - LOG() << "Found no render node corresponding to GPU node " - << gpuNode << std::endl; - LOG() << "Check your device permissions" << std::endl; - return -1; -} - TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { TEST_START(TESTPROFILE_RUNALL) diff --git a/tests/kfdtest/src/KFDGraphicsInterop.hpp b/tests/kfdtest/src/KFDGraphicsInterop.hpp index 9b94062520..260044cebf 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.hpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.hpp @@ -22,9 +22,6 @@ */ #include "KFDMemoryTest.hpp" -extern "C" { -#include -} #ifndef __KFD_GRAPHICS_INTEROP_TEST__H__ #define __KFD_GRAPHICS_INTEROP_TEST__H__ @@ -36,23 +33,6 @@ class KFDGraphicsInterop : public KFDMemoryTest public: KFDGraphicsInterop(void) {}; ~KFDGraphicsInterop(void) {}; -protected: - virtual void SetUp(); - virtual void TearDown(); - -protected: -#define MAX_RENDER_NODES 64 - struct { - int fd; - uint32_t major_version; - uint32_t minor_version; - amdgpu_device_handle device_handle; - uint32_t bdf; - } m_RenderNodes[MAX_RENDER_NODES]; - -// @brief Finds DRM Render node corresponding to gpuNode -// @return DRM Render Node if successful or -1 on failure -int FindDRMRenderNode(int gpuNode); }; #endif From 3167e3b9645315a14131d28395ff454d973aca35 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 1 Aug 2018 14:22:17 -0400 Subject: [PATCH 0368/1247] KFDEvictTest: change buffer size and add GFX vram allocation This is to coordinate kfd kernel vram limit change, and adding GFX vram allocation with submission of command nop is to trigger eviction. Change-Id: I18615cd13cfde034aae09c188ae3a82babde97b9 Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDEvictTest.cpp | 223 +++++++++++++++++++++++++---- tests/kfdtest/src/KFDEvictTest.hpp | 4 +- 2 files changed, 197 insertions(+), 30 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 156dabbf34..70b733b566 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -34,33 +34,13 @@ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) -HSAint32 KFDEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) { - HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT; - HSAuint64 sysMemSize = GetSysMemSize(); - HSAuint64 size, sizeInPages; - HSAuint32 count; - - LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; - - /* use one third of total system memory for eviction buffer to test - * limit max allocate size to duoble of vramSize - * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) - */ - size = sysMemSize/3 + vramSize; - size = size > vramSize<<1 ? vramSize<<1 : size; - sizeInPages = size >> PAGE_SHIFT; - count = sizeInPages / (vramBufSizeInPages * N_PROCESSES); - - return count; -} - void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, std::vector &pBuffers) { HSAuint64 totalMB; totalMB = N_PROCESSES*count*(vramBufSize>>20); if (m_IsParent) { - LOG() << "Testing " << N_PROCESSES << "*" << count << "*" << (vramBufSize>>20) << "(="<< totalMB << ")MB" << std::endl; + LOG() << "Allocating " << N_PROCESSES << "*" << count << "*" << (vramBufSize>>20) << "(="<< totalMB << ")MB VRAM in KFD" << std::endl; } HSAKMT_STATUS ret; @@ -106,6 +86,164 @@ void KFDEvictTest::FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufS } } +void KFDEvictTest::AllocAmdgpuBo(int rn, HSAuint64 vramBufSize, amdgpu_bo_handle &handle) { + struct amdgpu_bo_alloc_request alloc; + + alloc.alloc_size = vramBufSize / N_PROCESSES; + alloc.phys_alignment = PAGE_SIZE; + alloc.preferred_heap = AMDGPU_GEM_DOMAIN_VRAM; + alloc.flags = AMDGPU_GEM_CREATE_VRAM_CLEARED; + + if (m_IsParent) { + LOG() << "Allocating " << N_PROCESSES << "*" << (vramBufSize >> 20) / N_PROCESSES << "(=" << (vramBufSize >> 20) << ")MB VRAM in GFX" << std::endl; + } + ASSERT_EQ(0, amdgpu_bo_alloc(m_RenderNodes[rn].device_handle, &alloc, &handle)); +} + +void KFDEvictTest::FreeAmdgpuBo(amdgpu_bo_handle handle) { + ASSERT_EQ(0, amdgpu_bo_free(handle)); +} + +static int +amdgpu_bo_alloc_and_map(amdgpu_device_handle dev, unsigned size, + unsigned alignment, unsigned heap, uint64_t flags, + amdgpu_bo_handle *bo, void **cpu, uint64_t *mc_address, + amdgpu_va_handle *va_handle) +{ + struct amdgpu_bo_alloc_request request = {}; + amdgpu_bo_handle buf_handle; + amdgpu_va_handle handle; + uint64_t vmc_addr; + int r; + + request.alloc_size = size; + request.phys_alignment = alignment; + request.preferred_heap = heap; + request.flags = flags; + + r = amdgpu_bo_alloc(dev, &request, &buf_handle); + if (r) + return r; + + r = amdgpu_va_range_alloc(dev, + amdgpu_gpu_va_range_general, + size, alignment, 0, &vmc_addr, + &handle, 0); + if (r) + goto error_va_alloc; + + r = amdgpu_bo_va_op(buf_handle, 0, size, vmc_addr, 0, AMDGPU_VA_OP_MAP); + if (r) + goto error_va_map; + + r = amdgpu_bo_cpu_map(buf_handle, cpu); + if (r) + goto error_cpu_map; + + *bo = buf_handle; + *mc_address = vmc_addr; + *va_handle = handle; + + return 0; + +error_cpu_map: + amdgpu_bo_cpu_unmap(buf_handle); + +error_va_map: + amdgpu_bo_va_op(buf_handle, 0, size, vmc_addr, 0, AMDGPU_VA_OP_UNMAP); + +error_va_alloc: + amdgpu_bo_free(buf_handle); + return r; +} + +static inline int +amdgpu_bo_unmap_and_free(amdgpu_bo_handle bo, amdgpu_va_handle va_handle, + uint64_t mc_addr, uint64_t size) +{ + amdgpu_bo_cpu_unmap(bo); + amdgpu_bo_va_op(bo, 0, size, mc_addr, 0, AMDGPU_VA_OP_UNMAP); + amdgpu_va_range_free(va_handle); + amdgpu_bo_free(bo); + + return 0; + +} + +static inline int +amdgpu_get_bo_list(amdgpu_device_handle dev, amdgpu_bo_handle bo1, + amdgpu_bo_handle bo2, amdgpu_bo_list_handle *list) +{ + amdgpu_bo_handle resources[] = {bo1, bo2}; + + return amdgpu_bo_list_create(dev, bo2 ? 2 : 1, resources, NULL, list); +} + +void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { + amdgpu_context_handle contextHandle; + amdgpu_bo_handle ibResultHandle; + void *ibResultCpu; + uint64_t ibResultMcAddress; + struct amdgpu_cs_request ibsRequest; + struct amdgpu_cs_ib_info ibInfo; + struct amdgpu_cs_fence fenceStatus; + amdgpu_bo_list_handle boList; + amdgpu_va_handle vaHandle; + uint32_t *ptr; + uint32_t expired; + + ASSERT_EQ(0, amdgpu_cs_ctx_create(m_RenderNodes[rn].device_handle, &contextHandle)); + + ASSERT_EQ(0, amdgpu_bo_alloc_and_map(m_RenderNodes[rn].device_handle, + PAGE_SIZE, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_GTT, 0, + &ibResultHandle, &ibResultCpu, + &ibResultMcAddress, &vaHandle)); + + ASSERT_EQ(0, amdgpu_get_bo_list(m_RenderNodes[rn].device_handle, ibResultHandle, NULL, + &boList)); + + /* Fill Nop cammands in IB */ + ptr = (uint32_t *)ibResultCpu; + for (int i = 0; i < 16; i++) + ptr[i] = 0xffff1000; + + memset(&ibInfo, 0, sizeof(struct amdgpu_cs_ib_info)); + ibInfo.ib_mc_address = ibResultMcAddress; + ibInfo.size = 16; + + memset(&ibsRequest, 0, sizeof(struct amdgpu_cs_request)); + ibsRequest.ip_type = AMDGPU_HW_IP_COMPUTE; + ibsRequest.ring = 0; + ibsRequest.number_of_ibs = 1; + ibsRequest.ibs = &ibInfo; + ibsRequest.resources = boList; + ibsRequest.fence_info.handle = NULL; + + memset(&fenceStatus, 0, sizeof(struct amdgpu_cs_fence)); + for (int i = 0; i < ALLOCATE_RETRY_TIMES; i++) { + ASSERT_EQ(0, amdgpu_cs_submit(contextHandle, 0, &ibsRequest, 1)); + sleep(1); + } + + fenceStatus.context = contextHandle; + fenceStatus.ip_type = AMDGPU_HW_IP_COMPUTE; + fenceStatus.ip_instance = 0; + fenceStatus.ring = 0; + fenceStatus.fence = ibsRequest.seq_no; + + ASSERT_EQ(0, amdgpu_cs_query_fence_status(&fenceStatus, + g_TestTimeOut, + 0, &expired)); + + ASSERT_EQ(0, amdgpu_bo_list_destroy(boList)); + + ASSERT_EQ(0, amdgpu_bo_unmap_and_free(ibResultHandle, vaHandle, + ibResultMcAddress, PAGE_SIZE)); + + ASSERT_EQ(0, amdgpu_cs_ctx_free(contextHandle)); +} + void KFDEvictTest::ForkChildProcesses(int nprocesses) { int i; @@ -196,21 +334,30 @@ TEST_F(KFDEvictTest, BasicTest) { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; } - HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); - if (count == 0) { - LOG() << "Not enough system memory, skipping the test" << std::endl; - return; - } + HSAint32 count = vramSize / vramBufSize / N_PROCESSES; + + LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; /* Fork the child processes */ ForkChildProcesses(N_PROCESSES); + int rn = FindDRMRenderNode(defaultGPUNode); + if (rn < 0) { + LOG() << "Skipping test" << std::endl; + return; + } + std::vector pBuffers; AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); - /* wait for other processes to finish allocation, then free buffer */ - sleep(ALLOCATE_RETRY_TIMES); + /* allocate gfx vram size of at most one third system memory */ + HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; + amdgpu_bo_handle handle; + AllocAmdgpuBo(rn, size, handle); + AmdgpuCommandSubmissionComputeNop(rn); + + FreeAmdgpuBo(handle); LOG() << m_psName << "free buffer" << std::endl; FreeBuffers(pBuffers, vramBufSize); @@ -388,7 +535,10 @@ TEST_F(KFDEvictTest, QueueTest) { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; } - HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); + HSAuint32 count = vramSize / vramBufSize / N_PROCESSES; + + LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; + if (count == 0) { LOG() << "Not enough system memory, skipping the test" << std::endl; return; @@ -401,6 +551,12 @@ TEST_F(KFDEvictTest, QueueTest) { /* Fork the child processes */ ForkChildProcesses(N_PROCESSES); + int rn = FindDRMRenderNode(defaultGPUNode); + if (rn < 0) { + LOG() << "Skipping test" << std::endl; + return; + } + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer addrBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer resultBuffer(PAGE_SIZE, defaultGPUNode); @@ -408,6 +564,13 @@ TEST_F(KFDEvictTest, QueueTest) { std::vector pBuffers; AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + /* allocate gfx vram size of at most one third system memory */ + HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; + amdgpu_bo_handle handle; + AllocAmdgpuBo(rn, size, handle); + + AmdgpuCommandSubmissionComputeNop(rn); + unsigned int wavefront_num = pBuffers.size(); LOG() << m_psName << "wavefront number " << wavefront_num << std::endl; @@ -439,6 +602,8 @@ TEST_F(KFDEvictTest, QueueTest) { dispatch0.SyncWithStatus(120000); ASSERT_SUCCESS(pm4Queue.Destroy()); + + FreeAmdgpuBo(handle); /* LOG() << m_psName << "free buffer" << std::endl; */ /* cleanup */ FreeBuffers(pBuffers, vramBufSize); diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index 3f500b2655..b98d275373 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -54,9 +54,11 @@ class KFDEvictTest : public KFDLocalMemoryTest { void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, std::vector &pBuffers); void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); + void AllocAmdgpuBo(int rn, HSAuint64 vramBufSize, amdgpu_bo_handle &handle); + void FreeAmdgpuBo(amdgpu_bo_handle handle); + void AmdgpuCommandSubmissionComputeNop(int rn); void ForkChildProcesses(int nprocesses); void WaitChildProcesses(); - HSAint32 GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize); protected: // members std::string m_psName; From 5c742f3e5e80d5d80020b08f5d782145d447800b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 3 Aug 2018 20:14:46 -0400 Subject: [PATCH 0369/1247] kfdtest: Blacklist Fragmentation test on all chips This test has been intermittently failing for various reasons and was already disabled on all chips except Ellesmere. It stresses memory management in unusual ways by having lots of memory allocated but +# not mapped, which is not relevant to compute applications over ROCr. Change-Id: I6b791ca7e2e0fcfe93fc720063b4b56acfded751 Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/kfdtest.exclude | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index c2f4772fb5..a804d3f846 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -1,9 +1,15 @@ # Permanent exclusions -# These tests are included for debugging, but are not executed in normal execution on any ASIC +# These tests are included for debugging, but are not executed in normal execution on any ASIC: +# +# Fragmentation test fails intermittently on different systems (see KFD-344). It +# stresses memory management in unusual ways by having lots of memory allocated but +# not mapped, which is not relevant to compute applications over ROCr. +# # CU Masking Linear/Split are not working correctly due to how the HW distributes work over CUs. -# They are available for testing but are not currently expected to pass on CI/VI/AI +# They are available for testing but are not currently expected to pass on CI/VI/AI. PERMANENT_BLACKLIST_ALL_ASICS=\ "-KFDEventTest.MeasureInterruptConsumption:"\ +"KFDLocalMemoryTest.Fragmentation:"\ "KFDQMTest.BasicCuMaskingLinear:"\ "RDMATest.GPUDirect" @@ -92,25 +98,20 @@ TONGA_TESTS_BLACKLIST=\ "KFDDBGTest.BasicAddressWatch:"\ "KFDQMTest.*" -# Fragmentation test intermittently failing on 4096 order 3 blocks. Investigating in conjunction with KFD-344 FIJI_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDLocalMemoryTest.Fragmentation:"\ "$SDMA_BLACKLIST" ELLESMERE_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST" -# Fragmentation test failing on 4096 order 3 blocks. Investigating in conjunction with KFD-344 BAFFIN_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDLocalMemoryTest.Fragmentation:"\ "$SDMA_BLACKLIST" # KFDDBG.BasicAddressWatch fails with message "E,INTERR: Setting ASIC VI (was previously set to GFX9), this changes the backend after we have already started parsing the shader, not currently supported by sp3, sorry." (KFD-317) # KFDExceptionTest.InvalidBadAddress causes some following tests to fail on SRIOV (KFD-343) -# KFDLocalMemoryTest.Fragmentation is still flaking out intermittently (KFD-344) # KFDQMTest.BasicCuMaskingEven fails intermittently (WIP) # KFDExceptionTest.InvalidWriteAddress is failing once we enable no-retry (KFD-380) VEGA10_TESTS_BLACKLIST=\ @@ -120,8 +121,7 @@ VEGA10_TESTS_BLACKLIST=\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ "KFDQMTest.GPUDoorbellWrite:"\ -"KFDMemoryTest.MemoryRegister:"\ -"KFDLocalMemoryTest.Fragmentation" +"KFDMemoryTest.MemoryRegister" VEGA20_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ @@ -130,8 +130,7 @@ VEGA20_TESTS_BLACKLIST=\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ "KFDQMTest.GPUDoorbellWrite:"\ -"KFDMemoryTest.MemoryRegister:"\ -"KFDLocalMemoryTest.Fragmentation" +"KFDMemoryTest.MemoryRegister" RAVEN_TESTS_BLACKLIST=\ From dd6f34b7f5700c4ae41e0a93137ecc5efcc69482 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 3 Aug 2018 20:17:09 -0400 Subject: [PATCH 0370/1247] libhsakmt: Fix pkg-config file paths Both the include and libpath were incorrect after recent build system changes. Use the proper GNUInstallDirs definitions in libhsakmt.pc.in to write the proper locations. This is needed for end users building KFDTest, which depends on correct pkg-config information. Change-Id: Ia45f36f054c2a607a77e7ecbcbd9eb7edd067348 Signed-off-by: Felix Kuehling --- libhsakmt.pc.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libhsakmt.pc.in b/libhsakmt.pc.in index 01d748bbc6..3093afc8dc 100644 --- a/libhsakmt.pc.in +++ b/libhsakmt.pc.in @@ -1,7 +1,7 @@ prefix=@CPACK_PACKAGING_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${prefix}/lib -includedir=${prefix}/include/libhsakmt +libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ Name: libhsakmt Description: HSA Kernel Mode Thunk library for AMD KFD support From c21927f4256fb537ffede46a993bac9b119aa122 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 3 Aug 2018 22:02:25 -0400 Subject: [PATCH 0371/1247] libhsakmt: Fix problems init_svm_apertures Unset ret_addr when unmapping the address space reservation. Otherwise it may try to unmap it again later. Remember the actual map_size and use it instead of len outside the reservation loops. Change-Id: I1a6b3fecfb59e22a713e5ed49c3ed37914cb6fb5 Signed-off-by: Felix Kuehling --- src/fmm.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 4bf6096259..ba810e088f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1569,7 +1569,7 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, bool disable_cache) { const HSAuint64 ADDR_INC = GPU_HUGE_PAGE_SIZE; - HSAuint64 len, alt_base, alt_size; + HSAuint64 len, map_size, alt_base, alt_size; bool found = false; void *addr, *ret_addr; @@ -1602,12 +1602,13 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, (HSAuint64)addr + ((len + 1) >> 1) - 1 <= limit; addr = (void *)((HSAuint64)addr + ADDR_INC)) { HSAuint64 top = MIN((HSAuint64)addr + len, limit+1); - HSAuint64 size = (top - (HSAuint64)addr) & - ~(HSAuint64)(PAGE_SIZE - 1); - if (size < SVM_MIN_VM_SIZE) + map_size = (top - (HSAuint64)addr) & + ~(HSAuint64)(PAGE_SIZE - 1); + if (map_size < SVM_MIN_VM_SIZE) break; - ret_addr = reserve_address(addr, size); + + ret_addr = reserve_address(addr, map_size); if (!ret_addr) break; if ((HSAuint64)ret_addr + ((len + 1) >> 1) - 1 <= limit) @@ -1616,7 +1617,8 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, * take it */ break; - munmap(ret_addr, size); + munmap(ret_addr, map_size); + ret_addr = NULL; } if (!ret_addr) { pr_warn("Failed to reserve %uGB for SVM ...\n", @@ -1626,9 +1628,10 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, if ((HSAuint64)ret_addr + SVM_MIN_VM_SIZE - 1 > limit) { /* addressable size is less than the minimum */ pr_warn("Got %uGB for SVM at %p with only %dGB usable ...\n", - (unsigned int)(len >> 30), ret_addr, + (unsigned int)(map_size >> 30), ret_addr, (int)((limit - (HSAint64)ret_addr) >> 30)); - munmap(ret_addr, len); + munmap(ret_addr, map_size); + ret_addr = NULL; continue; } else { found = true; @@ -1642,11 +1645,11 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, } base = (HSAuint64)ret_addr; - if (base + len - 1 > limit) + if (base + map_size - 1 > limit) /* trim the tail that's not GPU-addressable */ - munmap((void *)(limit + 1), base + len - 1 - limit); + munmap((void *)(limit + 1), base + map_size - 1 - limit); else - limit = base + len - 1; + limit = base + map_size - 1; /* init aperture */ svm.dgpu_aperture.base = dgpu_shared_aperture_base = ret_addr; From 4bb90d048c6b8bc5548f667bd5d3e4f99eff83ed Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 2 Aug 2018 13:25:44 -0400 Subject: [PATCH 0372/1247] Remove the use of IS_DGPU() The information can be obtained directly from node id. Also improve the whole logic for future compatibility. Change-Id: I130733be4e7930d5953d5e81409905e60c2ec35e Signed-off-by: Yong Zhao --- src/libhsakmt.h | 6 ++--- src/pmc_table.c | 2 +- src/queues.c | 64 ++++++++++++++++++++++++------------------------- src/topology.c | 9 ++++++- 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 2c215d7789..6177b1b3a8 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -100,13 +100,13 @@ enum asic_family_type { CHIP_RAVEN, CHIP_VEGA20 }; -#define IS_DGPU(chip) ((chip) != CHIP_KAVERI && (chip) != CHIP_CARRIZO && \ - (chip) != CHIP_RAVEN) + #define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); -uint16_t get_device_id_by_node(HSAuint32 node_id); +bool prefer_ats(HSAuint32 node_id); +uint16_t get_device_id_by_node_id(HSAuint32 node_id); uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, diff --git a/src/pmc_table.c b/src/pmc_table.c index 7c964821a2..b0d1a3f00f 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1712,7 +1712,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, enum perf_block_id block_id, struct perf_counter_block *block) { - uint16_t dev_id = get_device_id_by_node(node_id); + uint16_t dev_id = get_device_id_by_node_id(node_id); enum asic_family_type asic; if (block_id > PERFCOUNTER_BLOCKID__MAX || diff --git a/src/queues.c b/src/queues.c index 0055e5c5c6..2011e21795 100644 --- a/src/queues.c +++ b/src/queues.c @@ -50,68 +50,68 @@ struct device_info { uint32_t doorbell_size; }; -struct device_info kaveri_device_info = { +const struct device_info kaveri_device_info = { .asic_family = CHIP_KAVERI, .eop_buffer_size = 0, .doorbell_size = DOORBELL_SIZE_GFX7, }; -struct device_info hawaii_device_info = { +const struct device_info hawaii_device_info = { .asic_family = CHIP_HAWAII, .eop_buffer_size = 0, .doorbell_size = DOORBELL_SIZE_GFX7, }; -struct device_info carrizo_device_info = { +const struct device_info carrizo_device_info = { .asic_family = CHIP_CARRIZO, .eop_buffer_size = 4096, .doorbell_size = DOORBELL_SIZE_GFX8, }; -struct device_info tonga_device_info = { +const struct device_info tonga_device_info = { .asic_family = CHIP_TONGA, .eop_buffer_size = TONGA_PAGE_SIZE, .doorbell_size = DOORBELL_SIZE_GFX8, }; -struct device_info fiji_device_info = { +const struct device_info fiji_device_info = { .asic_family = CHIP_FIJI, .eop_buffer_size = TONGA_PAGE_SIZE, .doorbell_size = DOORBELL_SIZE_GFX8, }; -struct device_info polaris10_device_info = { +const struct device_info polaris10_device_info = { .asic_family = CHIP_POLARIS10, .eop_buffer_size = TONGA_PAGE_SIZE, .doorbell_size = DOORBELL_SIZE_GFX8, }; -struct device_info polaris11_device_info = { +const struct device_info polaris11_device_info = { .asic_family = CHIP_POLARIS11, .eop_buffer_size = TONGA_PAGE_SIZE, .doorbell_size = DOORBELL_SIZE_GFX8, }; -struct device_info vega10_device_info = { +const struct device_info vega10_device_info = { .asic_family = CHIP_VEGA10, .eop_buffer_size = 4096, .doorbell_size = DOORBELL_SIZE_GFX9, }; -struct device_info raven_device_info = { +const struct device_info raven_device_info = { .asic_family = CHIP_RAVEN, .eop_buffer_size = 4096, .doorbell_size = DOORBELL_SIZE_GFX9, }; -struct device_info vega20_device_info = { +const struct device_info vega20_device_info = { .asic_family = CHIP_VEGA20, .eop_buffer_size = 4096, .doorbell_size = DOORBELL_SIZE_GFX9, }; -static struct device_info *dev_lookup_table[] = { +static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, [CHIP_CARRIZO] = &carrizo_device_info, @@ -124,11 +124,6 @@ static struct device_info *dev_lookup_table[] = { [CHIP_RAVEN] = &raven_device_info }; -struct device_id { - uint16_t dev_id; - struct device_info *dev_info; -}; - struct queue { uint32_t queue_id; uint64_t wptr; @@ -138,6 +133,7 @@ struct queue { uint32_t ctx_save_restore_size; uint32_t ctl_stack_size; const struct device_info *dev_info; + bool use_ats; /* This queue structure is allocated from GPU with page aligned size * but only small bytes are used. We use the extra space in the end for * cu_mask bits array. @@ -180,7 +176,7 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) return ret; } -static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) +static const struct device_info *get_device_info_by_dev_id(uint16_t dev_id) { enum asic_family_type asic; @@ -193,7 +189,7 @@ static struct device_info *get_device_info_by_dev_id(uint16_t dev_id) static void get_doorbell_map_info(uint16_t dev_id, struct process_doorbells *doorbell) { - struct device_info *dev_info; + const struct device_info *dev_info; dev_info = get_device_info_by_dev_id(dev_id); @@ -301,7 +297,7 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, return HSAKMT_STATUS_SUCCESS; } - get_doorbell_map_info(get_device_id_by_node(NodeId), + get_doorbell_map_info(get_device_id_by_node_id(NodeId), &doorbells[NodeId]); if (doorbells[NodeId].use_gpuvm) { @@ -414,20 +410,20 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) * Allocates memory aligned to sysconf(_SC_PAGESIZE) */ static void *allocate_exec_aligned_memory(uint32_t size, - enum asic_family_type type, + bool use_ats, uint32_t NodeId, bool DeviceLocal) { - if (IS_DGPU(type)) + if (!use_ats) return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId, DeviceLocal, DeviceLocal); return allocate_exec_aligned_memory_cpu(size); } static void free_exec_aligned_memory(void *addr, uint32_t size, uint32_t align, - enum asic_family_type type) + bool use_ats) { - if (IS_DGPU(type)) + if (!use_ats) free_exec_aligned_memory_gpu(addr, size, align); else munmap(addr, size); @@ -438,11 +434,11 @@ static void free_queue(struct queue *q) if (q->eop_buffer) free_exec_aligned_memory(q->eop_buffer, q->dev_info->eop_buffer_size, - PAGE_SIZE, q->dev_info->asic_family); + PAGE_SIZE, q->use_ats); if (q->ctx_save_restore) free_exec_aligned_memory(q->ctx_save_restore, q->ctx_save_restore_size, - PAGE_SIZE, q->dev_info->asic_family); + PAGE_SIZE, q->use_ats); free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); } @@ -460,7 +456,7 @@ static int handle_concrete_asic(struct queue *q, if (dev_info->eop_buffer_size > 0) { q->eop_buffer = allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, - dev_info->asic_family, + q->use_ats, NodeId, true); if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; @@ -476,7 +472,7 @@ static int handle_concrete_asic(struct queue *q, args->ctl_stack_size = q->ctl_stack_size; q->ctx_save_restore = allocate_exec_aligned_memory(q->ctx_save_restore_size, - dev_info->asic_family, + q->use_ats, NodeId, false); if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; @@ -507,10 +503,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, uint16_t dev_id; uint64_t doorbell_mmap_offset; unsigned int doorbell_offset; - struct device_info *dev_info; + const struct device_info *dev_info; int err; HsaNodeProperties props; uint32_t cu_num, i; + bool use_ats; CHECK_KFD_OPEN(); @@ -522,17 +519,22 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, if (result != HSAKMT_STATUS_SUCCESS) return result; - dev_id = get_device_id_by_node(NodeId); + use_ats = prefer_ats(NodeId); + + dev_id = get_device_id_by_node_id(NodeId); dev_info = get_device_info_by_dev_id(dev_id); struct queue *q = allocate_exec_aligned_memory(sizeof(*q), - dev_info->asic_family, + use_ats, NodeId, false); if (!q) return HSAKMT_STATUS_NO_MEMORY; memset(q, 0, sizeof(*q)); + q->use_ats = use_ats; + q->dev_info = dev_info; + /* By default, CUs are all turned on. Initialize cu_mask to '1 * for all CU bits. */ @@ -550,8 +552,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, args.gpu_id = gpu_id; - q->dev_info = dev_info; - switch (Type) { case HSA_QUEUE_COMPUTE: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE; diff --git a/src/topology.c b/src/topology.c index fab47c44fd..eb2367e41f 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1939,7 +1939,7 @@ out: return err; } -uint16_t get_device_id_by_node(HSAuint32 node_id) +uint16_t get_device_id_by_node_id(HSAuint32 node_id) { if (!g_props || !_system || _system->NumNodes <= node_id) return 0; @@ -1947,6 +1947,13 @@ uint16_t get_device_id_by_node(HSAuint32 node_id) return g_props[node_id].node.DeviceId; } +bool prefer_ats(HSAuint32 node_id) +{ + return g_props[node_id].node.Capability.ui32.HSAMMUPresent + && g_props[node_id].node.NumCPUCores + && g_props[node_id].node.NumFComputeCores; +} + uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; From 9d6d0911e410dc169a94359c9027218eb3fd7d4b Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Thu, 2 Aug 2018 17:13:26 +0800 Subject: [PATCH 0373/1247] kfdtest: make p2ptest go through all gpus Implement sDMA copy packet broadcast. Each time sDMA will copy its local vram to sysbuf and next GPU's vram. That will verify where the p2p link is broken. Currently we just test push of p2p. test result on 2 cpus, 4 gpus, numa enabled system. [ RUN ] KFDQMTest.P2PTest [ ] Test 2 -> 3 [ ] PASS 2 -> 3 [ ] Test 3 -> 4 [ ] PASS 3 -> 4 [ ] Test 4 -> 5 [ ] PASS 4 -> 5 [ ] Test 5 -> 0 [ ] PASS 5 -> 0 [ OK ] KFDQMTest.P2PTest (190 ms) Change-Id: Ie6fb2604109e39465b8a873b3bb42abc6259825a --- tests/kfdtest/include/sdma_pkt_struct.h | 29 +++-- tests/kfdtest/src/KFDQMTest.cpp | 153 +++++++++++++++--------- tests/kfdtest/src/SDMAPacket.cpp | 41 +++++-- tests/kfdtest/src/SDMAPacket.hpp | 1 + 4 files changed, 143 insertions(+), 81 deletions(-) diff --git a/tests/kfdtest/include/sdma_pkt_struct.h b/tests/kfdtest/include/sdma_pkt_struct.h index aa13006256..bc37b5189f 100644 --- a/tests/kfdtest/include/sdma_pkt_struct.h +++ b/tests/kfdtest/include/sdma_pkt_struct.h @@ -102,23 +102,26 @@ typedef struct SDMA_PKT_COPY_LINEAR_TAG unsigned int DW_4_DATA; } SRC_ADDR_HI_UNION; - union + struct { - struct + union { - unsigned int dst_addr_31_0:32; - }; - unsigned int DW_5_DATA; - } DST_ADDR_LO_UNION; + struct + { + unsigned int dst_addr_31_0:32; + }; + unsigned int DW_5_DATA; + } DST_ADDR_LO_UNION; - union - { - struct + union { - unsigned int dst_addr_63_32:32; - }; - unsigned int DW_6_DATA; - } DST_ADDR_HI_UNION; + struct + { + unsigned int dst_addr_63_32:32; + }; + unsigned int DW_6_DATA; + } DST_ADDR_HI_UNION; + } DST_ADDR[0]; } SDMA_PKT_COPY_LINEAR, *PSDMA_PKT_COPY_LINEAR; /* diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index d88e33c7c1..25e4c8df73 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1352,6 +1352,31 @@ TEST_F(KFDQMTest, mGPUShareBO) { TEST_END } + +static void sdma_copy(HSAint32 node, void *src, void *const dst[], int n, unsigned int size) { + ROUTINE_START; + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(node)); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(dst, src, n, size)); + sdmaQueue.Wait4PacketConsumption(); + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + ROUTINE_END; +} + +static void sdma_fill(HSAint32 node, void *dst, unsigned int data, unsigned int size) { + ROUTINE_START; + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(node)); + sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(dst, data, size)); + sdmaQueue.Wait4PacketConsumption(); + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + ROUTINE_END; +} + TEST_F(KFDQMTest, P2PTest) { TEST_START(TESTPROFILE_RUNALL); if (!is_dgpu()) { @@ -1364,92 +1389,106 @@ TEST_F(KFDQMTest, P2PTest) { LOG() << "Skipping test: Need at least two GPUs" << std::endl; return; } - HSAint32 gpuNode1 = m_NodeInfo.HsaDefaultGPUNode(); - HSAint32 gpuNode2 = 0; + std::vector nodes; /* This test simulates RT team's P2P part in IPCtest: * - * +--------------------------------------------+ - * | gpu1 gpu2 | - * |gpu1 mem ----> gpu2 mem ----> system buffer | - * +--------------------------------------------+ + * +------------------------------------------------+ + * | gpu1 gpu2 gpuX | + * |gpu1 mem ----> gpu2 mem ----> gpuX mem | + * | \ \ \ | + * | \ \ \ | + * | system buffer system buffer system buffer| + * +------------------------------------------------+ * - * Copy data from GPU-1 memory to GPU-2 memory using GPU-1, then GPU-2 - * memory to system buffer using GPU-2. Verify the system buffer - * (initialized with 0) has the same content as gpu1 memory (0x5). + * Copy data from current GPU memory to next GPU memory and system memory + * Using current GPU, aka p2p push. + * Verify the system buffer has the expected content after each push. */ /* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */ if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { - gpuNode1 = g_TestNodeId; - gpuNode2 = g_TestDstNodeId; - } - - /* GPU-2 must have public memory(large bar) to do GPU-to-GPU copy. If - * not specified in the command line, find one. - */ - gpuNode2 = m_NodeInfo.FindLargeBarGPUNode(); - if (gpuNode2 < 0) { - LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; - return; - } - if (gpuNode1 == gpuNode2) { - for (unsigned i = 0; i < gpuNodes.size(); i++) { - if (gpuNodes.at(i) != gpuNode2) { - gpuNode1 = gpuNodes.at(i); - break; - } + nodes.push_back(g_TestNodeId); + nodes.push_back(g_TestDstNodeId); + if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[1])) { + LOG() << "Skipping test: Dst GPU is not a large bar GPU" << std::endl; + return; + } + if (nodes[0] == nodes[1]) { + LOG() << "Skipping test: Need different GPUs specified" << std::endl; + return; + } + } else { + HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); + nodes.push_back(defaultGPU); + for (unsigned i = 0; i < gpuNodes.size(); i++) + if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && gpuNodes.at(i) != defaultGPU) + nodes.push_back(gpuNodes.at(i)); + if (nodes.size() < 2) { + LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + return; } } HSAuint32 *sysBuf; - HSAuint32 size = 0xc00000; // bigger than 4MB to test non-contiguous memory + HSAuint32 size = 16ULL<<20; // bigger than 16MB to test non-contiguous memory HsaMemFlags memFlags = {0}; HsaMemMapFlags mapFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 1; + memFlags.ui32.NonPaged = 1; + unsigned int end = size / sizeof(HSAuint32) - 1; - /* 1. Allocate a system buffer and allow the access to GPU-2 */ + /* 1. Allocate a system buffer and allow the access to GPUs */ EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, (void **)&sysBuf)); EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL, - mapFlags, 1, (HSAuint32*)&gpuNode2)); + mapFlags, nodes.size(), &nodes[0])); +#define MAGIC_NUM 0xdeadbeaf - /* 2.- Allocate local memory on GPU-1 - * - Allocate local memory on GPU-2 and allow access to both GPUs - */ - HsaMemoryBuffer gpu1Mem(size, gpuNode1, false, true /*isLocal*/); - HsaMemoryBuffer gpu2Mem(size, gpuNode2, false, true); + /* First GPU fills mem with MAGIC_NUM*/ + void *src, *dst; + HSAuint32 cur = nodes[0], next; + ASSERT_SUCCESS(hsaKmtAllocMemory(cur, size, memFlags, (void**)&src)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(src, size, NULL)); + sdma_fill(cur, src, MAGIC_NUM, size); - SDMAQueue sdmaQueue1, sdmaQueue2; - ASSERT_SUCCESS(sdmaQueue1.Create(gpuNode1)); - ASSERT_SUCCESS(sdmaQueue2.Create(gpuNode2)); + for (unsigned i = 1; i <= nodes.size(); i++) { + int n; + memset(sysBuf, 0, size); - /* initialize sysBuf as 0 and fill up gpu1 mem with 0x5 */ - memset(sysBuf, 0, size); - sdmaQueue1.PlaceAndSubmitPacket(SDMAFillDataPacket(gpu1Mem.As(), - 0x5, size)); - sdmaQueue1.Wait4PacketConsumption(); + /* Last GPU just copy mem to sysBuf*/ + if (i == nodes.size()) { + n = 1; + next = 0;/*system memory node*/ + dst = 0; + } else { + n = 2; + next = nodes[i]; + ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, (void**)&dst)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dst, size, NULL)); + } - /* 3. Copy data from gpuNode1 to gpuNode2 */ - sdmaQueue1.PlaceAndSubmitPacket(SDMACopyDataPacket(gpu2Mem.As(), - gpu1Mem.As(), size)); - sdmaQueue1.Wait4PacketConsumption(); + LOG() << "Test " << cur << " -> " << next << std::endl; + /* copy to sysBuf and next GPU*/ + void *dst_array[] = {sysBuf, dst}; + sdma_copy(cur, src, dst_array, n, size); - /* 4. Copy data from gpuNode2 to system buffer */ - sdmaQueue2.PlaceAndSubmitPacket(SDMACopyDataPacket(sysBuf, - gpu2Mem.As(), size)); - sdmaQueue2.Wait4PacketConsumption(); + /* verify the data*/ + ASSERT_EQ(sysBuf[0], MAGIC_NUM); + ASSERT_EQ(sysBuf[end], MAGIC_NUM); - /* 5. Verify the data */ - ASSERT_EQ(sysBuf[0], 0x5); - unsigned int end = size / sizeof(HSAuint32) - 1; - ASSERT_EQ(sysBuf[end], 0x5); + LOG() << "PASS " << cur << " -> " << next << std::endl; + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(src)); + EXPECT_SUCCESS(hsaKmtFreeMemory(src, size)); + + cur = next; + src = dst; + } EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(sysBuf)); EXPECT_SUCCESS(hsaKmtFreeMemory(sysBuf, size)); - EXPECT_SUCCESS(sdmaQueue1.Destroy()); - EXPECT_SUCCESS(sdmaQueue2.Destroy()); TEST_END } diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index e7d1020546..45c7a6cd3e 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -72,14 +72,24 @@ void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int ndw, memcpy(&packetData->DATA0_UNION.DW_4_DATA, data, ndw*sizeof(unsigned int)); } -#define TWO_MEG (1 << 21) +#define BITS (21) +#define TWO_MEG (1 << BITS) SDMACopyDataPacket::~SDMACopyDataPacket(void) { free(packetData); } -SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsize) { - int32_t size = 0; - packetSize = ((surfsize + TWO_MEG - 1) >> 21) * sizeof(SDMA_PKT_COPY_LINEAR); +SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, unsigned int surfsize) { + int32_t size = 0, i; + void **dst = (void**)malloc(sizeof(void*) * n); + const int singlePacketSize = sizeof(SDMA_PKT_COPY_LINEAR) + + sizeof(SDMA_PKT_COPY_LINEAR::DST_ADDR[0]) * n; + + if (n > 2) + WARN() << "SDMACopyDataPacket does not support more than 2 dst addresses!" << std::endl; + + memcpy(dst, dsts, sizeof(void*) * n); + + packetSize = ((surfsize + TWO_MEG - 1) >> BITS) * singlePacketSize; SDMA_PKT_COPY_LINEAR *pSDMA = (SDMA_PKT_COPY_LINEAR *)malloc(packetSize); packetData = pSDMA; @@ -91,22 +101,31 @@ SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsi else size = surfsize; - memset(pSDMA, 0, sizeof(SDMA_PKT_COPY_LINEAR)); + memset(pSDMA, 0, singlePacketSize); pSDMA->HEADER_UNION.op = SDMA_OP_COPY; pSDMA->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR; + pSDMA->HEADER_UNION.broadcast = n > 1 ? 1 : 0; pSDMA->COUNT_UNION.count = SDMA_COUNT(size); SplitU64(reinterpret_cast(src), pSDMA->SRC_ADDR_LO_UNION.DW_3_DATA, // src_addr_31_0 pSDMA->SRC_ADDR_HI_UNION.DW_4_DATA); // src_addr_63_32 - SplitU64(reinterpret_cast(dst), - pSDMA->DST_ADDR_LO_UNION.DW_5_DATA, // dst_addr_31_0 - pSDMA->DST_ADDR_HI_UNION.DW_6_DATA); // dst_addr_63_32 - pSDMA++; + for (i = 0; i < n; i++) + SplitU64(reinterpret_cast(dst[i]), + pSDMA->DST_ADDR[i].DST_ADDR_LO_UNION.DW_5_DATA, // dst_addr_31_0 + pSDMA->DST_ADDR[i].DST_ADDR_HI_UNION.DW_6_DATA); // dst_addr_63_32 + + pSDMA = (SDMA_PKT_COPY_LINEAR *)((char *)pSDMA + singlePacketSize); + for (i = 0; i < n; i++) + dst[i] = (char *)dst[i] + size; src = (char *)src + size; - dst = (char *)dst + size; surfsize -= size; } + free(dst); +} + +SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsize) { + new (this)SDMACopyDataPacket(&dst, src, 1, surfsize); } SDMAFillDataPacket::~SDMAFillDataPacket() { @@ -118,7 +137,7 @@ SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned in SDMA_PKT_CONSTANT_FILL *pSDMA; /* SDMA support maximum 0x3fffe0 byte in one copy. Use 2M copy_size */ - m_PacketSize = ((size + TWO_MEG - 1) >> 21) * sizeof(SDMA_PKT_CONSTANT_FILL); + m_PacketSize = ((size + TWO_MEG - 1) >> BITS) * sizeof(SDMA_PKT_CONSTANT_FILL); pSDMA = (SDMA_PKT_CONSTANT_FILL *)calloc(1, m_PacketSize); m_PacketData = pSDMA; diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index 89ad4cac07..f7ef53f733 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -64,6 +64,7 @@ class SDMACopyDataPacket : public SDMAPacket { public: // this contructor will also init the packet, no need for adittional calls SDMACopyDataPacket(void *dest, void *src, unsigned int size); + SDMACopyDataPacket(void *const dst[], void *src, int n, unsigned int surfsize); virtual ~SDMACopyDataPacket(void); From fe04dd6890029fec096ba3cfb42a832cbbb5b343 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 3 Aug 2018 03:01:22 -0400 Subject: [PATCH 0374/1247] Calculate and store the first gpu mem during initializaiton Previously we used the first dgpu mem, but after careful examination, we found it only needs to be a GPU, so we modify the code to reflect that as well. Change-Id: I069d9b8e247aed55c1f885b79f743ea8e03ddf93 Signed-off-by: Yong Zhao --- src/fmm.c | 63 +++++++++++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ba810e088f..e2374a403f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -171,6 +171,8 @@ typedef struct { */ static gpu_mem_t *gpu_mem; static unsigned int gpu_mem_count; +static gpu_mem_t *g_first_gpu_mem; + static bool hsa_debug; static void *dgpu_shared_aperture_base; static void *dgpu_shared_aperture_limit; @@ -238,24 +240,6 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, void *address); static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_array_size); -static int32_t find_first_dgpu(HSAuint32 *gpu_id) -{ - uint32_t i; - - *gpu_id = NON_VALID_GPU_ID; - - for (i = 0; i < gpu_mem_count; i++) { - if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) - continue; - if (!topology_is_dgpu(gpu_mem[i].device_id)) - continue; - *gpu_id = gpu_mem[i].gpu_id; - return i; - } - - return -1; -} - static vm_area_t *vm_create_and_init_area(void *start, void *end) { vm_area_t *area = (vm_area_t *) malloc(sizeof(vm_area_t)); @@ -1213,14 +1197,16 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, uint64_t mmap_offset; uint32_t ioc_flags; uint64_t size; - int32_t gpu_mem_id; + int32_t gpu_drm_fd; uint32_t gpu_id; vm_object_t *vm_obj = NULL; - gpu_mem_id = find_first_dgpu(&gpu_id); - if (gpu_mem_id < 0) + if (!g_first_gpu_mem) return NULL; + gpu_id = g_first_gpu_mem->gpu_id; + gpu_drm_fd = g_first_gpu_mem->drm_render_fd; + size = MemorySizeInBytes; ioc_flags = 0; if (flags.ui32.CoarseGrain) @@ -1296,8 +1282,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, ioc_flags, &vm_obj); if (mem && flags.ui32.HostAccess) { - int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : - gpu_mem[gpu_mem_id].drm_render_fd; + int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_drm_fd; void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); @@ -1709,7 +1694,7 @@ static void fmm_init_rbtree(void) HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { - uint32_t i = 0; + uint32_t i; int32_t gpu_mem_id = 0; uint32_t gpu_id; HsaNodeProperties props; @@ -1748,6 +1733,9 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1) guardPages = 1; + gpu_mem_count = 0; + g_first_gpu_mem = NULL; + /* Trade off - NumNodes includes GPU nodes + CPU Node. So in * systems with CPU node, slightly more memory is allocated than * necessary @@ -1760,10 +1748,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * set to 0 by calloc. This is necessary because this function * gets called before hsaKmtAcquireSystemProperties() is called. */ - gpu_mem_count = 0; + pacc = pci_alloc(); pci_init(pacc); - while (i < NumNodes) { + for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) @@ -1790,9 +1778,12 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) get_vm_alignment(props.DeviceId); gpu_mem[gpu_mem_count].gpuvm_aperture.guard_pages = guardPages; pthread_mutex_init(&gpu_mem[gpu_mem_count].gpuvm_aperture.fmm_mutex, NULL); + + if (!g_first_gpu_mem) + g_first_gpu_mem = &gpu_mem[gpu_mem_count]; + gpu_mem_count++; } - i++; } pci_cleanup(pacc); @@ -2592,7 +2583,6 @@ static HSAuint8 fmm_check_user_memory(const void *addr, HSAuint64 size) static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_object_t **obj_ret) { - int32_t i; HSAuint32 gpu_id; manageable_aperture_t *aperture; void *svm_addr = NULL; @@ -2601,10 +2591,12 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj HSAuint64 aligned_addr = (HSAuint64)addr - page_offset; HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); - /* Find first dGPU for creating the userptr BO */ - i = find_first_dgpu(&gpu_id); - if (i < 0) + /* Find first GPU for creating the userptr BO */ + if (!g_first_gpu_mem) return HSAKMT_STATUS_ERROR; + + gpu_id = g_first_gpu_mem->gpu_id; + aperture = &svm.dgpu_aperture; /* Check if this address was already registered */ @@ -2856,12 +2848,13 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, if (r != HSAKMT_STATUS_SUCCESS) return r; if (!gpu_id && is_dgpu) { - /* Sharing non paged system memory. Use first dgpu which was + /* Sharing non paged system memory. Use first GPU which was * used during allocation. See fmm_allocate_host_gpu() */ - r = find_first_dgpu(&gpu_id); - if (r != HSAKMT_STATUS_SUCCESS) - return r; + if (!g_first_gpu_mem) + return HSAKMT_STATUS_ERROR; + + gpu_id = g_first_gpu_mem->gpu_id; } exportArgs.handle = obj->handle; exportArgs.gpu_id = gpu_id; From 8fbf4a26ec7bdbcfef30865c7a951144f41b7474 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Thu, 9 Aug 2018 17:41:03 +0800 Subject: [PATCH 0375/1247] thunk: fix a vm area release issue On some asics, like tonga, the memory alignment size is as big as 0x8000. fmm_allocate* alloc vm area with size passed in which is not aligned mostly. But __fmm_release free vm area with vm_object_t->size which is aligned. That might cause aperture_release_area fail to free the vm area as the size might be bigger than zone itself or it just free another vm area nearby unexpected. This patch somehow will alloc more space than it needed on tonga. gfx900+ is not affected. Change-Id: I5a88c92b08c4e6f6bc05881798f769b55d6debe9 Signed-off-by: xinhui pan --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index e2374a403f..1b43140435 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -509,7 +509,7 @@ static bool aperture_is_valid(void *app_base, void *app_limit) */ static uint64_t vm_align_area_size(manageable_aperture_t *app, uint64_t size) { - return ALIGN_UP(size + (uint64_t)app->guard_pages * PAGE_SIZE, + return ALIGN_UP(ALIGN_UP(size, app->align) + (uint64_t)app->guard_pages * PAGE_SIZE, app->align); } From 110e754f64437fb0c8a9042a7410586df3f45a3d Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 7 Aug 2018 16:03:24 -0400 Subject: [PATCH 0376/1247] Differentiate gfx700 and improve the logic by introducing is_gfx700() Because gfx700 has local memory but other APUs don't, we should reflect that in the code. Meanwhile, fix a bug that on gfx902 svm aperture is not added when calling hsaKmtGetNodeMemoryProperties(). Change-Id: Id840f2db0b14fda9ee713b219a9474c15f8a9771 Signed-off-by: Yong Zhao --- src/libhsakmt.h | 1 + src/memory.c | 2 +- src/topology.c | 15 +++++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 6177b1b3a8..17dce385fc 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -107,6 +107,7 @@ HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); bool prefer_ats(HSAuint32 node_id); uint16_t get_device_id_by_node_id(HSAuint32 node_id); +bool is_kaveri(HSAuint32 node_id); uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, diff --git a/src/memory.c b/src/memory.c index 8b1b07599b..6c1996365e 100644 --- a/src/memory.c +++ b/src/memory.c @@ -49,7 +49,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, pr_debug("[%s] node %d; default %d; alternate %d\n", __func__, Node, DefaultPolicy, AlternatePolicy); - if (is_dgpu) + if (!is_kaveri(Node)) /* This is a legacy API useful on Kaveri only. On dGPU * the alternate aperture is setup and used * automatically for coherent allocations. Don't let diff --git a/src/topology.c b/src/topology.c index eb2367e41f..53ea9b5b0b 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1780,7 +1780,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; uint32_t i, gpu_id; HSAuint64 aperture_limit; - bool nodeIsDGPU; if (!MemoryProperties) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -1816,8 +1815,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, if (gpu_id == 0) goto out; - nodeIsDGPU = topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)); - /*Add LDS*/ if (i < NumBanks && fmm_get_aperture_base_and_limit(FMM_LDS, gpu_id, @@ -1831,7 +1828,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, * For dGPU the topology node contains Local Memory and it is added by * the for loop above */ - if (!nodeIsDGPU && i < NumBanks && g_props[NodeId].node.LocalMemSize > 0 && + if (is_kaveri(NodeId) && i < NumBanks && g_props[NodeId].node.LocalMemSize > 0 && fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; @@ -1848,8 +1845,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, i++; } - /* On dGPUs add SVM aperture */ - if (nodeIsDGPU && i < NumBanks && + /* Add SVM aperture */ + if (topology_is_svm_needed(get_device_id_by_gpu_id(gpu_id)) && i < NumBanks && fmm_get_aperture_base_and_limit( FMM_SVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { @@ -1954,6 +1951,12 @@ bool prefer_ats(HSAuint32 node_id) && g_props[node_id].node.NumFComputeCores; } +bool is_kaveri(HSAuint32 node_id) +{ + return g_props[node_id].node.EngineId.ui32.Major == 7 + && g_props[node_id].node.EngineId.ui32.Minor == 0; +} + uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; From eb5539fb1061bd4165e8aef92e07424de512b30f Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Fri, 10 Aug 2018 15:39:54 +0800 Subject: [PATCH 0377/1247] thunk: fix a memory leak Hit queue create failure when do kfdtest with --gtest_repet=-1 fix: 4bb90d04("Remove the use of IS_DGPU()") Change-Id: I04fa73f90cef13a5517dbaceb89c41dc0f821a79 Signed-off-by: xinhui pan --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 2011e21795..95259bdb30 100644 --- a/src/queues.c +++ b/src/queues.c @@ -440,7 +440,7 @@ static void free_queue(struct queue *q) q->ctx_save_restore_size, PAGE_SIZE, q->use_ats); - free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->dev_info->asic_family); + free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->use_ats); } static int handle_concrete_asic(struct queue *q, From 3f7b6356fdc122847de1a32de4c00a8ddbd97575 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Fri, 10 Aug 2018 16:40:34 +0800 Subject: [PATCH 0378/1247] kfdtest: fix a memory leak issue in MMapLarge test When mapMemoryToGpu fails, we need unregister it with user address as the gpu address is not available. Change-Id: I4418eeaa7aa37008f5bffa144e2c2171f0d238fd Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDMemoryTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 6f12caed65..8282106420 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -168,7 +168,7 @@ TEST_F(KFDMemoryTest, MMapLarge) { break; if (hsaKmtMapMemoryToGPUNodes(addr + i, s - i, &AlternateVAGPU[i], mapFlags, 1, (HSAuint32 *)&defaultGPUNode)) { - hsaKmtDeregisterMemory((void*)AlternateVAGPU[i]); + hsaKmtDeregisterMemory(addr + i); break; } } From dffac0a97eada398dcc68dce3df00a6b35a499d8 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 13 Aug 2018 09:03:31 -0400 Subject: [PATCH 0379/1247] kfdtest: Style cleanup Clean up the KFDTest style via CPPLint. Some warnings remain regarding volatile variables being cast to void*. This is the command used: cpplint.py --linelength=120 --filter=-readability/multiline_string,-readability/todo,-build/include,-runtime/references multiline_string is due to using ISA code todo is to avoid errors that we don't have TODO(username) instead of TODO include is about including the folder in the header includes references is regarding non-const references '&' being const or using pointers. That can be addressed later Change-Id: I3c6622da0a13dd33ab29b2bfff48be25e763b750 --- tests/kfdtest/src/BaseQueue.cpp | 3 +- tests/kfdtest/src/Dispatch.cpp | 28 ++-- tests/kfdtest/src/Dispatch.hpp | 4 +- tests/kfdtest/src/GoogleTestExtension.hpp | 2 +- tests/kfdtest/src/IndirectBuffer.cpp | 3 +- tests/kfdtest/src/IsaGenerator.hpp | 9 +- tests/kfdtest/src/KFDBaseComponentTest.hpp | 18 +- tests/kfdtest/src/KFDCWSRTest.cpp | 3 +- tests/kfdtest/src/KFDDBGTest.cpp | 15 +- tests/kfdtest/src/KFDEventTest.cpp | 5 +- tests/kfdtest/src/KFDEvictTest.cpp | 109 ++++++------ tests/kfdtest/src/KFDExceptionTest.cpp | 2 +- tests/kfdtest/src/KFDGraphicsInterop.hpp | 9 +- tests/kfdtest/src/KFDIPCTest.cpp | 25 +-- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 6 +- tests/kfdtest/src/KFDMemoryTest.cpp | 186 +++++++++++---------- tests/kfdtest/src/KFDOpenCloseKFDTest.hpp | 3 +- tests/kfdtest/src/KFDPNPTest.cpp | 3 +- tests/kfdtest/src/KFDPerfCounters.cpp | 9 +- tests/kfdtest/src/KFDQMTest.cpp | 88 +++++----- tests/kfdtest/src/KFDQMTest.hpp | 4 +- tests/kfdtest/src/KFDTestFlags.hpp | 12 +- tests/kfdtest/src/KFDTestMain.cpp | 5 +- tests/kfdtest/src/KFDTestUtil.cpp | 32 ++-- tests/kfdtest/src/KFDTestUtil.hpp | 10 +- tests/kfdtest/src/KFDTopologyTest.cpp | 19 ++- tests/kfdtest/src/KFDTopologyTest.hpp | 4 +- tests/kfdtest/src/LinuxOSWrapper.cpp | 12 +- tests/kfdtest/src/OSWrapper.hpp | 10 +- tests/kfdtest/src/PM4Packet.cpp | 31 ++-- tests/kfdtest/src/PM4Packet.hpp | 2 +- tests/kfdtest/src/SDMAPacket.cpp | 26 +-- tests/kfdtest/src/SDMAPacket.hpp | 2 +- 33 files changed, 371 insertions(+), 328 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index f64a02fc9f..3af41aebe3 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -125,7 +125,8 @@ bool BaseQueue::AllPacketsSubmitted() { } void BaseQueue::PlacePacket(const BasePacket &packet) { - ASSERT_EQ(packet.PacketType(), PacketTypeSupported()) << "Cannot add a packet since packet type doesn't match queue"; + ASSERT_EQ(packet.PacketType(), PacketTypeSupported()) + << "Cannot add a packet since packet type doesn't match queue"; unsigned int readPtr = Rptr(); unsigned int writePtr = m_pendingWptr; diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index d3c50f7e60..e17911d65a 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -57,7 +57,7 @@ void Dispatch::SetDim(unsigned int x, unsigned int y, unsigned int z) { m_DimZ = z; } -void Dispatch::SetScratch(int numWaves, int waveSize, unsigned long long scratch_base) { +void Dispatch::SetScratch(int numWaves, int waveSize, HSAuint64 scratch_base) { m_ComputeTmpringSize = ((waveSize << 12) | (numWaves)); m_ScratchEn = true; m_scratch_base = scratch_base; @@ -98,7 +98,7 @@ int Dispatch::SyncWithStatus(unsigned int timeout) { } void Dispatch::BuildIb() { - unsigned long long shiftedIsaAddr = m_IsaBuf.As() >> 8; + HSAuint64 shiftedIsaAddr = m_IsaBuf.As() >> 8; unsigned int arg0, arg1, arg2, arg3; SplitU64(reinterpret_cast(m_pArg1), arg0, arg1); SplitU64(reinterpret_cast(m_pArg2), arg2, arg3); @@ -118,7 +118,7 @@ void Dispatch::BuildIb() { unsigned int pgmRsrc2 = 0; pgmRsrc2 |= (m_ScratchEn << COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT) & COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK; - pgmRsrc2 |= ((m_scratch_base ? 6 : 4 ) << COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT) + pgmRsrc2 |= ((m_scratch_base ? 6 : 4) << COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT) & COMPUTE_PGM_RSRC2__USER_SGPR_MASK; pgmRsrc2 |= (1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT) & COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK; @@ -132,7 +132,9 @@ void Dispatch::BuildIb() { & COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK; const unsigned int COMPUTE_PGM_RSRC[] = { - 0x000c0084 | ((m_SpiPriority & 3) << 10), // PGM_RSRC1 = { VGPRS: 16 SGPRS: 16 PRIORITY: m_SpiPriority FLOAT_MODE: c0 PRIV: 0 DX10_CLAMP: 0 DEBUG_MODE: 0 IEEE_MODE: 0 BULKY: 0 CDBG_USER: 0 } + // PGM_RSRC1 = { VGPRS: 16 SGPRS: 16 PRIORITY: m_SpiPriority FLOAT_MODE: c0 PRIV: 0 + // DX10_CLAMP: 0 DEBUG_MODE: 0 IEEE_MODE: 0 BULKY: 0 CDBG_USER: 0 } + 0x000c0084 | ((m_SpiPriority & 3) << 10), pgmRsrc2 }; @@ -200,18 +202,24 @@ void Dispatch::BuildIb() { m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket()); - m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_START_X, COMPUTE_DISPATCH_DIMS_VALUES, ARRAY_SIZE(COMPUTE_DISPATCH_DIMS_VALUES))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_START_X, COMPUTE_DISPATCH_DIMS_VALUES, + ARRAY_SIZE(COMPUTE_DISPATCH_DIMS_VALUES))); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO, (g_TestGPUFamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8, (g_TestGPUFamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8))); - m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC, ARRAY_SIZE(COMPUTE_PGM_RSRC))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC, + ARRAY_SIZE(COMPUTE_PGM_RSRC))); - m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_RESOURCE_LIMITS, COMPUTE_RESOURCE_LIMITS, ARRAY_SIZE(COMPUTE_RESOURCE_LIMITS))); - m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_TMPRING_SIZE, COMPUTE_TMPRING_SIZE, ARRAY_SIZE(COMPUTE_TMPRING_SIZE))); - m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_RESTART_X, COMPUTE_RESTART_VALUES, ARRAY_SIZE(COMPUTE_RESTART_VALUES))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_RESOURCE_LIMITS, COMPUTE_RESOURCE_LIMITS, + ARRAY_SIZE(COMPUTE_RESOURCE_LIMITS))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_TMPRING_SIZE, COMPUTE_TMPRING_SIZE, + ARRAY_SIZE(COMPUTE_TMPRING_SIZE))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_RESTART_X, COMPUTE_RESTART_VALUES, + ARRAY_SIZE(COMPUTE_RESTART_VALUES))); - m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_USER_DATA_0, COMPUTE_USER_DATA_VALUES, ARRAY_SIZE(COMPUTE_USER_DATA_VALUES))); + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_USER_DATA_0, COMPUTE_USER_DATA_VALUES, + ARRAY_SIZE(COMPUTE_USER_DATA_VALUES))); m_IndirectBuf.AddPacket(PM4DispatchDirectPacket(m_DimX, m_DimY, m_DimZ, DISPATCH_INIT_VALUE)); diff --git a/tests/kfdtest/src/Dispatch.hpp b/tests/kfdtest/src/Dispatch.hpp index 36983554fb..b89306c02d 100644 --- a/tests/kfdtest/src/Dispatch.hpp +++ b/tests/kfdtest/src/Dispatch.hpp @@ -42,7 +42,7 @@ class Dispatch { int SyncWithStatus(unsigned int timeout); - void SetScratch(int numWaves, int waveSize, unsigned long long scratch_base); + void SetScratch(int numWaves, int waveSize, HSAuint64 scratch_base); void SetSpiPriority(unsigned int priority); @@ -68,7 +68,7 @@ class Dispatch { bool m_ScratchEn; unsigned int m_ComputeTmpringSize; - unsigned long long m_scratch_base; + HSAuint64 m_scratch_base; unsigned int m_SpiPriority; }; diff --git a/tests/kfdtest/src/GoogleTestExtension.hpp b/tests/kfdtest/src/GoogleTestExtension.hpp index 5633a156bf..7b888b1677 100644 --- a/tests/kfdtest/src/GoogleTestExtension.hpp +++ b/tests/kfdtest/src/GoogleTestExtension.hpp @@ -34,7 +34,7 @@ enum LOGTYPE { }; class KFDLog{}; -std::ostream& operator << (KFDLog log ,LOGTYPE level); +std::ostream& operator << (KFDLog log, LOGTYPE level); // @brief log additional details, to be displayed in the same format as other google test outputs // currently not supported by google test diff --git a/tests/kfdtest/src/IndirectBuffer.cpp b/tests/kfdtest/src/IndirectBuffer.cpp index fb31a9203c..3fa70f27f6 100644 --- a/tests/kfdtest/src/IndirectBuffer.cpp +++ b/tests/kfdtest/src/IndirectBuffer.cpp @@ -29,7 +29,8 @@ IndirectBuffer::IndirectBuffer(PACKETTYPE type, unsigned int sizeInDWords, unsigned int NodeId) :m_NumOfPackets(0), m_MaxSize(sizeInDWords), m_ActualSize(0), m_PacketTypeAllowed(type) { - m_IndirectBuf = new HsaMemoryBuffer(sizeInDWords*sizeof(unsigned int), NodeId, true/*zero*/, false/*local*/, true/*exec*/); + m_IndirectBuf = new HsaMemoryBuffer(sizeInDWords*sizeof(unsigned int), NodeId, true/*zero*/, + false/*local*/, true/*exec*/); } IndirectBuffer::~IndirectBuffer(void) { diff --git a/tests/kfdtest/src/IsaGenerator.hpp b/tests/kfdtest/src/IsaGenerator.hpp index 11f1d0ca5b..4b9c49ad9e 100644 --- a/tests/kfdtest/src/IsaGenerator.hpp +++ b/tests/kfdtest/src/IsaGenerator.hpp @@ -28,8 +28,7 @@ /* isa generation class - interface */ class IsaGenerator { - -public: + public: static IsaGenerator* Create(unsigned int familyId); virtual ~IsaGenerator() {} @@ -43,11 +42,11 @@ public: void CompileShader(const char* shaderCode, const char* shaderName, HsaMemoryBuffer& rBuf); -protected: + protected: virtual const std::string& GetAsicName() = 0; -private: + private: static const std::string ADDRESS_WATCH_SP3; }; -#endif //_ISAGENERATOR_H_ +#endif // _ISAGENERATOR_H_ diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 9a12e545ce..f7a2852e65 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -24,9 +24,6 @@ #define __KFD_BASE_COMPONENT_TEST__H__ #include -#include "hsakmt.h" -#include "OSWrapper.hpp" -#include "KFDTestUtil.hpp" #include #include #include @@ -34,6 +31,9 @@ #include #include #include +#include "hsakmt.h" +#include "OSWrapper.hpp" +#include "KFDTestUtil.hpp" // @class KFDBaseComponentTest class KFDBaseComponentTest : public testing::Test { @@ -63,13 +63,17 @@ class KFDBaseComponentTest : public testing::Test { HsaMemFlags m_MemoryFlags; HsaNodeInfo m_NodeInfo; - // @brief SetUpTestCase function run before the first test that uses KFDOpenCloseKFDTest class fixture, and opens KFD. + // @brief SetUpTestCase function run before the first test that uses + // KFDOpenCloseKFDTest class fixture, and opens KFD. static void SetUpTestCase(); - // @brief TearDownTestCase function run after the last test from KFDOpenCloseKFDTest class fixture and calls close KFD. + // @brief TearDownTestCase function run after the last test from + // KFDOpenCloseKFDTest class fixture and calls close KFD. static void TearDownTestCase(); - // @brief SetUp function run before every test that uses KFDOpenCloseKFDTest class fixture, sets all common settings for the tests. + // @brief SetUp function run before every test that uses + // KFDOpenCloseKFDTest class fixture, sets all common settings for the tests. virtual void SetUp(); - // @brief TearDown function run after every test that uses KFDOpenCloseKFDTest class fixture. + // @brief TearDown function run after every test that uses + // KFDOpenCloseKFDTest class fixture. virtual void TearDown(); }; diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 594500c032..ace560e5f7 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -89,7 +89,8 @@ void KFDCWSRTest::SetUp() { m_pIsaGen = IsaGenerator::Create(m_FamilyId); - // TODO: Seems in the ISA, I can not get the workitem_id as expected, so I can not set the destination based on workitem_id. + // TODO: Seems in the ISA, I can not get the workitem_id as expected, so I can not + // set the destination based on workitem_id. // Set the wave_num to 1 for now as a workarpound. Will set it to 8 or even 256 in the future. wave_number = 1; diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index abff218160..52d904ef74 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -140,18 +140,17 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { ASSERT_SUCCESS(hsaKmtDbgRegister(defaultGPUNode)); AddressWatchSuccess = hsaKmtDbgAddressWatch( - defaultGPUNode, // IN - 2, // # watch points - &WatchMode[0], // IN - (void **) &WatchAddress[0], // IN - &WatchMask[0], // IN, optional - NULL // IN, optional - ); + defaultGPUNode, // IN + 2, // # watch points + &WatchMode[0], // IN + reinterpret_cast(&WatchAddress[0]), // IN + &WatchMask[0], // IN, optional + NULL); // IN, optional EXPECT_EQ(AddressWatchSuccess, HSAKMT_STATUS_SUCCESS); Dispatch dispatch(isaBuf); - dispatch.SetArgs(dstBuf.As(), (void *)secDstBuf); + dispatch.SetArgs(dstBuf.As(), reinterpret_cast(secDstBuf)); dispatch.SetDim(1, 1, 1); // TODO: use Memory ordering rules w/ atomics diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index 200fe638b4..7d266046bc 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -136,7 +136,7 @@ class QueueAndSignalBenchmark { uint64_t startTime; PM4Queue queue; - HsaEvent** pHsaEvent = (HsaEvent**) calloc(eventCount, sizeof(HsaEvent*)); + HsaEvent** pHsaEvent = reinterpret_cast(calloc(eventCount, sizeof(HsaEvent*))); size_t packetSize = PM4ReleaseMemoryPacket(false, 0, 0).SizeInBytes(); int qSize = fmax(PAGE_SIZE, pow2_round_up(packetSize*eventCount + 1)); @@ -268,7 +268,8 @@ TEST_F(KFDEventTest, SignalMultipleEventsWaitForAll) { unsigned int pktSizeDwords = 0; for (i = 0; i < EVENT_NUMBER; i++) { - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, pHsaEvent[i]->EventData.HWData2, + pHsaEvent[i]->EventId)); queue.Wait4PacketConsumption(); Delay(WAIT_BETWEEN_SUBMISSIONS_MS); diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 70b733b566..3f7e3537a1 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -40,7 +40,8 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui totalMB = N_PROCESSES*count*(vramBufSize>>20); if (m_IsParent) { - LOG() << "Allocating " << N_PROCESSES << "*" << count << "*" << (vramBufSize>>20) << "(="<< totalMB << ")MB VRAM in KFD" << std::endl; + LOG() << "Allocating " << N_PROCESSES << "*" << count << "*" << (vramBufSize>>20) << "(=" + << totalMB << ")MB VRAM in KFD" << std::endl; } HSAKMT_STATUS ret; @@ -95,7 +96,8 @@ void KFDEvictTest::AllocAmdgpuBo(int rn, HSAuint64 vramBufSize, amdgpu_bo_handle alloc.flags = AMDGPU_GEM_CREATE_VRAM_CLEARED; if (m_IsParent) { - LOG() << "Allocating " << N_PROCESSES << "*" << (vramBufSize >> 20) / N_PROCESSES << "(=" << (vramBufSize >> 20) << ")MB VRAM in GFX" << std::endl; + LOG() << "Allocating " << N_PROCESSES << "*" << (vramBufSize >> 20) / N_PROCESSES << "(=" + << (vramBufSize >> 20) << ")MB VRAM in GFX" << std::endl; } ASSERT_EQ(0, amdgpu_bo_alloc(m_RenderNodes[rn].device_handle, &alloc, &handle)); } @@ -104,79 +106,72 @@ void KFDEvictTest::FreeAmdgpuBo(amdgpu_bo_handle handle) { ASSERT_EQ(0, amdgpu_bo_free(handle)); } -static int -amdgpu_bo_alloc_and_map(amdgpu_device_handle dev, unsigned size, - unsigned alignment, unsigned heap, uint64_t flags, - amdgpu_bo_handle *bo, void **cpu, uint64_t *mc_address, - amdgpu_va_handle *va_handle) -{ - struct amdgpu_bo_alloc_request request = {}; - amdgpu_bo_handle buf_handle; - amdgpu_va_handle handle; - uint64_t vmc_addr; - int r; +static int amdgpu_bo_alloc_and_map(amdgpu_device_handle dev, unsigned size, + unsigned alignment, unsigned heap, uint64_t flags, + amdgpu_bo_handle *bo, void **cpu, uint64_t *mc_address, + amdgpu_va_handle *va_handle) { + struct amdgpu_bo_alloc_request request = {}; + amdgpu_bo_handle buf_handle; + amdgpu_va_handle handle; + uint64_t vmc_addr; + int r; - request.alloc_size = size; - request.phys_alignment = alignment; - request.preferred_heap = heap; - request.flags = flags; + request.alloc_size = size; + request.phys_alignment = alignment; + request.preferred_heap = heap; + request.flags = flags; - r = amdgpu_bo_alloc(dev, &request, &buf_handle); - if (r) - return r; + r = amdgpu_bo_alloc(dev, &request, &buf_handle); + if (r) + return r; - r = amdgpu_va_range_alloc(dev, - amdgpu_gpu_va_range_general, - size, alignment, 0, &vmc_addr, - &handle, 0); - if (r) - goto error_va_alloc; + r = amdgpu_va_range_alloc(dev, + amdgpu_gpu_va_range_general, + size, alignment, 0, &vmc_addr, + &handle, 0); + if (r) + goto error_va_alloc; - r = amdgpu_bo_va_op(buf_handle, 0, size, vmc_addr, 0, AMDGPU_VA_OP_MAP); - if (r) - goto error_va_map; + r = amdgpu_bo_va_op(buf_handle, 0, size, vmc_addr, 0, AMDGPU_VA_OP_MAP); + if (r) + goto error_va_map; - r = amdgpu_bo_cpu_map(buf_handle, cpu); - if (r) - goto error_cpu_map; + r = amdgpu_bo_cpu_map(buf_handle, cpu); + if (r) + goto error_cpu_map; - *bo = buf_handle; - *mc_address = vmc_addr; - *va_handle = handle; + *bo = buf_handle; + *mc_address = vmc_addr; + *va_handle = handle; - return 0; + return 0; error_cpu_map: - amdgpu_bo_cpu_unmap(buf_handle); + amdgpu_bo_cpu_unmap(buf_handle); error_va_map: - amdgpu_bo_va_op(buf_handle, 0, size, vmc_addr, 0, AMDGPU_VA_OP_UNMAP); + amdgpu_bo_va_op(buf_handle, 0, size, vmc_addr, 0, AMDGPU_VA_OP_UNMAP); error_va_alloc: - amdgpu_bo_free(buf_handle); - return r; + amdgpu_bo_free(buf_handle); + return r; } -static inline int -amdgpu_bo_unmap_and_free(amdgpu_bo_handle bo, amdgpu_va_handle va_handle, - uint64_t mc_addr, uint64_t size) -{ - amdgpu_bo_cpu_unmap(bo); - amdgpu_bo_va_op(bo, 0, size, mc_addr, 0, AMDGPU_VA_OP_UNMAP); - amdgpu_va_range_free(va_handle); - amdgpu_bo_free(bo); - - return 0; +static inline int amdgpu_bo_unmap_and_free(amdgpu_bo_handle bo, amdgpu_va_handle va_handle, + uint64_t mc_addr, uint64_t size) { + amdgpu_bo_cpu_unmap(bo); + amdgpu_bo_va_op(bo, 0, size, mc_addr, 0, AMDGPU_VA_OP_UNMAP); + amdgpu_va_range_free(va_handle); + amdgpu_bo_free(bo); + return 0; } -static inline int -amdgpu_get_bo_list(amdgpu_device_handle dev, amdgpu_bo_handle bo1, - amdgpu_bo_handle bo2, amdgpu_bo_list_handle *list) -{ - amdgpu_bo_handle resources[] = {bo1, bo2}; +static inline int amdgpu_get_bo_list(amdgpu_device_handle dev, amdgpu_bo_handle bo1, + amdgpu_bo_handle bo2, amdgpu_bo_list_handle *list) { + amdgpu_bo_handle resources[] = {bo1, bo2}; - return amdgpu_bo_list_create(dev, bo2 ? 2 : 1, resources, NULL, list); + return amdgpu_bo_list_create(dev, bo2 ? 2 : 1, resources, NULL, list); } void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { @@ -204,7 +199,7 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { &boList)); /* Fill Nop cammands in IB */ - ptr = (uint32_t *)ibResultCpu; + ptr = reinterpret_cast(ibResultCpu); for (int i = 0; i < 16; i++) ptr[i] = 0xffff1000; diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 1a4a715d97..22ea893fe0 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -85,7 +85,7 @@ void KFDExceptionTest::TestMemoryException(int defaultGPUNode, HSAuint64 pSrc, } dispatch.SetDim(dimX, dimY, dimZ); - dispatch.SetArgs((void *)pSrc, (void *)pDst); + dispatch.SetArgs(reinterpret_cast(pSrc), reinterpret_cast(pDst)); dispatch.Submit(queue); m_ChildStatus = hsaKmtWaitOnEvent(vmFaultEvent, g_TestTimeOut); diff --git a/tests/kfdtest/src/KFDGraphicsInterop.hpp b/tests/kfdtest/src/KFDGraphicsInterop.hpp index 260044cebf..3c4001ed41 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.hpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.hpp @@ -28,11 +28,10 @@ // @class KFDGraphicsInteropTest // Adds access to graphics device for interoperability testing -class KFDGraphicsInterop : public KFDMemoryTest -{ -public: - KFDGraphicsInterop(void) {}; - ~KFDGraphicsInterop(void) {}; +class KFDGraphicsInterop : public KFDMemoryTest { + public: + KFDGraphicsInterop(void) {} + ~KFDGraphicsInterop(void) {} }; #endif diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index 4f7e90adac..6a95f51c9a 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -77,10 +77,10 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { HSAuint32 *sharedLocalBuffer = NULL; /* Read from Pipe the shared Handle. Import shared Local Memory */ - ASSERT_GE(read(pipefd[0], (void*)&sharedHandleLM, sizeof(sharedHandleLM)), 0); + ASSERT_GE(read(pipefd[0], reinterpret_cast(&sharedHandleLM), sizeof(sharedHandleLM)), 0); ASSERT_SUCCESS(hsaKmtRegisterSharedHandle(&sharedHandleLM, - (void**)&sharedLocalBuffer, &sharedSize)); + reinterpret_cast(&sharedLocalBuffer), &sharedSize)); ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(sharedLocalBuffer, sharedSize, NULL)); /* Check for pattern in the shared Local Memory */ @@ -128,7 +128,7 @@ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pip /* Share it with the child process */ ASSERT_SUCCESS(hsaKmtShareMemory(toShareLocalBuffer.As(), size, &sharedHandleLM)); - ASSERT_GE(write(pipefd[1], (void*)&sharedHandleLM, sizeof(sharedHandleLM)), 0); + ASSERT_GE(write(pipefd[1], reinterpret_cast(&sharedHandleLM), sizeof(sharedHandleLM)), 0); /* Wait for the child to finish */ waitpid(cpid, &status, 0); @@ -413,7 +413,7 @@ static int read_non_block(int fd, void *buf, int size) { int total_bytes = 0, cur_bytes = 0; int retries = 5; struct timespec tm = { 0, 100000000ULL }; - char *ptr = (char *)buf; + char *ptr = reinterpret_cast(buf); do { cur_bytes = read(fd, ptr, (size - total_bytes)); @@ -439,7 +439,7 @@ static int read_non_block(int fd, void *buf, int size) { /* Send HsaMemoryRange to another process that is connected via writePipe */ CMA_TEST_STATUS KFDCMAArray::sendCMAArray(int writePipe) { - if (write_non_block(writePipe, (void*)&m_HsaMemoryRange, sizeof(m_HsaMemoryRange)) != + if (write_non_block(writePipe, reinterpret_cast(&m_HsaMemoryRange), sizeof(m_HsaMemoryRange)) != sizeof(m_HsaMemoryRange)) return CMA_IPC_PIPE_ERROR; return CMA_TEST_SUCCESS; @@ -449,7 +449,7 @@ CMA_TEST_STATUS KFDCMAArray::sendCMAArray(int writePipe) { CMA_TEST_STATUS KFDCMAArray::recvCMAArray(int readPipe) { int i; - if (read_non_block(readPipe, (void*)&m_HsaMemoryRange, sizeof(m_HsaMemoryRange)) != + if (read_non_block(readPipe, reinterpret_cast(&m_HsaMemoryRange), sizeof(m_HsaMemoryRange)) != sizeof(m_HsaMemoryRange)) return CMA_IPC_PIPE_ERROR; @@ -704,10 +704,13 @@ TEST_F(KFDIPCTest, CMABasicTest) { HSAuint32 expected_pattern; srcRange.MemoryAddress = testLocalBuffer.As(); - srcRange.SizeInBytes = size; /* Deliberately set to value > unaligned_size. Only unaligned_size - * should be copied since dstRange.SizeInBytes == unaligned_size - */ - dstRange.MemoryAddress = (void *)(testLocalBuffer.As() + (size / 2) + unaligned_offset); + + /* Deliberately set to value > unaligned_size. Only unaligned_size + * should be copied since dstRange.SizeInBytes == unaligned_size + */ + srcRange.SizeInBytes = size; + + dstRange.MemoryAddress = reinterpret_cast(testLocalBuffer.As() + (size / 2) + unaligned_offset); dstRange.SizeInBytes = unaligned_size; ASSERT_SUCCESS(hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied)); ASSERT_EQ(copied, unaligned_size); @@ -719,7 +722,7 @@ TEST_F(KFDIPCTest, CMABasicTest) { /* Test3. Test overflow and expect failure */ srcRange.MemoryAddress = testLocalBuffer.As(); srcRange.SizeInBytes = size; - dstRange.MemoryAddress = (void *)(testLocalBuffer.As() + 4); + dstRange.MemoryAddress = reinterpret_cast(testLocalBuffer.As() + 4); dstRange.SizeInBytes = size; /* This should overflow since offset is VA + 4 */ status = hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied); EXPECT_NE(status, HSAKMT_STATUS_SUCCESS); diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index c6119156bb..986ee4a85c 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -316,7 +316,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { break; } - void *bufferEnd = (void *)((unsigned long)pages[order].pointers[p] + void *bufferEnd = reinterpret_cast(reinterpret_cast(pages[order].pointers[p]) + size - sizeof(unsigned)); sysBuffer.As()[0] = ++value; @@ -340,7 +340,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { Dispatch dispatch3(isaBuffer); dispatch3.SetArgs(bufferEnd, - (void *)&(sysBuffer.As()[1])); + reinterpret_cast(&(sysBuffer.As()[1]))); dispatch3.Submit(queue); dispatch3.Sync(g_TestTimeOut); EXPECT_EQ(value, sysBuffer.As()[1]); @@ -349,7 +349,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { } LOG() << " Got " << pages[order].nPages << ", end of last block addr: " - << (void *)((unsigned long)pages[order].pointers[p-1] + size - 1) + << reinterpret_cast(reinterpret_cast(pages[order].pointers[p-1]) + size - 1) << std::endl; // Now free half the memory diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 8282106420..15a1c20051 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -94,14 +94,14 @@ shader ReadMemory\n\ asic(GFX9)\n\ type(CS)\n\ /* Assume src address in s0, s1 and dst address in s2, s3*/\n\ - s_movk_i32 s18, 0x5678\n\ - LOOP:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 LOOP\n\ - s_store_dword s18, s[2:3], 0x0 glc\n\ - s_endpgm\n\ - end\n\ + s_movk_i32 s18, 0x5678\n\ + LOOP:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 LOOP\n\ + s_store_dword s18, s[2:3], 0x0 glc\n\ + s_endpgm\n\ + end\n\ "; void KFDMemoryTest::SetUp() { @@ -127,7 +127,7 @@ void KFDMemoryTest::TearDown() { } #include -#define GB(x) ((x)<<30) +#define GB(x) ((x) << 30) /* * try to map as much as possible system memory to gpu. @@ -147,17 +147,17 @@ TEST_F(KFDMemoryTest, MMapLarge) { HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - const unsigned long nObjects = 1<<14; + const HSAuint64 nObjects = 1<<14; HSAuint64 *AlternateVAGPU = new HSAuint64[nObjects]; - ASSERT_NE((unsigned long)AlternateVAGPU, 0); + ASSERT_NE((HSAuint64)AlternateVAGPU, 0); HsaMemMapFlags mapFlags = {0}; - unsigned long s; + HSAuint64 s; char *addr; - unsigned long flags = MAP_ANONYMOUS | MAP_PRIVATE; + HSAuint64 flags = MAP_ANONYMOUS | MAP_PRIVATE; /* Test up to 1TB memory*/ s = GB(1024ULL) / nObjects; - addr = (char*)mmap(0, s, PROT_READ | PROT_WRITE, flags, -1, 0); + addr = reinterpret_cast(mmap(0, s, PROT_READ | PROT_WRITE, flags, -1, 0)); ASSERT_NE(addr, MAP_FAILED); memset(addr, 0, s); @@ -167,7 +167,7 @@ TEST_F(KFDMemoryTest, MMapLarge) { if (hsaKmtRegisterMemory(addr + i, s - i)) break; if (hsaKmtMapMemoryToGPUNodes(addr + i, s - i, - &AlternateVAGPU[i], mapFlags, 1, (HSAuint32 *)&defaultGPUNode)) { + &AlternateVAGPU[i], mapFlags, 1, reinterpret_cast(&defaultGPUNode))) { hsaKmtDeregisterMemory(addr + i); break; } @@ -177,8 +177,8 @@ TEST_F(KFDMemoryTest, MMapLarge) { << "GB system memory to gpu" << std::endl; while (i--) { - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU((void*)AlternateVAGPU[i])); - ASSERT_SUCCESS(hsaKmtDeregisterMemory((void*)AlternateVAGPU[i])); + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(AlternateVAGPU[i]))); + ASSERT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(AlternateVAGPU[i]))); } munmap(addr, s); @@ -268,7 +268,8 @@ TEST_F(KFDMemoryTest , MapMemoryToGPU) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode /* system */, PAGE_SIZE, m_MemoryFlags, (void**)&pDb)); + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode /* system */, PAGE_SIZE, m_MemoryFlags, + reinterpret_cast(&pDb))); // verify that pDb is not null before it's being used ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(pDb, PAGE_SIZE, NULL)); @@ -292,7 +293,8 @@ TEST_F(KFDMemoryTest, ZeroMemorySizeAlloc) { TEST_START(TESTPROFILE_RUNALL) unsigned int* pDb = NULL; - EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtAllocMemory(0 /* system */, 0, m_MemoryFlags, (void**)&pDb)); + EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtAllocMemory(0 /* system */, 0, m_MemoryFlags, + reinterpret_cast(&pDb))); TEST_END } @@ -302,7 +304,7 @@ TEST_F(KFDMemoryTest, MemoryAlloc) { TEST_START(TESTPROFILE_RUNALL) unsigned int* pDb = NULL; - EXPECT_SUCCESS(hsaKmtAllocMemory(0 /* system */, PAGE_SIZE, m_MemoryFlags, (void**)&pDb)); + EXPECT_SUCCESS(hsaKmtAllocMemory(0 /* system */, PAGE_SIZE, m_MemoryFlags, reinterpret_cast(&pDb))); TEST_END } @@ -340,7 +342,7 @@ TEST_F(KFDMemoryTest, AccessPPRMem) { * consumed by IOMMU HW. Because of that, a kernel driver workaround * is put in place to address that, so we don't need to wait here. */ - //sleep(5); + // sleep(5); VirtualFreeMemory(destBuf, PAGE_SIZE); @@ -472,10 +474,10 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { EXPECT_SUCCESS(hsaKmtMapMemoryToGPU((void *)&mem[0], sizeof(HSAuint32), &gpuva2)); EXPECT_TRUE(gpuva1 != gpuva2); - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva1)); - EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva1)); - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva2)); - EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva2)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva1))); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(gpuva1))); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva2))); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(gpuva2))); /* Same address, same size */ HsaMemMapFlags memFlags = {0}; @@ -496,20 +498,20 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { sizeof(HSAuint32) * 2, &gpuva2, memFlags, nGPU, nodes)); EXPECT_EQ(gpuva1, gpuva2); - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva1)); - EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva1)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva1))); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(gpuva1))); /* Confirm that we still have access to the memory, mem[2] */ PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); mem[2] = 0x0; - queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)gpuva2, + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(reinterpret_cast(gpuva2), 0xdeadbeef)); queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(true, 0, 0)); queue.Wait4PacketConsumption(); - EXPECT_EQ(true, WaitOnValue((unsigned int *)&mem[2], 0xdeadbeef)); + EXPECT_EQ(true, WaitOnValue((unsigned int *)(&mem[2]), 0xdeadbeef)); EXPECT_SUCCESS(queue.Destroy()); - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU((void *)gpuva2)); - EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)gpuva2)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva2))); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(gpuva2))); TEST_END } @@ -535,7 +537,8 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - HsaMemoryBuffer scratchBuffer(SCRATCH_SIZE, defaultGPUNode, false/*zero*/, false/*local*/, false/*exec*/, true /*scratch*/); + HsaMemoryBuffer scratchBuffer(SCRATCH_SIZE, defaultGPUNode, false/*zero*/, false/*local*/, + false/*exec*/, true /*scratch*/); // Unmap scratch for sub-allocation mapping tests ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As())); @@ -576,7 +579,8 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { if (pNodeProperties != NULL) { // Get the aperture of the scratch buffer HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; - EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(defaultGPUNode, pNodeProperties->NumMemoryBanks, memoryProperties)); + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(defaultGPUNode, pNodeProperties->NumMemoryBanks, + memoryProperties)); for (unsigned int bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) { if (memoryProperties[bank].HeapType == HSA_HEAPTYPE_GPU_SCRATCH) { @@ -684,14 +688,14 @@ void KFDMemoryTest::BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granular sizeMB = (lowMB + highMB) / 2; size = sizeMB * 1024 * 1024; ret = hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, - (void**)&pDb); + reinterpret_cast(&pDb)); if (ret) { highMB = sizeMB; continue; } ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, - mapFlags, 1, (HSAuint32 *)&defaultGPUNode); + mapFlags, 1, reinterpret_cast(&defaultGPUNode)); if (ret) { ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); highMB = sizeMB; @@ -740,14 +744,14 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, sizeMB = (lowMB + highMB) / 2; size = sizeMB * 1024 * 1024; ret = hsaKmtAllocMemory(defaultGPUNode, size, memFlags, - (void**)&pDb); + reinterpret_cast(&pDb)); if (ret) { highMB = sizeMB; continue; } ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, - mapFlags, 1, (HSAuint32 *)&defaultGPUNode); + mapFlags, 1, reinterpret_cast(&defaultGPUNode)); if (ret) { ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); highMB = sizeMB; @@ -810,13 +814,13 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { do { ret = hsaKmtAllocMemory(0 /* system */, block_size, m_MemoryFlags, - (void**)&pDb_array[i]); + reinterpret_cast(&pDb_array[i])); if (ret) { break; } ret = hsaKmtMapMemoryToGPUNodes(pDb_array[i], block_size, - &AlternateVAGPU, mapFlags, 1, (HSAuint32 *)&defaultGPUNode); + &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode)); if (ret) { ASSERT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); break; @@ -849,7 +853,7 @@ TEST_F(KFDMemoryTest, MMBench) { #define TEST_SDMA(index) (((index / nSizes) >> 1) & 0x1) void *bufs[nBufs]; - unsigned long long start, end; + HSAuint64 start, end; unsigned i; HSAKMT_STATUS ret; HsaMemFlags memFlags = {0}; @@ -898,7 +902,7 @@ TEST_F(KFDMemoryTest, MMBench) { unsigned bufSize = TEST_BUFSIZE(testIndex); unsigned memType = TEST_MEMTYPE(testIndex); bool interleaveSDMA = TEST_SDMA(testIndex); - unsigned long long allocTime, map1Time, unmap1Time, mapAllTime, unmapAllTime, freeTime; + HSAuint64 allocTime, map1Time, unmap1Time, mapAllTime, unmapAllTime, freeTime; HSAuint32 allocNode; if ((testIndex & (nSizes-1)) == 0) @@ -1033,16 +1037,16 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { EXPECT_EQ(ptrInfo.SizeInBytes, (HSAuint64)localBuffer.Size()); HSAuint32 *addr = localBuffer.As() + 4; - EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)addr, &ptrInfo)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(reinterpret_cast(addr), &ptrInfo)); EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)localBuffer.As()); } /** Registered memory: user pointer */ static volatile HSAuint32 mem[4]; // 8 bytes for register only and // 8 bytes for register to nodes - HsaMemoryBuffer hsaBuffer((void *)&mem[0], sizeof(HSAuint32)*2); + HsaMemoryBuffer hsaBuffer((void *)(&mem[0]), sizeof(HSAuint32)*2); if (is_dgpu()) { // APU doesn't use userptr - EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)&mem[0], &ptrInfo)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[0]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)hsaBuffer.As()); @@ -1053,29 +1057,29 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { HSAuint32 nodes[nGPU]; for (unsigned int i = 0; i < nGPU; i++) nodes[i] = gpuNodes.at(i); - EXPECT_SUCCESS(hsaKmtRegisterMemoryToNodes((void *)&mem[2], + EXPECT_SUCCESS(hsaKmtRegisterMemoryToNodes((void *)(&mem[2]), sizeof(HSAuint32)*2, nGPU, nodes)); - EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)&mem[2], &ptrInfo)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[2]), &ptrInfo)); EXPECT_EQ(ptrInfo.NRegisteredNodes, nGPU); - EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)&mem[2])); + EXPECT_SUCCESS(hsaKmtDeregisterMemory((void *)(&mem[2]))); } /* Not a starting address, but an address inside the memory range * should also get the memory information */ HSAuint32 *address = hostBuffer.As() + 1; - EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)address, &ptrInfo)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(reinterpret_cast(address), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED); EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As()); if (is_dgpu()) { - EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)&mem[1], &ptrInfo)); + EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[1]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); } /*** Set user data ***/ char userData[16] = "This is a test."; - EXPECT_SUCCESS(hsaKmtSetMemoryUserData(hostBuffer.As(), (void *)userData)); + EXPECT_SUCCESS(hsaKmtSetMemoryUserData(hostBuffer.As(), reinterpret_cast(userData))); EXPECT_SUCCESS(hsaKmtQueryPointerInfo(hostBuffer.As(), &ptrInfo)); EXPECT_EQ(ptrInfo.UserData, (void *)userData); @@ -1106,16 +1110,16 @@ TEST_F(KFDMemoryTest, PtraceAccess) { // Offset in the VRAM buffer to test crossing non-contiguous // buffer boundaries. The second access starting from offset - // sizeof(long)+1 will cross a node boundary in a single access, + // sizeof(HSAint64)+1 will cross a node boundary in a single access, // for node sizes of 4MB or smaller. - const HSAuint64 VRAM_OFFSET = (4 << 20) - 2 * sizeof(long); + const HSAuint64 VRAM_OFFSET = (4 << 20) - 2 * sizeof(HSAint64); // alloc system memory from node 0 and initialize it memFlags.ui32.NonPaged = 0; ASSERT_SUCCESS(hsaKmtAllocMemory(0, PAGE_SIZE*2, memFlags, &mem[0])); - for (i = 0; i < 4*sizeof(long) + 4; i++) { - ((HSAuint8 *)mem[0])[i] = i; // source - ((HSAuint8 *)mem[0])[PAGE_SIZE+i] = 0; // destination + for (i = 0; i < 4*sizeof(HSAint64) + 4; i++) { + (reinterpret_cast(mem[0]))[i] = i; // source + (reinterpret_cast(mem[0]))[PAGE_SIZE+i] = 0; // destination } // try to alloc local memory from GPU node @@ -1123,10 +1127,10 @@ TEST_F(KFDMemoryTest, PtraceAccess) { if (m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode)) { EXPECT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE*2 + (4 << 20), memFlags, &mem[1])); - mem[1] = (void *)((HSAuint8 *)mem[1] + VRAM_OFFSET); - for (i = 0; i < 4*sizeof(long) + 4; i++) { - ((HSAuint8 *)mem[1])[i] = i; - ((HSAuint8 *)mem[1])[PAGE_SIZE+i] = 0; + mem[1] = reinterpret_cast(reinterpret_cast(mem[1]) + VRAM_OFFSET); + for (i = 0; i < 4*sizeof(HSAint64) + 4; i++) { + (reinterpret_cast(mem[1]))[i] = i; + (reinterpret_cast(mem[1]))[PAGE_SIZE+i] = 0; } } else { LOG() << "Not testing local memory, it's invisible" << std::endl; @@ -1168,22 +1172,22 @@ TEST_F(KFDMemoryTest, PtraceAccess) { for (i = 0; i < 4; i++) { // Test 4 different (mis-)alignments, leaving 1-byte // gaps between longs - HSAuint8 *addr = (HSAuint8 *)((long *)mem[0] + i) + i; + HSAuint8 *addr = reinterpret_cast(reinterpret_cast(mem[0]) + i) + i; errno = 0; long data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); EXPECT_EQ(0, errno); EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, - (void *)data)); + reinterpret_cast(data))); if (mem[1] == NULL) continue; - addr = (HSAuint8 *)((long *)mem[1] + i) + i; + addr = reinterpret_cast(reinterpret_cast(mem[1]) + i) + i; errno = 0; data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); EXPECT_EQ(0, errno); EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, - (void *)data)); + reinterpret_cast(data))); } } catch (...) { err = 1; @@ -1204,32 +1208,31 @@ TEST_F(KFDMemoryTest, PtraceAccess) { } // Clear gaps in the source that should not have been copied - ((uint8_t*)mem[0])[ sizeof(long) ] = 0; - ((uint8_t*)mem[0])[2*sizeof(long) + 1] = 0; - ((uint8_t*)mem[0])[3*sizeof(long) + 2] = 0; - ((uint8_t*)mem[0])[4*sizeof(long) + 3] = 0; + (reinterpret_cast(mem[0]))[ sizeof(long) ] = 0; + (reinterpret_cast(mem[0]))[2*sizeof(long) + 1] = 0; + (reinterpret_cast(mem[0]))[3*sizeof(long) + 2] = 0; + (reinterpret_cast(mem[0]))[4*sizeof(long) + 3] = 0; // Check results - EXPECT_EQ(0, memcmp(mem[0], (HSAuint8 *)mem[0] + PAGE_SIZE, + EXPECT_EQ(0, memcmp(mem[0], reinterpret_cast(mem[0]) + PAGE_SIZE, sizeof(long)*4 + 4)); // Free memory EXPECT_SUCCESS(hsaKmtFreeMemory(mem[0], PAGE_SIZE*2)); if (mem[1]) { - ((uint8_t*)mem[1])[ sizeof(long) ] = 0; - ((uint8_t*)mem[1])[2*sizeof(long) + 1] = 0; - ((uint8_t*)mem[1])[3*sizeof(long) + 2] = 0; - ((uint8_t*)mem[1])[4*sizeof(long) + 3] = 0; - EXPECT_EQ(0, memcmp(mem[1], (HSAuint8 *)mem[1] + PAGE_SIZE, - sizeof(long)*4 + 4)); - mem[1] = (void *)((HSAuint8 *)mem[1] - VRAM_OFFSET); + (reinterpret_cast(mem[1]))[ sizeof(HSAint64) ] = 0; + (reinterpret_cast(mem[1]))[2*sizeof(HSAint64) + 1] = 0; + (reinterpret_cast(mem[1]))[3*sizeof(HSAint64) + 2] = 0; + (reinterpret_cast(mem[1]))[4*sizeof(HSAint64) + 3] = 0; + EXPECT_EQ(0, memcmp(mem[1], reinterpret_cast(mem[1]) + PAGE_SIZE, + sizeof(HSAint64)*4 + 4)); + mem[1] = reinterpret_cast(reinterpret_cast(mem[1]) - VRAM_OFFSET); EXPECT_SUCCESS(hsaKmtFreeMemory(mem[1], PAGE_SIZE*2)); } TEST_END } -TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) -{ +TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { char *hsaDebug = getenv("HSA_DEBUG"); if (!is_dgpu()) { @@ -1266,8 +1269,8 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) /* set the word before 4M boundary to 0xdeadbeefdeadbeef * and the word after 4M boundary to 0xcafebabecafebabe */ - mem0 = (void *)((HSAuint8 *)mem + VRAM_OFFSET); - mem1 = (void *)((HSAuint8 *)mem + VRAM_OFFSET + sizeof(HSAuint64)); + mem0 = reinterpret_cast(reinterpret_cast(mem) + VRAM_OFFSET); + mem1 = reinterpret_cast(reinterpret_cast(mem) + VRAM_OFFSET + sizeof(HSAuint64)); PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)mem0, @@ -1313,17 +1316,17 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) /* peek the memory */ errno = 0; - long data0 = ptrace(PTRACE_PEEKDATA, tracePid, mem0, NULL); + HSAint64 data0 = ptrace(PTRACE_PEEKDATA, tracePid, mem0, NULL); EXPECT_EQ(0, errno); EXPECT_EQ(data[0], data0); - long data1 = ptrace(PTRACE_PEEKDATA, tracePid, mem1, NULL); + HSAint64 data1 = ptrace(PTRACE_PEEKDATA, tracePid, mem1, NULL); EXPECT_EQ(0, errno); EXPECT_EQ(data[1], data1); /* swap mem0 and mem1 by poking */ - EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, (void *)data[1])); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, reinterpret_cast(data[1]))); EXPECT_EQ(0, errno); - EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, (void *)data[0])); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, reinterpret_cast(data[0]))); EXPECT_EQ(0, errno); } catch (...) { err = 1; @@ -1345,10 +1348,10 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) /* Use shader to read back data to check poke results */ HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - //dstBuffer is cpu accessible gtt memory + // dstBuffer is cpu accessible gtt memory HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? gfx9_ScratchCopyDword : gfx8_ScratchCopyDword, - "ScratchCopyDword",isaBuffer); + "ScratchCopyDword", isaBuffer); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(mem0, dstBuffer.As()); dispatch0.Submit(queue); @@ -1405,7 +1408,7 @@ TEST_F(KFDMemoryTest, SignalHandling) { */ size = (sysMemSize >> 2) & ~(HSAuint64)(PAGE_SIZE - 1); - ASSERT_SUCCESS(hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, (void**)&pDb)); + ASSERT_SUCCESS(hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, reinterpret_cast(&pDb))); // verify that pDb is not null before it's being used ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; @@ -1468,7 +1471,7 @@ TEST_F(KFDMemoryTest, CheckZeroInitializationSysMem) { while (count--) { ret = hsaKmtAllocMemory(0 /* system */, sysBufSize, m_MemoryFlags, - (void**)&pDb); + reinterpret_cast(&pDb)); if (ret) { LOG() << "Failed to allocate system buffer of" << std::dec << sysBufSizeMB << "MB" << std::endl; @@ -1496,8 +1499,7 @@ TEST_F(KFDMemoryTest, CheckZeroInitializationSysMem) { TEST_END } -static inline void access(volatile void *sd, int size, int rw) -{ +static inline void access(volatile void *sd, int size, int rw) { /* Most like sit in cache*/ static struct DUMMY { char dummy[1024]; @@ -1531,7 +1533,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { #define _TEST_MEMTYPE(index) ((index / nSizes) % nMemTypes) void *bufs[nBufs]; - unsigned long long start; + HSAuint64 start; unsigned i; HSAKMT_STATUS ret; HsaMemFlags memFlags = {0}; @@ -1545,7 +1547,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode) || !vramSizeMB) { - LOG() << "not a largebar system, skip!"<NumCounters << " counter IDs" << std::endl; - block = (HsaCounterBlockProperties *)&block->Counters[block->NumCounters]; + block = reinterpret_cast(&block->Counters[block->NumCounters]); } TEST_END @@ -161,7 +162,7 @@ TEST_F(KFDPerfCountersTest, RegisterTrace) { priv_block_found = true; break; } - block = (HsaCounterBlockProperties *)&block->Counters[block->NumCounters]; + block = reinterpret_cast(&block->Counters[block->NumCounters]); } if (!priv_block_found) { @@ -202,7 +203,7 @@ TEST_F(KFDPerfCountersTest, StartStopQueryTrace) { priv_block_found = true; break; } - block = (HsaCounterBlockProperties *)&block->Counters[block->NumCounters]; + block = reinterpret_cast(&block->Counters[block->NumCounters]); } if (!priv_block_found) { diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 25e4c8df73..c1dab79916 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -271,7 +271,8 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) { // don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); - ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; + ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) + << "Packet executed even though the queue is supposed to be disabled!"; ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); @@ -311,7 +312,8 @@ TEST_F(KFDQMTest, DisableSdmaQueueByUpdateWithNullAddress) { // don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); - ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; + ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) + << "Packet executed even though the queue is supposed to be disabled!"; ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); @@ -357,7 +359,8 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) { // don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); - ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; + ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) + << "Packet executed even though the queue is supposed to be disabled!"; ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); @@ -373,13 +376,13 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) { TEST_F(KFDQMTest, CreateQueueStressSingleThreaded) { TEST_START(TESTPROFILE_RUNALL) - static const unsigned long long TEST_TIME_SEC = 15; + static const HSAuint64 TEST_TIME_SEC = 15; - unsigned long long initialTime = GetSystemTickCountInMicroSec(); + HSAuint64 initialTime = GetSystemTickCountInMicroSec(); unsigned int numIter = 0; - unsigned long long timePassed = 0; + HSAuint64 timePassed = 0; int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -404,7 +407,7 @@ TEST_F(KFDQMTest, CreateQueueStressSingleThreaded) { delete queues[1]; ++numIter; - unsigned long long curTime = GetSystemTickCountInMicroSec(); + HSAuint64 curTime = GetSystemTickCountInMicroSec(); timePassed = (curTime - initialTime) / 1000000; } while (timePassed < TEST_TIME_SEC); @@ -553,7 +556,7 @@ s_waitcnt lgkmcnt(0)\n\ end\n\ "; -long long KFDQMTest::TimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count) { +HSAint64 KFDQMTest::TimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer dstBuffer(PAGE_SIZE, node, true, false, false); HsaMemoryBuffer ctlBuffer(PAGE_SIZE, node, true, false, false); @@ -580,9 +583,9 @@ long long KFDQMTest::TimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t m } /* To cover for outliers, allow us to get the Average time based on a specified number of iterations */ -long long KFDQMTest::GetAverageTimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count, int iterations) { - long long timeArray[iterations]; - long long timeTotal = 0; +HSAint64 KFDQMTest::GetAverageTimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count, int iterations) { + HSAint64 timeArray[iterations]; + HSAint64 timeTotal = 0; if (iterations < 1) { LOG() << "ERROR: At least 1 iteration must be performed" << std::endl; return 0; @@ -599,9 +602,11 @@ long long KFDQMTest::GetAverageTimeConsumedwithCUMask(int node, uint32_t* mask, } for (int x = 0; x < iterations; x++) { - long long variance = timeArray[x] / (timeTotal / iterations); + HSAint64 variance = timeArray[x] / (timeTotal / iterations); if (variance < CuNegVariance || variance > CuPosVariance) - LOG() << "WARNING: Measurement #" << x << "/" << iterations << " (" << timeArray[x] << ") is at least " << CuVariance*100 << "% away from the mean (" << timeTotal/iterations << ")" << std::endl; + LOG() << "WARNING: Measurement #" << x << "/" << iterations << " (" << timeArray[x] + << ") is at least " << CuVariance*100 << "% away from the mean (" << timeTotal/iterations << ")" + << std::endl; } return timeTotal / iterations; @@ -625,7 +630,7 @@ TEST_F(KFDQMTest, BasicCuMaskingLinear) { LOG() << std::hex << "# SIMDs per CPU: 0x" << pNodeProperties->NumSIMDPerCU << std::endl; LOG() << std::hex << "# Shader engines: 0x" << numSEs << std::endl; LOG() << std::hex << "# Active CUs: 0x" << ActiveCU << std::endl; - long long TimewithCU1, TimewithCU; + HSAint64 TimewithCU1, TimewithCU; uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */ uint32_t maskNumBits = maskNumDwords * 32; uint32_t mask[maskNumDwords]; @@ -646,10 +651,11 @@ TEST_F(KFDQMTest, BasicCuMaskingLinear) { mask[maskIndex] |= 1 << ((nCUs - 1) % 32); TimewithCU = TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); - ratio = (double)TimewithCU1 / ((double)TimewithCU * nCUs); + ratio = (double)(TimewithCU1) / ((double)(TimewithCU) * nCUs); LOG() << "Expected performance of " << nCUs << " CUs vs 1 CU:" << std::endl; - LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; + LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) + << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); } @@ -685,7 +691,7 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { LOG() << std::hex << "# SIMDs per CPU: 0x" << pNodeProperties->NumSIMDPerCU << std::endl; LOG() << std::hex << "# Shader engines: 0x" << numShaderEngines << std::endl; LOG() << std::hex << "# Active CUs: 0x" << ActiveCU << std::endl; - long long TimewithCU1, TimewithCU; + HSAint64 TimewithCU1, TimewithCU; uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */ uint32_t maskNumBits = maskNumDwords * 32; uint32_t mask[maskNumDwords]; @@ -716,10 +722,11 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { int nCUs = numShaderEngines * (x + 1); TimewithCU = TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); - ratio = (double)TimewithCU1 / ((double)TimewithCU * nCUs); + ratio = (double)(TimewithCU1) / ((double)(TimewithCU) * nCUs); LOG() << "Expected performance of " << nCUs << " CUs vs 1 CU:" << std::endl; - LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; + LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) + << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); } @@ -945,10 +952,10 @@ TEST_F(KFDQMTest, MultipleCpQueuesStressDispatch) { unsigned int* src = srcBuffer.As(); unsigned int* dst = destBuffer.As(); - static const unsigned long long TEST_TIME_SEC = 15; - unsigned long long initialTime, curTime; + static const HSAuint64 TEST_TIME_SEC = 15; + HSAuint64 initialTime, curTime; unsigned int numIter = 0; - unsigned long long timePassed = 0; + HSAuint64 timePassed = 0; unsigned int i; PM4Queue queues[MAX_CP_QUEUES]; @@ -1019,7 +1026,8 @@ TEST_F(KFDQMTest, CpuWriteCoherence) { EXPECT_EQ(0, queue.Rptr()); - // now that the GPU has cached the PQ contents, we modify them in CPU cache and ensure that the GPU sees the updated value: + // now that the GPU has cached the PQ contents, we modify them in CPU cache and + // ensure that the GPU sees the updated value: queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0x42, 0x42)); queue.Wait4PacketConsumption(); @@ -1046,7 +1054,7 @@ TEST_F(KFDQMTest, CreateAqlCpQueue) { TEST_END } -#define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define ALIGN_UP(x, align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) #define CounterToNanoSec(x) ((x) * 1000 / (is_dgpu() ? 27 : 100)) #include @@ -1056,7 +1064,7 @@ TEST_F(KFDQMTest, QueueLatency) { PM4Queue queue; const int queueSize = PAGE_SIZE * 2; - const int packetSize = PM4ReleaseMemoryPacket(0,0,0,0,0).SizeInBytes(); + const int packetSize = PM4ReleaseMemoryPacket(0, 0, 0, 0, 0).SizeInBytes(); /* We always leave one NOP(dword) empty after packet which is required by ring itself. * We also place NOPs when queue wraparound to avoid crossing buffer end. See PlacePacket(). * So the worst case is that we need two packetSize space to place one packet. @@ -1067,16 +1075,16 @@ TEST_F(KFDQMTest, QueueLatency) { */ const int reservedSpace = packetSize + queueSize % packetSize; const int slots = (queueSize - reservedSpace) / packetSize; - long queue_latency_avg = 0, queue_latency_min, queue_latency_max, queue_latency_med; - long overhead, workload; - long *queue_latency_arr = (long*)calloc(slots, sizeof(long)); + HSAint64 queue_latency_avg = 0, queue_latency_min, queue_latency_max, queue_latency_med; + HSAint64 overhead, workload; + HSAint64 *queue_latency_arr = reinterpret_cast(calloc(slots, sizeof(HSAint64))); const int skip = 2; const char *fs[skip] = {"1st", "2nd"}; HsaClockCounters *ts; HSAuint64 *qts; int i = 0; - ASSERT_NE((unsigned long)queue_latency_arr, 0); + ASSERT_NE((unsigned HSAint64)queue_latency_arr, 0); int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -1102,7 +1110,7 @@ TEST_F(KFDQMTest, QueueLatency) { i = 0; do { queue.PlacePacket(PM4ReleaseMemoryPacket(true, - (unsigned long)&qts[i], + (unsigned HSAint64)&qts[i], 0, true, 1)); @@ -1114,7 +1122,7 @@ TEST_F(KFDQMTest, QueueLatency) { /* Calculate timing which includes workload and overhead*/ i = 0; do { - long queue_latency = qts[i] - ts[i].GPUClockCounter; + HSAint64 queue_latency = qts[i] - ts[i].GPUClockCounter; ASSERT_GE(queue_latency, 0); @@ -1129,7 +1137,7 @@ TEST_F(KFDQMTest, QueueLatency) { i = 0; do { queue.PlacePacket(PM4ReleaseMemoryPacket(true, - (unsigned long)&qts[i], + (unsigned HSAint64)&qts[i], 0, true, 1)); @@ -1151,7 +1159,7 @@ TEST_F(KFDQMTest, QueueLatency) { do { /* The queue_latency is not that correct as the workload and overhead are average*/ queue_latency_arr[i] -= workload + overhead; - /* The First submit takes a long time*/ + /* The First submit takes a HSAint64 time*/ if (i < skip) LOG() << "Queue Latency " << fs[i] << ": \t" << CounterToNanoSec(queue_latency_arr[i]) << std::endl; } while (++i < slots); @@ -1243,13 +1251,13 @@ TEST_F(KFDQMTest, SdmaQueueWraparound) { } struct AtomicIncThreadParams { - long* pDest; + HSAint64* pDest; volatile unsigned int count; volatile bool stop; }; unsigned int AtomicIncThread(void* pCtx) { - AtomicIncThreadParams* pArgs = (AtomicIncThreadParams*)pCtx; + AtomicIncThreadParams* pArgs = reinterpret_cast(pCtx); while (pArgs->stop) {} @@ -1288,7 +1296,7 @@ TEST_F(KFDQMTest, Atomics) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); AtomicIncThreadParams params; - params.pDest = destBuf.As(); + params.pDest = destBuf.As(); params.stop = true; params.count = 0; @@ -1441,7 +1449,7 @@ TEST_F(KFDQMTest, P2PTest) { /* 1. Allocate a system buffer and allow the access to GPUs */ EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, - (void **)&sysBuf)); + reinterpret_cast(&sysBuf))); EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL, mapFlags, nodes.size(), &nodes[0])); #define MAGIC_NUM 0xdeadbeaf @@ -1449,7 +1457,7 @@ TEST_F(KFDQMTest, P2PTest) { /* First GPU fills mem with MAGIC_NUM*/ void *src, *dst; HSAuint32 cur = nodes[0], next; - ASSERT_SUCCESS(hsaKmtAllocMemory(cur, size, memFlags, (void**)&src)); + ASSERT_SUCCESS(hsaKmtAllocMemory(cur, size, memFlags, reinterpret_cast(&src))); ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(src, size, NULL)); sdma_fill(cur, src, MAGIC_NUM, size); @@ -1465,7 +1473,7 @@ TEST_F(KFDQMTest, P2PTest) { } else { n = 2; next = nodes[i]; - ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, (void**)&dst)); + ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, reinterpret_cast(&dst))); ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dst, size, NULL)); } @@ -1506,7 +1514,7 @@ TEST_F(KFDQMTest, SdmaEventInterrupt) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - queue.PlaceAndSubmitPacket(SDMAFencePacket((void*)event->EventData.HWData2, event->EventId)); + queue.PlaceAndSubmitPacket(SDMAFencePacket(reinterpret_cast(event->EventData.HWData2), event->EventId)); queue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); diff --git a/tests/kfdtest/src/KFDQMTest.hpp b/tests/kfdtest/src/KFDQMTest.hpp index 1834d4b506..9f120f83cf 100644 --- a/tests/kfdtest/src/KFDQMTest.hpp +++ b/tests/kfdtest/src/KFDQMTest.hpp @@ -43,8 +43,8 @@ class KFDQMTest : public KFDBaseComponentTest { void SyncDispatch(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf, int node = -1); // void SyncDispatchWithSleep(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf); - long long TimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count); - long long GetAverageTimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count, int iterations); + HSAint64 TimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count); + HSAint64 GetAverageTimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count, int iterations); protected: // members /* Acceptable performance for CU Masking should be within 5% of linearly-predicted performance */ const double CuVariance = 0.15; diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index 11321f9b88..d98d99262a 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -54,12 +54,12 @@ enum ENVCAPS{ enum KfdFamilyId { FAMILY_UNKNOWN = 0, - FAMILY_CI, // Sea Islands: Hawaii (P), Maui (P), Bonaire (M) - FAMILY_KV, // Fusion Kaveri: Spectre, Spooky; Fusion Kabini: Kalindi - FAMILY_VI, // Volcanic Islands: Iceland (V), Tonga (M) - FAMILY_CZ, // Carrizo, Nolan, Amur - FAMILY_AI, // Arctic Islands - FAMILY_RV, // Raven + FAMILY_CI, // Sea Islands: Hawaii (P), Maui (P), Bonaire (M) + FAMILY_KV, // Fusion Kaveri: Spectre, Spooky; Fusion Kabini: Kalindi + FAMILY_VI, // Volcanic Islands: Iceland (V), Tonga (M) + FAMILY_CZ, // Carrizo, Nolan, Amur + FAMILY_AI, // Arctic Islands + FAMILY_RV, // Raven }; #endif // __KFD_TEST_FLAGS__H__ diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index 7dfd24674e..847582c2bc 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -42,7 +42,7 @@ std::ostream& operator << (std::ostream& out, TESTPROFILE profile) { break; default: out << "INVALID"; - }; + } return out; } @@ -71,7 +71,8 @@ GTEST_API_ int main(int argc, char **argv) { bool success = GetCommandLineArguments(argc, argv, args); if (success) { - if ((GetHwCapabilityHWS() || args.HwsEnabled == HWCAP__FORCE_ENABLED) && (args.HwsEnabled != HWCAP__FORCE_DISABLED)) + if ((GetHwCapabilityHWS() || args.HwsEnabled == HWCAP__FORCE_ENABLED) && + (args.HwsEnabled != HWCAP__FORCE_DISABLED)) g_TestENVCaps |= ENVCAPS_HWSCHEDULING; g_TestRunProfile = args.TestProfile; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index f8b36c01f0..2e357583b4 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -43,7 +43,7 @@ bool WaitOnValue(const volatile unsigned int *buf, unsigned int value) { return *buf == value; } -void SplitU64(const unsigned long long value, unsigned int& rLoPart, unsigned int& rHiPart) { +void SplitU64(const HSAuint64 value, unsigned int& rLoPart, unsigned int& rHiPart) { rLoPart = static_cast(value); rHiPart = static_cast(value >> 32); } @@ -125,7 +125,8 @@ bool isTonga(const HsaNodeProperties *props) { const HsaMemoryBuffer HsaMemoryBuffer::Null; -HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, bool isScratch, bool isReadOnly) +HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, + bool isScratch, bool isReadOnly) :m_Size(size), m_pUser(NULL), m_pBuf(NULL), @@ -153,7 +154,7 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b if (isReadOnly) m_Flags.ui32.ReadOnly = 1; - EXPECT_SUCCESS(hsaKmtAllocMemory( m_Node, m_Size, m_Flags, &m_pBuf)); + EXPECT_SUCCESS(hsaKmtAllocMemory(m_Node, m_Size, m_Flags, &m_pBuf)); if (is_dgpu()) { EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(m_pBuf, m_Size, NULL)); m_MappedNodes = 1 << m_Node; @@ -189,9 +190,9 @@ void HsaMemoryBuffer::Fill(unsigned char value, HSAuint64 offset, HSAuint64 size ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; if (m_pUser != NULL) - memset((char *)m_pUser + offset, value, size); + memset(reinterpret_cast(m_pUser) + offset, value, size); else if (m_pBuf != NULL) - memset((char *)m_pBuf + offset, value, size); + memset(reinterpret_cast(m_pBuf) + offset, value, size); else ASSERT_TRUE(0) << "Invalid HsaMemoryBuffer"; } @@ -207,9 +208,9 @@ void HsaMemoryBuffer::Fill(HSAuint32 value, HSAuint64 offset, HSAuint64 size) { ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; if (m_pUser != NULL) - ptr = (HSAuint32 *)((char *)m_pUser + offset); + ptr = reinterpret_cast(reinterpret_cast(m_pUser) + offset); else if (m_pBuf != NULL) - ptr = (HSAuint32 *)((char *)m_pBuf + offset); + ptr = reinterpret_cast(reinterpret_cast(m_pBuf) + offset); ASSERT_NOTNULL(ptr); @@ -229,8 +230,8 @@ void HsaMemoryBuffer::Fill(HSAuint32 value, BaseQueue& baseQueue, HSAuint64 offs size = size ? size : m_Size; ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; - baseQueue.PlacePacket(SDMAFillDataPacket((void *)(this->As() + offset), value, size)); - baseQueue.PlacePacket(SDMAFencePacket((void*)event->EventData.HWData2, event->EventId)); + baseQueue.PlacePacket(SDMAFillDataPacket((reinterpret_cast(this->As() + offset)), value, size)); + baseQueue.PlacePacket(SDMAFencePacket(reinterpret_cast(event->EventData.HWData2), event->EventId)); baseQueue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); ASSERT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); @@ -250,9 +251,9 @@ bool HsaMemoryBuffer::IsPattern(HSAuint64 location, HSAuint32 pattern) { return false; if (m_pUser != NULL) - ptr = (HSAuint32 *)m_pUser; + ptr = reinterpret_cast(m_pUser); else if (m_pBuf != NULL) - ptr = (HSAuint32 *)m_pBuf; + ptr = reinterpret_cast(m_pBuf); else return false; @@ -284,9 +285,9 @@ bool HsaMemoryBuffer::IsPattern(HSAuint64 location, HSAuint32 pattern, BaseQueue *tmp = ~pattern; baseQueue.PlacePacket(SDMACopyDataPacket((void *)tmp, - (void *)(this->As() + location), + reinterpret_cast(this->As() + location), sizeof(HSAuint32))); - baseQueue.PlacePacket(SDMAFencePacket((void*)event->EventData.HWData2, + baseQueue.PlacePacket(SDMAFencePacket(reinterpret_cast(event->EventData.HWData2), event->EventId)); baseQueue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); @@ -394,14 +395,15 @@ HsaMemoryBuffer::~HsaMemoryBuffer() { m_pBuf = NULL; } -HsaInteropMemoryBuffer::HsaInteropMemoryBuffer(unsigned long long device_handle, unsigned long long buffer_handle, unsigned long long size, unsigned int node) +HsaInteropMemoryBuffer::HsaInteropMemoryBuffer(HSAuint64 device_handle, HSAuint64 buffer_handle, + HSAuint64 size, unsigned int node) :m_Size(0), m_pBuf(NULL), m_graphic_handle(0), m_Node(node) { HSAuint64 flat_address; EXPECT_SUCCESS(hsaKmtMapGraphicHandle(m_Node, device_handle, buffer_handle, 0, size, &flat_address)); - m_pBuf = (void*)flat_address; + m_pBuf = reinterpret_cast(flat_address); } HsaInteropMemoryBuffer::~HsaInteropMemoryBuffer() { diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 8f71b7ecb6..241c1a0fd3 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -36,7 +36,7 @@ class BaseQueue; // @brief: waits until the value is written to the buffer or until time out if received through args bool WaitOnValue(const volatile unsigned int *buf, unsigned int value); -void SplitU64(const unsigned long long value, unsigned int& rLoPart, unsigned int& rHiPart); +void SplitU64(const HSAuint64 value, unsigned int& rLoPart, unsigned int& rHiPart); bool GetHwCapabilityHWS(); @@ -106,14 +106,14 @@ class HsaMemoryBuffer { void* m_pBuf; bool m_Local; unsigned int m_Node; - unsigned short m_MappedNodes; + HSAuint64 m_MappedNodes; }; class HsaInteropMemoryBuffer { public: - HsaInteropMemoryBuffer(unsigned long long device_handle, unsigned long long buffer_handle, unsigned long long size, unsigned int node); + HsaInteropMemoryBuffer(HSAuint64 device_handle, HSAuint64 buffer_handle, HSAuint64 size, unsigned int node); template RetType As() { @@ -135,9 +135,9 @@ class HsaInteropMemoryBuffer { const HsaInteropMemoryBuffer& operator=(const HsaInteropMemoryBuffer&); private: - unsigned long long m_Size; + HSAuint64 m_Size; void* m_pBuf; - unsigned long long m_graphic_handle; + HSAuint64 m_graphic_handle; unsigned int m_Node; }; diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index 877899dcaa..b302f166b3 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -27,8 +27,8 @@ // @todo complete topology test according to whats in: hsathk\source\windows\kmt_topology.cpp -const unsigned long long KFDTopologyTest::c_4Gigabyte = (1ull << 32) - 1; -const unsigned long long KFDTopologyTest::c_40BitAddressSpace = (1ull << 40); +const HSAuint64 KFDTopologyTest::c_4Gigabyte = (1ull << 32) - 1; +const HSAuint64 KFDTopologyTest::c_40BitAddressSpace = (1ull << 40); TEST_F(KFDTopologyTest , BasicTest) { TEST_START(TESTPROFILE_RUNALL) @@ -41,12 +41,14 @@ TEST_F(KFDTopologyTest , BasicTest) { if (pNodeProperties != NULL) { // checking for cpu core only if it's a cpu only node or if its KAVERY apu. if (pNodeProperties->DeviceId == 0 || FamilyIdFromNode(pNodeProperties) == FAMILY_KV) { - EXPECT_GT(pNodeProperties->NumCPUCores, HSAuint32(0)) << "Node index: " << node << " No CPUs core are connected for node index"; + EXPECT_GT(pNodeProperties->NumCPUCores, HSAuint32(0)) << "Node index: " << node + << " No CPUs core are connected for node index"; } // if it's not a cpu only node, look for a gpu core if (pNodeProperties->DeviceId != 0) { - EXPECT_GT(pNodeProperties->NumFComputeCores, HSAuint32(0)) << "Node index: " << node << "No GPUs core are connected."; + EXPECT_GT(pNodeProperties->NumFComputeCores, HSAuint32(0)) << "Node index: " << node + << "No GPUs core are connected."; // EngineId only applies to GPU, not CPU-only nodes EXPECT_GT(pNodeProperties->EngineId.ui32.uCode, 0) << "uCode version is 0"; EXPECT_GE(pNodeProperties->EngineId.ui32.Major, 7) << "Major Version is less than 7"; @@ -118,7 +120,8 @@ TEST_F(KFDTopologyTest, GpuvmApertureValidate) { return; } HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; - EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(GpuNodes.at(i), pNodeProperties->NumMemoryBanks, memoryProperties)); + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(GpuNodes.at(i), pNodeProperties->NumMemoryBanks, + memoryProperties)); bool GpuVMHeapFound = false; for (unsigned int bank = 0 ; bank < pNodeProperties->NumMemoryBanks ; bank++) { // Check for either private (small-bar/APU) or public (large-bar) @@ -145,9 +148,11 @@ TEST_F(KFDTopologyTest, GetNodeCacheProperties) { pNodeProperties = m_NodeInfo.GetNodeProperties(node); if (pNodeProperties != NULL) { HsaCacheProperties *cacheProperties = new HsaCacheProperties[pNodeProperties->NumCaches]; - EXPECT_SUCCESS(hsaKmtGetNodeCacheProperties(node, pNodeProperties->CComputeIdLo, pNodeProperties->NumCaches, cacheProperties)); + EXPECT_SUCCESS(hsaKmtGetNodeCacheProperties(node, pNodeProperties->CComputeIdLo, + pNodeProperties->NumCaches, cacheProperties)); if (pNodeProperties->NumCPUCores > 0) { // this is a CPU node - LOG() << "CPU Node " << std::dec << node << ": " << pNodeProperties->NumCaches << " caches" << std::endl; + LOG() << "CPU Node " << std::dec << node << ": " << pNodeProperties->NumCaches << " caches" + << std::endl; for (unsigned n = 0; n < pNodeProperties->NumCaches; n++) { LOG()<< n << " - Level " << cacheProperties[n].CacheLevel << " Type " << cacheProperties[n].CacheType.Value << diff --git a/tests/kfdtest/src/KFDTopologyTest.hpp b/tests/kfdtest/src/KFDTopologyTest.hpp index 2ab631f8a4..005de1c04d 100644 --- a/tests/kfdtest/src/KFDTopologyTest.hpp +++ b/tests/kfdtest/src/KFDTopologyTest.hpp @@ -33,8 +33,8 @@ class KFDTopologyTest : public KFDBaseComponentTest { public: KFDTopologyTest(void) {} ~KFDTopologyTest(void) {} - static const unsigned long long c_4Gigabyte; - static const unsigned long long c_40BitAddressSpace; + static const HSAuint64 c_4Gigabyte; + static const HSAuint64 c_40BitAddressSpace; }; #endif // __KFD_TOPOLOGY_TEST__H__ diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/tests/kfdtest/src/LinuxOSWrapper.cpp index 90ea87b88f..74a0e65201 100644 --- a/tests/kfdtest/src/LinuxOSWrapper.cpp +++ b/tests/kfdtest/src/LinuxOSWrapper.cpp @@ -41,7 +41,9 @@ #include #include -static int protection_flags[8] = {int(PROT_NONE), int(PROT_READ), int(PROT_WRITE), int(PROT_READ | PROT_WRITE), int(PROT_EXEC), int(PROT_EXEC | PROT_READ), int(PROT_EXEC | PROT_WRITE), int(PROT_EXEC | PROT_WRITE | PROT_READ)}; +static int protection_flags[8] = {PROT_NONE, PROT_READ, PROT_WRITE, PROT_READ | PROT_WRITE, + PROT_EXEC, PROT_EXEC | PROT_READ, PROT_EXEC | PROT_WRITE, + PROT_EXEC | PROT_WRITE | PROT_READ}; void SetConsoleTextColor(TEXTCOLOR color) { // TODO complete @@ -69,8 +71,8 @@ bool VirtualFreeMemory(void *address, unsigned int size) { return false; } -unsigned long GetLastErrorNo() { - return errno; +HSAuint64 GetLastErrorNo() { + return errno; } bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProcess) { @@ -78,7 +80,7 @@ bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProc return false; } -unsigned long long GetSystemTickCountInMicroSec() { +HSAuint64 GetSystemTickCountInMicroSec() { struct timeval t; gettimeofday(&t, 0); return t.tv_sec * 1000000ULL + t.tv_usec; @@ -229,7 +231,7 @@ bool WaitForThread(uint64_t threadId) { return 0 == pthread_join((pthread_t)threadId, NULL); } -long AtomicInc(volatile long* pValue) { +HSAint64 AtomicInc(volatile HSAint64* pValue) { return __sync_add_and_fetch(pValue, 1); } diff --git a/tests/kfdtest/src/OSWrapper.hpp b/tests/kfdtest/src/OSWrapper.hpp index bcba47bb63..6b2f500258 100644 --- a/tests/kfdtest/src/OSWrapper.hpp +++ b/tests/kfdtest/src/OSWrapper.hpp @@ -26,6 +26,7 @@ #include #include "KFDTestFlags.hpp" +#include "hsakmt.h" #ifndef __OS__WRAPPER__H__ #define __OS__WRAPPER__H__ @@ -82,19 +83,20 @@ void *VirtualAllocMemory(void *address, unsigned int size, int memProtection = M // @brief replacement for windows FreeVirtual func bool VirtualFreeMemory(void *address, unsigned int size); // @brief retrieve the last error number -unsigned long GetLastErrorNo(); +HSAuint64 GetLastErrorNo(); -long AtomicInc(volatile long* pValue); +HSAint64 AtomicInc(volatile HSAint64* pValue); void MemoryBarrier(); // @brief: runs the selected test case number of times required, each in a separate process -// @params testToRun : can be a specific test testcase like TestCase.TestName or if you want to run all tests in a test case: TestCase.* and so on +// @params testToRun : can be a specific test testcase like TestCase.TestName or if you want +// to run all tests in a test case: TestCase.* and so on // @params numOfProcesses : how many processes to run in parallel // @params runsPerProcess : how many iteration a test should do per process, must be a positive number bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProcess = 1); -unsigned long long GetSystemTickCountInMicroSec(); +HSAuint64 GetSystemTickCountInMicroSec(); /**Put the system to S3/S4 power state and bring it back to S0. @return 'true' on success, 'false' on failure. diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index c0c48337a5..f8e57aa4ed 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -21,11 +21,11 @@ * */ -#include "PM4Packet.hpp" -#include "hsakmttypes.h" #include #include #include +#include "PM4Packet.hpp" +#include "hsakmttypes.h" #include "asic_reg/gfx_7_2_enum.h" @@ -52,7 +52,7 @@ unsigned int PM4WriteDataPacket::SizeInBytes() const { } void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) { - m_pPacketData = (PM4WRITE_DATA_CI *)calloc(1, SizeInBytes()); + m_pPacketData = reinterpret_cast(calloc(1, SizeInBytes())); // verify that the memory is allocated successfully, cannot use assert here EXPECT_NOTNULL(m_pPacketData); @@ -84,7 +84,7 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, PM4_RELEASE_MEM_CI *pkt; m_packetSize = sizeof(PM4_RELEASE_MEM_CI); - pkt = (PM4_RELEASE_MEM_CI *)calloc(1, m_packetSize); + pkt = reinterpret_cast(calloc(1, m_packetSize)); m_pPacketData = pkt; EXPECT_NOTNULL(m_pPacketData); @@ -147,7 +147,7 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, PM4MEC_RELEASE_MEM_AI *pkt; m_packetSize = sizeof(PM4MEC_RELEASE_MEM_AI); - pkt = (PM4MEC_RELEASE_MEM_AI *)calloc(1, m_packetSize); + pkt = reinterpret_cast(calloc(1, m_packetSize)); m_pPacketData = pkt; EXPECT_NOTNULL(m_pPacketData); @@ -233,7 +233,8 @@ PM4SetShaderRegPacket::PM4SetShaderRegPacket(void) : m_packetDataAllocated(false) { } -PM4SetShaderRegPacket::PM4SetShaderRegPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs) +PM4SetShaderRegPacket::PM4SetShaderRegPacket(unsigned int baseOffset, const unsigned int regValues[], + unsigned int numRegs) : m_packetDataAllocated(false) { InitPacket(baseOffset, regValues, numRegs); } @@ -243,11 +244,15 @@ PM4SetShaderRegPacket::~PM4SetShaderRegPacket(void) { free(m_pPacketData); } -void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs) { - m_packetSize = sizeof(PM4SET_SH_REG) + (numRegs-1)*sizeof(uint32_t); // 1st register is a part of the packet struct. +void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned int regValues[], + unsigned int numRegs) { + // 1st register is a part of the packet struct. + m_packetSize = sizeof(PM4SET_SH_REG) + (numRegs-1)*sizeof(uint32_t); - // allocating the size of the packet, since the packet is assembled from a struct followed by an additional DWORD data - m_pPacketData = (PM4SET_SH_REG *)malloc(m_packetSize); + /* allocating the size of the packet, since the packet is assembled from a struct + * followed by an additional DWORD data + */ + m_pPacketData = reinterpret_cast(malloc(m_packetSize)); ASSERT_NOTNULL(m_pPacketData); @@ -262,11 +267,13 @@ void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned i memcpy(m_pPacketData->reg_data, regValues, numRegs*sizeof(uint32_t)); } -PM4DispatchDirectPacket::PM4DispatchDirectPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, unsigned int dispatchInit) { +PM4DispatchDirectPacket::PM4DispatchDirectPacket(unsigned int dimX, unsigned int dimY, + unsigned int dimZ, unsigned int dispatchInit) { InitPacket(dimX, dimY, dimZ, dispatchInit); } -void PM4DispatchDirectPacket::InitPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, unsigned int dispatchInit) { +void PM4DispatchDirectPacket::InitPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, + unsigned int dispatchInit) { memset(&m_packetData, 0, SizeInBytes()); InitPM4Header(m_packetData.header, IT_DISPATCH_DIRECT); diff --git a/tests/kfdtest/src/PM4Packet.hpp b/tests/kfdtest/src/PM4Packet.hpp index cc09d43b68..a7ab06ab5a 100644 --- a/tests/kfdtest/src/PM4Packet.hpp +++ b/tests/kfdtest/src/PM4Packet.hpp @@ -112,7 +112,7 @@ class PM4IndirectBufPacket : public PM4Packet { // empty constructor, befor using the packet call the init func PM4IndirectBufPacket(void) {} // this contructor will also init the packet, no need for adittional calls - PM4IndirectBufPacket(IndirectBuffer *pIb); + explicit PM4IndirectBufPacket(IndirectBuffer *pIb); virtual ~PM4IndirectBufPacket(void) {} // @returns the packet size in bytes diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index 45c7a6cd3e..d1120590ab 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -59,12 +59,12 @@ void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int ndw, void *data) { packetSize = sizeof(SDMA_PKT_WRITE_UNTILED) + (ndw - 1) * sizeof(unsigned int); - packetData = (SDMA_PKT_WRITE_UNTILED *)calloc(1, packetSize); + packetData = reinterpret_cast(calloc(1, packetSize)); packetData->HEADER_UNION.op = SDMA_OP_WRITE; packetData->HEADER_UNION.sub_op = SDMA_SUBOP_WRITE_LINEAR; - SplitU64(reinterpret_cast(destAddr), + SplitU64(reinterpret_cast(destAddr), packetData->DST_ADDR_LO_UNION.DW_1_DATA, // dst_addr_31_0 packetData->DST_ADDR_HI_UNION.DW_2_DATA); // dst_addr_63_32 @@ -80,7 +80,7 @@ SDMACopyDataPacket::~SDMACopyDataPacket(void) { SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, unsigned int surfsize) { int32_t size = 0, i; - void **dst = (void**)malloc(sizeof(void*) * n); + void **dst = reinterpret_cast(malloc(sizeof(void*) * n)); const int singlePacketSize = sizeof(SDMA_PKT_COPY_LINEAR) + sizeof(SDMA_PKT_COPY_LINEAR::DST_ADDR[0]) * n; @@ -91,7 +91,7 @@ SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, uns packetSize = ((surfsize + TWO_MEG - 1) >> BITS) * singlePacketSize; - SDMA_PKT_COPY_LINEAR *pSDMA = (SDMA_PKT_COPY_LINEAR *)malloc(packetSize); + SDMA_PKT_COPY_LINEAR *pSDMA = reinterpret_cast(malloc(packetSize)); packetData = pSDMA; while (surfsize > 0) { @@ -106,19 +106,19 @@ SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, uns pSDMA->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR; pSDMA->HEADER_UNION.broadcast = n > 1 ? 1 : 0; pSDMA->COUNT_UNION.count = SDMA_COUNT(size); - SplitU64(reinterpret_cast(src), + SplitU64(reinterpret_cast(src), pSDMA->SRC_ADDR_LO_UNION.DW_3_DATA, // src_addr_31_0 pSDMA->SRC_ADDR_HI_UNION.DW_4_DATA); // src_addr_63_32 for (i = 0; i < n; i++) - SplitU64(reinterpret_cast(dst[i]), + SplitU64(reinterpret_cast(dst[i]), pSDMA->DST_ADDR[i].DST_ADDR_LO_UNION.DW_5_DATA, // dst_addr_31_0 pSDMA->DST_ADDR[i].DST_ADDR_HI_UNION.DW_6_DATA); // dst_addr_63_32 - pSDMA = (SDMA_PKT_COPY_LINEAR *)((char *)pSDMA + singlePacketSize); + pSDMA = reinterpret_cast(reinterpret_cast(pSDMA) + singlePacketSize); for (i = 0; i < n; i++) - dst[i] = (char *)dst[i] + size; - src = (char *)src + size; + dst[i] = reinterpret_cast(dst[i]) + size; + src = reinterpret_cast(src) + size; surfsize -= size; } free(dst); @@ -138,7 +138,7 @@ SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned in /* SDMA support maximum 0x3fffe0 byte in one copy. Use 2M copy_size */ m_PacketSize = ((size + TWO_MEG - 1) >> BITS) * sizeof(SDMA_PKT_CONSTANT_FILL); - pSDMA = (SDMA_PKT_CONSTANT_FILL *)calloc(1, m_PacketSize); + pSDMA = reinterpret_cast(calloc(1, m_PacketSize)); m_PacketData = pSDMA; while (size > 0) { @@ -158,14 +158,14 @@ SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned in pSDMA->COUNT_UNION.count = SDMA_COUNT(copy_size); - SplitU64(reinterpret_cast(dst), + SplitU64(reinterpret_cast(dst), pSDMA->DST_ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ pSDMA->DST_ADDR_HI_UNION.DW_2_DATA); /*dst_addr_63_32*/ pSDMA->DATA_UNION.DW_3_DATA = data; pSDMA++; - dst = (char *)dst + copy_size; + dst = reinterpret_cast(dst) + copy_size; size -= copy_size; } } @@ -185,7 +185,7 @@ void SDMAFencePacket::InitPacket(void* destAddr, unsigned int data) { packetData.HEADER_UNION.op = SDMA_OP_FENCE; - SplitU64(reinterpret_cast(destAddr), + SplitU64(reinterpret_cast(destAddr), packetData.ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ packetData.ADDR_HI_UNION.DW_2_DATA); /*dst_addr_63_32*/ diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index f7ef53f733..17b9027b5e 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -125,7 +125,7 @@ class SDMAFencePacket : public SDMAPacket { class SDMATrapPacket : public SDMAPacket { public: // empty constructor, befor using the packet call the init func - SDMATrapPacket(unsigned int eventID = 0); + explicit SDMATrapPacket(unsigned int eventID = 0); virtual ~SDMATrapPacket(void); From cb019f00cd8bc7335cc21d24c6b3f3a12f184963 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 13 Aug 2018 09:33:22 -0400 Subject: [PATCH 0380/1247] kfdtest: Consolidate indentation of multi-line function calls Make indentation consistent, which is that subsequent lines are aligned with the variables declared above Change-Id: I590f7768d93565145b986ad1fb6ac8e82f9c0d58 --- tests/kfdtest/src/BaseQueue.hpp | 2 +- tests/kfdtest/src/KFDEvictTest.cpp | 2 +- tests/kfdtest/src/KFDEvictTest.hpp | 2 +- tests/kfdtest/src/KFDIPCTest.cpp | 6 +++--- tests/kfdtest/src/KFDIPCTest.hpp | 4 ++-- tests/kfdtest/src/KFDTestUtil.hpp | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index 730807f97e..d926cd34e2 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -45,7 +45,7 @@ class BaseQueue { * @param pointers is used only for creating AQL queues. Otherwise it is omitted. */ virtual HSAKMT_STATUS Create(unsigned int NodeId, unsigned int size = DEFAULT_QUEUE_SIZE, - HSAuint64 *pointers = NULL); + HSAuint64 *pointers = NULL); /** Update the queue. * @see hsaKmtUpdateQueue * @param percent the new queue percentage diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 3f7e3537a1..94d5e216ca 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -35,7 +35,7 @@ #define ALLOCATE_RETRY_TIMES (3) void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, - std::vector &pBuffers) { + std::vector &pBuffers) { HSAuint64 totalMB; totalMB = N_PROCESSES*count*(vramBufSize>>20); diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index b98d275373..15fee8c1bc 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -52,7 +52,7 @@ class KFDEvictTest : public KFDLocalMemoryTest { protected: std::string CreateShader(); void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, - std::vector &pBuffers); + std::vector &pBuffers); void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); void AllocAmdgpuBo(int rn, HSAuint64 vramBufSize, amdgpu_bo_handle &handle); void FreeAmdgpuBo(amdgpu_bo_handle handle); diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index 6a95f51c9a..f3ca46b489 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -462,7 +462,7 @@ CMA_TEST_STATUS KFDCMAArray::recvCMAArray(int readPipe) { CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachChildProcess(int defaultGPUNode, int writePipe, - int readPipe, CMA_TEST_TYPE testType) { + int readPipe, CMA_TEST_TYPE testType) { KFDCMAArray cmaLocalArray; char msg[16]; int testNo; @@ -516,8 +516,8 @@ CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachChildProcess(int defaultGPUNode, in CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachParentProcess(int defaultGPUNode, pid_t cid, - int writePipe, int readPipe, - CMA_TEST_TYPE testType) { + int writePipe, int readPipe, + CMA_TEST_TYPE testType) { KFDCMAArray cmaLocalArray, cmaRemoteArray; HSAuint64 copied = 0; int testNo; diff --git a/tests/kfdtest/src/KFDIPCTest.hpp b/tests/kfdtest/src/KFDIPCTest.hpp index 2cb208e3da..50ff9508dc 100644 --- a/tests/kfdtest/src/KFDIPCTest.hpp +++ b/tests/kfdtest/src/KFDIPCTest.hpp @@ -123,9 +123,9 @@ class KFDIPCTest : public KFDBaseComponentTest { /* For CMA testing */ CMA_TEST_STATUS CrossMemoryAttachChildProcess(int defaultGPUNode, int writePipe, - int readPipe, CMA_TEST_TYPE testType); + int readPipe, CMA_TEST_TYPE testType); CMA_TEST_STATUS CrossMemoryAttachParentProcess(int defaultGPUNode, pid_t cid, - int writePipe, int readPipe, CMA_TEST_TYPE testType); + int writePipe, int readPipe, CMA_TEST_TYPE testType); protected: pid_t m_ChildPid; }; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 241c1a0fd3..57cf2898f1 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -53,7 +53,7 @@ class HsaMemoryBuffer { public: HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero = true, bool isLocal = false, - bool isExec = false, bool isScratch = false, bool isReadOnly = false); + bool isExec = false, bool isScratch = false, bool isReadOnly = false); HsaMemoryBuffer(void *addr, HSAuint64 size); template RetType As() { From f2bd7e1d52d300619a3f2764ba7f9f44031238d3 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 13 Aug 2018 10:18:04 -0400 Subject: [PATCH 0381/1247] kfdtest: Consolidate log messages for skipped tests When skipping a test, the output should be: Skipping test: . This will allow for easier identification, automation and general readability Change-Id: I98bda1c068f9dbc83aeea74f642b6101121f234d --- tests/kfdtest/src/KFDCWSRTest.cpp | 2 +- tests/kfdtest/src/KFDDBGTest.cpp | 2 +- tests/kfdtest/src/KFDEvictTest.cpp | 12 ++++++------ tests/kfdtest/src/KFDExceptionTest.cpp | 14 +++++++------- tests/kfdtest/src/KFDGraphicsInterop.cpp | 12 ++++++------ tests/kfdtest/src/KFDIPCTest.cpp | 6 +++--- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 12 ++++++------ tests/kfdtest/src/KFDMemoryTest.cpp | 24 ++++++++++++------------ tests/kfdtest/src/KFDPerfCounters.cpp | 7 +++---- tests/kfdtest/src/KFDQMTest.cpp | 24 ++++++++++++------------ tests/kfdtest/src/KFDTopologyTest.cpp | 2 +- 11 files changed, 58 insertions(+), 59 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index ace560e5f7..7dffc56ef9 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -178,7 +178,7 @@ TEST_F(KFDCWSRTest, BasicTest) { delete dispatch2; } else { - LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + LOG() << "Skipping test: No CWSR present for family ID 0x" << m_FamilyId << "." << std::endl; } TEST_END diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 52d904ef74..00d7922f2c 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -169,7 +169,7 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { ASSERT_SUCCESS(queue.Destroy()); ASSERT_SUCCESS(queue_flush.Destroy()); } else { - LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; } TEST_END } diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 94d5e216ca..a32590ac77 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -323,7 +323,7 @@ TEST_F(KFDEvictTest, BasicTest) { HSAuint64 vramSize = GetVramSize(defaultGPUNode); if (!vramSize) { - LOG() << "No VRAM found, skipping the test" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } else { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; @@ -338,7 +338,7 @@ TEST_F(KFDEvictTest, BasicTest) { int rn = FindDRMRenderNode(defaultGPUNode); if (rn < 0) { - LOG() << "Skipping test" << std::endl; + LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; return; } @@ -516,7 +516,7 @@ TEST_F(KFDEvictTest, QueueTest) { /* Skip test for chip it doesn't have CWSR, which the test depends on */ if (m_FamilyId < FAMILY_VI || isTonga(pNodeProperties)) { - LOG() << std::hex << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + LOG() << std::hex << "Skipping test: No CWSR present for family ID 0x" << m_FamilyId << "." << std::endl; return; } @@ -524,7 +524,7 @@ TEST_F(KFDEvictTest, QueueTest) { HSAuint64 vramSize = GetVramSize(defaultGPUNode); if (!vramSize) { - LOG() << "No VRAM found, skipping the test" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } else { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; @@ -535,7 +535,7 @@ TEST_F(KFDEvictTest, QueueTest) { LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; if (count == 0) { - LOG() << "Not enough system memory, skipping the test" << std::endl; + LOG() << "Skipping test: Not enough system memory available." << std::endl; return; } /* assert all buffer address can be stored within one page @@ -548,7 +548,7 @@ TEST_F(KFDEvictTest, QueueTest) { int rn = FindDRMRenderNode(defaultGPUNode); if (rn < 0) { - LOG() << "Skipping test" << std::endl; + LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 22ea893fe0..fca646bd4e 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -124,7 +124,7 @@ TEST_F(KFDExceptionTest, InvalidBadAddress) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (m_FamilyId == FAMILY_RV) { - LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; return; } @@ -172,12 +172,12 @@ TEST_F(KFDExceptionTest, InvalidPPRWriteProtection) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (is_dgpu()) { - LOG() << "Not an APU, no PPR available, skip the test" << std::endl; + LOG() << "Skipping test: Test requires APU." << std::endl; return; } if (m_FamilyId == FAMILY_RV) { - LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; return; } @@ -223,12 +223,12 @@ TEST_F(KFDExceptionTest, InvalidPPRReadProtection) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (is_dgpu()) { - LOG() << "Not an APU, no PPR available, skip the test" << std::endl; + LOG() << "Skipping test: Test requires APU." << std::endl; return; } if (m_FamilyId == FAMILY_RV) { - LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; return; } @@ -275,7 +275,7 @@ TEST_F(KFDExceptionTest, InvalidWriteAddress) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (m_FamilyId == FAMILY_RV) { - LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; return; } @@ -323,7 +323,7 @@ TEST_F(KFDExceptionTest, FaultStorm) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (m_FamilyId == FAMILY_RV) { - LOG() << "Skip the test on Raven due to IOMMU issues" << std::endl; + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp index f44130aea8..4faea041f6 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.cpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -36,7 +36,7 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { const HSAuint32 familyID = FamilyIdFromNode(pNodeProps); if (isTonga(pNodeProps)) { - LOG() << "Skipping test: Tonga workaround in thunk returns incorrect allocation size" << std::endl; + LOG() << "Skipping test: Tonga workaround in thunk returns incorrect allocation size." << std::endl; return; } @@ -47,7 +47,7 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { int rn = FindDRMRenderNode(defaultGPUNode); if (rn < 0) { - LOG() << "Skipping test" << std::endl; + LOG() << "Skipping test: Could not find render node for default GPU node." << std::endl; return; } @@ -141,13 +141,13 @@ TEST_F(KFDGraphicsInterop, RegisterForeignDeviceMem) { TEST_START(TESTPROFILE_RUNALL) if (!is_dgpu()) { - LOG() << "Skipping test: Supports only multi-dGPU system" << std::endl; + LOG() << "Skipping test: Only supported on multi-dGPU system." << std::endl; return; } const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); if (gpuNodes.size() < 2) { - LOG() << "Skipping test: Need at least two GPUs" << std::endl; + LOG() << "Skipping test: At least two GPUs are required." << std::endl; return; } @@ -159,7 +159,7 @@ TEST_F(KFDGraphicsInterop, RegisterForeignDeviceMem) { gpuNode2 = m_NodeInfo.FindLargeBarGPUNode(); if (gpuNode2 < 0) { - LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + LOG() << "Skipping test: At least one large bar GPU is required." << std::endl; return; } if (gpuNode1 == gpuNode2) { @@ -177,7 +177,7 @@ TEST_F(KFDGraphicsInterop, RegisterForeignDeviceMem) { int rn = FindDRMRenderNode(gpuNode2); if (rn < 0) { - LOG() << "Skipping test" << std::endl; + LOG() << "Skipping test: Cound not find render node for 2nd GPU." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index f3ca46b489..d85f7c59e3 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -167,7 +167,7 @@ TEST_F(KFDIPCTest, BasicTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (!GetVramSize(defaultGPUNode)) { - LOG() << "Skip the test because of no VRAM found" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } @@ -606,7 +606,7 @@ TEST_F(KFDIPCTest, CrossMemoryAttachTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (!GetVramSize(defaultGPUNode)) { - LOG() << "Skip the test because of no VRAM found" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } @@ -670,7 +670,7 @@ TEST_F(KFDIPCTest, CMABasicTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (!GetVramSize(defaultGPUNode)) { - LOG() << "Skip the test because of no VRAM found" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index 986ee4a85c..2c72814357 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -64,7 +64,7 @@ TEST_F(KFDLocalMemoryTest, BasicTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (!GetVramSize(defaultGPUNode)) { - LOG() << "No VRAM found, skipping the test" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } @@ -119,7 +119,7 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (!GetVramSize(defaultGPUNode)) { - LOG() << "No VRAM found, skipping the test" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } @@ -239,7 +239,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { fbSize = GetVramSize(defaultGPUNode); if (!fbSize) { - LOG() << "No VRAM found, skipping test." << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } else { LOG() << "Found VRAM of " << std::dec << (fbSize >> 20) << "MB." << std::endl; @@ -396,7 +396,7 @@ TEST_F(KFDLocalMemoryTest, CheckZeroInitializationVram) { HSAuint64 vramSizeMB = GetVramSize(defaultGPUNode) >> 20; if (!vramSizeMB) { - LOG() << "No VRAM found, skipping the test" << std::endl; + LOG() << "Skipping test: No VRAM found." << std::endl; return; } @@ -450,7 +450,7 @@ TEST_F(KFDLocalMemoryTest, MapVramToGPUNodesTest) { const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); if (gpuNodes.size() < 2) { - LOG() << "Skipping test: Need at least two GPUs" << std::endl; + LOG() << "Skipping test: Test requires at least two GPUs." << std::endl; return; } @@ -462,7 +462,7 @@ TEST_F(KFDLocalMemoryTest, MapVramToGPUNodesTest) { dst_node = m_NodeInfo.FindLargeBarGPUNode(); if (dst_node < 0) { - LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 15a1c20051..95cb14d543 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -141,7 +141,7 @@ TEST_F(KFDMemoryTest, MMapLarge) { TEST_START(TESTPROFILE_RUNALL) if (!is_dgpu()) { - LOG() << "Skip the test on APU" << std::endl; + LOG() << "Skipping test: Test not supported on APU." << std::endl; return; } @@ -207,13 +207,13 @@ TEST_F(KFDMemoryTest, MMapLarge) { TEST_F(KFDMemoryTest, MapUnmapToNodes) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId != FAMILY_AI) { - LOG() << "Skipping test: Test uses gfx9-based shader, skip on other ASICs" << std::endl; + LOG() << "Skipping test: GFX9-based shader not supported on other ASICs." << std::endl; return; } const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); if (gpuNodes.size() < 2) { - LOG() << "Skipping test: Need at least two GPUs" << std::endl; + LOG() << "Skipping test: At least two GPUs are required." << std::endl; return; } HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); @@ -316,7 +316,7 @@ TEST_F(KFDMemoryTest, AccessPPRMem) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; if (is_dgpu()) { - LOG() << "Not an APU, no PPR available, skip the test" << std::endl; + LOG() << "Skipping test: Test requires APU." << std::endl; return; } @@ -353,7 +353,7 @@ TEST_F(KFDMemoryTest, AccessPPRMem) { TEST_F(KFDMemoryTest, MemoryRegister) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); if (isTonga(pNodeProperties)) { - LOG() << "Skipping test: Workaround in thunk for Tonga causes failure:" << std::endl; + LOG() << "Skipping test: Workaround in thunk for Tonga causes failure." << std::endl; return; } @@ -453,7 +453,7 @@ TEST_F(KFDMemoryTest, MemoryRegister) { TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { if (!is_dgpu()) { - LOG() << "Skipping test: Will run on APU once APU+dGPU supported:" << std::endl; + LOG() << "Skipping test: Will run on APU once APU+dGPU supported." << std::endl; return; } @@ -529,7 +529,7 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { TEST_F(KFDMemoryTest, FlatScratchAccess) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId == FAMILY_CI || m_FamilyId == FAMILY_KV) { - LOG() << "Skipping test: Test uses VI-based shader, fails on CI" << std::endl; + LOG() << "Skipping test: VI-based shader not supported on other ASICs." << std::endl; return; } @@ -784,7 +784,7 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, * the onerous memory swap operation. So we limit the buffer size that way.*/ TEST_F(KFDMemoryTest, BigBufferStressTest) { if (!is_dgpu()) { - LOG() << "Skipping test: Running on APU fails and locks the system" << std::endl; + LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl; return; } TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); @@ -1236,12 +1236,12 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { char *hsaDebug = getenv("HSA_DEBUG"); if (!is_dgpu()) { - LOG() << "Skipping test: No VRAM on APU" << std::endl; + LOG() << "Skipping test: There is no VRAM on APU." << std::endl; return; } if (!hsaDebug || !strcmp(hsaDebug, "0")) { - LOG() << "Skipping test: HSA_DEBUG environment variable not set" << std::endl; + LOG() << "Skipping test: HSA_DEBUG environment variable not set." << std::endl; return; } @@ -1382,7 +1382,7 @@ TEST_F(KFDMemoryTest, SignalHandling) { TEST_START(TESTPROFILE_RUNALL) if (!is_dgpu()) { - LOG() << "Skip the test on APU" << std::endl; + LOG() << "Skipping test: Test not supported on APU." << std::endl; return; } @@ -1547,7 +1547,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode) || !vramSizeMB) { - LOG() << "not a largebar system, skip!" << std::endl; + LOG() << "Skipping test: Test requires a large bar GPU." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDPerfCounters.cpp b/tests/kfdtest/src/KFDPerfCounters.cpp index 1ecf61041b..22341458b6 100644 --- a/tests/kfdtest/src/KFDPerfCounters.cpp +++ b/tests/kfdtest/src/KFDPerfCounters.cpp @@ -166,7 +166,7 @@ TEST_F(KFDPerfCountersTest, RegisterTrace) { } if (!priv_block_found) { - LOG() << "No privileged block is found. Skipping RegisterTrace test." + LOG() << "Skipping test: No privileged block is found." << std::endl; return; } @@ -207,14 +207,13 @@ TEST_F(KFDPerfCountersTest, StartStopQueryTrace) { } if (!priv_block_found) { - LOG() << "No privileged block is found. Skipping StartStopQuery test." + LOG() << "Skipping test: No privileged block is found." << std::endl; return; } if (getuid()) { /* Non-root */ - LOG() << "Privileged counters requires the user as root." << std::endl - << "Skipping StartStopQuery test." << std::endl; + LOG() << "Skipping test: Privileged counters requires the user as root." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index c1dab79916..1d0eb6fe38 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -417,7 +417,7 @@ TEST_F(KFDQMTest, CreateQueueStressSingleThreaded) { TEST_F(KFDQMTest, OverSubscribeCpQueues) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId == FAMILY_CI || m_FamilyId == FAMILY_KV) { - LOG() << "Skipping test: CI doesn't have HW scheduling" << std::endl; + LOG() << "Skipping test: CI doesn't have HW scheduling." << std::endl; return; } @@ -660,7 +660,7 @@ TEST_F(KFDQMTest, BasicCuMaskingLinear) { ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); } } else { - LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl; } TEST_END @@ -683,7 +683,7 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { uint32_t ActiveCU = (pNodeProperties->NumFComputeCores / pNodeProperties->NumSIMDPerCU); uint32_t numShaderEngines = pNodeProperties->NumShaderBanks; if (numShaderEngines == 1) { - LOG() << "Only 1 Shader Engine. Skipping test." << std::endl; + LOG() << "Skipping test: Only 1 Shader Engine present." << std::endl; return; } @@ -731,7 +731,7 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); } } else { - LOG() << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl; } TEST_END @@ -741,7 +741,7 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { TEST_START(TESTPROFILE_RUNALL); if (m_FamilyId < FAMILY_VI) { - LOG() << "Skipping test: Shader won't run on CI" << std::endl; + LOG() << "Skipping test: Shader won't run on CI." << std::endl; return; } @@ -806,7 +806,7 @@ TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { TEST_START(TESTPROFILE_RUNALL); if (m_FamilyId < FAMILY_VI) { - LOG() << "Skipping test: Shader won't run on CI" << std::endl; + LOG() << "Skipping test: Shader won't run on CI." << std::endl; return; } @@ -1274,7 +1274,7 @@ TEST_F(KFDQMTest, Atomics) { TEST_START(TESTPROFILE_RUNALL); /* CI doesn't support Atomics. KV does, but gets its own FAMILY_KV def */ if (m_FamilyId == FAMILY_CI) { - LOG() << "Skipping test: CI doesn't support Atomics" << std::endl; + LOG() << "Skipping test: CI doesn't support Atomics." << std::endl; return; } int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); @@ -1388,13 +1388,13 @@ static void sdma_fill(HSAint32 node, void *dst, unsigned int data, unsigned int TEST_F(KFDQMTest, P2PTest) { TEST_START(TESTPROFILE_RUNALL); if (!is_dgpu()) { - LOG() << "Skipping test: Can't have 2 APUs on the same system" << std::endl; + LOG() << "Skipping test: Two GPUs are required, but no dGPUs are present." << std::endl; return; } const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); if (gpuNodes.size() < 2) { - LOG() << "Skipping test: Need at least two GPUs" << std::endl; + LOG() << "Skipping test: At least two GPUs are required." << std::endl; return; } std::vector nodes; @@ -1419,11 +1419,11 @@ TEST_F(KFDQMTest, P2PTest) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[1])) { - LOG() << "Skipping test: Dst GPU is not a large bar GPU" << std::endl; + LOG() << "Skipping test: Dst GPU is not a large bar GPU." << std::endl; return; } if (nodes[0] == nodes[1]) { - LOG() << "Skipping test: Need different GPUs specified" << std::endl; + LOG() << "Skipping test: Different GPUs must be specified (2 GPUs required)." << std::endl; return; } } else { @@ -1433,7 +1433,7 @@ TEST_F(KFDQMTest, P2PTest) { if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && gpuNodes.at(i) != defaultGPU) nodes.push_back(gpuNodes.at(i)); if (nodes.size() < 2) { - LOG() << "Skipping test: Need at least one large bar GPU" << std::endl; + LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; return; } } diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index b302f166b3..43e84cc242 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -116,7 +116,7 @@ TEST_F(KFDTopologyTest, GpuvmApertureValidate) { pNodeProperties = m_NodeInfo.GetNodeProperties(GpuNodes.at(i)); if (pNodeProperties != NULL) { if (!is_dgpu() && !(FamilyIdFromNode(pNodeProperties) == FAMILY_KV)) { - LOG() << "Skipping test: GPUVM framebuffer heap not exposed on APU except Kaveri" << std::endl; + LOG() << "Skipping test: GPUVM framebuffer heap not exposed on APU except Kaveri." << std::endl; return; } HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; From d3fdaaca3a0fdcba1df0eb07f25dc7f9102b4cc4 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 10 Aug 2018 22:43:06 -0400 Subject: [PATCH 0382/1247] kfdtest: Enable more tests for gfx900 A lot of tests were disabled on gfx900 for historical reasons that are no longer valid. The only remaining one that won't work on gfx900 is BasicAddressWatch. Change-Id: I11507de0dfd31262713127d6cb15cc09c14b8b9f Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/kfdtest.exclude | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index a804d3f846..9fdd2e19f9 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -110,18 +110,10 @@ BAFFIN_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST" -# KFDDBG.BasicAddressWatch fails with message "E,INTERR: Setting ASIC VI (was previously set to GFX9), this changes the backend after we have already started parsing the shader, not currently supported by sp3, sorry." (KFD-317) -# KFDExceptionTest.InvalidBadAddress causes some following tests to fail on SRIOV (KFD-343) -# KFDQMTest.BasicCuMaskingEven fails intermittently (WIP) -# KFDExceptionTest.InvalidWriteAddress is failing once we enable no-retry (KFD-380) +# KFDDBG.BasicAddressWatch is not supported on GFXv9 and later. FIXME: it should disable itself VEGA10_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDDBGTest.BasicAddressWatch:"\ -"KFDExceptionTest.InvalidBadAddress:"\ -"KFDExceptionTest.InvalidWriteAddress:"\ -"KFDQMTest.CreateQueueStressSingleThreaded:"\ -"KFDQMTest.GPUDoorbellWrite:"\ -"KFDMemoryTest.MemoryRegister" +"KFDDBGTest.BasicAddressWatch" VEGA20_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ From 842359a826682e46ecb6f3a9335d47693e628ff4 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 4 Aug 2018 15:58:23 -0400 Subject: [PATCH 0383/1247] libhsakmt: Remove aperture offset parameter This parameter was used for non-canonical GPUVM allocations on GFX7/8 APUs only, to prevent getting NULL pointers from valid allocation after subtracting the aperture base. The same can be achieved less intrusively by reserving address space at the start of the aperture during initialization. Change-Id: I0aae773f069c2b228824ba464b0612a4d8b489ce Signed-off-by: Felix Kuehling --- src/fmm.c | 59 ++++++++++++++++++++++--------------------------------- 1 file changed, 24 insertions(+), 35 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 1b43140435..4939f42a56 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -554,7 +554,6 @@ static void aperture_release_area(manageable_aperture_t *app, void *address, */ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, uint64_t MemorySizeInBytes, - uint64_t offset, uint64_t align) { vm_area_t *cur, *next; @@ -574,8 +573,7 @@ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, /* Find a big enough "hole" in the address space */ cur = NULL; next = app->vm_ranges; - start = (void *)ALIGN_UP((uint64_t)VOID_PTR_ADD(app->base, offset), - align); + start = (void *)ALIGN_UP((uint64_t)app->base, align); while (next) { if (next->start > start && VOID_PTRS_SUB(next->start, start) >= MemorySizeInBytes) @@ -612,10 +610,9 @@ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, return start; } static void *aperture_allocate_area(manageable_aperture_t *app, - uint64_t MemorySizeInBytes, - uint64_t offset) + uint64_t MemorySizeInBytes) { - return aperture_allocate_area_aligned(app, MemorySizeInBytes, offset, app->align); + return aperture_allocate_area_aligned(app, MemorySizeInBytes, app->align); } /* returns 0 on success. Assumes, that fmm_mutex is locked on entry */ @@ -933,7 +930,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) pthread_mutex_lock(&svm.dgpu_aperture.fmm_mutex); mem = aperture_allocate_area_aligned( &svm.dgpu_aperture, - aligned_size, 0, SCRATCH_ALIGN); + aligned_size, SCRATCH_ALIGN); pthread_mutex_unlock(&svm.dgpu_aperture.fmm_mutex); } else { uint64_t aligned_padded_size = aligned_size + @@ -974,7 +971,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) } static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, - manageable_aperture_t *aperture, uint64_t offset, uint64_t *mmap_offset, + manageable_aperture_t *aperture, uint64_t *mmap_offset, uint32_t flags, vm_object_t **vm_obj) { void *mem = NULL; @@ -986,8 +983,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area(aperture, - MemorySizeInBytes, offset); + mem = aperture_allocate_area(aperture, MemorySizeInBytes); pthread_mutex_unlock(&aperture->fmm_mutex); /* @@ -1014,16 +1010,11 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, return mem; } -/* - * The offset from GPUVM aperture base address to ensure that address 0 - * (after base subtraction) won't be used - */ -#define GPUVM_APP_OFFSET 0x10000 void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { manageable_aperture_t *aperture; int32_t gpu_mem_id; - uint32_t ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_VRAM, offset; + uint32_t ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_VRAM; uint64_t size, mmap_offset; void *mem; vm_object_t *vm_obj = NULL; @@ -1042,20 +1033,17 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla if (topology_is_svm_needed(get_device_id_by_gpu_id(gpu_id))) { aperture = &svm.dgpu_aperture; - offset = 0; if (flags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; - offset = GPUVM_APP_OFFSET; } if (aperture->is_coherent) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - mem = __fmm_allocate_device(gpu_id, size, - aperture, offset, &mmap_offset, - ioc_flags, &vm_obj); + mem = __fmm_allocate_device(gpu_id, size, aperture, &mmap_offset, + ioc_flags, &vm_obj); if (mem && vm_obj) { pthread_mutex_lock(&aperture->fmm_mutex); @@ -1104,9 +1092,8 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, - aperture, 0, NULL, - ioc_flags, &vm_obj); + mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, NULL, + ioc_flags, &vm_obj); if (mem && vm_obj) { HsaMemFlags flags; @@ -1231,7 +1218,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area(aperture, size, 0); + mem = aperture_allocate_area(aperture, size); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) return NULL; @@ -1277,9 +1264,8 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, } } else { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; - mem = __fmm_allocate_device(gpu_id, size, - aperture, 0, &mmap_offset, - ioc_flags, &vm_obj); + mem = __fmm_allocate_device(gpu_id, size, aperture, + &mmap_offset, ioc_flags, &vm_obj); if (mem && flags.ui32.HostAccess) { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_drm_fd; @@ -1862,6 +1848,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_base); gpu_mem[gpu_mem_id].gpuvm_aperture.limit = PORT_UINT64_TO_VPTR(process_apertures[i].gpuvm_limit); + /* Reserve space at the start of the + * aperture. After subtracting the base, we + * don't want valid pointers to become NULL. + */ + aperture_allocate_area( + &gpu_mem[gpu_mem_id].gpuvm_aperture, + gpu_mem[gpu_mem_id].gpuvm_aperture.align); } /* Acquire the VM from the DRM render node for KFD use */ @@ -2615,7 +2608,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj fmm_check_user_memory(addr, size); /* Allocate BO, userptr address is passed in mmap_offset */ - svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, 0, + svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE, &obj); @@ -2728,7 +2721,6 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, void *metadata; void *mem, *aperture_base; int32_t gpu_mem_id; - uint64_t offset; int r; HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; static const uint64_t IMAGE_ALIGN = 256*1024; @@ -2763,16 +2755,14 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { aperture = &svm.dgpu_aperture; aperture_base = NULL; - offset = 0; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; aperture_base = aperture->base; - offset = GPUVM_APP_OFFSET; } if (!aperture_is_valid(aperture->base, aperture->limit)) goto error_free_metadata; pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area_aligned(aperture, infoArgs.size, offset, + mem = aperture_allocate_area_aligned(aperture, infoArgs.size, MAX(aperture->align, IMAGE_ALIGN)); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) @@ -2900,8 +2890,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo pthread_mutex_lock(&aperture->fmm_mutex); reservedMem = aperture_allocate_area(aperture, - (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), - 0); + (SharedMemoryStruct->SizeInPages << PAGE_SHIFT)); pthread_mutex_unlock(&aperture->fmm_mutex); if (!reservedMem) { err = HSAKMT_STATUS_NO_MEMORY; From 9d96af0150623a306fc0e1e1d7c1a634b9566ee7 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 4 Aug 2018 16:56:15 -0400 Subject: [PATCH 0384/1247] libhsakmt: scratch is not a manageable aperture Only scratch_physical, for scratch-backing memory is managed by the Thunk. Change-Id: I4716981aa908d9569584dc35f40ffd270a2f9014 Signed-off-by: Felix Kuehling --- src/fmm.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 4939f42a56..67e7f63f3b 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -138,7 +138,7 @@ typedef struct { uint32_t node_id; uint64_t local_mem_size; aperture_t lds_aperture; - manageable_aperture_t scratch_aperture; + aperture_t scratch_aperture; manageable_aperture_t scratch_physical; /* For dGPU, scratch physical is allocated from * dgpu_aperture. When requested by RT, each * GPU will get a differnt range @@ -826,7 +826,7 @@ void fmm_print(uint32_t gpu_id) pr_info("GPUVM aperture:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].gpuvm_aperture); pr_info("Scratch aperture:\n"); - manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_aperture); + aperture_print(&gpu_mem[gpu_mem_id].scratch_aperture); pr_info("Scratch backing memory:\n"); manageable_aperture_print(&gpu_mem[gpu_mem_id].scratch_physical); } @@ -1669,8 +1669,6 @@ static void fmm_init_rbtree(void) } while (i--) { - rbtree_init(&gpu_mem[i].scratch_aperture.tree); - rbtree_init(&gpu_mem[i].scratch_aperture.user_tree); rbtree_init(&gpu_mem[i].scratch_physical.tree); rbtree_init(&gpu_mem[i].scratch_physical.user_tree); rbtree_init(&gpu_mem[i].gpuvm_aperture.tree); @@ -1758,8 +1756,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_count].node_id = i; gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_physical.fmm_mutex, NULL); - gpu_mem[gpu_mem_count].scratch_aperture.align = PAGE_SIZE; - pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_aperture.fmm_mutex, NULL); gpu_mem[gpu_mem_count].gpuvm_aperture.align = get_vm_alignment(props.DeviceId); gpu_mem[gpu_mem_count].gpuvm_aperture.guard_pages = guardPages; @@ -3318,7 +3314,6 @@ void fmm_clear_all_mem(void) for (i = 0; i < gpu_mem_count; i++) { fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); - fmm_clear_aperture(&gpu_mem[i].scratch_aperture); fmm_clear_aperture(&gpu_mem[i].scratch_physical); } From 2d2181b4781ace2fc04e0dbb32f922d52d4cef16 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 4 Aug 2018 19:45:54 -0400 Subject: [PATCH 0385/1247] libhsakmt: Move unmapping into aperture_release_area This prepares the code for an alternative aperture management method that needs to unmap memory differently. Change-Id: I5494aa5420f85edb8f7857f00c17e1d2e6479a51 Signed-off-by: Felix Kuehling --- src/fmm.c | 62 ++++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 67e7f63f3b..19a6f3d2c6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -55,7 +55,8 @@ .guard_pages = 1, \ .vm_ranges = NULL, \ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER, \ - .is_coherent = false \ + .is_coherent = false, \ + .is_cpu_accessible = false \ } #define container_of(ptr, type, member) ({ \ @@ -125,6 +126,7 @@ typedef struct { rbtree_t user_tree; pthread_mutex_t fmm_mutex; bool is_coherent; + bool is_cpu_accessible; } manageable_aperture_t; typedef struct { @@ -546,6 +548,28 @@ static void aperture_release_area(manageable_aperture_t *app, void *address, else vm_split_area(app, area, address, MemorySizeInBytes); } + + if (app->is_cpu_accessible) { + void *mmap_ret; + + /* Reset NUMA policy */ + mbind(address, MemorySizeInBytes, MPOL_DEFAULT, NULL, 0, 0); + + /* Remove any CPU mapping, but keep the address range reserved */ + mmap_ret = mmap(address, MemorySizeInBytes, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + if (mmap_ret == MAP_FAILED && errno == ENOMEM) { + /* When mmap count reaches max_map_count, any mmap will + * fail. Reduce the count with munmap then map it as + * NORESERVE immediately. + */ + munmap(address, MemorySizeInBytes); + mmap(address, MemorySizeInBytes, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + } + } } /* @@ -957,6 +981,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) /* Remember scratch backing aperture for later */ aperture_phy->base = mem; aperture_phy->limit = VOID_PTR_ADD(mem, aligned_size-1); + aperture_phy->is_cpu_accessible = true; /* Program SH_HIDDEN_PRIVATE_BASE */ args.gpu_id = gpu_id; @@ -1156,26 +1181,6 @@ static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, return mem; } -/* Remove any CPU mapping, but keep the address range reserved */ -static void munmap_and_reserve_address(void *address, uint64_t size) -{ - void *mmap_ret; - - mmap_ret = mmap(address, size, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, - -1, 0); - if (mmap_ret == MAP_FAILED && errno == ENOMEM) { - /* When mmap count reaches max_map_count, any mmap will - * fail. Reduce the count with munmap then map it as - * NORESERVE immediately. - */ - munmap(address, size); - mmap(address, size, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, - -1, 0); - } -} - static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) { @@ -1259,7 +1264,6 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, size); pthread_mutex_unlock(&aperture->fmm_mutex); - munmap_and_reserve_address(mem, MemorySizeInBytes); return NULL; } } else { @@ -1310,7 +1314,6 @@ void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args = {0}; - void *address; if (!object) return; @@ -1325,16 +1328,7 @@ static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); - address = object->start; - - if (address >= dgpu_shared_aperture_base && - address <= dgpu_shared_aperture_limit) { - /* Reset NUMA policy */ - mbind(address, object->size, MPOL_DEFAULT, NULL, 0, 0); - munmap_and_reserve_address(address, object->size); - } - - aperture_release_area(aperture, address, object->size); + aperture_release_area(aperture, object->start, object->size); vm_remove_object(aperture, object); pthread_mutex_unlock(&aperture->fmm_mutex); @@ -1628,6 +1622,7 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, svm.dgpu_aperture.align = align; svm.dgpu_aperture.guard_pages = guard_pages; svm.dgpu_aperture.is_coherent = disable_cache; + svm.dgpu_aperture.is_cpu_accessible = true; /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. @@ -1643,6 +1638,7 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, svm.dgpu_alt_aperture.align = align; svm.dgpu_alt_aperture.guard_pages = guard_pages; svm.dgpu_alt_aperture.is_coherent = true; + svm.dgpu_alt_aperture.is_cpu_accessible = true; svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); From d57026f447dd7a9bfac18951a50be4e52e5b68cd Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 10 Aug 2018 17:05:42 -0400 Subject: [PATCH 0386/1247] libhsakmt: Allow dgpu and dgpu_alt aperture to be the same Make dgpu_aperture and dgpu_alt_aperture pointers that can point to the same actual aperture. This will be useful on GFXv9 and later, where the MType is not defined by the aperture and we want to have a single aperture covering the entire virtual address space. aperture->is_coherent can no longer be a reliable indicator of coherency. Replace it with different conditions based on mem flags and svm.disable_cache (from HSA_DISABLE_CACHE environment). Change-Id: Iefc415b87b8abd96e3916586485a0a55d9b27c19 Signed-off-by: Felix Kuehling --- src/fmm.c | 262 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 136 insertions(+), 126 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 19a6f3d2c6..c6ddb55607 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -55,7 +55,6 @@ .guard_pages = 1, \ .vm_ranges = NULL, \ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER, \ - .is_coherent = false, \ .is_cpu_accessible = false \ } @@ -125,7 +124,6 @@ typedef struct { rbtree_t tree; rbtree_t user_tree; pthread_mutex_t fmm_mutex; - bool is_coherent; bool is_cpu_accessible; } manageable_aperture_t; @@ -149,23 +147,31 @@ typedef struct { int drm_render_fd; } gpu_mem_t; +enum svm_aperture_type { + SVM_DEFAULT = 0, + SVM_COHERENT, + SVM_APERTURE_NUM +}; + /* The main structure for dGPU Shared Virtual Memory Management */ typedef struct { - /* used for non-coherent system and invisible device mem on dGPU. - * This aperture is shared by all dGPUs - */ - manageable_aperture_t dgpu_aperture; + /* Two apertures can have different MTypes (for coherency) */ + manageable_aperture_t apertures[SVM_APERTURE_NUM]; - /* used for coherent (fine-grain) system memory on dGPU, - * This aperture is shared by all dGPUs + /* Pointers to apertures, may point to the same aperture on + * GFXv9 and later, where MType is not based on apertures */ - manageable_aperture_t dgpu_alt_aperture; + manageable_aperture_t *dgpu_aperture; + manageable_aperture_t *dgpu_alt_aperture; /* whether to use userptr for paged memory */ bool userptr_for_paged_mem; /* whether to check userptrs on registration */ bool check_userptr; + + /* whether all memory is coherent (GPU cache disabled) */ + bool disable_cache; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -180,10 +186,13 @@ static void *dgpu_shared_aperture_base; static void *dgpu_shared_aperture_limit; static svm_t svm = { - INIT_MANAGEABLE_APERTURE(0, 0), - INIT_MANAGEABLE_APERTURE(0, 0), - true, - false + .apertures = {INIT_MANAGEABLE_APERTURE(0, 0), + INIT_MANAGEABLE_APERTURE(0, 0)}, + .dgpu_aperture = NULL, + .dgpu_alt_aperture = NULL, + .userptr_for_paged_mem = false, + .check_userptr = false, + .disable_cache = false }; /* On APU, for memory allocated on the system memory that GPU doesn't access @@ -677,9 +686,9 @@ static manageable_aperture_t *fmm_get_aperture(HsaApertureInfo info) { switch (info.type) { case HSA_APERTURE_DGPU: - return &svm.dgpu_aperture; + return svm.dgpu_aperture; case HSA_APERTURE_DGPU_ALT: - return &svm.dgpu_alt_aperture; + return svm.dgpu_alt_aperture; case HSA_APERTURE_GPUVM: return &gpu_mem[info.idx].gpuvm_aperture; case HSA_APERTURE_CPUVM: @@ -713,26 +722,26 @@ static manageable_aperture_t *fmm_find_aperture(const void *address, HsaApertureInfo _info = { .type = HSA_APERTURE_UNSUPPORTED, .idx = 0}; if (is_dgpu) { - if (address >= svm.dgpu_aperture.base && - address <= svm.dgpu_aperture.limit) { + if (address >= svm.dgpu_aperture->base && + address <= svm.dgpu_aperture->limit) { aperture = fmm_is_scratch_aperture(address); if (!aperture) { - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; } - } else if (address >= svm.dgpu_alt_aperture.base && - address <= svm.dgpu_alt_aperture.limit) { - aperture = &svm.dgpu_alt_aperture; + } else if (address >= svm.dgpu_alt_aperture->base && + address <= svm.dgpu_alt_aperture->limit) { + aperture = svm.dgpu_alt_aperture; _info.type = HSA_APERTURE_DGPU_ALT; } else { /* Not in SVM, it can be system memory registered by userptr */ - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; } } else { /* APU */ - if (address >= svm.dgpu_aperture.base && address <= svm.dgpu_aperture.limit) { - aperture = &svm.dgpu_aperture; + if (address >= svm.dgpu_aperture->base && address <= svm.dgpu_aperture->limit) { + aperture = svm.dgpu_aperture; _info.type = HSA_APERTURE_DGPU; } else { /* gpuvm_aperture */ @@ -856,10 +865,12 @@ void fmm_print(uint32_t gpu_id) } pr_info("dGPU aperture:\n"); - manageable_aperture_print(&svm.dgpu_aperture); + manageable_aperture_print(svm.dgpu_aperture); pr_info("dGPU alt aperture:\n"); - manageable_aperture_print(&svm.dgpu_alt_aperture); - + if (svm.dgpu_aperture == svm.dgpu_alt_aperture) + pr_info("\t Alias of dGPU aperture\n"); + else + manageable_aperture_print(svm.dgpu_alt_aperture); } #else void fmm_print(uint32_t gpu_id) @@ -900,11 +911,11 @@ static void fmm_release_scratch(uint32_t gpu_id) pthread_mutex_unlock(&aperture->fmm_mutex); /* release address space */ - pthread_mutex_lock(&svm.dgpu_aperture.fmm_mutex); - aperture_release_area(&svm.dgpu_aperture, + pthread_mutex_lock(&svm.dgpu_aperture->fmm_mutex); + aperture_release_area(svm.dgpu_aperture, gpu_mem[gpu_mem_id].scratch_physical.base, size); - pthread_mutex_unlock(&svm.dgpu_aperture.fmm_mutex); + pthread_mutex_unlock(&svm.dgpu_aperture->fmm_mutex); } else /* release address space */ munmap(gpu_mem[gpu_mem_id].scratch_physical.base, size); @@ -951,11 +962,11 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) /* Allocate address space for scratch backing, 64KB aligned */ if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { - pthread_mutex_lock(&svm.dgpu_aperture.fmm_mutex); + pthread_mutex_lock(&svm.dgpu_aperture->fmm_mutex); mem = aperture_allocate_area_aligned( - &svm.dgpu_aperture, + svm.dgpu_aperture, aligned_size, SCRATCH_ALIGN); - pthread_mutex_unlock(&svm.dgpu_aperture.fmm_mutex); + pthread_mutex_unlock(&svm.dgpu_aperture->fmm_mutex); } else { uint64_t aligned_padded_size = aligned_size + SCRATCH_ALIGN - PAGE_SIZE; @@ -1057,14 +1068,14 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); if (topology_is_svm_needed(get_device_id_by_gpu_id(gpu_id))) { - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; if (flags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } - if (aperture->is_coherent) + if (!flags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; mem = __fmm_allocate_device(gpu_id, size, aperture, &mmap_offset, @@ -1112,7 +1123,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, return NULL; /* Use fine-grained aperture */ - aperture = &svm.dgpu_alt_aperture; + aperture = svm.dgpu_alt_aperture; ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; @@ -1202,11 +1213,11 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, size = MemorySizeInBytes; ioc_flags = 0; if (flags.ui32.CoarseGrain) - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; else - aperture = &svm.dgpu_alt_aperture; /* always coherent */ + aperture = svm.dgpu_alt_aperture; /* always coherent */ - if (aperture->is_coherent) + if (!flags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); @@ -1357,12 +1368,12 @@ HSAKMT_STATUS fmm_release(void *address) } if (!aperture) { - if (address >= svm.dgpu_aperture.base && - address <= svm.dgpu_aperture.limit) { - aperture = &svm.dgpu_aperture; - } else if (address >= svm.dgpu_alt_aperture.base && - address <= svm.dgpu_alt_aperture.limit) { - aperture = &svm.dgpu_alt_aperture; + if (address >= svm.dgpu_aperture->base && + address <= svm.dgpu_aperture->limit) { + aperture = svm.dgpu_aperture; + } else if (address >= svm.dgpu_alt_aperture->base && + address <= svm.dgpu_alt_aperture->limit) { + aperture = svm.dgpu_alt_aperture; } } @@ -1530,8 +1541,7 @@ static void *reserve_address(void *addr, unsigned long long int len) #define IS_CANONICAL_ADDR(a) ((a) < (1ULL << 47)) static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, - HSAuint32 align, HSAuint32 guard_pages, - bool disable_cache) + HSAuint32 align, HSAuint32 guard_pages) { const HSAuint64 ADDR_INC = GPU_HUGE_PAGE_SIZE; HSAuint64 len, map_size, alt_base, alt_size; @@ -1616,36 +1626,37 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, else limit = base + map_size - 1; - /* init aperture */ - svm.dgpu_aperture.base = dgpu_shared_aperture_base = ret_addr; - svm.dgpu_aperture.limit = dgpu_shared_aperture_limit = (void *)limit; - svm.dgpu_aperture.align = align; - svm.dgpu_aperture.guard_pages = guard_pages; - svm.dgpu_aperture.is_coherent = disable_cache; - svm.dgpu_aperture.is_cpu_accessible = true; + /* init two apertures for non-coherent and coherent memory */ + svm.apertures[SVM_DEFAULT].base = dgpu_shared_aperture_base = ret_addr; + svm.apertures[SVM_DEFAULT].limit = dgpu_shared_aperture_limit = (void *)limit; + svm.apertures[SVM_DEFAULT].align = align; + svm.apertures[SVM_DEFAULT].guard_pages = guard_pages; + svm.apertures[SVM_DEFAULT].is_cpu_accessible = true; /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. * Base and size must be 64KB aligned. */ - alt_base = (HSAuint64)svm.dgpu_aperture.base; - alt_size = (VOID_PTRS_SUB(svm.dgpu_aperture.limit, - svm.dgpu_aperture.base) + 1) >> 2; + alt_base = (HSAuint64)svm.apertures[SVM_DEFAULT].base; + alt_size = (VOID_PTRS_SUB(svm.apertures[SVM_DEFAULT].limit, + svm.apertures[SVM_DEFAULT].base) + 1) >> 2; alt_base = (alt_base + 0xffff) & ~0xffffULL; alt_size = (alt_size + 0xffff) & ~0xffffULL; - svm.dgpu_alt_aperture.base = (void *)alt_base; - svm.dgpu_alt_aperture.limit = (void *)(alt_base + alt_size - 1); - svm.dgpu_alt_aperture.align = align; - svm.dgpu_alt_aperture.guard_pages = guard_pages; - svm.dgpu_alt_aperture.is_coherent = true; - svm.dgpu_alt_aperture.is_cpu_accessible = true; + svm.apertures[SVM_COHERENT].base = (void *)alt_base; + svm.apertures[SVM_COHERENT].limit = (void *)(alt_base + alt_size - 1); + svm.apertures[SVM_COHERENT].align = align; + svm.apertures[SVM_COHERENT].guard_pages = guard_pages; + svm.apertures[SVM_COHERENT].is_cpu_accessible = true; - svm.dgpu_aperture.base = VOID_PTR_ADD(svm.dgpu_alt_aperture.limit, 1); + svm.apertures[SVM_DEFAULT].base = VOID_PTR_ADD(svm.apertures[SVM_COHERENT].limit, 1); pr_info("SVM alt (coherent): %12p - %12p\n", - svm.dgpu_alt_aperture.base, svm.dgpu_alt_aperture.limit); + svm.apertures[SVM_COHERENT].base, svm.apertures[SVM_COHERENT].limit); pr_info("SVM (non-coherent): %12p - %12p\n", - svm.dgpu_aperture.base, svm.dgpu_aperture.limit); + svm.apertures[SVM_DEFAULT].base, svm.apertures[SVM_DEFAULT].limit); + + svm.dgpu_aperture = &svm.apertures[SVM_DEFAULT]; + svm.dgpu_alt_aperture = &svm.apertures[SVM_COHERENT]; return HSAKMT_STATUS_SUCCESS; } @@ -1656,10 +1667,10 @@ static void fmm_init_rbtree(void) int i = gpu_mem_count; if (once++ == 0) { - rbtree_init(&svm.dgpu_aperture.tree); - rbtree_init(&svm.dgpu_aperture.user_tree); - rbtree_init(&svm.dgpu_alt_aperture.tree); - rbtree_init(&svm.dgpu_alt_aperture.user_tree); + rbtree_init(&svm.apertures[SVM_DEFAULT].tree); + rbtree_init(&svm.apertures[SVM_DEFAULT].user_tree); + rbtree_init(&svm.apertures[SVM_COHERENT].tree); + rbtree_init(&svm.apertures[SVM_COHERENT].user_tree); rbtree_init(&cpuvm_aperture.tree); rbtree_init(&cpuvm_aperture.user_tree); } @@ -1693,8 +1704,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ disableCache = getenv("HSA_DISABLE_CACHE"); - if (disableCache && strcmp(disableCache, "0") == 0) - disableCache = NULL; + svm.disable_cache = (disableCache && strcmp(disableCache, "0")); /* If HSA_USERPTR_FOR_PAGED_MEM is set to a non-0 value, * enable userptr for all paged memory allocations @@ -1861,7 +1871,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * space. Set up SVM apertures shared by all such GPUs */ ret = init_svm_apertures(svm_base, svm_limit, svm_alignment, - guardPages, disableCache); + guardPages); if (ret != HSAKMT_STATUS_SUCCESS) goto init_svm_failed; @@ -1874,11 +1884,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) continue; /* Set memory policy to match the SVM apertures */ - alt_base = (uintptr_t)svm.dgpu_alt_aperture.base; - alt_size = VOID_PTRS_SUB(svm.dgpu_alt_aperture.limit, - svm.dgpu_alt_aperture.base) + 1; + alt_base = (uintptr_t)svm.dgpu_alt_aperture->base; + alt_size = VOID_PTRS_SUB(svm.dgpu_alt_aperture->limit, + svm.dgpu_alt_aperture->base) + 1; err = fmm_set_memory_policy(process_apertures[i].gpu_id, - svm.dgpu_aperture.is_coherent ? + svm.disable_cache ? KFD_IOC_CACHE_POLICY_COHERENT : KFD_IOC_CACHE_POLICY_NONCOHERENT, KFD_IOC_CACHE_POLICY_COHERENT, @@ -1956,10 +1966,10 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA /* Report single SVM aperture, starting at base of * fine-grained, ending at limit of coarse-grained */ - if (aperture_is_valid(svm.dgpu_alt_aperture.base, - svm.dgpu_aperture.limit)) { - *aperture_base = PORT_VPTR_TO_UINT64(svm.dgpu_alt_aperture.base); - *aperture_limit = PORT_VPTR_TO_UINT64(svm.dgpu_aperture.limit); + if (aperture_is_valid(svm.dgpu_alt_aperture->base, + svm.dgpu_aperture->limit)) { + *aperture_base = PORT_VPTR_TO_UINT64(svm.dgpu_alt_aperture->base); + *aperture_limit = PORT_VPTR_TO_UINT64(svm.dgpu_aperture->limit); } break; @@ -2211,7 +2221,7 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); int ret; - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; /* Find the start address in SVM space for GPU mapping */ if (!object) @@ -2265,15 +2275,15 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) address, size, gpuvm_address); } - if ((address >= svm.dgpu_aperture.base) && - (address <= svm.dgpu_aperture.limit)) + if ((address >= svm.dgpu_aperture->base) && + (address <= svm.dgpu_aperture->limit)) /* map it */ - return _fmm_map_to_gpu(&svm.dgpu_aperture, + return _fmm_map_to_gpu(svm.dgpu_aperture, address, size, NULL, NULL, 0); - else if ((address >= svm.dgpu_alt_aperture.base) && - (address <= svm.dgpu_alt_aperture.limit)) + else if ((address >= svm.dgpu_alt_aperture->base) && + (address <= svm.dgpu_alt_aperture->limit)) /* map it */ - return _fmm_map_to_gpu(&svm.dgpu_alt_aperture, + return _fmm_map_to_gpu(svm.dgpu_alt_aperture, address, size, NULL, NULL, 0); /* @@ -2442,7 +2452,7 @@ static int _fmm_unmap_from_gpu_userptr(void *addr) vm_object_t *obj; void *svm_addr; - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; /* Find the start address in SVM space for GPU unmapping */ pthread_mutex_lock(&aperture->fmm_mutex); @@ -2480,15 +2490,15 @@ int fmm_unmap_from_gpu(void *address) address, NULL, 0, NULL); } - if ((address >= svm.dgpu_aperture.base) && - (address <= svm.dgpu_aperture.limit)) + if ((address >= svm.dgpu_aperture->base) && + (address <= svm.dgpu_aperture->limit)) /* unmap it */ - return _fmm_unmap_from_gpu(&svm.dgpu_aperture, + return _fmm_unmap_from_gpu(svm.dgpu_aperture, address, NULL, 0, NULL); - else if ((address >= svm.dgpu_alt_aperture.base) && - (address <= svm.dgpu_alt_aperture.limit)) + else if ((address >= svm.dgpu_alt_aperture->base) && + (address <= svm.dgpu_alt_aperture->limit)) /* unmap it */ - return _fmm_unmap_from_gpu(&svm.dgpu_alt_aperture, + return _fmm_unmap_from_gpu(svm.dgpu_alt_aperture, address, NULL, 0, NULL); /* @@ -2525,12 +2535,12 @@ bool fmm_get_handle(void *address, uint64_t *handle) } if (!aperture) { - if ((address >= svm.dgpu_aperture.base) && - (address <= svm.dgpu_aperture.limit)) { - aperture = &svm.dgpu_aperture; - } else if ((address >= svm.dgpu_alt_aperture.base) && - (address <= svm.dgpu_alt_aperture.limit)) { - aperture = &svm.dgpu_alt_aperture; + if ((address >= svm.dgpu_aperture->base) && + (address <= svm.dgpu_aperture->limit)) { + aperture = svm.dgpu_aperture; + } else if ((address >= svm.dgpu_alt_aperture->base) && + (address <= svm.dgpu_alt_aperture->limit)) { + aperture = svm.dgpu_alt_aperture; } } @@ -2582,7 +2592,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj gpu_id = g_first_gpu_mem->gpu_id; - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; /* Check if this address was already registered */ pthread_mutex_lock(&aperture->fmm_mutex); @@ -2635,12 +2645,12 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; - if ((address >= svm.dgpu_aperture.base) && - (address <= svm.dgpu_aperture.limit)) - aperture = &svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture.base) && - (address <= svm.dgpu_alt_aperture.limit)) - aperture = &svm.dgpu_alt_aperture; + if ((address >= svm.dgpu_aperture->base) && + (address <= svm.dgpu_aperture->limit)) + aperture = svm.dgpu_aperture; + else if ((address >= svm.dgpu_alt_aperture->base) && + (address <= svm.dgpu_alt_aperture->limit)) + aperture = svm.dgpu_alt_aperture; else { /* * If address isn't SVM address, we assume that this @@ -2651,7 +2661,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, return ret; if (gpu_id_array_size == 0) return HSAKMT_STATUS_SUCCESS; - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; /* fall through */ } @@ -2745,7 +2755,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, if (gpu_mem_id < 0) goto error_free_metadata; if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; aperture_base = NULL; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; @@ -2954,7 +2964,7 @@ static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) manageable_aperture_t *aperture; vm_object_t *obj; - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; /* Find the size and start address in SVM space */ pthread_mutex_lock(&aperture->fmm_mutex); @@ -2978,12 +2988,12 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) unsigned int i; HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); - if ((address >= svm.dgpu_aperture.base) && - (address <= svm.dgpu_aperture.limit)) - aperture = &svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture.base) && - (address <= svm.dgpu_alt_aperture.limit)) - aperture = &svm.dgpu_alt_aperture; + if ((address >= svm.dgpu_aperture->base) && + (address <= svm.dgpu_aperture->limit)) + aperture = svm.dgpu_aperture; + else if ((address >= svm.dgpu_alt_aperture->base) && + (address <= svm.dgpu_alt_aperture->limit)) + aperture = svm.dgpu_alt_aperture; else for (i = 0; i < gpu_mem_count; i++) { if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && @@ -3077,14 +3087,14 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_INVALID_PARAMETER; /* Find object by address */ - if ((address >= svm.dgpu_aperture.base) && - (address <= svm.dgpu_aperture.limit)) - aperture = &svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture.base) && - (address <= svm.dgpu_alt_aperture.limit)) - aperture = &svm.dgpu_alt_aperture; + if ((address >= svm.dgpu_aperture->base) && + (address <= svm.dgpu_aperture->limit)) + aperture = svm.dgpu_aperture; + else if ((address >= svm.dgpu_alt_aperture->base) && + (address <= svm.dgpu_alt_aperture->limit)) + aperture = svm.dgpu_alt_aperture; else { - aperture = &svm.dgpu_aperture; + aperture = svm.dgpu_aperture; userptr = true; } @@ -3314,8 +3324,8 @@ void fmm_clear_all_mem(void) } if (dgpu_shared_aperture_limit) { - fmm_clear_aperture(&svm.dgpu_aperture); - fmm_clear_aperture(&svm.dgpu_alt_aperture); + fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); + fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); /* Use the same dgpu range as the parent. If failed, then set * is_dgpu_mem_init to false. Later on dgpu_mem_init will try From d79b9c1a291ab3a528bbc217d3f08bbe158f9a9b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 4 Aug 2018 20:04:02 -0400 Subject: [PATCH 0387/1247] libhsakmt: Make VA management scheme configurable per aperture Change-Id: Ib70b038b4ef6465b03545317c6494a4e4950c107 Signed-off-by: Felix Kuehling --- src/fmm.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index c6ddb55607..b580f6a7c4 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -55,7 +55,8 @@ .guard_pages = 1, \ .vm_ranges = NULL, \ .fmm_mutex = PTHREAD_MUTEX_INITIALIZER, \ - .is_cpu_accessible = false \ + .is_cpu_accessible = false, \ + .ops = &reserved_aperture_ops \ } #define container_of(ptr, type, member) ({ \ @@ -115,7 +116,36 @@ struct vm_area { typedef struct vm_area vm_area_t; /* Memory manager for an aperture */ +typedef struct manageable_aperture manageable_aperture_t; + +/* Aperture management function pointers to allow different management + * schemes. + */ typedef struct { + void *(*allocate_area_aligned)(manageable_aperture_t *aper, + uint64_t size, uint64_t align); + void (*release_area)(manageable_aperture_t *aper, + void *addr, uint64_t size); +} manageable_aperture_ops_t; + +/* Reserved aperture type managed by its own address allocator */ +static void *reserved_aperture_allocate_aligned(manageable_aperture_t *aper, + uint64_t size, uint64_t align); +static void reserved_aperture_release(manageable_aperture_t *aper, + void *addr, uint64_t size); +static const manageable_aperture_ops_t reserved_aperture_ops = { + reserved_aperture_allocate_aligned, + reserved_aperture_release +}; + +#if 0 +/* Unreserved aperture type using mmap to allocate virtual address space */ +static const manageable_aperture_ops_t mmap_aperture_ops = { + NULL, NULL /* TODO */ +}; +#endif + +struct manageable_aperture { void *base; void *limit; uint64_t align; @@ -125,7 +155,8 @@ typedef struct { rbtree_t user_tree; pthread_mutex_t fmm_mutex; bool is_cpu_accessible; -} manageable_aperture_t; + const manageable_aperture_ops_t *ops; +}; typedef struct { void *base; @@ -527,8 +558,9 @@ static uint64_t vm_align_area_size(manageable_aperture_t *app, uint64_t size) /* * Assumes that fmm_mutex is locked on entry. */ -static void aperture_release_area(manageable_aperture_t *app, void *address, - uint64_t MemorySizeInBytes) +static void reserved_aperture_release(manageable_aperture_t *app, + void *address, + uint64_t MemorySizeInBytes) { vm_area_t *area; uint64_t SizeOfRegion; @@ -585,9 +617,9 @@ static void aperture_release_area(manageable_aperture_t *app, void *address, * returns allocated address or NULL. Assumes, that fmm_mutex is locked * on entry. */ -static void *aperture_allocate_area_aligned(manageable_aperture_t *app, - uint64_t MemorySizeInBytes, - uint64_t align) +static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, + uint64_t MemorySizeInBytes, + uint64_t align) { vm_area_t *cur, *next; void *start; @@ -642,10 +674,23 @@ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, return start; } + +/* Wrapper functions to call aperture-specific VA management functions */ +static void *aperture_allocate_area_aligned(manageable_aperture_t *app, + uint64_t MemorySizeInBytes, + uint64_t align) +{ + return app->ops->allocate_area_aligned(app, MemorySizeInBytes, align); +} static void *aperture_allocate_area(manageable_aperture_t *app, uint64_t MemorySizeInBytes) { - return aperture_allocate_area_aligned(app, MemorySizeInBytes, app->align); + return app->ops->allocate_area_aligned(app, MemorySizeInBytes, app->align); +} +static void aperture_release_area(manageable_aperture_t *app, void *address, + uint64_t MemorySizeInBytes) +{ + app->ops->release_area(app, address, MemorySizeInBytes); } /* returns 0 on success. Assumes, that fmm_mutex is locked on entry */ @@ -1632,6 +1677,7 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, svm.apertures[SVM_DEFAULT].align = align; svm.apertures[SVM_DEFAULT].guard_pages = guard_pages; svm.apertures[SVM_DEFAULT].is_cpu_accessible = true; + svm.apertures[SVM_DEFAULT].ops = &reserved_aperture_ops; /* Use the first 1/4 of the dGPU aperture as * alternate aperture for coherent access. @@ -1647,6 +1693,7 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, svm.apertures[SVM_COHERENT].align = align; svm.apertures[SVM_COHERENT].guard_pages = guard_pages; svm.apertures[SVM_COHERENT].is_cpu_accessible = true; + svm.apertures[SVM_COHERENT].ops = &reserved_aperture_ops; svm.apertures[SVM_DEFAULT].base = VOID_PTR_ADD(svm.apertures[SVM_COHERENT].limit, 1); @@ -1760,11 +1807,15 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; gpu_mem[gpu_mem_count].node_id = i; + gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; + gpu_mem[gpu_mem_count].scratch_physical.ops = &reserved_aperture_ops; pthread_mutex_init(&gpu_mem[gpu_mem_count].scratch_physical.fmm_mutex, NULL); + gpu_mem[gpu_mem_count].gpuvm_aperture.align = get_vm_alignment(props.DeviceId); gpu_mem[gpu_mem_count].gpuvm_aperture.guard_pages = guardPages; + gpu_mem[gpu_mem_count].gpuvm_aperture.ops = &reserved_aperture_ops; pthread_mutex_init(&gpu_mem[gpu_mem_count].gpuvm_aperture.fmm_mutex, NULL); if (!g_first_gpu_mem) From 40c46cc6cb3801ce13cea4f83cbb08fc00071a0a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 10 Aug 2018 22:30:19 -0400 Subject: [PATCH 0388/1247] libhsakmt: Fix assumptions about userptrs relative to apertures So far we have assumed that userptrs are always memory outside reserved SVM apertures that are mapped into the SVM aperture for GPU access. With an unreserved SVM aperture that covers the entire virtual address range, this distinction will no longer be true. Userptrs will generally be inside the unreserved SVM aperture. Take that into consideration when registering, mapping and unmapping virtual addresses. We now need a retry logic when looking up buffers from addresses. If it is not found by its GPU address, try it as a userptr. We also need to consider the new possibility that a userptr is registered at the same address for CPU and GPU access. So a buffer found by its GPU address may also turn out to be a userptr. In that case use a stricter lookup using the userptr and size (if the size is known), to identify the correct one of multiple overlapping objects. Change-Id: Ia43633aaa40f9fd2a74918ae969a631d2ff68419 Signed-off-by: Felix Kuehling --- src/fmm.c | 127 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 35 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index b580f6a7c4..26fd4f8fdb 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -138,12 +138,10 @@ static const manageable_aperture_ops_t reserved_aperture_ops = { reserved_aperture_release }; -#if 0 /* Unreserved aperture type using mmap to allocate virtual address space */ static const manageable_aperture_ops_t mmap_aperture_ops = { NULL, NULL /* TODO */ }; -#endif struct manageable_aperture { void *base; @@ -2304,8 +2302,12 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) { + manageable_aperture_t *aperture; + vm_object_t *object; + bool userptr = false; uint32_t i; uint64_t pi; + int ret; /* Find an aperture the requested address belongs to */ for (i = 0; i < gpu_mem_count; i++) { @@ -2328,23 +2330,48 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if ((address >= svm.dgpu_aperture->base) && (address <= svm.dgpu_aperture->limit)) - /* map it */ - return _fmm_map_to_gpu(svm.dgpu_aperture, - address, size, NULL, NULL, 0); + aperture = svm.dgpu_aperture; else if ((address >= svm.dgpu_alt_aperture->base) && (address <= svm.dgpu_alt_aperture->limit)) - /* map it */ - return _fmm_map_to_gpu(svm.dgpu_alt_aperture, - address, size, NULL, NULL, 0); + aperture = svm.dgpu_alt_aperture; + else { + aperture = svm.dgpu_aperture; + userptr = true; + } - /* - * If address isn't an SVM memory address, we assume that this - * is system memory address. On dGPU we need to map it, - * assuming it was previously registered. - */ - if (is_dgpu) - /* TODO: support mixed APU and dGPU configurations */ - return _fmm_map_to_gpu_userptr(address, size, gpuvm_address, NULL); + pthread_mutex_lock(&aperture->fmm_mutex); + if (userptr && is_dgpu) + object = vm_find_object_by_userptr(aperture, address, size); + else { + object = vm_find_object_by_address(aperture, address, 0); + /* If the object wasn't found in an unreserved + * aperture, it may be a userptr + */ + if (!object && aperture->ops == &mmap_aperture_ops) { + object = vm_find_object_by_userptr(aperture, address, + size); + userptr = true; + } else if (object && object->userptr == address) { + /* We found a userptr, but make sure we get + * the right one + */ + object = vm_find_object_by_userptr(aperture, address, + size); + userptr = true; + } + } + + if (object) { + if (userptr && is_dgpu) + ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); + else + ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0); + + pthread_mutex_unlock(&aperture->fmm_mutex); + return ret; + } + + pthread_mutex_unlock(&aperture->fmm_mutex); /* * On an APU a system memory address is accessed through @@ -2521,7 +2548,9 @@ static int _fmm_unmap_from_gpu_userptr(void *addr) int fmm_unmap_from_gpu(void *address) { + manageable_aperture_t *aperture = NULL; uint32_t i; + int ret; /* Find the aperture the requested address belongs to */ for (i = 0; i < gpu_mem_count; i++) { @@ -2543,14 +2572,20 @@ int fmm_unmap_from_gpu(void *address) if ((address >= svm.dgpu_aperture->base) && (address <= svm.dgpu_aperture->limit)) - /* unmap it */ - return _fmm_unmap_from_gpu(svm.dgpu_aperture, - address, NULL, 0, NULL); + aperture = svm.dgpu_aperture; else if ((address >= svm.dgpu_alt_aperture->base) && (address <= svm.dgpu_alt_aperture->limit)) - /* unmap it */ - return _fmm_unmap_from_gpu(svm.dgpu_alt_aperture, - address, NULL, 0, NULL); + aperture = svm.dgpu_alt_aperture; + + if (aperture) { + ret = _fmm_unmap_from_gpu(aperture, address, NULL, 0, NULL); + /* If unmap failed for an address in a reserved + * aperture, it can't be a userptr address + */ + if (!ret || aperture->ops == &reserved_aperture_ops) + return ret; + /* fall through: try userptr */ + } /* * If address isn't an SVM address, we assume that this is @@ -2689,7 +2724,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size) { - manageable_aperture_t *aperture; + manageable_aperture_t *aperture = NULL; vm_object_t *object = NULL; HSAKMT_STATUS ret; @@ -2702,7 +2737,21 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, else if ((address >= svm.dgpu_alt_aperture->base) && (address <= svm.dgpu_alt_aperture->limit)) aperture = svm.dgpu_alt_aperture; - else { + /* else it's probably a userptr, handled later */ + + if (aperture) { + pthread_mutex_lock(&aperture->fmm_mutex); + object = vm_find_object_by_address(aperture, address, 0); + /* If the SVM aperture is reserved, it can't be a userptr */ + if (!object && aperture->ops == &reserved_aperture_ops) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_NOT_SUPPORTED; + } + } + + if (!object) { + if (aperture) + pthread_mutex_unlock(&aperture->fmm_mutex); /* * If address isn't SVM address, we assume that this * is system memory address. @@ -2713,16 +2762,8 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (gpu_id_array_size == 0) return HSAKMT_STATUS_SUCCESS; aperture = svm.dgpu_aperture; - /* fall through */ - } - - pthread_mutex_lock(&aperture->fmm_mutex); - if (!object) - object = vm_find_object_by_address(aperture, address, 0); - - if (!object) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_NOT_SUPPORTED; + pthread_mutex_lock(&aperture->fmm_mutex); + /* fall through for registered device ID array setup */ } if (object->registered_device_id_array_size > 0) { @@ -3152,8 +3193,24 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, pthread_mutex_lock(&aperture->fmm_mutex); if (userptr && is_dgpu) object = vm_find_object_by_userptr(aperture, address, size); - else + else { object = vm_find_object_by_address(aperture, address, 0); + /* If the object wasn't found in an unreserved + * aperture, it may be a userptr + */ + if (!object && aperture->ops == &mmap_aperture_ops) { + object = vm_find_object_by_userptr(aperture, address, + size); + userptr = true; + } else if (object && object->userptr == address) { + /* We found a userptr, but make sure we get + * the right one + */ + object = vm_find_object_by_userptr(aperture, address, + size); + userptr = true; + } + } if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); From 62f7dc2a48d54294933ea72eb40d4a0cbb9b7e1b Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 15 Aug 2018 12:06:34 -0400 Subject: [PATCH 0389/1247] kfdtest: Do not set GTEST_FLAG throw_on_failure The flag makes EXPECT_* to behave like ASSERT_*, which actually work against our favor, so disable the flag. Change-Id: I2ea1dfeaf916b396593a504d081148abdac0fc70 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDMemoryTest.cpp | 43 +++++++++++++---------------- tests/kfdtest/src/KFDTestMain.cpp | 2 -- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 95cb14d543..df193ff754 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1153,12 +1153,10 @@ TEST_F(KFDMemoryTest, PtraceAccess) { int traceStatus; int err = 0, r; - // Child process: don't use ASSERTs after attaching to parent - // process because terminating without detaching from the - // traced process leaves it stopped. Unfortunately, main() - // sets throw_on_failure to true, which seems to affect EXPECT - // as well. So we catch any exceptions and detach before - // terminating. + /* Child process: we catch any exceptions to make sure we detach + * from the traced process, because terminating without detaching + * leaves the traced process stopped. + */ r = ptrace(PTRACE_ATTACH, tracePid, NULL, NULL); if (r) { WARN() << "PTRACE_ATTACH failed: " << r << std::endl; @@ -1175,8 +1173,8 @@ TEST_F(KFDMemoryTest, PtraceAccess) { HSAuint8 *addr = reinterpret_cast(reinterpret_cast(mem[0]) + i) + i; errno = 0; long data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); - EXPECT_EQ(0, errno); - EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, + ASSERT_EQ(0, errno); + ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, reinterpret_cast(data))); if (mem[1] == NULL) @@ -1185,8 +1183,8 @@ TEST_F(KFDMemoryTest, PtraceAccess) { addr = reinterpret_cast(reinterpret_cast(mem[1]) + i) + i; errno = 0; data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); - EXPECT_EQ(0, errno); - EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, + ASSERT_EQ(0, errno); + ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, reinterpret_cast(data))); } } catch (...) { @@ -1297,12 +1295,9 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { int traceStatus; int err = 0, r; - /* Child process: don't use ASSERTs after attaching to parent - * process because terminating without detaching from the - * traced process leaves it stopped. Unfortunately, main() - * sets throw_on_failure to true, which seems to affect EXPECT - * as well. So we catch any exceptions and detach before - * terminating. + /* Child process: we catch any exceptions to make sure we detach + * from the traced process, because terminating without detaching + * leaves the traced process stopped. */ r = ptrace(PTRACE_ATTACH, tracePid, NULL, NULL); if (r) { @@ -1317,17 +1312,17 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { /* peek the memory */ errno = 0; HSAint64 data0 = ptrace(PTRACE_PEEKDATA, tracePid, mem0, NULL); - EXPECT_EQ(0, errno); - EXPECT_EQ(data[0], data0); + ASSERT_EQ(0, errno); + ASSERT_EQ(data[0], data0); HSAint64 data1 = ptrace(PTRACE_PEEKDATA, tracePid, mem1, NULL); - EXPECT_EQ(0, errno); - EXPECT_EQ(data[1], data1); + ASSERT_EQ(0, errno); + ASSERT_EQ(data[1], data1); /* swap mem0 and mem1 by poking */ - EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, reinterpret_cast(data[1]))); - EXPECT_EQ(0, errno); - EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, reinterpret_cast(data[0]))); - EXPECT_EQ(0, errno); + ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, reinterpret_cast(data[1]))); + ASSERT_EQ(0, errno); + ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, reinterpret_cast(data[0]))); + ASSERT_EQ(0, errno); } catch (...) { err = 1; } diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index 847582c2bc..5021445099 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -61,8 +61,6 @@ GTEST_API_ int main(int argc, char **argv) { g_TestENVCaps = ENVCAPS_NOADDEDCAPS | ENVCAPS_64BITLINUX; g_TestTimeOut = KFD_TEST_DEFAULT_TIMEOUT; - // every fatal fail ( = assert that failed ) will throw an exption - testing::GTEST_FLAG(throw_on_failure) = true; testing::InitGoogleTest(&argc, argv); CommandLineArguments args; From 163fa2f3aa40e5327e6c87c5a4b4f0faf21b0417 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Thu, 16 Aug 2018 15:32:08 +0800 Subject: [PATCH 0390/1247] kfdtest: use HSAuint64 instead of unsigned HSAint64 This should fix gtest compile errors. code like below has trouble, typedef char char8; typedef unsigned char uchar8; ASSERT_NE((uchar8)1, 0); ASSERT_NE((unsigned char8)1, 0); // compile error here or ASSERT_NE((unsigned char8)1, 0); ASSERT_NE((uchar8)1, 0); // compile error here HSA[u]int64 are alias. So ASSERT_XX((unsigned HSAint64)..) with ASSERT_XX((HSAuint64)..) fail to compile. Change-Id: I4c24bc699a69bd4f37c4bc8aaaa9f1a92a24a33e Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDQMTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 1d0eb6fe38..30bc5416ed 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1084,7 +1084,7 @@ TEST_F(KFDQMTest, QueueLatency) { HSAuint64 *qts; int i = 0; - ASSERT_NE((unsigned HSAint64)queue_latency_arr, 0); + ASSERT_NE((HSAuint64)queue_latency_arr, 0); int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -1110,7 +1110,7 @@ TEST_F(KFDQMTest, QueueLatency) { i = 0; do { queue.PlacePacket(PM4ReleaseMemoryPacket(true, - (unsigned HSAint64)&qts[i], + (HSAuint64)&qts[i], 0, true, 1)); @@ -1137,7 +1137,7 @@ TEST_F(KFDQMTest, QueueLatency) { i = 0; do { queue.PlacePacket(PM4ReleaseMemoryPacket(true, - (unsigned HSAint64)&qts[i], + (HSAuint64)&qts[i], 0, true, 1)); From 80f2cc644cdad7b31b1e6ad906bae7f496c7eada Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 4 Aug 2018 21:20:27 -0400 Subject: [PATCH 0391/1247] libhsakmt: Add mmap-based aperture management for GFXv9 and later If the GPU virtual address space is >= 47 bits, don't reserve virtual address space at startup and use mmap to allocate virtual addresses. Change-Id: Ic935b03c8e78271829fc8e6cfd0e543184aff818 Signed-off-by: Felix Kuehling --- src/fmm.c | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 142 insertions(+), 4 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 26fd4f8fdb..805d26200d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -139,8 +139,13 @@ static const manageable_aperture_ops_t reserved_aperture_ops = { }; /* Unreserved aperture type using mmap to allocate virtual address space */ +static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, + uint64_t size, uint64_t align); +static void mmap_aperture_release(manageable_aperture_t *aper, + void *addr, uint64_t size); static const manageable_aperture_ops_t mmap_aperture_ops = { - NULL, NULL /* TODO */ + mmap_aperture_allocate_aligned, + mmap_aperture_release }; struct manageable_aperture { @@ -673,6 +678,84 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, return start; } +static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, + uint64_t size, uint64_t align) +{ + uint64_t aligned_padded_size, guard_size; + void *addr, *aligned_addr, *aligned_end, *mapping_end; + + if (!aper->is_cpu_accessible) { + pr_err("MMap Aperture must be CPU accessible\n"); + return NULL; + } + + if (align < aper->align) + align = aper->align; + + /* Align big buffers to the next power-of-2 up to huge page + * size for flexible fragment size TLB optimizations + */ + while (align < GPU_HUGE_PAGE_SIZE && size >= (align << 1)) + align <<= 1; + + /* Align memory size to match aperture requirements */ + size = ALIGN_UP(size, aper->align); + + /* Add padding to guarantee proper alignment and leave guard + * pages on both sides + */ + guard_size = (uint64_t)aper->guard_pages * PAGE_SIZE; + aligned_padded_size = size + align + + 2*guard_size - PAGE_SIZE; + + /* Map memory */ + addr = mmap(0, aligned_padded_size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); + if (addr == MAP_FAILED) { + pr_err("mmap failed: %s\n", strerror(errno)); + return NULL; + } + + /* Adjust for alignment and guard pages, range-check the reslt */ + aligned_addr = (void *)ALIGN_UP((uint64_t)addr + guard_size, align); + if (aligned_addr < aper->base || + VOID_PTR_ADD(aligned_addr, size - 1) > aper->limit) { + pr_err("mmap returned %p, out of range %p-%p\n", aligned_addr, + aper->base, aper->limit); + munmap(addr, aligned_padded_size); + return NULL; + } + + /* Unmap padding and guard pages */ + if (aligned_addr > addr) + munmap(addr, VOID_PTRS_SUB(aligned_addr, addr)); + + aligned_end = VOID_PTR_ADD(aligned_addr, size); + mapping_end = VOID_PTR_ADD(addr, aligned_padded_size); + if (mapping_end > aligned_end) + munmap(aligned_end, VOID_PTRS_SUB(mapping_end, aligned_end)); + + return aligned_addr; +} + +static void mmap_aperture_release(manageable_aperture_t *aper, + void *addr, uint64_t size) +{ + if (!aper->is_cpu_accessible) { + pr_err("MMap Aperture must be CPU accessible\n"); + return; + } + + /* Align memory size to match aperture requirements */ + size = ALIGN_UP(size, aper->align); + + /* Reset NUMA policy */ + mbind(addr, size, MPOL_DEFAULT, NULL, 0, 0); + + /* Unmap memory */ + munmap(addr, size); +} + /* Wrapper functions to call aperture-specific VA management functions */ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, uint64_t MemorySizeInBytes, @@ -1561,6 +1644,43 @@ static HSAKMT_STATUS acquire_vm(uint32_t gpu_id, int fd) return HSAKMT_STATUS_SUCCESS; } +static HSAKMT_STATUS init_mmap_apertures(HSAuint64 base, HSAuint64 limit, + HSAuint32 align, HSAuint32 guard_pages) +{ + void *addr; + + /* Set up one SVM aperture */ + svm.apertures[SVM_DEFAULT].base = (void *)base; + svm.apertures[SVM_DEFAULT].limit = (void *)limit; + svm.apertures[SVM_DEFAULT].align = align; + svm.apertures[SVM_DEFAULT].guard_pages = guard_pages; + svm.apertures[SVM_DEFAULT].is_cpu_accessible = true; + svm.apertures[SVM_DEFAULT].ops = &mmap_aperture_ops; + + svm.apertures[SVM_COHERENT].base = svm.apertures[SVM_COHERENT].limit = + NULL; + + /* Try to allocate one page. If it fails, we'll fall back to + * managing our own reserved address range. + */ + addr = aperture_allocate_area(&svm.apertures[SVM_DEFAULT], PAGE_SIZE); + if (addr) { + aperture_release_area(&svm.apertures[SVM_DEFAULT], addr, + PAGE_SIZE); + + svm.dgpu_aperture = svm.dgpu_alt_aperture = + &svm.apertures[SVM_DEFAULT]; + pr_info("Initialized unreserved SVM apertures: %p - %p\n", + svm.apertures[SVM_DEFAULT].base, + svm.apertures[SVM_DEFAULT].limit); + } else { + pr_info("Failed to allocate unreserved SVM address space.\n"); + pr_info("Falling back to reserved SVM apertures.\n"); + } + + return addr ? HSAKMT_STATUS_SUCCESS : HSAKMT_STATUS_ERROR; +} + static void *reserve_address(void *addr, unsigned long long int len) { void *ret_addr; @@ -1597,8 +1717,26 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, if (dgpu_shared_aperture_limit) return HSAKMT_STATUS_SUCCESS; + /* Align base and limit to huge page size */ base = ALIGN_UP(base, GPU_HUGE_PAGE_SIZE); limit = ((limit + 1) & ~(HSAuint64)(GPU_HUGE_PAGE_SIZE - 1)) - 1; + + /* If the limit is greater or equal 47-bits of address space, + * it means we have GFXv9 or later GPUs only. We don't need + * apertures to determine the MTYPE and the virtual address + * space of the GPUs covers the full CPU address range (on + * x86_64) or at least mmap is unlikely to run out of + * addresses the GPUs can handle. + */ + if (limit >= (1ULL << 47) - 1) { + HSAKMT_STATUS status = init_mmap_apertures(base, limit, align, + guard_pages); + + if (status == HSAKMT_STATUS_SUCCESS) + return status; + /* fall through: fall back to reserved address space */ + } + if (limit > SVM_RESERVATION_LIMIT) limit = SVM_RESERVATION_LIMIT; if (base >= limit) { @@ -3431,10 +3569,10 @@ void fmm_clear_all_mem(void) fmm_clear_aperture(&gpu_mem[i].scratch_physical); } - if (dgpu_shared_aperture_limit) { - fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); - fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); + fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); + fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); + if (dgpu_shared_aperture_limit) { /* Use the same dgpu range as the parent. If failed, then set * is_dgpu_mem_init to false. Later on dgpu_mem_init will try * to get a new range From ab181c46c0af4aaceaa981919ba37fd52c44a479 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sun, 5 Aug 2018 13:56:27 -0400 Subject: [PATCH 0392/1247] libhsakmt: Try to use CPU addr as GPU addr for userptrs If the CPU addr of a userptr is accessible by the GPU, try to use it instead of allocating a different GPU address. If something else is already registered with an overlapping address range, we still need to allocate a GPU address, because KFD does not support overlapping GPUVM mappings. Change-Id: I452963ee45a454f735755a0b43122b9aee5d55be Signed-off-by: Felix Kuehling --- src/fmm.c | 69 ++++++++++++++++++++++++++++++++++++++++++++----- src/libhsakmt.h | 4 ++- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 805d26200d..12c0be415d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -502,6 +502,38 @@ loop: return cur; /* NULL if not found */ } +/* Returns true if there is any object in GPU VM address space + * overlapping the specified address range + */ +static bool vm_exists_overlapping_object(manageable_aperture_t *aper, + const void *addr, uint64_t size) +{ + unsigned long start_addr = (unsigned long)addr; + unsigned long end_addr = start_addr + size; + rbtree_t *tree = vm_object_tree(aper, 0); + rbtree_key_t start_key, end_key; + rbtree_node_t *rn_start, *ln_end; + vm_object_t *cur; + + start_key = rbtree_key(start_addr, 0); + rn_start = rbtree_lookup_nearest(tree, &start_key, LKP_ALL, RIGHT); + if (rn_start) { + cur = vm_object_entry(rn_start, 0); + if ((unsigned long)cur->start < end_addr) + return true; + } + + end_key = rbtree_key(end_addr, 0); + ln_end = rbtree_lookup_nearest(tree, &end_key, LKP_ALL, LEFT); + if (ln_end) { + cur = vm_object_entry(ln_end, 0); + if ((unsigned long)cur->start + cur->size > start_addr) + return true; + } + + return false; +} + static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app, const void *address, uint64_t size) { @@ -1136,7 +1168,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t *mmap_offset, uint32_t flags, vm_object_t **vm_obj) { - void *mem = NULL; + void *mem = NULL, *userpage = NULL; vm_object_t *obj; /* Check that aperture is properly initialized/supported */ @@ -1145,9 +1177,27 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area(aperture, MemorySizeInBytes); + /* If it's a userptr within the GPU-addressable address space, + * and no object is registered yet in the same address range, + * then use the CPU address as GPU address + */ + if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) && mmap_offset) { + userpage = (void *)*mmap_offset; + if (userpage >= aperture->base && + VOID_PTR_ADD(userpage, MemorySizeInBytes - 1) <= aperture->limit && + !vm_exists_overlapping_object(aperture, userpage, MemorySizeInBytes)) + mem = userpage; + } + if (!mem) + mem = aperture_allocate_area(aperture, MemorySizeInBytes); pthread_mutex_unlock(&aperture->fmm_mutex); + if (!mem) { + pr_err("Failed to allocate %ld bytes virtual address space\n", + MemorySizeInBytes); + return NULL; + } + /* * Now that we have the area reserved, allocate memory in the device * itself @@ -1159,9 +1209,11 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, * allocation of memory in device failed. * Release region in aperture */ - pthread_mutex_lock(&aperture->fmm_mutex); - aperture_release_area(aperture, mem, MemorySizeInBytes); - pthread_mutex_unlock(&aperture->fmm_mutex); + if (mem != userpage) { + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, MemorySizeInBytes); + pthread_mutex_unlock(&aperture->fmm_mutex); + } /* Assign NULL to mem to indicate failure to calling function */ mem = NULL; @@ -1465,7 +1517,12 @@ static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); - aperture_release_area(aperture, object->start, object->size); + /* Userptrs with CPU addr == GPU addr don't have aperture + * space allocated to them + */ + if (!object->userptr || + (void *)PAGE_ALIGN_DOWN(object->userptr) != object->start) + aperture_release_area(aperture, object->start, object->size); vm_remove_object(aperture, object); pthread_mutex_unlock(&aperture->fmm_mutex); diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 17dce385fc..3186787c26 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -65,8 +65,10 @@ extern int PAGE_SHIFT; do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define ALIGN_DOWN(x,align) ((uint64_t)(x) & ~(uint64_t)((align)-1)) #define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1)) -#define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) +#define PAGE_ALIGN_UP(x) ALIGN_UP((x),PAGE_SIZE) +#define PAGE_ALIGN_DOWN(x) ALIGN_DOWN((x),PAGE_SIZE) #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) From 9271e69ddff1b75cb92315f389886c91b3fdc78d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 17 Aug 2018 11:46:45 -0400 Subject: [PATCH 0393/1247] libhsakmt: Fix processing of memory fault events AMDKFD_IOC_WAIT_EVENTS with multiple events and wait_for_all = 0 returns success after any of the events have signaled. So we can't blindly assume that a memory fault event that was in the list has actually signaled. Check the gpu_id as an indicator whether there really was a memory fault before processing it further. Change-Id: I6cc311bfc184c631beaf684027176a6ca42e05c1 Signed-off-by: Felix Kuehling --- src/events.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/events.c b/src/events.c index 2e9d126fed..43d99b8da8 100644 --- a/src/events.c +++ b/src/events.c @@ -313,7 +313,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], else { result = HSAKMT_STATUS_SUCCESS; for (HSAuint32 i = 0; i < NumEvents; i++) { - if (Events[i]->EventData.EventType == HSA_EVENTTYPE_MEMORY) { + if (Events[i]->EventData.EventType == HSA_EVENTTYPE_MEMORY && + event_data[i].memory_exception_data.gpu_id) { Events[i]->EventData.EventData.MemoryAccessFault.VirtualAddress = event_data[i].memory_exception_data.va; result = gpuid_to_nodeid(event_data[i].memory_exception_data.gpu_id, &Events[i]->EventData.EventData.MemoryAccessFault.NodeId); if (result != HSAKMT_STATUS_SUCCESS) From db92d5af2308231490bfb63697f2ec83497fcc75 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Fri, 13 Jul 2018 14:05:27 -0400 Subject: [PATCH 0394/1247] Add GFX debug trap control code Add initial support for the kfd debugger trap support for GFX9 chips. - Adding support for Enable/Disable trap support - Setting debug trap support data - Setting wave launch trap override - Setting wave launch mode Change-Id: If39f2395c4b6cf56249cf76f1c44cfcbdcef891c Signed-off-by: Philip Cox --- include/hsakmt.h | 341 ++++++++++++++++++++++++++++++++++++++ include/hsakmttypes.h | 58 ++++++- include/linux/kfd_ioctl.h | 36 +++- src/debug.c | 87 ++++++++++ src/libhsakmt.ver | 5 + 5 files changed, 519 insertions(+), 8 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 9084079882..eb5307e84c 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -610,6 +610,347 @@ hsaKmtDbgAddressWatch( HsaEvent* WatchEvent[] //IN, optional ); +/** + Suspend the execution of a set of queues. A queue that is suspended + allows the context save state to be inspected and modified. If a + queue is already suspended it remains suspended. A suspended queue + can be resumed by hsaKmtDbgQueueResume(). + + If NoGracePeriod is false then the default grace period used for + waiting for waves to complete before context switching is used. If + NoGracePeriod is true then no grace period us used and waves are + context saved as soon as possible. + + If MemFence is true all queues being suspended will perform a + sequentially consistent system scope release that synchronizes with + a sequentially consistent system scope acquire performed by this + call. This ensures any memory updates performed by the suspended + queues are visible to the thread calling this operation. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueueSuspend( + HSAuint32 NumQueues, //IN + HSA_QUEUEID QueueId[], //IN + bool NoGracePeriod, //IN + bool MemFence //IN + ); + +/** + Resume the execution of a set of queues. If a queue is not + suspended by hsaKmtDbgQueueSuspend() then it remains executing. + + If MemFence is true this call will perform a sequentially + consistent system scope release that synchronizes with a + sequentially consistent system scope acquire performed by all + queues being resumed. This ensures any memory updates performed by + the thread calling this operation are visible to the resumed + queues. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueueResume( + HSAuint32 NumQueues, //IN + HSA_QUEUEID QueueId[], //IN + bool MemFence //IN + ); + +/** + Enable debug trap for NodeId. If QueueId is INVALID_QUEUEID then + enable for all queues on NodeId, otherwise enable only for QueueId. + + When debug trap is enabled the trap handler behavior changes + depending on architecture of the node and can include the following: + + - Initialize Trap Temp Registers: All new waves are launched with + specific trap temp registers initialized with: + + - HSA dispatch packet address of the wave. + + - X, Y, Z grid and work-group position of the wave within the + dispatch. + + - The value of TrapData registers. hsaKmtEnableDebugTrap() sets + these to 0 and they can be changed by hsaKmtSetDebugTrapData2(). + + - The scratch backing memory address. + + - Enable wave launch trap override. hsaKmtEnableDebugTrap() sets the + TrapMask to 0 and the TrapOverride to HSA_DBG_TRAP_OVERRIDE_OR and + they can be changed by hsaKmtSetWaveLaunchTrapOverride(). + + If debug trap is already enabled for NodeId, any features controlled + by it are still reset to their default values as defined above. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_INVALID_HANDLE if: + + - NodeId is invalid + + - QueueId is not INVALID_QUEUE, or is not a valid queue of + NodeId. + + - HSAKMT_STATUS_UNAVAILABLE if debugging is not available to this + process. For example, there may be a limit on number of + processes that can perform debugging at the same time. + + - HSAKMT_STATUS_NOT_SUPPORTED if debug trap is not supported by + NodeId, or if QueueId is not INVALID_QUEUEID and NodeId does not + support per queue enabling. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtEnableDebugTrap( + HSAuint32 NodeId, //IN + HSA_QUEUEID QueueId //IN + ); + +/** + Disable debug trap enabled by hsaKmtEnableDebugTrap(). If debug trap + is not currently enabled not action is taken. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. + + - HSAKMT_STATUS_NOT_SUPPORTED if debug trap not supported for NodeId. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDisableDebugTrap( + HSAuint32 NodeId //IN + ); + +/** + Set the value to use to initialize the TrapData used when + initializing trap temp registers for NodeId when debug trap is enabled. + + An error is returned if debug trap is not currently enabled for + NodeId. Debug trap is enabled by hsaKmtEnableDebugTrap() which + initializes TrapData to 0. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_NOT_SUPPORTED if debug trap data is not supported + by NodeId. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. + + - HSAKMT_STATUS_INVALID_PARAMETER if TrapDataIndex is larger than + trap-data-count - 1. + + - HSAKMT_STATUS_ERROR if debug trap is not currently enabled by + hsaKmtEnableDebugTrap() for NodeId. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetDebugTrapData2( + HSAuint32 NodeId, //IN + HSAuint32 TrapData0, //IN + HSAuint32 TrapData1 //IN + ); + +/** + Set the trap override mask. When debug trap is enabled by + hsaKmtEnableDebugTrap() each wave launched has its initial + MODE.excp_en register overriden by TrapMask as specified by + TrapOverride. + + An error is returned if debug trap is not currently enabled for + NodeId. Debug trap is enabled by hsaKmtEnableDebugTrap() which + initializes TrapMask to 0 and TrapOverride to + HSA_DBG_TRAP_OVERRIDE_OR. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_NOT_SUPPORTED if wave launch trap override is not + supported by NodeId. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. + + - HSAKMT_STATUS_INVALID_PARAMETER if TrapOverride is invalid. + + - HSAKMT_STATUS_ERROR if debug trap is not currently enabled by + hsaKmtEnableDebugTrap() for NodeId. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetWaveLaunchTrapOverride( + HSAuint32 NodeId, //IN + HSA_DBG_TRAP_OVERRIDE TrapOverride, //IN + HSA_DBG_TRAP_MASK TrapMask //IN + ); + +/** + Set the mode in which all future waves will be launched for + NodeId. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_UNAVAILABLE if debugging is not available to this + process. For example, there may be a limit on number of + processes that can perform debugging at the same time. + + - HSAKMT_STATUS_NOT_SUPPORTED if the WaveLaunchMode requested is + not supported by the NodeId. Different implementations and + different nodes within an implementation can support different + sets of launch modes. Only HSA_DBG_WAVE_LAUNCH_MODE_NORMAL mode + is supported by all. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId is not a valid node. + + - HSAKMT_STATUS_INVALID_PARAMETER if WaveLaunchMode is not a valid + value. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetWaveLaunchMode( + HSAuint32 NodeId, //IN + HSA_DBG_WAVE_LAUNCH_MODE WaveLaunchMode //IN + ); + +/** + Set a debug memory access watch point. A memory access of the kind + specified by WatchMode to an matching address will cause the trap + handler to be entered. An address matches if, after ANDing the + watch-addr-mask-lo..watch-addr-mask-hi bits of WatchAddrMask, it + equals the WatchAddress with the bottom watch-addr-mask-lo bits + cleared. + + WatchId will be in the range 0 to watch-count - 1. The WatchId + value will match the address watch exception reported to the trap + handler. + + hsaKmtGetNodeProperties() can be used to obtain HsaNodeProperties. + watch-addr-mask-lo and watch-addr-mask-hi can be obtained from + HsaNodeProperties.Capabilities.WatchAddrMaskLoBit and + HsaNodeProperties.Capabilities.WatchAddrMaskHiBit respectively. + watch-count can be obtained from + 2^HsaNodeProperties.Capabilities.WatchPointsTotalBits. + + To cause debug memory address watch points to be reported to the + trap handler the address watch exception must be enabled. This can + be accomplished by using hsaKmtSetWaveLaunchTrapOverride() with a + TrapMask that includes HSA_DBG_TRAP_MASK_DBG_ADDRESS_WATCH. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_NOT_SUPPORTED if debug memory watch points are + not supported for NodeId. + + - HSAKMT_STATUS_UNAVAILABLE if debugging is not available to this + process. For example, there may be a limit on number of + processes that can perform debugging at the same time. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId or WatchId* is invalid. + + - HSAKMT_STATUS_INVALID_PARAMETER if: + + - WatchAddrMask contains non-0 bits outside the inclusive range + watch-addr-mask-lo to watch-addr-mask-hi. + + - If WatchAddress contain non-0 bits in the inclusive range 0 to + watch-addr-mask-lo. + + - If WatchMode is not one of the values of HSA_DBG_WATCH_MODE. + + - WatchId is NULL. + + - HSAKMT_STATUS_OUT_OF_RESOURCES if no more watch points are + available to set currently. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetAddressWatch( + HSAuint32 NodeId, //IN + HSA_DBG_WATCH_MODE WatchMode, //IN + void* WatchAddress, //IN + HSAuint64 WatchAddrMask, //IN + HSAuint32* WatchId //OUT + ); + +/** + Clear a debug memory access watch point set by + hsaKmtSetAddressWatch(). + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_NOT_SUPPORTED if debug memory watch points are + not supported for NodeId. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid or WatchId is not valid for this + NodeId. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtClearAddressWatch( + HSAuint32 NodeId, //IN + HSAuint32 WatchId //IN + ); + +/** + Enable precise memory operations. + + When precise memory operations are enabled a wave waits for each + memory operation to complete before executing further + operations. This results in more precise reporting of memory related + events such as memory violation or address watch points. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_UNAVAILABLE if precise memory operations is not + available to this process. For example, the feature may require + specific privileges. + + - HSAKMT_STATUS_NOT_SUPPORTED if precise memory operations is not + supported by NodeId. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtEnablePreciseMemoryOperations( + HSAuint32 NodeId //IN + ); + +/** + Disable precise memory operations enabled by + hsaKmtEnablePreciseMemoryOperations(). If precise memory operations + are not currently enabled no action is taken. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful. + + - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. + + - HSAKMT_STATUS_NOT_SUPPORTED if precise memory operations is not + supported by NodeId. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDisablePreciseMemoryOperations( + HSAuint32 NodeId //IN + ); + /** Gets GPU and CPU clock counters for particular Node */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 107703e20d..c43e48911e 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -72,6 +72,8 @@ extern "C" { typedef void* HSA_HANDLE; typedef HSAuint64 HSA_QUEUEID; +// An HSA_QUEUEID that is never a valid queue ID. +#define INVALID_QUEUEID 0xFFFFFFFFFFFFFFFFULL // This is included in order to force the alignments to be 4 bytes so that // it avoids extra padding added by the compiler when a 64-bit binary is generated. @@ -97,13 +99,14 @@ typedef enum _HSAKMT_STATUS HSAKMT_STATUS_NOT_IMPLEMENTED = 10, // KFD function is not implemented for this set of paramters HSAKMT_STATUS_NOT_SUPPORTED = 11, // KFD function is not supported on this node HSAKMT_STATUS_UNAVAILABLE = 12, // KFD function is not available currently on this node (but - // may be at a later time) + // may be at a later time) + HSAKMT_STATUS_OUT_OF_RESOURCES = 13, // KFD function request exceeds the resources currently available. HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED = 20, // KFD driver path not opened HSAKMT_STATUS_KERNEL_COMMUNICATION_ERROR = 21, // user-kernel mode communication failure HSAKMT_STATUS_KERNEL_ALREADY_OPENED = 22, // KFD driver path already opened HSAKMT_STATUS_HSAMMU_UNAVAILABLE = 23, // ATS/PRI 1.1 (Address Translation Services) not available - // (IOMMU driver not installed or not-available) + // (IOMMU driver not installed or not-available) HSAKMT_STATUS_WAIT_FAILURE = 30, // The wait operation failed HSAKMT_STATUS_WAIT_TIMEOUT = 31, // The wait operation timed out @@ -190,13 +193,20 @@ typedef union unsigned int VALimit : 1; // This node GPU has limited VA range for platform // (typical 40bit). Affects shared VM use for 64bit apps unsigned int WatchPointsSupported: 1; // Indicates if Watchpoints are available on the node. - unsigned int WatchPointsTotalBits: 4; // ld(Watchpoints) available. To determine the number use 2^value + unsigned int WatchPointsTotalBits: 4; // Watchpoints available. To determine the number use 2^value unsigned int DoorbellType : 2; // 0: This node has pre-1.0 doorbell characteristic // 1: This node has 1.0 doorbell characteristic // 2,3: reserved for future use - unsigned int AQLQueueDoubleMap : 1; // The unit needs a VA “double map” - unsigned int Reserved : 17; + unsigned int AQLQueueDoubleMap : 1; // The unit needs a VA “double map” + unsigned int WatchAddrMaskLoBit : 4; // Only bits WatchAddrMaskLoBit..WatchAddrMaskHiBit of the + unsigned int WatchAddrMaskHiBit : 6; // watch address mask are used. 0 is the least significant bit. + unsigned int DebugTrapSupported : 1; // Indicates if Debug Trap is supported on the node. + unsigned int TrapDataCount : 2; // Number of 32 bit TrapData registers supported. + unsigned int WaveLaunchTrapOverrideSupported: 1; // Indicates if Wave Launch Trap Override is supported on the node. + unsigned int WaveLaunchModeSupported: 1; // Indicates if Wave Launch Mode is supported on the node. + unsigned int PreciseMemoryOperationsSupported: 1; // Indicates if Precise Memory Operations are supported on the node. + unsigned int Reserved : 1; } ui32; } HSA_CAPABILITY; @@ -690,10 +700,44 @@ typedef enum _HSA_DBG_WATCH_MODE HSA_DBG_WATCH_NONREAD = 1, //Write or Atomic operations only HSA_DBG_WATCH_ATOMIC = 2, //Atomic Operations only HSA_DBG_WATCH_ALL = 3, //Read, Write or Atomic operations - HSA_DBG_WATCH_NUM, - HSA_DBG_WATCH_SIZE = 0xFFFFFFFF + HSA_DBG_WATCH_NUM } HSA_DBG_WATCH_MODE; +typedef enum _HSA_DBG_TRAP_OVERRIDE +{ + HSA_DBG_TRAP_OVERRIDE_OR = 0, // Bitwise OR exception mask with HSA_DBG_TRAP_MASK + HSA_DBG_TRAP_OVERRIDE_REPLACE = 1, // Replace exception mask with HSA_DBG_TRAP_MASK + HSA_DBG_TRAP_OVERRIDE_NUM +} HSA_DBG_TRAP_OVERRIDE; + +typedef enum _HSA_DBG_TRAP_MASK +{ + HSA_DBG_TRAP_MASK_FP_INVALID = 1, // Floating point invalid operation + HSA_DBG_TRAP_MASK_FP_INPUT_DENOMAL = 2, // Floating point input denormal + HSA_DBG_TRAP_MASK_FP_DIVIDE_BY_ZERO = 4, // Floating point divide by zero + HSA_DBG_TRAP_MASK_FP_OVERFLOW = 8, // Floating point overflow + HSA_DBG_TRAP_MASK_FP_UNDERFLOW = 16, // Floating point underflow + HSA_DBG_TRAP_MASK_FP_INEXACT = 32, // Floating point inexact + HSA_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO = 64, // Integer divide by zero + HSA_DBG_TRAP_MASK_DBG_ADDRESS_WATCH = 128, // Debug address watch + HSA_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION = 256 // Memory violation +} HSA_DBG_TRAP_MASK; + +typedef enum _HSA_DBG_WAVE_LAUNCH_MODE +{ + HSA_DBG_WAVE_LAUNCH_MODE_NORMAL = 0, // Wavefront launched normally. + HSA_DBG_WAVE_LAUNCH_MODE_HALT = 1, // Wavefront launched in halted mode. + HSA_DBG_WAVE_LAUNCH_MODE_KILL = 2, // Wavefront is launched but immediately + // terminated before executing any instructions. + HSA_DBG_WAVE_LAUNCH_MODE_SINGLE_STEP = 3, // Wavefront is launched in single step (debug) + // mode. If debug trap is enabled by + // hsaKmtDbgEnableDebugTrap() then causes a + // trap after executing each instruction, + // otherwise behaves the same as + // HSA_DBG_WAVE_LAUNCH_MODE_NORMAL. + HSA_DBG_WAVE_LAUNCH_MODE_DISABLE = 4, // Disable launching any new waves. + HSA_DBG_WAVE_LAUNCH_MODE_NUM +} HSA_DBG_WAVE_LAUNCH_MODE; //This structure is hardware specific and may change in the future typedef struct _HsaDbgWaveMsgAMDGen2 diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index fdedc63c30..4597e045f8 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -186,6 +186,37 @@ struct kfd_ioctl_dbg_wave_control_args { __u32 buf_size_in_bytes; /*including gpu_id and buf_size */ }; +/* KFD_IOC_DBG_TRAP_ENABLE: + * data1: 0=disable, 1=enable + * data2: queue ID (for future use) + */ +#define KFD_IOC_DBG_TRAP_ENABLE 0 + +/* KFD_IOC_DBG_TRAP_SET_TRAP_DATA: + * data1: SPI_GDBG_TRAP_DATA0 + * data2: SPI_GDBG_TRAP_DATA1 + */ +#define KFD_IOC_DBG_TRAP_SET_TRAP_DATA 1 + +/* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE: + * data1: override mode: 0=OR, 1=REPLACE + * data2: mask + */ +#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 2 + +/* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE: + * data1: 0=normal, 1=halt, 2=kill, 3=singlestep, 4=disable + * data2: unused + */ +#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 3 + +struct kfd_ioctl_dbg_trap_args { + __u32 gpu_id; /* to KFD */ + __u32 op; /* to KFD */ + __u32 data1; /* to KFD */ + __u32 data2; /* to KFD */ +}; + /* Matching HSA_EVENTTYPE */ #define KFD_IOC_EVENT_SIGNAL 0 #define KFD_IOC_EVENT_NODECHANGE 1 @@ -547,7 +578,10 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x20, struct kfd_ioctl_cross_memory_copy_args) +#define AMDKFD_IOC_DBG_TRAP \ + AMDKFD_IOW(0x21, struct kfd_ioctl_dbg_trap_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x21 +#define AMDKFD_COMMAND_END 0x22 #endif diff --git a/src/debug.c b/src/debug.c index 34dcc8970c..fc32b925d8 100644 --- a/src/debug.c +++ b/src/debug.c @@ -267,3 +267,90 @@ int debug_get_reg_status(uint32_t node_id, bool *is_debugged) *is_debugged = is_device_debugged[node_id]; return 0; } + +static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, + HSAuint32 op, + HSAuint32 data1, + HSAuint32 data2) +{ + uint32_t gpu_id; + HSAKMT_STATUS result; + HsaNodeProperties NodeProperties = {0}; + struct kfd_ioctl_dbg_trap_args args = {0}; + + CHECK_KFD_OPEN(); + + if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) + return HSAKMT_STATUS_INVALID_HANDLE; + + result = hsaKmtGetNodeProperties(NodeId, &NodeProperties); + + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + if (!NodeProperties.Capability.ui32.DebugTrapSupported) + return HSAKMT_STATUS_NOT_SUPPORTED; + + memset(&args, 0x00, sizeof(args)); + args.gpu_id = gpu_id; + args.op = op; + args.data1 = data1; + args.data2 = data2; + + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); + + if (err == 0) + result = HSAKMT_STATUS_SUCCESS; + else + result = HSAKMT_STATUS_ERROR; + + return result; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32 NodeId, + HSA_QUEUEID QueueId) +{ + if (QueueId != INVALID_QUEUEID) + return HSAKMT_STATUS_NOT_SUPPORTED; + + return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 1, QueueId); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId) +{ + return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 0, 0); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, + HSAuint32 TrapData0, + HSAuint32 TrapData1) +{ + return debug_trap(NodeId, + KFD_IOC_DBG_TRAP_SET_TRAP_DATA, + TrapData0, + TrapData1); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( + HSAuint32 NodeId, + HSA_DBG_TRAP_OVERRIDE TrapOverride, + HSA_DBG_TRAP_MASK TrapMask) +{ + if (TrapOverride >= HSA_DBG_TRAP_OVERRIDE_NUM) + return HSAKMT_STATUS_INVALID_PARAMETER; + + return debug_trap(NodeId, + KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE, + TrapOverride, + TrapMask); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode( + HSAuint32 NodeId, + HSA_DBG_WAVE_LAUNCH_MODE WaveLaunchMode) +{ + return debug_trap(NodeId, + KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE, + WaveLaunchMode, + 0); +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 436e33bf8c..91b7721764 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -56,6 +56,11 @@ hsaKmtGetTileConfig; hsaKmtQueryPointerInfo; hsaKmtSetMemoryUserData; hsaKmtGetQueueInfo; +hsaKmtEnableDebugTrap; +hsaKmtDisableDebugTrap; +hsaKmtSetDebugTrapData2; +hsaKmtSetWaveLaunchTrapOverride; +hsaKmtSetWaveLaunchMode; local: *; }; From 414042abf76f9a95cd63490e7bbed57fb40b53c6 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 14 Aug 2018 09:52:31 -0400 Subject: [PATCH 0395/1247] kfdtest: Clean up comments Consolidate style (use /* */ for multi-line), fix typos, use dword instad of DWORD/DWord Change-Id: I620e45c1687550db41127e45641b7d79d28223a1 --- tests/kfdtest/src/AqlQueue.cpp | 2 +- tests/kfdtest/src/AqlQueue.hpp | 10 +- tests/kfdtest/src/BasePacket.hpp | 14 +- tests/kfdtest/src/BaseQueue.cpp | 10 +- tests/kfdtest/src/BaseQueue.hpp | 24 ++-- tests/kfdtest/src/Dispatch.cpp | 24 ++-- tests/kfdtest/src/GoogleTestExtension.cpp | 9 +- tests/kfdtest/src/GoogleTestExtension.hpp | 18 +-- tests/kfdtest/src/IndirectBuffer.hpp | 20 +-- tests/kfdtest/src/KFDBaseComponentTest.cpp | 10 +- tests/kfdtest/src/KFDBaseComponentTest.hpp | 12 +- tests/kfdtest/src/KFDCWSRTest.cpp | 13 +- tests/kfdtest/src/KFDCWSRTest.hpp | 2 +- tests/kfdtest/src/KFDDBGTest.cpp | 10 +- tests/kfdtest/src/KFDDBGTest.hpp | 2 +- tests/kfdtest/src/KFDEventTest.cpp | 4 +- tests/kfdtest/src/KFDEventTest.hpp | 4 +- tests/kfdtest/src/KFDEvictTest.cpp | 53 ++++---- tests/kfdtest/src/KFDEvictTest.hpp | 4 +- tests/kfdtest/src/KFDExceptionTest.cpp | 9 +- tests/kfdtest/src/KFDExceptionTest.hpp | 2 +- tests/kfdtest/src/KFDGraphicsInterop.cpp | 5 +- tests/kfdtest/src/KFDIPCTest.cpp | 13 +- tests/kfdtest/src/KFDIPCTest.hpp | 15 ++- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 6 +- tests/kfdtest/src/KFDLocalMemoryTest.hpp | 2 +- tests/kfdtest/src/KFDMemoryTest.cpp | 141 +++++++++++---------- tests/kfdtest/src/KFDMemoryTest.hpp | 7 +- tests/kfdtest/src/KFDOpenCloseKFDTest.cpp | 17 ++- tests/kfdtest/src/KFDOpenCloseKFDTest.hpp | 5 +- tests/kfdtest/src/KFDPMTest.cpp | 4 +- tests/kfdtest/src/KFDPNPTest.cpp | 2 +- tests/kfdtest/src/KFDQMTest.cpp | 60 +++++---- tests/kfdtest/src/KFDQMTest.hpp | 3 +- tests/kfdtest/src/KFDTestFlags.hpp | 11 +- tests/kfdtest/src/KFDTestMain.cpp | 4 +- tests/kfdtest/src/KFDTestUtil.cpp | 22 ++-- tests/kfdtest/src/KFDTestUtil.hpp | 22 ++-- tests/kfdtest/src/KFDTopologyTest.cpp | 28 ++-- tests/kfdtest/src/KFDTopologyTest.hpp | 7 +- tests/kfdtest/src/LinuxOSWrapper.cpp | 16 +-- tests/kfdtest/src/OSWrapper.hpp | 27 ++-- tests/kfdtest/src/PM4Packet.cpp | 8 +- tests/kfdtest/src/PM4Packet.hpp | 62 ++++----- tests/kfdtest/src/PM4Queue.cpp | 8 +- tests/kfdtest/src/PM4Queue.hpp | 4 +- tests/kfdtest/src/RDMATest.hpp | 2 +- tests/kfdtest/src/SDMAPacket.cpp | 9 +- tests/kfdtest/src/SDMAPacket.hpp | 52 ++++---- tests/kfdtest/src/SDMAQueue.cpp | 15 ++- tests/kfdtest/src/SDMAQueue.hpp | 8 +- 51 files changed, 427 insertions(+), 414 deletions(-) diff --git a/tests/kfdtest/src/AqlQueue.cpp b/tests/kfdtest/src/AqlQueue.cpp index d8e9b43ab6..1c3abd5691 100644 --- a/tests/kfdtest/src/AqlQueue.cpp +++ b/tests/kfdtest/src/AqlQueue.cpp @@ -45,7 +45,7 @@ unsigned int AqlQueue::RptrWhenConsumed() { } void AqlQueue::SubmitPacket() { - // m_pending Wptr is in DWORDs + // m_pending Wptr is in dwords *m_Resources.Queue_write_ptr = m_pendingWptr; *(m_Resources.Queue_DoorBell) = Wptr(); } diff --git a/tests/kfdtest/src/AqlQueue.hpp b/tests/kfdtest/src/AqlQueue.hpp index 45395f70eb..33dc0589b8 100644 --- a/tests/kfdtest/src/AqlQueue.hpp +++ b/tests/kfdtest/src/AqlQueue.hpp @@ -31,14 +31,14 @@ class AqlQueue : public BaseQueue { AqlQueue(); virtual ~AqlQueue(); - // @brief update queue write pointer and sets the queue doorbell to the queue write pointer + // @brief Updates queue write pointer and sets the queue doorbell to the queue write pointer virtual void SubmitPacket(); - // @ return read pointer are in DWORDs + // @return Read pointer in dwords virtual unsigned int Rptr(); - // @ return write pointer are in DWORDs + // @return Write pointer in dwords virtual unsigned int Wptr(); - // @ return expected m_Resources.Queue_read_ptr when all packets consumed + // @return Expected m_Resources.Queue_read_ptr when all packets are consumed virtual unsigned int RptrWhenConsumed(); protected: @@ -47,4 +47,4 @@ class AqlQueue : public BaseQueue { virtual _HSA_QUEUE_TYPE GetQueueType() { return HSA_QUEUE_COMPUTE_AQL; } }; -#endif +#endif // __KFD_AQL_QUEUE__H__ diff --git a/tests/kfdtest/src/BasePacket.hpp b/tests/kfdtest/src/BasePacket.hpp index b2cc6e7da6..1802f901df 100644 --- a/tests/kfdtest/src/BasePacket.hpp +++ b/tests/kfdtest/src/BasePacket.hpp @@ -25,8 +25,8 @@ #define __KFD_BASE_PACKET__H__ /** - * all packets profiles must be defined here - * every type defined here has sub-types + * All packets profiles must be defined here + * Every type defined here has sub-types */ enum PACKETTYPE { PACKETTYPE_PM4, @@ -40,13 +40,13 @@ class BasePacket { BasePacket(void) {} virtual ~BasePacket(void) {} - // @returns the packet type + // @returns Packet type virtual PACKETTYPE PacketType() const = 0; - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const = 0; - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const = 0; - // @returns the packet size in DWORDS + // @returns Packet size in dwordS unsigned int SizeInDWords() const { return SizeInBytes()/sizeof(unsigned int); } void Dump() const; @@ -54,4 +54,4 @@ class BasePacket { protected: }; -#endif +#endif // __KFD_BASE_PACKET__H__ diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 3af41aebe3..50ca76f778 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -41,7 +41,7 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 HSAKMT_STATUS status; if (m_QueueBuf != NULL) { - // queue already exist, one queue per object + // Queue already exists, one queue per object Destroy(); } @@ -78,7 +78,7 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 status = HSAKMT_STATUS_ERROR; } - // needs to match the queue write ptr + // Needs to match the queue write ptr m_pendingWptr = 0; m_pendingWptr64 = 0; m_Node = NodeId; @@ -137,7 +137,7 @@ void BaseQueue::PlacePacket(const BasePacket &packet) { unsigned int queueSizeInDWord = m_QueueBuf->Size() / sizeof(uint32_t); if (writePtr + packetSizeInDwords > queueSizeInDWord) { - // wraparound expected. We need enough room to also place NOPs to avoid crossing the buffer end. + // Wraparound expected. We need enough room to also place NOPs to avoid crossing the buffer end. dwordsRequired += queueSizeInDWord - writePtr; } @@ -147,14 +147,14 @@ void BaseQueue::PlacePacket(const BasePacket &packet) { ASSERT_GE(queueSizeInDWord, packetSizeInDwords) << "Cannot add a packet, packet size too large"; if (writePtr + packetSizeInDwords >= queueSizeInDWord) { - // wraparound + // Wraparound while (writePtr + packetSizeInDwords > queueSizeInDWord) { m_QueueBuf->As()[writePtr] = CMD_NOP; writePtr = (writePtr + 1) % queueSizeInDWord; writePtr64++; } - // not updating Wptr since we might want to do place packet without submission + // Not updating Wptr since we might want to place the packet without submission m_pendingWptr = (writePtr % queueSizeInDWord); m_pendingWptr64 = writePtr64; } diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index d926cd34e2..a45c44d8b9 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -48,10 +48,10 @@ class BaseQueue { HSAuint64 *pointers = NULL); /** Update the queue. * @see hsaKmtUpdateQueue - * @param percent the new queue percentage - * @param priority the new queue priority + * @param percent New queue percentage + * @param priority New queue priority * @param nullifyBuffer - * if 'true', set the new buffer address to NULL and the size to 0. Otherwise + * If 'true', set the new buffer address to NULL and the size to 0. Otherwise * don't change the queue buffer address/size. */ virtual HSAKMT_STATUS Update(unsigned int percent, HSA_QUEUE_PRIORITY priority, bool nullifyBuffer); @@ -64,17 +64,17 @@ class BaseQueue { * Note that all packets being consumed is not the same as all packets being processed. */ virtual void Wait4PacketConsumption(); - /** @brief place packet and submit it in one go + /** @brief Place packet and submit it in one function */ virtual void PlaceAndSubmitPacket(const BasePacket &packet); - /** @brief copy packet to queue and update write pointer + /** @brief Copy packet to queue and update write pointer */ virtual void PlacePacket(const BasePacket &packet); - /** @brief update queue write pointer and sets the queue doorbell to the queue write pointer + /** @brief Update queue write pointer and set the queue doorbell to the queue write pointer */ virtual void SubmitPacket() = 0; - /** @brief checkes if all packets in queue already processed - * compares queue read and write pointers + /** @brief Check if all packets in queue are already processed + * Compare queue read and write pointers */ bool AllPacketsSubmitted(); @@ -100,11 +100,11 @@ class BaseQueue { HsaMemoryBuffer *m_QueueBuf; unsigned int m_Node; - // @ return write pointer modulo queue size in DWORDs + // @return Write pointer modulo queue size in dwords virtual unsigned int Wptr() = 0; - // @ return read pointer modulo queue size in DWORDs + // @return Read pointer modulo queue size in dwords virtual unsigned int Rptr() = 0; - // @ return expected m_Resources.Queue_read_ptr when all packets consumed + // @return Expected m_Resources.Queue_read_ptr when all packets consumed virtual unsigned int RptrWhenConsumed() = 0; virtual PACKETTYPE PacketTypeSupported() = 0; @@ -131,4 +131,4 @@ class QueueArray { void Destroy(); }; -#endif +#endif // __KFD_BASE_QUEUE__H__ diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index e17911d65a..6fc9ffe56c 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -90,7 +90,7 @@ void Dispatch::Sync(unsigned int timeout) { ASSERT_SUCCESS(hsaKmtWaitOnEvent(m_pEop, timeout)); } -// returning with status in order to allow to take actions before proc termination +// Returning with status in order to allow actions to be performed before process termination int Dispatch::SyncWithStatus(unsigned int timeout) { int stat; @@ -103,7 +103,7 @@ void Dispatch::BuildIb() { SplitU64(reinterpret_cast(m_pArg1), arg0, arg1); SplitU64(reinterpret_cast(m_pArg2), arg2, arg3); - // starts at COMPUTE_START_X + // Starts at COMPUTE_START_X const unsigned int COMPUTE_DISPATCH_DIMS_VALUES[] = { 0, // START_X 0, // START_Y @@ -138,14 +138,14 @@ void Dispatch::BuildIb() { pgmRsrc2 }; - // starts at COMPUTE_PGM_LO + // Starts at COMPUTE_PGM_LO const unsigned int COMPUTE_PGM_VALUES_GFX8[] = { static_cast(shiftedIsaAddr), // PGM_LO static_cast(shiftedIsaAddr >> 32) // PGM_HI | (is_dgpu() ? 0 : (1<<8)) // including PGM_ATC=? }; - // starts at COMPUTE_PGM_LO + // Starts at COMPUTE_PGM_LO const unsigned int COMPUTE_PGM_VALUES_GFX9[] = { static_cast(shiftedIsaAddr), // PGM_LO static_cast(shiftedIsaAddr >> 32) // PGM_HI @@ -156,17 +156,17 @@ void Dispatch::BuildIb() { static_cast(m_scratch_base >> 40) }; - // starts at COMPUTE_RESOURCE_LIMITS + // Starts at COMPUTE_RESOURCE_LIMITS const unsigned int COMPUTE_RESOURCE_LIMITS[] = { 0, // COMPUTE_RESOURCE_LIMITS }; - // starts at COMPUTE_TMPRING_SIZE + // Starts at COMPUTE_TMPRING_SIZE const unsigned int COMPUTE_TMPRING_SIZE[] = { m_ComputeTmpringSize, // COMPUTE_TMPRING_SIZE }; - // starts at COMPUTE_RESTART_X + // Starts at COMPUTE_RESTART_X const unsigned int COMPUTE_RESTART_VALUES[] = { 0, // COMPUTE_RESTART_X 0, // COMPUTE_RESTART_Y @@ -174,13 +174,13 @@ void Dispatch::BuildIb() { 0 // COMPUTE_THREAD_TRACE_ENABLE }; - // starts at COMPUTE_USER_DATA_0 + // Starts at COMPUTE_USER_DATA_0 const unsigned int COMPUTE_USER_DATA_VALUES[] = { // Reg name - use in KFDtest - use in ABI - arg0, // COMPUTE_USER_DATA_0 - arg0 - resource descriptor for the scratch buffer - 1st DWORD - arg1, // COMPUTE_USER_DATA_1 - arg1 - resource descriptor for the scratch buffer - 2nd DWORD - arg2, // COMPUTE_USER_DATA_2 - arg2 - resource descriptor for the scratch buffer - 3rd DWORD - arg3, // COMPUTE_USER_DATA_3 - arg3 - resource descriptor for the scratch buffer - 4th DWORD + arg0, // COMPUTE_USER_DATA_0 - arg0 - resource descriptor for the scratch buffer - 1st dword + arg1, // COMPUTE_USER_DATA_1 - arg1 - resource descriptor for the scratch buffer - 2nd dword + arg2, // COMPUTE_USER_DATA_2 - arg2 - resource descriptor for the scratch buffer - 3rd dword + arg3, // COMPUTE_USER_DATA_3 - arg3 - resource descriptor for the scratch buffer - 4th dword static_cast(m_scratch_base), // COMPUTE_USER_DATA_4 - flat_scratch_lo static_cast(m_scratch_base >> 32), // COMPUTE_USER_DATA_4 - flat_scratch_hi 0, // COMPUTE_USER_DATA_6 - - AQL queue address, low part diff --git a/tests/kfdtest/src/GoogleTestExtension.cpp b/tests/kfdtest/src/GoogleTestExtension.cpp index 9e510e11e2..d1e5cad7da 100644 --- a/tests/kfdtest/src/GoogleTestExtension.cpp +++ b/tests/kfdtest/src/GoogleTestExtension.cpp @@ -27,7 +27,6 @@ bool Ok2Run(unsigned int testProfile) { bool testMatchProfile = true; if ((testProfile & g_TestRunProfile) == 0) { - // display msg to notify a test that is not running WARN() << "Test is skipped beacuse profile does not match current run mode" << std::endl; testMatchProfile = false; } @@ -35,11 +34,10 @@ bool Ok2Run(unsigned int testProfile) { return testMatchProfile; } -// This predication is used when specific HW capabilites must exist for the test to succeed. +// This predication is used when specific HW capabilities must exist for the test to succeed. bool TestReqEnvCaps(unsigned int envCaps) { bool testMatchEnv = true; if ((envCaps & g_TestENVCaps) != envCaps) { - // display msg to notify a test that is not running WARN() << "Test is skipped due to HW capability issues" << std::endl; testMatchEnv = false; } @@ -47,12 +45,11 @@ bool TestReqEnvCaps(unsigned int envCaps) { return testMatchEnv; } -// This predication is used when specific HW capabilites must abscent for the test to succeed. -// e.g testing capabilites not supported by HW scheduling +// This predication is used when specific HW capabilities must be absent for the test to succeed. +// e.g Testing capabilities not supported by HW scheduling bool TestReqNoEnvCaps(unsigned int envCaps) { bool testMatchEnv = true; if ((envCaps & g_TestENVCaps) != 0) { - // display msg to notify a test that is not running WARN() << "Test is skipped due to HW capability issues" << std::endl; testMatchEnv = false; } diff --git a/tests/kfdtest/src/GoogleTestExtension.hpp b/tests/kfdtest/src/GoogleTestExtension.hpp index 7b888b1677..b53ce6bf84 100644 --- a/tests/kfdtest/src/GoogleTestExtension.hpp +++ b/tests/kfdtest/src/GoogleTestExtension.hpp @@ -29,24 +29,24 @@ #include "KFDTestFlags.hpp" enum LOGTYPE { - LOGTYPE_INFO, // msg header in green + LOGTYPE_INFO, // msg header in green LOGTYPE_WARNING // msg header in yellow }; class KFDLog{}; std::ostream& operator << (KFDLog log, LOGTYPE level); -// @brief log additional details, to be displayed in the same format as other google test outputs -// currently not supported by google test -// should be used like cout: LOG() << "message" << value << std::endl; +// @brief Log additional details, to be displayed in the same format as other google test outputs +// Currently not supported by gtest +// Should be used like cout: LOG() << "message" << value << std::endl; #define LOG() KFDLog() << LOGTYPE_INFO #define WARN() KFDLog() << LOGTYPE_WARNING -// all test MUST be in a try catch since google test flag to throw exception on any fatal fail is on +// All tests MUST be in a try catch since the gtest flag to throw an exception on any fatal failure is enabled #define TEST_START(testProfile) if (Ok2Run(testProfile)) try { #define TEST_END } catch (...) {} -// used to wrape setup and teardown functions, anything that is build-in gtest and is not a test +// Used to wrap setup and teardown functions, anything that is built-in gtest and is not a test #define ROUTINE_START try { #define ROUTINE_END }catch(...) {} @@ -59,13 +59,13 @@ std::ostream& operator << (KFDLog log, LOGTYPE level); #define ASSERT_NOTNULL(_val) ASSERT_NE((void *)NULL, _val) #define EXPECT_NOTNULL(_val) EXPECT_NE((void *)NULL, _val) -// @brief determines if its ok to run a test given input flags +// @brief Determines if it is ok to run a test given input flags bool Ok2Run(unsigned int testProfile); -// @brief checks if all HW capabilities needed for a test to run exist +// @brief Checks if all HW capabilities needed for a test to run exist bool TestReqEnvCaps(unsigned int hwCaps); -// @brief checks if all HW capabilities that prevents a test from running are non existing +// @brief Checks if all HW capabilities that prevents a test from running are absent bool TestReqNoEnvCaps(unsigned int hwCaps); #endif diff --git a/tests/kfdtest/src/IndirectBuffer.hpp b/tests/kfdtest/src/IndirectBuffer.hpp index 14fa1e75f8..c049402558 100644 --- a/tests/kfdtest/src/IndirectBuffer.hpp +++ b/tests/kfdtest/src/IndirectBuffer.hpp @@ -28,32 +28,32 @@ #include "KFDTestUtil.hpp" /** @class IndirectBuffer - * when working with indirect buffer, create IndirectBuffer, fill it with all the packets you want + * When working with an indirect buffer, create IndirectBuffer, fill it with all the packets you want, * create an indirect packet to point to it, and submit the packet to queue */ class IndirectBuffer { public: - // @param[size] queue max size in DWords - // @param[type] packets type allowed in queue + // @param[size] Queue max size in DWords + // @param[type] Packet type allowed in queue IndirectBuffer(PACKETTYPE type, unsigned int sizeInDWords, unsigned int NodeId); ~IndirectBuffer(void); - // @brief add packet to queue, all validations are done with gtest ASSERT and EXPECT + // @brief Add packet to queue, all validations are done with gtest ASSERT and EXPECT void AddPacket(const BasePacket &packet); - // @returns the actual size of the indirect queue in DWord, equivalent to write pointer + // @returns Actual size of the indirect queue in DWords, equivalent to write pointer unsigned int SizeInDWord() { return m_ActualSize; } - // @returns indirect queue address + // @returns Indirect queue address unsigned int *Addr() { return m_IndirectBuf->As(); } protected: - // how many packets in queue + // Number of packets in the queue unsigned int m_NumOfPackets; - // max size of queue in DWords + // Max size of queue in DWords unsigned int m_MaxSize; - // current size of queue in DWords + // Current size of queue in DWords unsigned int m_ActualSize; HsaMemoryBuffer *m_IndirectBuf; - // defines what packets are supported in this queue + // What packets are supported in this queue PACKETTYPE m_PacketTypeAllowed; }; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index ff7ce063c2..8d806670fe 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -38,9 +38,9 @@ void KFDBaseComponentTest::SetUp() { memset( &m_SystemProperties, 0, sizeof(m_SystemProperties) ); memset(m_RenderNodes, 0, sizeof(m_RenderNodes)); - /** in order to be correctly testing the KFD interfaces and ensure + /** In order to be correctly testing the KFD interfaces and ensure * that the KFD acknowledges relevant node parameters - * for the rest of the tests and used for more specific topology tests + * for the rest of the tests and used for more specific topology tests, * call to GetSystemProperties for a system snapshot of the topology here */ ASSERT_SUCCESS(hsaKmtAcquireSystemProperties(&m_SystemProperties)); @@ -53,8 +53,8 @@ void KFDBaseComponentTest::SetUp() { m_MemoryFlags.ui32.CachePolicy = HSA_CACHING_NONCACHED; // Non cached m_MemoryFlags.ui32.ReadOnly = 0; // Read/Write m_MemoryFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; // 4KB page - m_MemoryFlags.ui32.HostAccess = 1; // host accessible - m_MemoryFlags.ui32.NoSubstitute = 0; // fall back to node 0 if needed + m_MemoryFlags.ui32.HostAccess = 1; // Host accessible + m_MemoryFlags.ui32.NoSubstitute = 0; // Fall back to node 0 if needed m_MemoryFlags.ui32.GDSMemory = 0; m_MemoryFlags.ui32.Scratch = 0; @@ -94,7 +94,7 @@ HSAuint64 KFDBaseComponentTest::GetSysMemSize() { for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { nodeProps = m_NodeInfo.GetNodeProperties(node); if (nodeProps != NULL && nodeProps->NumCPUCores > 0 && nodeProps->NumMemoryBanks > 0) { - /* For NUMA nodes, memory is distributed among differnt nodes. + /* For NUMA nodes, memory is distributed among different nodes. * Compute total system memory size. KFD driver also computes * the system memory (si_meminfo) similarly */ diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index f7a2852e65..768664e3d6 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -63,17 +63,13 @@ class KFDBaseComponentTest : public testing::Test { HsaMemFlags m_MemoryFlags; HsaNodeInfo m_NodeInfo; - // @brief SetUpTestCase function run before the first test that uses - // KFDOpenCloseKFDTest class fixture, and opens KFD. + // @brief Executed before the first test that uses KFDOpenCloseKFDTest class and opens KFD. static void SetUpTestCase(); - // @brief TearDownTestCase function run after the last test from - // KFDOpenCloseKFDTest class fixture and calls close KFD. + // @brief Executed after the last test from KFDOpenCloseKFDTest class and closes KFD. static void TearDownTestCase(); - // @brief SetUp function run before every test that uses - // KFDOpenCloseKFDTest class fixture, sets all common settings for the tests. + // @brief Executed before every test that uses KFDOpenCloseKFDTest class and sets all common settings for the tests. virtual void SetUp(); - // @brief TearDown function run after every test that uses - // KFDOpenCloseKFDTest class fixture. + // @brief Executed after every test that uses KFDOpenCloseKFDTest class. virtual void TearDown(); }; diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 7dffc56ef9..e1750a6390 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -89,9 +89,10 @@ void KFDCWSRTest::SetUp() { m_pIsaGen = IsaGenerator::Create(m_FamilyId); - // TODO: Seems in the ISA, I can not get the workitem_id as expected, so I can not - // set the destination based on workitem_id. - // Set the wave_num to 1 for now as a workarpound. Will set it to 8 or even 256 in the future. + /* TODO: In the ISA, the workitem_id is not obtained as expected, so the destination cannot + * be set based on workitem_id. Set the wave_num to 1 for now as a workarpound. + * Will set it to 8 or even 256 in the future. + */ wave_number = 1; ROUTINE_END @@ -149,15 +150,15 @@ TEST_F(KFDCWSRTest, BasicTest) { iter[0] = 40000000; iter[1] = 20000000; - // submit the shader, queue1 + // Submit the shader, queue1 dispatch1->Submit(queue1); // Create queue2 during queue1 still running will trigger the CWSR EXPECT_SUCCESS(queue2.Create(defaultGPUNode)); - // submit the shader + // Submit the shader dispatch2->Submit(queue2); dispatch1->Sync(); dispatch2->Sync(); - // ensure all the waves complete as expected + // Ensure all the waves complete as expected int i; for (i = 0 ; i < wave_number; ++i) { if (result[i] != iter[0]) { diff --git a/tests/kfdtest/src/KFDCWSRTest.hpp b/tests/kfdtest/src/KFDCWSRTest.hpp index 40526cc7fb..779180ea3d 100644 --- a/tests/kfdtest/src/KFDCWSRTest.hpp +++ b/tests/kfdtest/src/KFDCWSRTest.hpp @@ -39,7 +39,7 @@ class KFDCWSRTest : public KFDBaseComponentTest { virtual void SetUp(); virtual void TearDown(); - protected: // members + protected: // Members unsigned wave_number; IsaGenerator* m_pIsaGen; }; diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 00d7922f2c..1f717476e6 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -79,7 +79,7 @@ void KFDDBGTest::TearDown() { delete m_pIsaGen; m_pIsaGen = NULL; - /* reset the user trap handler */ + /* Reset the user trap handler */ hsaKmtSetTrapHandler(m_NodeInfo.HsaDefaultGPUNode(), 0, 0, 0, 0); KFDBaseComponentTest::TearDown(); @@ -118,7 +118,7 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { ASSERT_SUCCESS(queue_flush.Create(defaultGPUNode)); // Set Address Watch Params - // TODO: Set atchMode[1] to Atomic in case we want to test this mode. + // TODO: Set WatchMode[1] to Atomic in case we want to test this mode. HSA_DBG_WATCH_MODE WatchMode[2]; HSAuint64 WatchAddress[2]; @@ -153,9 +153,9 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { dispatch.SetArgs(dstBuf.As(), reinterpret_cast(secDstBuf)); dispatch.SetDim(1, 1, 1); - // TODO: use Memory ordering rules w/ atomics - // for host-GPU memory syncs. - // set to: std::memory_order_seq_cst + /* TODO: Use Memory ordering rules w/ atomics for host-GPU memory syncs. + * Set to std::memory_order_seq_cst + */ dispatch.Submit(queue); diff --git a/tests/kfdtest/src/KFDDBGTest.hpp b/tests/kfdtest/src/KFDDBGTest.hpp index 331b9017f8..c4b46b296b 100644 --- a/tests/kfdtest/src/KFDDBGTest.hpp +++ b/tests/kfdtest/src/KFDDBGTest.hpp @@ -38,7 +38,7 @@ class KFDDBGTest : public KFDBaseComponentTest { virtual void SetUp(); virtual void TearDown(); - protected: // members + protected: // Members IsaGenerator* m_pIsaGen; }; diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index 7d266046bc..35adeee898 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -41,7 +41,7 @@ void KFDEventTest::SetUp() { void KFDEventTest::TearDown() { ROUTINE_START - // not all tests create event, destroy only if there is one + // Not all tests create an event, destroy only if there is one if (m_pHsaEvent != NULL) { // hsaKmtDestroyEvent moved to TearDown to make sure it is being called EXPECT_SUCCESS(hsaKmtDestroyEvent(m_pHsaEvent)); @@ -58,7 +58,7 @@ TEST_F(KFDEventTest, CreateDestroyEvent) { ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_NodeInfo.HsaDefaultGPUNode(), &m_pHsaEvent)); EXPECT_NE(0, m_pHsaEvent->EventData.HWData2); - // destroy event is being called in test TearDown + // Destroy event is being called in test TearDown TEST_END; } diff --git a/tests/kfdtest/src/KFDEventTest.hpp b/tests/kfdtest/src/KFDEventTest.hpp index 43b6935f7f..e0bca47288 100644 --- a/tests/kfdtest/src/KFDEventTest.hpp +++ b/tests/kfdtest/src/KFDEventTest.hpp @@ -31,9 +31,9 @@ class KFDEventTest : public KFDBaseComponentTest { KFDEventTest(void) {} ~KFDEventTest(void) {} - // @brief SetUp function runs before every test in KFDEventTest. + // @brief Executed before every test in KFDEventTest. virtual void SetUp(); - // @brief TearDown function runs after every test in KFDEventTest. + // @brief Executed after every test in KFDEventTest. virtual void TearDown(); protected: diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index a32590ac77..5765acc578 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -30,7 +30,7 @@ #include "SDMAQueue.hpp" #include "Dispatch.hpp" -#define N_PROCESSES (8) /* number of processes running in parallel, at least 2 */ +#define N_PROCESSES (8) /* Number of processes running in parallel, must be at least 2 */ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) @@ -70,7 +70,7 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui break; } - /* wait for 1 second to try allocate again */ + /* Wait for 1 second to try allocate again */ sleep(1); } } @@ -268,7 +268,7 @@ void KFDEvictTest::ForkChildProcesses(int nprocesses) { void KFDEvictTest::WaitChildProcesses() { if (m_IsParent) { - /* only run by parent process */ + /* Only run by parent process */ int childStatus; int childExitOkNum = 0; int size = m_ChildPids.size(); @@ -286,7 +286,7 @@ void KFDEvictTest::WaitChildProcesses() { ASSERT_EQ(childExitOkNum, size); } - /* child process or parent process finished successfullly */ + /* Child process or parent process finished successfully */ m_ChildStatus = HSAKMT_STATUS_SUCCESS; } @@ -296,19 +296,19 @@ void KFDEvictTest::WaitChildProcesses() { * * ALLOCATE_BUF_SIZE_MB buf allocation size * - * number of buf is equal to (vramSizeMB / (vramBufSizeMB * N_PROCESSES) ) + 8 + * buf is equal to (vramSizeMB / (vramBufSizeMB * N_PROCESSES) ) + 8 * Total vram all processes allocated: 8GB for 4GB Fiji, and 20GB for 16GB Vega10 * - * many times of eviction and restore will happen: - * ttm will evict buffers of another process if not enough free vram + * Eviction and restore will happen many times: + * ttm will evict buffers of another process if there is not enough free vram * process restore will evict buffers of another process * - * Sometimes the allocate may fail (maybe that is normal) + * Sometimes the allocation may fail (maybe that is normal) * ALLOCATE_RETRY_TIMES max retry times to allocate * - * This is basic test, no queue so vram are not used by GPU during test + * This is basic test with no queue, so vram is not used by the GPU during test * - * Todo: + * TODO: * - Synchronization between the processes, so they know for sure when * they are done allocating memory */ @@ -345,7 +345,7 @@ TEST_F(KFDEvictTest, BasicTest) { std::vector pBuffers; AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); - /* allocate gfx vram size of at most one third system memory */ + /* Allocate gfx vram size of at most one third system memory */ HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); @@ -365,12 +365,12 @@ TEST_F(KFDEvictTest, BasicTest) { * until address buffer is filled with specific value 0x5678 by host program, * then each wavefront fills value 0x5678 at corresponding result buffer and quit * - * initial state: + * Initial state: * s[0:1] - address buffer base address * s[2:3] - result buffer base address * s4 - workgroup id * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 - * registers: + * Registers: * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 @@ -514,7 +514,7 @@ TEST_F(KFDEvictTest, QueueTest) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); - /* Skip test for chip it doesn't have CWSR, which the test depends on */ + /* Skip test for chip if it doesn't have CWSR, which the test depends on */ if (m_FamilyId < FAMILY_VI || isTonga(pNodeProperties)) { LOG() << std::hex << "Skipping test: No CWSR present for family ID 0x" << m_FamilyId << "." << std::endl; return; @@ -538,7 +538,7 @@ TEST_F(KFDEvictTest, QueueTest) { LOG() << "Skipping test: Not enough system memory available." << std::endl; return; } - /* assert all buffer address can be stored within one page + /* Assert all buffer address can be stored within one page * because only one page host memory srcBuf is allocated */ ASSERT_LE(count, PAGE_SIZE/sizeof(unsigned int *)); @@ -559,7 +559,7 @@ TEST_F(KFDEvictTest, QueueTest) { std::vector pBuffers; AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); - /* allocate gfx vram size of at most one third system memory */ + /* Allocate gfx vram size of at most one third system memory */ HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); @@ -583,27 +583,32 @@ TEST_F(KFDEvictTest, QueueTest) { Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(localBufAddr, result); dispatch0.SetDim(wavefront_num, 1, 1); - /* submit the packet and start shader */ + /* Submit the packet and start shader */ dispatch0.Submit(pm4Queue); - /* doing evict/restore queue test for 5 seconds while queue is running */ + /* Doing evict/restore queue test for 5 seconds while queue is running */ sleep(5); - /* LOG() << m_psName << "notify shader to quit" << std::endl; */ - /* fill address buffer so shader quits */ + /* Uncomment this line for debugging */ + // LOG() << m_psName << "notify shader to quit" << std::endl; + + /* Fill address buffer so shader quits */ addrBuffer.Fill(0x5678); - /* wait for shader to finish or timeout if shade has vm page fault */ + /* Wait for shader to finish or timeout if shader has vm page fault */ dispatch0.SyncWithStatus(120000); ASSERT_SUCCESS(pm4Queue.Destroy()); FreeAmdgpuBo(handle); - /* LOG() << m_psName << "free buffer" << std::endl; */ - /* cleanup */ + + /* Uncomment this line for debugging */ + // LOG() << m_psName << "free buffer" << std::endl; + + /* Cleanup */ FreeBuffers(pBuffers, vramBufSize); - /* check if all wavefronts finish successfully */ + /* Check if all wavefronts finished successfully */ for (i = 0; i < wavefront_num; i++) ASSERT_EQ(0x5678, *(result + i)); diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index 15fee8c1bc..0ab4630763 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -38,7 +38,7 @@ class KFDEvictTest : public KFDLocalMemoryTest { ~KFDEvictTest(void) { if (!m_IsParent) { - /* child process has to exit + /* Child process has to exit * otherwise gtest will continue other tests */ exit(m_ChildStatus); @@ -60,7 +60,7 @@ class KFDEvictTest : public KFDLocalMemoryTest { void ForkChildProcesses(int nprocesses); void WaitChildProcesses(); - protected: // members + protected: // Members std::string m_psName; std::vector m_ChildPids; HsaMemFlags m_Flags; diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index fca646bd4e..f7e79aaf0f 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -28,8 +28,6 @@ #include "SDMAQueue.hpp" #include "Dispatch.hpp" -// All tests are marked by their serial number in the QCM FDD - void KFDExceptionTest::SetUp() { ROUTINE_START @@ -198,7 +196,7 @@ TEST_F(KFDExceptionTest, InvalidPPRWriteProtection) { TestMemoryException(defaultGPUNode, srcBuffer.As(), (HSAuint64)pDst); - /* Wait enough time here to ensure this process got killed by kernel + /* Wait for enough time here to ensure this process got killed by kernel * due to PPR exception. */ sleep(5); @@ -213,8 +211,7 @@ TEST_F(KFDExceptionTest, InvalidPPRWriteProtection) { TEST_END } -/* TODO: Same as previous test InvalidPPRWriteProtection - */ +/* TODO: Same as previous test InvalidPPRWriteProtection */ TEST_F(KFDExceptionTest, InvalidPPRReadProtection) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -249,7 +246,7 @@ TEST_F(KFDExceptionTest, InvalidPPRReadProtection) { TestMemoryException(defaultGPUNode, (HSAuint64)pSrc, dstBuffer.As()); - /* Wait enough time here to ensure this process got killed by kernel + /* Wait for enough time here to ensure this process got killed by kernel * due to PPR exception. */ sleep(5); diff --git a/tests/kfdtest/src/KFDExceptionTest.hpp b/tests/kfdtest/src/KFDExceptionTest.hpp index 24bf433437..bf6bd7812b 100644 --- a/tests/kfdtest/src/KFDExceptionTest.hpp +++ b/tests/kfdtest/src/KFDExceptionTest.hpp @@ -55,7 +55,7 @@ class KFDExceptionTest : public KFDBaseComponentTest { unsigned int dimX = 1, unsigned int dimY = 1, unsigned int dimZ = 1); - protected: // members + protected: // Members pid_t m_ChildPid; HSAKMT_STATUS m_ChildStatus; diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp index 4faea041f6..51232216e2 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.cpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -83,8 +83,9 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { ASSERT_SUCCESS(hsaKmtRegisterGraphicsHandleToNodes(dmabufFd, &info, 1, nodes)); - // DMA buffer handle and GEM handle are no longer needed, KFD - // should have taken a reference to the BO + /* DMA buffer handle and GEM handle are no longer needed, KFD + * should have taken a reference to the BO + */ ASSERT_EQ(0, close(dmabufFd)); ASSERT_EQ(0, amdgpu_bo_free(handle)); diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index d85f7c59e3..b68940e2a6 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -60,7 +60,7 @@ KFDIPCTest::~KFDIPCTest(void) { exit(0); } -/* Imort shared Local Memory from parent process. Check for the pattern +/* Import shared Local Memory from parent process. Check for the pattern * filled in by the parent process. Then fill a new pattern. */ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { @@ -101,9 +101,9 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { ASSERT_SUCCESS(hsaKmtDeregisterMemory(sharedLocalBuffer)); } -/* Fill a pattern in to Local Memory and share with the child process. +/* Fill a pattern into Local Memory and share with the child process. * Then wait until Child process to exit and check for the new pattern - * fill in by the child process. + * filled in by the child process. */ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pipefd) { @@ -213,7 +213,7 @@ TEST_F(KFDIPCTest, BasicTest) { * dstBuf3[0x800-0x1000] is expected to be 0xAAAAAAAA * and dstBuf4[0x0-0x1000] is expected to be 0xAAAAAAAA * - * For this CMA test after copy only the first and the last of dstBuf is checked + * For this CMA test, after copying only the first and the last of dstBuf is checked */ static testMemoryDescriptor srcRange[CMA_TEST_COUNT][CMA_MEMORY_TEST_ARRAY_SIZE] = { @@ -490,7 +490,7 @@ CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachChildProcess(int defaultGPUNode, in break; } - /* Wait till the test is over */ + /* Wait until the test is over */ memset(msg, 0, sizeof(msg)); if (read_non_block(readPipe, msg, 4) < 0) { status = CMA_IPC_PIPE_ERROR; @@ -523,7 +523,7 @@ CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachParentProcess(int defaultGPUNode, p int testNo; CMA_TEST_STATUS status; - /* Recevie buffer array from child and then initialize and fill in Local Buffer Array. + /* Receive buffer array from child and then initialize and fill in Local Buffer Array. * READ_TEST: Copy remote buffer array into Local Buffer Array and then check * for the new pattern. * WRITE_TEST: Write Local Buffer Array into remote buffer array. Notify child to @@ -615,7 +615,6 @@ TEST_F(KFDIPCTest, CrossMemoryAttachTest) { ASSERT_EQ(pipe2(pipePtoC, O_NONBLOCK), 0); /* Create a child process and share the above Local Memory with it */ - m_ChildPid = fork(); if (m_ChildPid == 0 && hsaKmtOpenKFD() == HSAKMT_STATUS_SUCCESS) { /* Child Process */ diff --git a/tests/kfdtest/src/KFDIPCTest.hpp b/tests/kfdtest/src/KFDIPCTest.hpp index 50ff9508dc..6a72f4a805 100644 --- a/tests/kfdtest/src/KFDIPCTest.hpp +++ b/tests/kfdtest/src/KFDIPCTest.hpp @@ -53,8 +53,9 @@ enum CMA_TEST_STATUS { CMA_TEST_HSA_WRITE_FAIL }; -// @struct testMemoryDescriptor -// Describes test buffers for Cross Memory Attach Test. +/* @struct testMemoryDescriptor + * @brief Describes test buffers for Cross Memory Attach Test. + */ struct testMemoryDescriptor { CMA_MEM_TYPE m_MemType; HSAuint64 m_MemSize; @@ -76,9 +77,10 @@ struct testMemoryDescriptor { ~testMemoryDescriptor(){} }; -// @class KFDCMAArray -// Array of buffers that will be passed between the parent and child -// process for Cross memory read and write tests +/* @class KFDCMAArray + * @brief Array of buffers that will be passed between the parent and child + * process for Cross memory read and write tests + */ class KFDCMAArray { /* Used to store the actual buffer array */ HsaMemoryBuffer* m_MemArray[CMA_MEMORY_TEST_ARRAY_SIZE]; @@ -130,5 +132,4 @@ class KFDIPCTest : public KFDBaseComponentTest { pid_t m_ChildPid; }; -#endif - +#endif // __KFD_MEMORY_TEST__H__ diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index 2c72814357..a2928b0c38 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -28,8 +28,6 @@ #include "SDMAQueue.hpp" #include "Dispatch.hpp" -// All tests are marked by their serial number in the QCM FDD - void KFDLocalMemoryTest::SetUp() { ROUTINE_START @@ -285,7 +283,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { unsigned value = 0; memset(pages, 0, sizeof(pages)); for (order = 0; order <= maxOrder; order++) { - // At maxOrder, block sizes is 1/4 of available memory + // At maxOrder, block size is 1/4 of available memory pages[order].nPages = 1UL << (maxOrder - order + 2); // At order != 0, 1/2 the memory is already allocated if (order > 0) @@ -467,7 +465,7 @@ TEST_F(KFDLocalMemoryTest, MapVramToGPUNodesTest) { } if (dst_node != defaultGPUNode) { - /* at least one node should be defaultGPUNode */ + /* At least one node should be defaultGPUNode */ src_node = defaultGPUNode; } else { for (auto node : gpuNodes) { diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.hpp b/tests/kfdtest/src/KFDLocalMemoryTest.hpp index 368e92b21f..519081cfc0 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.hpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.hpp @@ -38,7 +38,7 @@ class KFDLocalMemoryTest : public KFDBaseComponentTest { virtual void SetUp(); virtual void TearDown(); - protected: // members + protected: // Members IsaGenerator* m_pIsaGen; }; diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index df193ff754..dfd279ef87 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -130,10 +130,10 @@ void KFDMemoryTest::TearDown() { #define GB(x) ((x) << 30) /* - * try to map as much as possible system memory to gpu. - * lets see if kfd support 1TB memory correctly or not. - * And after this test case, we can observe if there is any sideeffect. - * NOTICE: there are memory usage limit checks in hsa/kfd according to the total + * Try to map as much as possible system memory to gpu + * to see if KFD supports 1TB memory correctly or not. + * After this test case, we can observe if there are any side effects. + * NOTICE: There are memory usage limit checks in hsa/kfd according to the total * physical system memory. */ TEST_F(KFDMemoryTest, MMapLarge) { @@ -187,19 +187,19 @@ TEST_F(KFDMemoryTest, MMapLarge) { TEST_END } -/* keep memory mapped to default node +/* Keep memory mapped to default node * Keep mapping/unmapping memory to/from non-default node - * A shader running on default node consistantly access - * memory - make sure memory is always accessible on default, - * i.e., there is no gpu vm fault. + * A shader running on default node consistantly accesses + * memory - make sure memory is always accessible by default, + * i.e. there is no gpu vm fault. * Synchronization b/t host program and shader: - * 1. host initialize src and dst buffer to 0 - * 2. shader keep reading src buffer and check value - * 3. host write src buffer to 0x5678 to indicate quit, polling dst until it becomes 0x5678 - * 4. shader write dst buffer to 0x5678 after src changed to 0x5678, quit - * 5. host program quit after dst becomes 0x5678 - * Need at least two gpu nodes to run the test. The defaut node has to be a gfx9 node. - * Otherwise, test is skipped. Use kfdtest --node=$$ to specify the defaut node + * 1. Host initializes src and dst buffer to 0 + * 2. Shader keeps reading src buffer and check value + * 3. Host writes src buffer to 0x5678 to indicate quit, polling dst until it becomes 0x5678 + * 4. Shader write dst buffer to 0x5678 after src changes to 0x5678, then quits + * 5. Host program quits after dst becomes 0x5678 + * Need at least two gpu nodes to run the test. The default node has to be a gfx9 node, + * otherwise, test is skipped. Use kfdtest --node=$$ to specify the default node * This test case is introduced as a side-result of investigation of SWDEV-134798, which * is a gpu vm fault while running rocr conformance test. Here we try to simulate the * same test behaviour. @@ -250,7 +250,7 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { hsaKmtMapMemoryToGPUNodes(srcBuffer.As(), PAGE_SIZE, NULL, memFlags, (i>>5)&1+1, mapNodes); } - /* fill src buffer so shader quits */ + /* Fill src buffer so shader quits */ srcBuffer.Fill(0x5678); WaitOnValue(dstBuffer.As(), 0x5678); ASSERT_EQ(*dstBuffer.As(), 0x5678); @@ -258,7 +258,7 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { TEST_END } -// basic test of hsaKmtMapMemoryToGPU and hsaKmtUnmapMemoryToGPU +// Basic test of hsaKmtMapMemoryToGPU and hsaKmtUnmapMemoryToGPU TEST_F(KFDMemoryTest , MapMemoryToGPU) { TEST_START(TESTPROFILE_RUNALL) @@ -280,7 +280,7 @@ TEST_F(KFDMemoryTest , MapMemoryToGPU) { TEST_END } -// following tests are for hsaKmtAllocMemory with invalid params +// Following tests are for hsaKmtAllocMemory with invalid params TEST_F(KFDMemoryTest, InvalidMemoryPointerAlloc) { TEST_START(TESTPROFILE_RUNALL) @@ -299,7 +299,7 @@ TEST_F(KFDMemoryTest, ZeroMemorySizeAlloc) { TEST_END } -// basic test for hsaKmtAllocMemory +// Basic test for hsaKmtAllocMemory TEST_F(KFDMemoryTest, MemoryAlloc) { TEST_START(TESTPROFILE_RUNALL) @@ -381,7 +381,8 @@ TEST_F(KFDMemoryTest, MemoryRegister) { HsaMemoryBuffer sdmaBuffer((void *)&stackData[sdmaOffset], sizeof(HSAuint32)); /* Create PM4 and SDMA queues before fork+COW to test queue - * eviction and restore */ + * eviction and restore + */ PM4Queue pm4Queue; SDMAQueue sdmaQueue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); @@ -392,7 +393,8 @@ TEST_F(KFDMemoryTest, MemoryRegister) { /* First submit just so the queues are not empty, and to get the * TLB populated (in case we need to flush TLBs somewhere after - * updating the page tables) */ + * updating the page tables) + */ Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(srcBuffer.As(), dstBuffer.As()); dispatch0.Submit(pm4Queue); @@ -410,7 +412,8 @@ TEST_F(KFDMemoryTest, MemoryRegister) { * make any write access to the stack because we want the * parent to make the first write access and get a new copy. A * busy loop is the safest way to do that, since any function - * call (e.g. sleep) would write to the stack. */ + * call (e.g. sleep) would write to the stack. + */ while (1) {} WARN() << "Shouldn't get here!" << std::endl; @@ -419,13 +422,15 @@ TEST_F(KFDMemoryTest, MemoryRegister) { /* Parent process writes to COW page(s) and gets a new copy. MMU * notifier needs to update the GPU mapping(s) for the test to - * pass. */ + * pass. + */ globalData = 0xD00BED00; stackData[dstOffset] = 0xdeadbeef; stackData[sdmaOffset] = 0xdeadbeef; /* Terminate the child process before a possible test failure that - * would leave it spinning in the background indefinitely. */ + * would leave it spinning in the background indefinitely. + */ int status; EXPECT_EQ(0, kill(pid, SIGTERM)); EXPECT_EQ(pid, waitpid(pid, &status, 0)); @@ -516,10 +521,11 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { TEST_END } -// FlatScratchAccess -// Since HsaMemoryBuffer has to be associated with a specific GPU node, this function in the current form -// will not work for multiple GPU nodes. For now test only one default GPU node. -// TODO: Generalize it to support multiple nodes +/* FlatScratchAccess + * Since HsaMemoryBuffer has to be associated with a specific GPU node, this function in the current form + * will not work for multiple GPU nodes. For now test only one default GPU node. + * TODO: Generalize it to support multiple nodes + */ #define SCRATCH_SLICE_SIZE 0x10000 #define SCRATCH_SLICE_NUM 3 @@ -558,24 +564,23 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { // Map everything for test below ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(scratchBuffer.As(), SCRATCH_SIZE, NULL)); - // source & destination memory buffers HsaMemoryBuffer srcMemBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer dstMemBuffer(PAGE_SIZE, defaultGPUNode); - // Initialize the srcBuffer to some fixed value srcMemBuffer.Fill(0x01010101); - // Initialize a buffer with a DWORD copy ISA + // Initialize a buffer with a dword copy ISA m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? gfx9_ScratchCopyDword : gfx8_ScratchCopyDword, "ScratchCopyDword", isaBuffer); const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); - // TODO: Add support to all GPU Nodes. - // The loop over the system nodes is removed as the test can be executed only on GPU nodes. This - // also requires changes to be made to all the HsaMemoryBuffer variables defined above, as - // HsaMemoryBuffer is now associated with a Node. + /* TODO: Add support to all GPU Nodes. + * The loop over the system nodes is removed as the test can be executed only on GPU nodes. This + * also requires changes to be made to all the HsaMemoryBuffer variables defined above, as + * HsaMemoryBuffer is now associated with a Node. + */ if (pNodeProperties != NULL) { // Get the aperture of the scratch buffer HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; @@ -585,7 +590,7 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { for (unsigned int bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) { if (memoryProperties[bank].HeapType == HSA_HEAPTYPE_GPU_SCRATCH) { int numWaves = 4; // WAVES must be >= # SE - int waveSize = 1; // amount of space used by each wave in units of 256 dwords... + int waveSize = 1; // Amount of space used by each wave in units of 256 dwords PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); @@ -595,25 +600,24 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { // Create a dispatch packet to copy Dispatch dispatchSrcToScratch(isaBuffer); - // setup the dispatch packet + // Setup the dispatch packet // Copying from the source Memory Buffer to the scratch buffer dispatchSrcToScratch.SetArgs(srcMemBuffer.As(), reinterpret_cast(scratchApertureAddr)); dispatchSrcToScratch.SetDim(1, 1, 1); dispatchSrcToScratch.SetScratch(numWaves, waveSize, scratchBuffer.As()); - // submit the packet + // Submit the packet dispatchSrcToScratch.Submit(queue); dispatchSrcToScratch.Sync(); // Create another dispatch packet to copy scratch buffer contents to destination buffer. Dispatch dispatchScratchToDst(isaBuffer); - // set the arguments to copy from the scratch buffer - // to the destination buffer + // Set the arguments to copy from the scratch buffer to the destination buffer dispatchScratchToDst.SetArgs(reinterpret_cast(scratchApertureAddr), dstMemBuffer.As()); dispatchScratchToDst.SetDim(1, 1, 1); dispatchScratchToDst.SetScratch(numWaves, waveSize, scratchBuffer.As()); - // submit the packet + // Submit the packet dispatchScratchToDst.Submit(queue); dispatchScratchToDst.Sync(); @@ -708,7 +712,7 @@ void KFDMemoryTest::BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granular lastTestedSize = sizeMB; } - /* Save the biggest allocated system buffer forsignal handling test */ + /* Save the biggest allocated system buffer for signal handling test */ LOG() << "The biggest allocated system buffer is " << std::dec << lastTestedSize << "MB" << std::endl; if (lastSize) @@ -781,7 +785,8 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, * is small. For example, on a typical Carrizo platform, the biggest allocated * system buffer could be more than 14G even though it only has 4G memory. * In that situation, it will take too much time to finish the test, because of - * the onerous memory swap operation. So we limit the buffer size that way.*/ + * the onerous memory swap operation. So we limit the buffer size that way. + */ TEST_F(KFDMemoryTest, BigBufferStressTest) { if (!is_dgpu()) { LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl; @@ -804,7 +809,8 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { BigBufferVRAM(defaultGPUNode, granularityMB, NULL); /* Repeatedly allocate and map big buffers in system memory until it fails, - * then unmap and free them. */ + * then unmap and free them. + */ #define ARRAY_ENTRIES 2048 int i = 0; @@ -875,7 +881,8 @@ TEST_F(KFDMemoryTest, MMBench) { /* Two SDMA queues to interleave user mode SDMA with memory * management on either SDMA engine. Make the queues long enough * to buffer at least nBufs x WriteData packets (7 dwords per - * packet). */ + * packet). + */ SDMAQueue sdmaQueue[2]; ASSERT_SUCCESS(sdmaQueue[0].Create(defaultGPUNode, PAGE_SIZE*8)); ASSERT_SUCCESS(sdmaQueue[1].Create(defaultGPUNode, PAGE_SIZE*8)); @@ -1094,7 +1101,8 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { * to access its memory like a debugger would. Child copies data in * the parent process using PTRACE_PEEKDATA and PTRACE_POKEDATA. After * the child terminates, the parent checks that the copy was - * successful. */ + * successful. + */ TEST_F(KFDMemoryTest, PtraceAccess) { TEST_START(TESTPROFILE_RUNALL) @@ -1108,13 +1116,14 @@ TEST_F(KFDMemoryTest, PtraceAccess) { void *mem[2]; unsigned i; - // Offset in the VRAM buffer to test crossing non-contiguous - // buffer boundaries. The second access starting from offset - // sizeof(HSAint64)+1 will cross a node boundary in a single access, - // for node sizes of 4MB or smaller. + /* Offset in the VRAM buffer to test crossing non-contiguous + * buffer boundaries. The second access starting from offset + * sizeof(HSAint64)+1 will cross a node boundary in a single access, + * for node sizes of 4MB or smaller. + */ const HSAuint64 VRAM_OFFSET = (4 << 20) - 2 * sizeof(HSAint64); - // alloc system memory from node 0 and initialize it + // Alloc system memory from node 0 and initialize it memFlags.ui32.NonPaged = 0; ASSERT_SUCCESS(hsaKmtAllocMemory(0, PAGE_SIZE*2, memFlags, &mem[0])); for (i = 0; i < 4*sizeof(HSAint64) + 4; i++) { @@ -1122,7 +1131,7 @@ TEST_F(KFDMemoryTest, PtraceAccess) { (reinterpret_cast(mem[0]))[PAGE_SIZE+i] = 0; // destination } - // try to alloc local memory from GPU node + // Try to alloc local memory from GPU node memFlags.ui32.NonPaged = 1; if (m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode)) { EXPECT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE*2 + (4 << 20), @@ -1137,13 +1146,14 @@ TEST_F(KFDMemoryTest, PtraceAccess) { mem[1] = NULL; } - // Allow any process to trace this one. If kernel is built without - // Yama, this is not needed, and this call will fail. + /* Allow any process to trace this one. If kernel is built without + * Yama, this is not needed, and this call will fail. + */ #ifdef PR_SET_PTRACER prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); #endif - // Find out my pid so the child can trace it + // Find current pid so the child can trace it pid_t tracePid = getpid(); // Fork the child @@ -1168,8 +1178,7 @@ TEST_F(KFDMemoryTest, PtraceAccess) { } while (!WIFSTOPPED(traceStatus)); for (i = 0; i < 4; i++) { - // Test 4 different (mis-)alignments, leaving 1-byte - // gaps between longs + // Test 4 different (mis-)alignments, leaving 1-byte gaps between longs HSAuint8 *addr = reinterpret_cast(reinterpret_cast(mem[0]) + i) + i; errno = 0; long data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); @@ -1264,7 +1273,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, size, memFlags, &mem)); ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(mem, size, NULL)); - /* set the word before 4M boundary to 0xdeadbeefdeadbeef + /* Set the word before 4M boundary to 0xdeadbeefdeadbeef * and the word after 4M boundary to 0xcafebabecafebabe */ mem0 = reinterpret_cast(reinterpret_cast(mem) + VRAM_OFFSET); @@ -1309,7 +1318,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { waitpid(tracePid, &traceStatus, 0); } while (!WIFSTOPPED(traceStatus)); - /* peek the memory */ + /* Peek the memory */ errno = 0; HSAint64 data0 = ptrace(PTRACE_PEEKDATA, tracePid, mem0, NULL); ASSERT_EQ(0, errno); @@ -1318,7 +1327,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { ASSERT_EQ(0, errno); ASSERT_EQ(data[1], data1); - /* swap mem0 and mem1 by poking */ + /* Swap mem0 and mem1 by poking */ ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, reinterpret_cast(data[1]))); ASSERT_EQ(0, errno); ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, reinterpret_cast(data[0]))); @@ -1404,7 +1413,7 @@ TEST_F(KFDMemoryTest, SignalHandling) { size = (sysMemSize >> 2) & ~(HSAuint64)(PAGE_SIZE - 1); ASSERT_SUCCESS(hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, reinterpret_cast(&pDb))); - // verify that pDb is not null before it's being used + // Verify that pDb is not null before it's being used ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; pid_t childPid = fork(); @@ -1473,7 +1482,7 @@ TEST_F(KFDMemoryTest, CheckZeroInitializationSysMem) { return; } - /* check the first 64 bit */ + /* Check the first 64 bits */ EXPECT_EQ(0, pDb[0]); pDb[0] = 1; @@ -1495,7 +1504,7 @@ TEST_F(KFDMemoryTest, CheckZeroInitializationSysMem) { } static inline void access(volatile void *sd, int size, int rw) { - /* Most like sit in cache*/ + /* Most likely sitting in cache*/ static struct DUMMY { char dummy[1024]; } dummy; @@ -1509,8 +1518,8 @@ static inline void access(volatile void *sd, int size, int rw) { } /* - * on large-ber system, test the visible vram access speed. - * kfd is not allowd to alloc visible vram on non-largebar system. + * On large-bar system, test the visible vram access speed. + * KFD is not allowed to alloc visible vram on non-largebar system. */ TEST_F(KFDMemoryTest, MMBandWidth) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); @@ -1571,7 +1580,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { memFlags.ui32.HostAccess = 1; memFlags.ui32.NonPaged = 0; } else { - /* alloc visible vram*/ + /* Alloc visible vram*/ allocNode = defaultGPUNode; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 1; diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp index 8982fcb2df..947968999e 100644 --- a/tests/kfdtest/src/KFDMemoryTest.hpp +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -27,9 +27,10 @@ #ifndef __KFD_MEMORY_TEST__H__ #define __KFD_MEMORY_TEST__H__ -// @class KFDTopologyTest -// this class has no additional features to KFDBaseComponentTest -// the separation was made so we are able to goup all memeory tests together +/* @class KFDTopologyTest + * This class has no additional features to KFDBaseComponentTest + * The separation was made so we are able to group all memory tests together + */ class KFDMemoryTest : public KFDBaseComponentTest { public: KFDMemoryTest(void) :m_pIsaGen(NULL) {} diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp b/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp index 68351c847b..86e485218a 100644 --- a/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp +++ b/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp @@ -24,7 +24,7 @@ #include "KFDOpenCloseKFDTest.hpp" #include "KFDTestUtil.hpp" -// before every test from this class fixture - open KFD +// Before every test from this class fixture, open KFD void KFDOpenCloseKFDTest::SetUp() { ROUTINE_START @@ -33,7 +33,7 @@ void KFDOpenCloseKFDTest::SetUp() { ROUTINE_END } -// after every test from this class fixture - close KFD +// After every test from this class fixture, close KFD void KFDOpenCloseKFDTest::TearDown() { ROUTINE_START @@ -42,9 +42,10 @@ void KFDOpenCloseKFDTest::TearDown() { ROUTINE_END } -// this test does not use class KFDOpenCloseKFDTest but is placed here -// since it's testing same topic as other test -// verify that calling hsaKmtCloseKFD on a closed KFD will return right status +/* This test does not use class KFDOpenCloseKFDTest but is placed here + * since it's testing same topic as other test + * Verify that calling hsaKmtCloseKFD on a closed KFD will return right status + */ TEST(KFDCloseKFDTest, CloseAClosedKfd ) { TEST_START(TESTPROFILE_RUNALL) @@ -53,8 +54,7 @@ TEST(KFDCloseKFDTest, CloseAClosedKfd ) { TEST_END } -// verify that calling hsaKmtCloseKFD on an already opened KFD will return -// right status +// Verify that calling hsaKmtCloseKFD on an already opened KFD will return right status TEST_F(KFDOpenCloseKFDTest, OpenAlreadyOpenedKFD ) { TEST_START(TESTPROFILE_RUNALL) @@ -65,8 +65,7 @@ TEST_F(KFDOpenCloseKFDTest, OpenAlreadyOpenedKFD ) { TEST_END } -// testing the normal scenario: open followed by close (done in the setup and -// teardown functions) +// Testing the normal scenario: open followed by close (done in the setup and teardown functions) TEST_F(KFDOpenCloseKFDTest, OpenCloseKFD ) { } diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp b/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp index 7e903163b0..96b48277f2 100644 --- a/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp +++ b/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp @@ -34,10 +34,9 @@ class KFDOpenCloseKFDTest : public testing::Test { ~KFDOpenCloseKFDTest(void) {} protected: - // @brief SetUp function run before every test that uses KFDOpenCloseKFDTest class fixture, - // sets all common settings for the tests. + // @brief Executed before every test that uses KFDOpenCloseKFDTest class, sets all common settings for the tests. virtual void SetUp(); - // @brief TearDown function run after every test that uses KFDOpenCloseKFDTest class fixture. + // @brief Executed after every test that uses KFDOpenCloseKFDTest class virtual void TearDown(); }; diff --git a/tests/kfdtest/src/KFDPMTest.cpp b/tests/kfdtest/src/KFDPMTest.cpp index 6e63f7ef9e..c45e190e22 100644 --- a/tests/kfdtest/src/KFDPMTest.cpp +++ b/tests/kfdtest/src/KFDPMTest.cpp @@ -35,7 +35,7 @@ void KFDPMTest::SetUpTestCase() { m_SetupSuccess = false; AcquirePrivilege(OS_SUSPEND); - // if AcquirePrivilege fails, it will throw and we will not reach here. + // If AcquirePrivilege fails, it will throw and we will not reach here. m_SetupSuccess = true; ROUTINE_END @@ -119,4 +119,4 @@ TEST_F(KFDPMTest, SuspendWithIdleQueueAfterWork) { TEST_END } -// TODO suspend while workload is being executed by a queue +// TODO: Suspend while workload is being executed by a queue diff --git a/tests/kfdtest/src/KFDPNPTest.cpp b/tests/kfdtest/src/KFDPNPTest.cpp index a379d390f4..f2a0bb7ad3 100644 --- a/tests/kfdtest/src/KFDPNPTest.cpp +++ b/tests/kfdtest/src/KFDPNPTest.cpp @@ -34,7 +34,7 @@ void KFDPNPTest::SetUpTestCase() { AcquirePrivilege(OS_DRIVER_OPERATIONS); - // if AcquirePrivilege fails, it will throw and we will not reach here. + // If AcquirePrivilege fails, it will throw and we will not reach here. m_SetupSuccess = true; ROUTINE_END diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 30bc5416ed..1e3fb65eca 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -32,8 +32,6 @@ #include "Dispatch.hpp" -// All tests are marked by their serial number in the QCM FDD - void KFDQMTest::SetUp() { ROUTINE_START @@ -155,7 +153,8 @@ TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { * Fiji and other VI/Polaris GPUs. This test typically hangs in a few * seconds. According to analysis done by HW engineers, the culprit * seems to be PCIe speed switching. The problem can be worked around - * by disabling the lowest DPM level on Fiji. */ + * by disabling the lowest DPM level on Fiji. + */ TEST_F(KFDQMTest, SdmaConcurrentCopies) { TEST_START(TESTPROFILE_RUNALL) @@ -189,14 +188,16 @@ TEST_F(KFDQMTest, SdmaConcurrentCopies) { srcBuf.As()+COPY_SIZE*j, COPY_SIZE)); queue.SubmitPacket(); - // Waste a variable amount of time. Submission timing - // while SDMA runs concurrently seems to be critical for - // reproducing the hang + /* Waste a variable amount of time. Submission timing + * while SDMA runs concurrently seems to be critical for + * reproducing the hang + */ for (int k = 0; k < (i & 0xfff); k++) memcpy(srcBuf.As()+PAGE_SIZE, srcBuf.As(), 1024); - // Wait for idle every 8 packets to allow the SDMA engine to - // run concurrently for a bit without getting too far ahead + /* Wait for idle every 8 packets to allow the SDMA engine to + * run concurrently for a bit without getting too far ahead + */ if ((i & 0x7) == 0) queue.Wait4PacketConsumption(); } @@ -268,7 +269,7 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) { queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 1, 1)); - // don't sync since we don't expect rptr to change when the queue is disabled. + // Don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) @@ -309,7 +310,7 @@ TEST_F(KFDQMTest, DisableSdmaQueueByUpdateWithNullAddress) { queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0)); - // don't sync since we don't expect rptr to change when the queue is disabled. + // Don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) @@ -356,7 +357,7 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) { queue.PlaceAndSubmitPacket(packet2); - // don't sync since we don't expect rptr to change when the queue is disabled. + // Don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) @@ -388,7 +389,7 @@ TEST_F(KFDQMTest, CreateQueueStressSingleThreaded) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; do { - // the following means we'll get the order 0,0 => 0,1 => 1,0 => 1,1 so we cover all options. + // The following means we'll get the order 0,0 => 0,1 => 1,0 => 1,1 so we cover all options. unsigned int firstToCreate = (numIter % 2 != 0) ? 1 : 0; unsigned int firstToDestroy = (numIter % 4 > 1) ? 1 : 0; @@ -440,7 +441,7 @@ TEST_F(KFDQMTest, OverSubscribeCpQueues) { unsigned int pktSizeDw = 0; for (unsigned int i = 0; i < MAX_PACKETS; i++) { PM4WriteDataPacket packet; - packet.InitPacket(destBuf.As()+qidx*2, qidx+i, qidx+i); // two DWORDs per packet + packet.InitPacket(destBuf.As()+qidx*2, qidx+i, qidx+i); // two dwords per packet queues[qidx].PlacePacket(packet); } } @@ -612,9 +613,9 @@ HSAint64 KFDQMTest::GetAverageTimeConsumedwithCUMask(int node, uint32_t* mask, u return timeTotal / iterations; } -/** +/* * Apply CU masking in a linear fashion, adding 1 CU per iteration - * until all Shader Engines are full ... + * until all Shader Engines are full */ TEST_F(KFDQMTest, BasicCuMaskingLinear) { TEST_START(TESTPROFILE_RUNALL); @@ -1026,8 +1027,9 @@ TEST_F(KFDQMTest, CpuWriteCoherence) { EXPECT_EQ(0, queue.Rptr()); - // now that the GPU has cached the PQ contents, we modify them in CPU cache and - // ensure that the GPU sees the updated value: + /* Now that the GPU has cached the PQ contents, we modify them in CPU cache and + * ensure that the GPU sees the updated value: + */ queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0x42, 0x42)); queue.Wait4PacketConsumption(); @@ -1130,10 +1132,10 @@ TEST_F(KFDQMTest, QueueLatency) { if (i >= skip) queue_latency_avg += queue_latency; } while (++i < slots); - /*Calculate avg from packet[skip, slots-1]*/ + /* Calculate avg from packet[skip, slots-1] */ queue_latency_avg /= (slots - skip); - /*workload of queue packet itself*/ + /* Workload of queue packet itself */ i = 0; do { queue.PlacePacket(PM4ReleaseMemoryPacket(true, @@ -1159,7 +1161,7 @@ TEST_F(KFDQMTest, QueueLatency) { do { /* The queue_latency is not that correct as the workload and overhead are average*/ queue_latency_arr[i] -= workload + overhead; - /* The First submit takes a HSAint64 time*/ + /* The First submit takes an HSAint64 time*/ if (i < skip) LOG() << "Queue Latency " << fs[i] << ": \t" << CounterToNanoSec(queue_latency_arr[i]) << std::endl; } while (++i < slots); @@ -1454,7 +1456,7 @@ TEST_F(KFDQMTest, P2PTest) { mapFlags, nodes.size(), &nodes[0])); #define MAGIC_NUM 0xdeadbeaf - /* First GPU fills mem with MAGIC_NUM*/ + /* First GPU fills mem with MAGIC_NUM */ void *src, *dst; HSAuint32 cur = nodes[0], next; ASSERT_SUCCESS(hsaKmtAllocMemory(cur, size, memFlags, reinterpret_cast(&src))); @@ -1478,11 +1480,11 @@ TEST_F(KFDQMTest, P2PTest) { } LOG() << "Test " << cur << " -> " << next << std::endl; - /* copy to sysBuf and next GPU*/ + /* Copy to sysBuf and next GPU*/ void *dst_array[] = {sysBuf, dst}; sdma_copy(cur, src, dst_array, n, size); - /* verify the data*/ + /* Verify the data*/ ASSERT_EQ(sysBuf[0], MAGIC_NUM); ASSERT_EQ(sysBuf[end], MAGIC_NUM); @@ -1558,7 +1560,8 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { #ifdef DOORBELL_WRITE_USE_SDMA /* Write the wptr and doorbell update using the GPU's SDMA * engine. This should submit the PM4 packet on the first - * queue. */ + * queue. + */ otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_write_ptr, pendingWptr)); otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_DoorBell, @@ -1566,7 +1569,8 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { #else /* Write the wptr and doorbell update using WRITE_DATA packets * on a second PM4 queue. This should submit the PM4 packet on - * the first queue. */ + * the first queue. + */ otherQueue.PlacePacket( PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_write_ptr, pendingWptr, false)); @@ -1582,7 +1586,8 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { #ifdef DOORBELL_WRITE_USE_SDMA /* Write the wptr and doorbell update using the GPU's SDMA * engine. This should submit the PM4 packet on the first - * queue. */ + * queue. + */ otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_write_ptr, 2, &pendingWptr64)); otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_DoorBell, @@ -1591,7 +1596,8 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { /* Write the 64-bit wptr and doorbell update using RELEASE_MEM * packets without IRQs on a second PM4 queue. RELEASE_MEM * should perform one atomic 64-bit access. This should submit - * the PM4 packet on the first queue. */ + * the PM4 packet on the first queue. + */ otherQueue.PlacePacket( PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_write_ptr, pendingWptr64, true)); diff --git a/tests/kfdtest/src/KFDQMTest.hpp b/tests/kfdtest/src/KFDQMTest.hpp index 9f120f83cf..b0d3f66073 100644 --- a/tests/kfdtest/src/KFDQMTest.hpp +++ b/tests/kfdtest/src/KFDQMTest.hpp @@ -42,10 +42,9 @@ class KFDQMTest : public KFDBaseComponentTest { virtual void TearDown(); void SyncDispatch(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf, int node = -1); -// void SyncDispatchWithSleep(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf); HSAint64 TimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count); HSAint64 GetAverageTimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count, int iterations); - protected: // members + protected: // Members /* Acceptable performance for CU Masking should be within 5% of linearly-predicted performance */ const double CuVariance = 0.15; const double CuNegVariance = 1.0 - CuVariance; diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index d98d99262a..04522a5b46 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -32,13 +32,12 @@ extern int g_TestDstNodeId; extern bool g_IsChildProcess; extern unsigned int g_TestGPUFamilyId; -// each test should call TEST_START with the test custome profile and HW scheduling - +// Each test should call TEST_START with the test custom profile and HW scheduling enum TESTPROFILE{ TESTPROFILE_DEV = 0x1, TESTPROFILE_PROMO = 0x2, - // 0x4 - 0x8000 - unsed flags - // can add any flag that will mark only part of the tests to run + // 0x4 - 0x8000 - unused flags + // Can add any flag that will mark only part of the tests to run TESTPROFILE_RUNALL = 0xFFFF }; @@ -48,8 +47,8 @@ enum ENVCAPS{ ENVCAPS_16BITPASID = 0x2, ENVCAPS_32BITLINUX = 0x4, ENVCAPS_64BITLINUX = 0x8 - // 0x8 - 0x8000 - unsed flags - // can add any flag that will mark specific hw limitation \ capability + // 0x8 - 0x8000 - unused flags + // Can add any flag that will mark specific hw limitation or capability }; enum KfdFamilyId { diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index 5021445099..98d186d9ee 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -56,11 +56,13 @@ bool g_IsChildProcess; unsigned int g_TestGPUFamilyId; GTEST_API_ int main(int argc, char **argv) { - // default values for run parameters + // Default values for run parameters g_TestRunProfile = TESTPROFILE_RUNALL; g_TestENVCaps = ENVCAPS_NOADDEDCAPS | ENVCAPS_64BITLINUX; g_TestTimeOut = KFD_TEST_DEFAULT_TIMEOUT; + // Every fatal fail ( = assert that failed ) will throw an exception + testing::GTEST_FLAG(throw_on_failure) = true; testing::InitGoogleTest(&argc, argv); CommandLineArguments args; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 2e357583b4..c50dcb811c 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -52,7 +52,7 @@ bool GetHwCapabilityHWS() { unsigned int value = 0; bool valExists = ReadDriverConfigValue(CONFIG_HWS, value); - /* HWS is enabled by default, so... */ + /* HWS is enabled by default */ return ( (!valExists) || ( value > 0)); } @@ -64,7 +64,7 @@ HSAKMT_STATUS CreateQueueTypeEvent( ) { HsaEventDescriptor Descriptor; -// TODO Create per-OS header with this sort of definitions +// TODO: Create per-OS header with this sort of definitions #ifdef _WIN32 Descriptor.EventType = HSA_EVENTTYPE_QUEUE_EVENT; #else @@ -369,7 +369,7 @@ void HsaMemoryBuffer::UnmapAllNodes() { } /* - * TODO: when thunk will be updated use hsaKmtRegisterToNodes. and then nodes will be used + * TODO: When thunk is updated, use hsaKmtRegisterToNodes. Then nodes will be used */ hsaKmtUnmapMemoryToGPU(m_pBuf); hsaKmtDeregisterMemory(m_pBuf); @@ -414,10 +414,10 @@ HsaInteropMemoryBuffer::~HsaInteropMemoryBuffer() { HsaNodeInfo::HsaNodeInfo() { } -// Init - Get and store information about all the HSA nodes from the Thunk Library. -// @NumOfNodes - Number to system nodes returned by hsaKmtAcquireSystemProperties -// @Return - false: if no node information is available -// +/* Init - Get and store information about all the HSA nodes from the Thunk Library. + * @NumOfNodes - Number to system nodes returned by hsaKmtAcquireSystemProperties + * @Return - false: if no node information is available + */ bool HsaNodeInfo::Init(int NumOfNodes) { HsaNodeProperties *nodeProperties; _HSAKMT_STATUS status; @@ -427,8 +427,9 @@ bool HsaNodeInfo::Init(int NumOfNodes) { nodeProperties = new HsaNodeProperties(); status = hsaKmtGetNodeProperties(i, nodeProperties); - /* this is not a fatal test (not using assert), since even when it fails for one node - * we want to get information regarding others. */ + /* This is not a fatal test (not using assert), since even when it fails for one node + * we want to get information regarding others. + */ EXPECT_SUCCESS(status) << "Node index: " << i << "hsaKmtGetNodeProperties returned status " << status; if (status == HSAKMT_STATUS_SUCCESS) { @@ -476,8 +477,7 @@ const int HsaNodeInfo::HsaDefaultGPUNode() const { return -1; if (g_TestNodeId >= 0) { - // Check if this is a valid Id, if so use this else use first - // available + // Check if this is a valid Id, if so use this else use first available for (unsigned int i = 0; i < m_NodesWithGPU.size(); i++) { if (g_TestNodeId == m_NodesWithGPU.at(i)) return g_TestNodeId; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 57cf2898f1..853153ef9b 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -92,7 +92,7 @@ class HsaMemoryBuffer { ~HsaMemoryBuffer(); private: - // disable copy + // Disable copy HsaMemoryBuffer(const HsaMemoryBuffer&); const HsaMemoryBuffer& operator=(const HsaMemoryBuffer&); @@ -130,7 +130,7 @@ class HsaInteropMemoryBuffer { ~HsaInteropMemoryBuffer(); private: - // disable copy + // Disable copy HsaInteropMemoryBuffer(const HsaInteropMemoryBuffer&); const HsaInteropMemoryBuffer& operator=(const HsaInteropMemoryBuffer&); @@ -141,8 +141,7 @@ class HsaInteropMemoryBuffer { unsigned int m_Node; }; -// Class HsaNodeInfo - Gather and store all HSA node information from Thunk. -// +// @class HsaNodeInfo - Gather and store all HSA node information from Thunk. class HsaNodeInfo { // List containing HsaNodeProperties of all Nodes available std::vector m_HsaNodeProps; @@ -159,16 +158,17 @@ class HsaNodeInfo { bool Init(int NumOfNodes); - // This function should be soon depricated. This for transistion purpose only - // Currently, KfdTest is designed to test only ONE node. This function acts - // as transistion. + /* This function should be deprecated soon. This for transistion purpose only + * Currently, KfdTest is designed to test only ONE node. This function acts + * as transition. + */ const HsaNodeProperties* HsaDefaultGPUNodeProperties() const; const int HsaDefaultGPUNode() const; - // Future use the following two functions to support multi-GPU. - // const std::vector& GpuNodes = GetNodesWithGPU() - // for (..GpuNodes.size()..) GetNodeProperties(GpuNodes.at(i)) - // + /* TODO: Use the following two functions to support multi-GPU. + * const std::vector& GpuNodes = GetNodesWithGPU() + * for (..GpuNodes.size()..) GetNodeProperties(GpuNodes.at(i)) + */ const std::vector& GetNodesWithGPU() const; // @param node index of the node we are looking at diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index 43e84cc242..16fe8cec99 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -25,8 +25,6 @@ #include #include -// @todo complete topology test according to whats in: hsathk\source\windows\kmt_topology.cpp - const HSAuint64 KFDTopologyTest::c_4Gigabyte = (1ull << 32) - 1; const HSAuint64 KFDTopologyTest::c_40BitAddressSpace = (1ull << 40); @@ -35,17 +33,17 @@ TEST_F(KFDTopologyTest , BasicTest) { const HsaNodeProperties *pNodeProperties; - // goes over all nodes in the sytem properties and check the basic info received + // Goes over all nodes in the sytem properties and check the basic info received for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { pNodeProperties = m_NodeInfo.GetNodeProperties(node); if (pNodeProperties != NULL) { - // checking for cpu core only if it's a cpu only node or if its KAVERY apu. + // Checking for cpu core only if it's a cpu only node or if its KAVERI apu. if (pNodeProperties->DeviceId == 0 || FamilyIdFromNode(pNodeProperties) == FAMILY_KV) { EXPECT_GT(pNodeProperties->NumCPUCores, HSAuint32(0)) << "Node index: " << node << " No CPUs core are connected for node index"; } - // if it's not a cpu only node, look for a gpu core + // If it's not a cpu only node, look for a gpu core if (pNodeProperties->DeviceId != 0) { EXPECT_GT(pNodeProperties->NumFComputeCores, HSAuint32(0)) << "Node index: " << node << "No GPUs core are connected."; @@ -64,7 +62,7 @@ TEST_F(KFDTopologyTest , BasicTest) { TEST_END } -// this test verify failure status on hsaKmtGetNodeProperties with invalid params +// This test verifies failure status on hsaKmtGetNodeProperties with invalid params TEST_F(KFDTopologyTest, GetNodePropertiesInvalidParams) { TEST_START(TESTPROFILE_RUNALL) @@ -73,7 +71,7 @@ TEST_F(KFDTopologyTest, GetNodePropertiesInvalidParams) { TEST_END } -// this test verify failure status on hsaKmtGetNodeProperties with invalid params +// This test verifies failure status on hsaKmtGetNodeProperties with invalid params TEST_F(KFDTopologyTest, GetNodePropertiesInvalidNodeNum) { TEST_START(TESTPROFILE_RUNALL) @@ -84,8 +82,8 @@ TEST_F(KFDTopologyTest, GetNodePropertiesInvalidNodeNum) { TEST_END } -// test that we can get memory property successfully per node -// @todo check validity of values returned +// Test that we can get memory properties successfully per node +// TODO: Check validity of values returned TEST_F(KFDTopologyTest, GetNodeMemoryProperties) { TEST_START(TESTPROFILE_RUNALL) const HsaNodeProperties *pNodeProperties; @@ -104,7 +102,7 @@ TEST_F(KFDTopologyTest, GetNodeMemoryProperties) { } -// test the GPU local memory aperture is valid. +// Test that the GPU local memory aperture is valid. TEST_F(KFDTopologyTest, GpuvmApertureValidate) { TEST_REQUIRE_NO_ENV_CAPABILITIES(ENVCAPS_32BITLINUX); @@ -137,8 +135,8 @@ TEST_F(KFDTopologyTest, GpuvmApertureValidate) { TEST_END } -// test that we can get cache property successfully per node -// @todo check validity of values returned +// Test that we can get cache property successfully per node +// TODO: Check validity of values returned TEST_F(KFDTopologyTest, GetNodeCacheProperties) { TEST_START(TESTPROFILE_RUNALL) @@ -179,8 +177,8 @@ TEST_F(KFDTopologyTest, GetNodeCacheProperties) { TEST_END } -// test that we can get NodeIoLink property successfully per node -// @todo check validity of values returned +// Test that we can get NodeIoLink property successfully per node +// TODO: Check validity of values returned // GetNodeIoLinkProperties is disabled for now, test fails due to bug in BIOS TEST_F(KFDTopologyTest, GetNodeIoLinkProperties) { TEST_START(TESTPROFILE_RUNALL) @@ -197,7 +195,7 @@ TEST_F(KFDTopologyTest, GetNodeIoLinkProperties) { HsaIoLinkProperties *IolinkProperties = new HsaIoLinkProperties[pNodeProperties->NumIOLinks]; EXPECT_SUCCESS(hsaKmtGetNodeIoLinkProperties(node, pNodeProperties->NumIOLinks, IolinkProperties)); if (pNodeProperties->NumIOLinks == 0) { - // No io_links. Just Print the node + // No io_links. Just print the node LOG() << "[" << node << "]" << std::endl; continue; } diff --git a/tests/kfdtest/src/KFDTopologyTest.hpp b/tests/kfdtest/src/KFDTopologyTest.hpp index 005de1c04d..1f9de9bcf6 100644 --- a/tests/kfdtest/src/KFDTopologyTest.hpp +++ b/tests/kfdtest/src/KFDTopologyTest.hpp @@ -26,9 +26,10 @@ #ifndef __KFD_TOPOLOGY_TEST__H__ #define __KFD_TOPOLOGY_TEST__H__ -// @class KFDTopologyTest -// this class has no additional features to KFDBaseComponentTest -// the separation was made so we are able to goup all topology tests together +/* @class KFDTopologyTest + * This class has no additional features to KFDBaseComponentTest + * The separation was made so we are able to group all topology tests together + */ class KFDTopologyTest : public KFDBaseComponentTest { public: KFDTopologyTest(void) {} diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/tests/kfdtest/src/LinuxOSWrapper.cpp index 74a0e65201..fd1e7b8aeb 100644 --- a/tests/kfdtest/src/LinuxOSWrapper.cpp +++ b/tests/kfdtest/src/LinuxOSWrapper.cpp @@ -46,11 +46,11 @@ static int protection_flags[8] = {PROT_NONE, PROT_READ, PROT_WRITE, PROT_READ | PROT_EXEC | PROT_WRITE | PROT_READ}; void SetConsoleTextColor(TEXTCOLOR color) { - // TODO complete + // TODO: Complete } void Delay(int delayCount) { - // usleeps accept time in microseconds + // usleep accepts time in microseconds usleep(delayCount * 1000); } @@ -76,7 +76,7 @@ HSAuint64 GetLastErrorNo() { } bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProcess) { - // TODO IMPLEMENT + // TODO: Implement return false; } @@ -87,20 +87,20 @@ HSAuint64 GetSystemTickCountInMicroSec() { } bool SuspendAndWakeUp() { - // TODO IMPLEMENT + // TODO: Implement return false; } void AcquirePrivilege(OS_PRIVILEGE priv) { - // TODO IMPLEMENT + // TODO: Implement } void DisableKfd() { - // TODO IMPLEMENT + // TODO: Implement } void EnableKfd() { - // TODO IMPLEMENT + // TODO: Implement } bool ReadDriverConfigValue(CONFIG_VALUE config, unsigned int& rValue) { @@ -145,7 +145,7 @@ bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs) if (c != 0) break; - /* If this option set a flag, do nothing else now. */ + /* If this option sets a flag, do nothing else. */ if (long_options[option_index].flag != 0) continue; diff --git a/tests/kfdtest/src/OSWrapper.hpp b/tests/kfdtest/src/OSWrapper.hpp index 6b2f500258..480b0c91de 100644 --- a/tests/kfdtest/src/OSWrapper.hpp +++ b/tests/kfdtest/src/OSWrapper.hpp @@ -72,35 +72,34 @@ struct CommandLineArguments { #define MEM_WRITE 0x02 #define MEM_EXECUTE 0x4 - - -// @brief change console text color +// @brief Change console text color void SetConsoleTextColor(TEXTCOLOR color); // @params delayCount : delay time in milliseconds void Delay(int delayCount); -// @brief replacement for windows VirtualAlloc func +// @brief Replacement for windows VirtualAlloc func void *VirtualAllocMemory(void *address, unsigned int size, int memProtection = MEM_READ | MEM_WRITE); -// @brief replacement for windows FreeVirtual func +// @brief Replacement for windows FreeVirtual func bool VirtualFreeMemory(void *address, unsigned int size); -// @brief retrieve the last error number +// @brief Retrieve the last error number HSAuint64 GetLastErrorNo(); HSAint64 AtomicInc(volatile HSAint64* pValue); void MemoryBarrier(); -// @brief: runs the selected test case number of times required, each in a separate process -// @params testToRun : can be a specific test testcase like TestCase.TestName or if you want -// to run all tests in a test case: TestCase.* and so on -// @params numOfProcesses : how many processes to run in parallel -// @params runsPerProcess : how many iteration a test should do per process, must be a positive number +/* @brief: Runs the selected test case number of times required, each in a separate process + * @params testToRun : Can be a specific test testcase like TestCase.TestName or if you want + * to run all tests in a test case: TestCase.* and so on + * @params numOfProcesses : How many processes to run in parallel + * @params runsPerProcess : How many iteration a test should do per process, must be a positive number + */ bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProcess = 1); HSAuint64 GetSystemTickCountInMicroSec(); -/**Put the system to S3/S4 power state and bring it back to S0. -@return 'true' on success, 'false' on failure. -*/ +/* Put the system to S3/S4 power state and bring it back to S0. + * @return 'true' on success, 'false' on failure. + */ bool SuspendAndWakeUp(); void AcquirePrivilege(OS_PRIVILEGE priv); diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index f8e57aa4ed..1bd70bebd1 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -53,7 +53,7 @@ unsigned int PM4WriteDataPacket::SizeInBytes() const { void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) { m_pPacketData = reinterpret_cast(calloc(1, SizeInBytes())); - // verify that the memory is allocated successfully, cannot use assert here + // Verify that the memory is allocated successfully, cannot use assert here EXPECT_NOTNULL(m_pPacketData); InitPM4Header(m_pPacketData->header, IT_WRITE_DATA); @@ -249,8 +249,8 @@ void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned i // 1st register is a part of the packet struct. m_packetSize = sizeof(PM4SET_SH_REG) + (numRegs-1)*sizeof(uint32_t); - /* allocating the size of the packet, since the packet is assembled from a struct - * followed by an additional DWORD data + /* Allocating the size of the packet, since the packet is assembled from a struct + * followed by an additional dword data */ m_pPacketData = reinterpret_cast(malloc(m_packetSize)); @@ -296,7 +296,7 @@ PM4PartialFlushPacket::PM4PartialFlushPacket(void) { } unsigned int PM4PartialFlushPacket::SizeInBytes() const { - // for PARTIAL_FLUSH_CS packets, the last 2 DWORDS don't exist. + // For PARTIAL_FLUSH_CS packets, the last 2 dwordS don't exist. return sizeof(PM4EVENT_WRITE) - sizeof(uint32_t)*2; } diff --git a/tests/kfdtest/src/PM4Packet.hpp b/tests/kfdtest/src/PM4Packet.hpp index a7ab06ab5a..8f2f5753b5 100644 --- a/tests/kfdtest/src/PM4Packet.hpp +++ b/tests/kfdtest/src/PM4Packet.hpp @@ -31,7 +31,7 @@ #include "pm4_pkt_struct_ai.h" #include "IndirectBuffer.hpp" -// @class PM4Packet: marks a group of all PM4 packets +// @class PM4Packet: Marks a group of all PM4 packets class PM4Packet : public BasePacket { public: PM4Packet(void) {} @@ -47,9 +47,9 @@ class PM4Packet : public BasePacket { // @class PM4WriteDataPacket class PM4WriteDataPacket : public PM4Packet { public: - // empty constructor, befor using the packet call the init func + // Empty constructor, before using the packet call the init func PM4WriteDataPacket(void): m_ndw(0), m_pPacketData(NULL) {} - // this contructor will also init the packet, no need for additional calls + // This contructor will also init the packet, no need for additional calls PM4WriteDataPacket(unsigned int *destBuf, unsigned int data1): m_ndw(1), m_pPacketData(NULL) {InitPacket(destBuf, &data1);} PM4WriteDataPacket(unsigned int *destBuf, unsigned int data1, unsigned int data2): @@ -59,11 +59,11 @@ class PM4WriteDataPacket : public PM4Packet { } virtual ~PM4WriteDataPacket(void); - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const; - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return m_pPacketData; } - // @brief initialise the packet + // @brief Initialise the packet void InitPacket(unsigned int *destBuf, unsigned int data1) { m_ndw = 1; InitPacket(destBuf, &data1); @@ -77,27 +77,27 @@ class PM4WriteDataPacket : public PM4Packet { protected: unsigned int m_ndw; - // PM4WRITE_DATA_CI struct contains all the packets data + // PM4WRITE_DATA_CI struct contains all the packet's data PM4WRITE_DATA_CI *m_pPacketData; }; // @class PM4ReleaseMemoryPacket class PM4ReleaseMemoryPacket : public PM4Packet { public: - // empty constructor, befor using the packet call the init func + // Empty constructor, before using the packet call the init func PM4ReleaseMemoryPacket(void): m_pPacketData(NULL) {} - // this contructor will also init the packet, no need for adittional calls + // This contructor will also init the packet, no need for additional calls PM4ReleaseMemoryPacket(bool isPolling, uint64_t address, uint64_t data, bool is64bit = false, bool isTimeStamp = false): m_pPacketData(NULL) { InitPacket(isPolling, address, data, is64bit, isTimeStamp); } virtual ~PM4ReleaseMemoryPacket(void); - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return m_packetSize; } - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return m_pPacketData; } - // @brief initialise the packet + // @brief Initialise the packet void InitPacket(bool isPolling, uint64_t address, uint64_t data, bool is64bit = false, bool isTimeStamp = false); @@ -109,21 +109,21 @@ class PM4ReleaseMemoryPacket : public PM4Packet { // @class PM4IndirectBufPacket class PM4IndirectBufPacket : public PM4Packet { public: - // empty constructor, befor using the packet call the init func + // Empty constructor, before using the packet call the init func PM4IndirectBufPacket(void) {} - // this contructor will also init the packet, no need for adittional calls + // This contructor will also init the packet, no need for additional calls explicit PM4IndirectBufPacket(IndirectBuffer *pIb); virtual ~PM4IndirectBufPacket(void) {} - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const; - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return &m_packetData; } - // @breif initialise the packet + // @breif Initialise the packet void InitPacket(IndirectBuffer *pIb); private: - // PM4MEC_INDIRECT_BUFFER struct contains all the packets data + // PM4MEC_INDIRECT_BUFFER struct contains all the packet's data PM4MEC_INDIRECT_BUFFER m_packetData; }; @@ -139,11 +139,11 @@ class PM4AcquireMemoryPacket : public PM4Packet { virtual const void *GetPacket() const { return &m_packetData; } private: - // PM4ACQUIRE_MEM struct contains all the packets data + // PM4ACQUIRE_MEM struct contains all the packet's data PM4ACQUIRE_MEM m_packetData; }; -// @class PM4SetShaderRegPacket packet that writes to consecutive registers starting at baseOffset. +// @class PM4SetShaderRegPacket Packet that writes to consecutive registers starting at baseOffset. class PM4SetShaderRegPacket : public PM4Packet { public: PM4SetShaderRegPacket(void); @@ -152,9 +152,9 @@ class PM4SetShaderRegPacket : public PM4Packet { virtual ~PM4SetShaderRegPacket(void); - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return m_packetSize; } - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return m_pPacketData; } void InitPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs); @@ -162,7 +162,7 @@ class PM4SetShaderRegPacket : public PM4Packet { private: unsigned int m_packetSize; bool m_packetDataAllocated; - // PM4SET_SH_REG struct contains all the packets data + // PM4SET_SH_REG struct contains all the packet's data PM4SET_SH_REG *m_pPacketData; }; @@ -175,15 +175,15 @@ class PM4DispatchDirectPacket : public PM4Packet { virtual ~PM4DispatchDirectPacket(void) {} - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const; - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return &m_packetData; } void InitPacket(unsigned int dimX, unsigned int dimY, unsigned int dimZ, unsigned int dispatchInit); private: - // PM4DISPATCH_DIRECT struct contains all the packets data + // PM4DISPATCH_DIRECT struct contains all the packet's data PM4DISPATCH_DIRECT m_packetData; }; @@ -193,13 +193,13 @@ class PM4PartialFlushPacket : public PM4Packet { PM4PartialFlushPacket(void); virtual ~PM4PartialFlushPacket(void) {} - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const; - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return &m_packetData; } private: - // PM4EVENT_WRITE struct contains all the packets data + // PM4EVENT_WRITE struct contains all the packet's data PM4EVENT_WRITE m_packetData; }; @@ -209,9 +209,9 @@ class PM4NopPacket : public PM4Packet { PM4NopPacket(void); virtual ~PM4NopPacket(void) {} - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const; - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return &m_packetData; } private: diff --git a/tests/kfdtest/src/PM4Queue.cpp b/tests/kfdtest/src/PM4Queue.cpp index f569c84ef1..d1152c0b34 100644 --- a/tests/kfdtest/src/PM4Queue.cpp +++ b/tests/kfdtest/src/PM4Queue.cpp @@ -36,7 +36,8 @@ PM4Queue::~PM4Queue(void) { unsigned int PM4Queue::Wptr() { /* Write pointer in dwords. Simulate 32-bit wptr that wraps at - * queue size even on Vega10 and later chips with 64-bit wptr. */ + * queue size even on Vega10 and later chips with 64-bit wptr. + */ return *m_Resources.Queue_write_ptr % (m_QueueBuf->Size() / 4); } @@ -48,12 +49,13 @@ unsigned int PM4Queue::Rptr() { unsigned int PM4Queue::RptrWhenConsumed() { /* On PM4 queues Rptr is always 32-bit in dword units and wraps at * queue size. The expected value when all packets are consumed is - * exactly the value returned by Wptr(). */ + * exactly the value returned by Wptr(). + */ return Wptr(); } void PM4Queue::SubmitPacket() { - // m_pending Wptr is in DWORDs + // m_pending Wptr is in dwords if (g_TestGPUFamilyId < FAMILY_AI) { // Pre-Vega10 uses 32-bit wptr and doorbell MemoryBarrier(); diff --git a/tests/kfdtest/src/PM4Queue.hpp b/tests/kfdtest/src/PM4Queue.hpp index 2acd767622..dd5d35b85f 100644 --- a/tests/kfdtest/src/PM4Queue.hpp +++ b/tests/kfdtest/src/PM4Queue.hpp @@ -35,9 +35,9 @@ class PM4Queue : public BaseQueue { // @brief update queue write pointer and sets the queue doorbell to the queue write pointer virtual void SubmitPacket(); - // @ return read pointer modulo queue size in DWORDs + // @ return read pointer modulo queue size in dwords virtual unsigned int Rptr(); - // @ return write pointer modulo queue size in DWORDs + // @ return write pointer modulo queue size in dwords virtual unsigned int Wptr(); // @ return expected m_Resources.Queue_read_ptr when all packets consumed virtual unsigned int RptrWhenConsumed(); diff --git a/tests/kfdtest/src/RDMATest.hpp b/tests/kfdtest/src/RDMATest.hpp index e15ea2395b..3e4b2331aa 100644 --- a/tests/kfdtest/src/RDMATest.hpp +++ b/tests/kfdtest/src/RDMATest.hpp @@ -38,7 +38,7 @@ class RDMATest : public KFDBaseComponentTest { virtual void SetUp(); virtual void TearDown(); - protected: // members + protected: // Members IsaGenerator* m_pIsaGen; }; diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index d1120590ab..2dbb449d9f 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -27,8 +27,9 @@ #include "SDMAPacket.hpp" #include "KFDTestUtil.hpp" -/* Byte/dword cound in many SDMA packets is 1-based in AI, meaning a - * count of 1 is encoded as 0. */ +/* Byte/dword count in many SDMA packets is 1-based in AI, meaning a + * count of 1 is encoded as 0. + */ #define SDMA_COUNT(c) (g_TestGPUFamilyId < FAMILY_AI ? (c) : (c)-1) SDMAWriteDataPacket::SDMAWriteDataPacket(void): @@ -95,7 +96,7 @@ SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, uns packetData = pSDMA; while (surfsize > 0) { - /* sdma support maximum 0x3fffe0 byte in one copy, take 2M here */ + /* SDMA support maximum 0x3fffe0 byte in one copy, take 2M here */ if (surfsize > TWO_MEG) size = TWO_MEG; else @@ -150,7 +151,7 @@ SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned in pSDMA->HEADER_UNION.op = SDMA_OP_CONST_FILL; pSDMA->HEADER_UNION.sub_op = 0; - /* If Both size and address are DW aligned, then use DW fill */ + /* If both size and address are DW aligned, then use DW fill */ if (!(copy_size & 0x3) && !((HSAuint64)dst & 0x3)) pSDMA->HEADER_UNION.fillsize = 2; /* DW Fill */ else diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index 17b9027b5e..074c52976e 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -27,7 +27,7 @@ #include "BasePacket.hpp" #include "sdma_pkt_struct.h" -// @class SDMSPacket: marks a group of all SDMA packets +// @class SDMAPacket: Marks a group of all SDMA packets class SDMAPacket : public BasePacket { public: SDMAPacket(void) {} @@ -38,44 +38,44 @@ class SDMAPacket : public BasePacket { class SDMAWriteDataPacket : public SDMAPacket { public: - // empty constructor, befor using the packet call the init func + // Empty constructor, before using the packet call the init func SDMAWriteDataPacket(void); - // this contructor will also init the packet, no need for adittional calls + // This contructor will also init the packet, no need for additional calls SDMAWriteDataPacket(void* destAddr, unsigned int data); SDMAWriteDataPacket(void* destAddr, unsigned int ndw, void *data); virtual ~SDMAWriteDataPacket(void); - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return packetData; } - // @breif initialise the packet + // @breif Initialise the packet void InitPacket(void* destAddr, unsigned int data); void InitPacket(void* destAddr, unsigned int ndw, void *data); - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return packetSize; } protected: - // SDMA_PKT_WRITE_UNTILED struct contains all the packets data + // SDMA_PKT_WRITE_UNTILED struct contains all the packet's data SDMA_PKT_WRITE_UNTILED *packetData; unsigned int packetSize; }; class SDMACopyDataPacket : public SDMAPacket { public: - // this contructor will also init the packet, no need for adittional calls + // This contructor will also init the packet, no need for additional calls SDMACopyDataPacket(void *dest, void *src, unsigned int size); SDMACopyDataPacket(void *const dst[], void *src, int n, unsigned int surfsize); virtual ~SDMACopyDataPacket(void); - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return packetData; } - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return packetSize; } protected: - // SDMA_PKT_COPY_LINEAR struct contains all the packets data + // SDMA_PKT_COPY_LINEAR struct contains all the packet's data SDMA_PKT_COPY_LINEAR *packetData; unsigned int packetSize; @@ -83,19 +83,19 @@ class SDMACopyDataPacket : public SDMAPacket { class SDMAFillDataPacket : public SDMAPacket { public: - // this contructor will also init the packet, no need for adittional calls + // This contructor will also init the packet, no need for additional calls SDMAFillDataPacket(void *dest, unsigned int data, unsigned int size); virtual ~SDMAFillDataPacket(void); - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return m_PacketData; } - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return m_PacketSize; } protected: - // SDMA_PKT_CONSTANT_FILL struct contains all the packets data + // SDMA_PKT_CONSTANT_FILL struct contains all the packet's data SDMA_PKT_CONSTANT_FILL *m_PacketData; unsigned int m_PacketSize; @@ -103,41 +103,41 @@ class SDMAFillDataPacket : public SDMAPacket { class SDMAFencePacket : public SDMAPacket { public: - // empty constructor, befor using the packet call the init func + // Empty constructor, before using the packet call the init func SDMAFencePacket(void); - // this contructor will also init the packet, no need for adittional calls + // This contructor will also init the packet, no need for additional calls SDMAFencePacket(void* destAddr, unsigned int data); virtual ~SDMAFencePacket(void); - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return &packetData; } - // @brief initialise the packet + // @brief Initialise the packet void InitPacket(void* destAddr, unsigned int data); - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return sizeof(SDMA_PKT_FENCE ); } protected: - // SDMA_PKT_FENCE struct contains all the packets data + // SDMA_PKT_FENCE struct contains all the packet's data SDMA_PKT_FENCE packetData; }; class SDMATrapPacket : public SDMAPacket { public: - // empty constructor, befor using the packet call the init func + // Empty constructor, before using the packet call the init func explicit SDMATrapPacket(unsigned int eventID = 0); virtual ~SDMATrapPacket(void); - // @returns a pointer to the packet + // @returns Pointer to the packet virtual const void *GetPacket() const { return &packetData; } - // @brief initialise the packet + // @brief Initialise the packet void InitPacket(unsigned int eventID); - // @returns the packet size in bytes + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return sizeof(SDMA_PKT_TRAP); } protected: - // SDMA_PKT_TRAP struct contains all the packets data + // SDMA_PKT_TRAP struct contains all the packet's data SDMA_PKT_TRAP packetData; }; diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/tests/kfdtest/src/SDMAQueue.cpp index 982d919556..6f352b2cef 100644 --- a/tests/kfdtest/src/SDMAQueue.cpp +++ b/tests/kfdtest/src/SDMAQueue.cpp @@ -32,20 +32,22 @@ SDMAQueue::~SDMAQueue(void) { unsigned int SDMAQueue::Wptr() { /* In SDMA queues write pointers are saved in bytes, convert the - * wptr value to DWORD to fit the way BaseQueue works. On Vega10 + * wptr value to dword to fit the way BaseQueue works. On Vega10 * the write ptr is 64-bit. We only read the low 32 bit (assuming * the queue buffer is smaller than 4GB) and modulo divide by the - * queue size to simulate a 32-bit read pointer. */ + * queue size to simulate a 32-bit read pointer. + */ return (*m_Resources.Queue_write_ptr % m_QueueBuf->Size()) / sizeof(unsigned int); } unsigned int SDMAQueue::Rptr() { /* In SDMA queues read pointers are saved in bytes, convert the - * read value to DWORD to fit the way BaseQueue works. On Vega10 + * read value to dword to fit the way BaseQueue works. On Vega10 * the read ptr is 64-bit. We only read the low 32 bit (assuming * the queue buffer is smaller than 4GB) and modulo divide by the - * queue size to simulate a 32-bit read pointer. */ + * queue size to simulate a 32-bit read pointer. + */ return (*m_Resources.Queue_read_ptr % m_QueueBuf->Size()) / sizeof(unsigned int); } @@ -53,12 +55,13 @@ unsigned int SDMAQueue::Rptr() { unsigned int SDMAQueue::RptrWhenConsumed() { /* Rptr is same size and byte units as Wptr. Here we only care * about the low 32-bits. When all packets are consumed, read and - * write pointers should have the same value. */ + * write pointers should have the same value. + */ return *m_Resources.Queue_write_ptr; } void SDMAQueue::SubmitPacket() { - // m_pending Wptr is in DWORDs + // m_pending Wptr is in dwords if (g_TestGPUFamilyId < FAMILY_AI) { // Pre-Vega10 uses 32-bit wptr and doorbell unsigned int wPtrInBytes = m_pendingWptr * sizeof(unsigned int); diff --git a/tests/kfdtest/src/SDMAQueue.hpp b/tests/kfdtest/src/SDMAQueue.hpp index 7cf931368a..fa762e3742 100644 --- a/tests/kfdtest/src/SDMAQueue.hpp +++ b/tests/kfdtest/src/SDMAQueue.hpp @@ -31,15 +31,15 @@ class SDMAQueue : public BaseQueue { SDMAQueue(void); virtual ~SDMAQueue(void); - // @brief update queue write pointer and sets the queue doorbell to the queue write pointer + // @brief Update queue write pointer and set the queue doorbell to the queue write pointer virtual void SubmitPacket(); protected: - // @ return write pointer modulo queue size in DWORDs + // @ return Write pointer modulo queue size in dwords virtual unsigned int Wptr(); - // @ return read pointer modulo queue size in DWORDs + // @ return Read pointer modulo queue size in dwords virtual unsigned int Rptr(); - // @ return expected m_Resources.Queue_read_ptr when all packets consumed + // @ return Expected m_Resources.Queue_read_ptr when all packets are consumed virtual unsigned int RptrWhenConsumed(); virtual PACKETTYPE PacketTypeSupported() { return PACKETTYPE_SDMA; } From fe33461622171d5f12c1fdcfca89e227e302b767 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 20 Aug 2018 09:54:26 -0400 Subject: [PATCH 0396/1247] kfdtest: Consolidate logic for ASSERT vs EXPECT ASSERT failures result in immediate termination of the test. EXPECT returns a failure but continues execution. Reserve ASSERT for required functionality (node initialization, queue creation, etc) where the rest of the test cannot run if that call fails. Use EXPECT everywhere else Change-Id: I1c11326fc3ae22b50fa83b07b3b49af1e1f4e69e --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 4 +- tests/kfdtest/src/KFDCWSRTest.cpp | 4 +- tests/kfdtest/src/KFDDBGTest.cpp | 8 +- tests/kfdtest/src/KFDEventTest.cpp | 11 +- tests/kfdtest/src/KFDEvictTest.cpp | 14 +-- tests/kfdtest/src/KFDExceptionTest.cpp | 32 +++--- tests/kfdtest/src/KFDGraphicsInterop.cpp | 32 +++--- tests/kfdtest/src/KFDIPCTest.cpp | 42 ++++---- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 20 ++-- tests/kfdtest/src/KFDMemoryTest.cpp | 116 ++++++++++----------- tests/kfdtest/src/KFDPMTest.cpp | 14 +-- tests/kfdtest/src/KFDPNPTest.cpp | 4 +- tests/kfdtest/src/KFDPerfCounters.cpp | 8 +- tests/kfdtest/src/KFDQMTest.cpp | 100 +++++++++--------- tests/kfdtest/src/KFDTestUtil.cpp | 2 +- tests/kfdtest/src/PM4Packet.cpp | 2 +- tests/kfdtest/src/RDMATest.cpp | 4 +- 17 files changed, 208 insertions(+), 209 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 8d806670fe..2f697ce8aa 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -79,8 +79,8 @@ void KFDBaseComponentTest::TearDown() { drmClose(m_RenderNodes[i].fd); } - ASSERT_SUCCESS(hsaKmtReleaseSystemProperties()); - ASSERT_SUCCESS(hsaKmtCloseKFD()); + EXPECT_SUCCESS(hsaKmtReleaseSystemProperties()); + EXPECT_SUCCESS(hsaKmtCloseKFD()); ROUTINE_END } diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index e1750a6390..497065b6a6 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -135,7 +135,7 @@ TEST_F(KFDCWSRTest, BasicTest) { PM4Queue queue1, queue2; - EXPECT_SUCCESS(queue1.Create(defaultGPUNode)); + ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); Dispatch *dispatch1, *dispatch2; @@ -153,7 +153,7 @@ TEST_F(KFDCWSRTest, BasicTest) { // Submit the shader, queue1 dispatch1->Submit(queue1); // Create queue2 during queue1 still running will trigger the CWSR - EXPECT_SUCCESS(queue2.Create(defaultGPUNode)); + ASSERT_SUCCESS(queue2.Create(defaultGPUNode)); // Submit the shader dispatch2->Submit(queue2); dispatch1->Sync(); diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 1f717476e6..47e9da7ebb 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -163,11 +163,11 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { dispatch.Sync(g_TestTimeOut); // Check that we got trap handler calls due to add watch triggers - ASSERT_GE(*(tmaBuf.As()+ TMA_TRAP_COUNT_OFFSET), TMA_TRAP_COUNT_VALUE); + EXPECT_GE(*(tmaBuf.As()+ TMA_TRAP_COUNT_OFFSET), TMA_TRAP_COUNT_VALUE); - ASSERT_SUCCESS(hsaKmtDbgUnregister(defaultGPUNode)); - ASSERT_SUCCESS(queue.Destroy()); - ASSERT_SUCCESS(queue_flush.Destroy()); + EXPECT_SUCCESS(hsaKmtDbgUnregister(defaultGPUNode)); + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue_flush.Destroy()); } else { LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; } diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index 35adeee898..3888899dde 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -99,9 +99,9 @@ TEST_F(KFDEventTest, SignalEvent) { queue.Wait4PacketConsumption(); - ASSERT_SUCCESS(hsaKmtWaitOnEvent(m_pHsaEvent, g_TestTimeOut)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pHsaEvent, g_TestTimeOut)); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END; } @@ -184,8 +184,7 @@ exit: uint64_t minTime = ULONG_MAX, maxTime = 0; uint64_t minLat = ULONG_MAX, maxLat = 0; - r = queueAndSignalEvents(node, mNumEvents, time, latency); - ASSERT_EQ(r, HSAKMT_STATUS_SUCCESS); + ASSERT_EQ(queueAndSignalEvents(node, mNumEvents, time, latency), HSAKMT_STATUS_SUCCESS); mTimeHistory[mHistorySlot%HISTORY_SIZE] = time; mLatHistory[mHistorySlot%HISTORY_SIZE] = latency; @@ -275,9 +274,9 @@ TEST_F(KFDEventTest, SignalMultipleEventsWaitForAll) { Delay(WAIT_BETWEEN_SUBMISSIONS_MS); } - ASSERT_SUCCESS(hsaKmtWaitOnMultipleEvents(pHsaEvent, EVENT_NUMBER, true, g_TestTimeOut)); + EXPECT_SUCCESS(hsaKmtWaitOnMultipleEvents(pHsaEvent, EVENT_NUMBER, true, g_TestTimeOut)); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); for (i = 0; i < EVENT_NUMBER; i++) EXPECT_SUCCESS(hsaKmtDestroyEvent(pHsaEvent[i])); diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 5765acc578..6fc6969f6a 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -227,16 +227,16 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { fenceStatus.ring = 0; fenceStatus.fence = ibsRequest.seq_no; - ASSERT_EQ(0, amdgpu_cs_query_fence_status(&fenceStatus, + EXPECT_EQ(0, amdgpu_cs_query_fence_status(&fenceStatus, g_TestTimeOut, 0, &expired)); - ASSERT_EQ(0, amdgpu_bo_list_destroy(boList)); + EXPECT_EQ(0, amdgpu_bo_list_destroy(boList)); - ASSERT_EQ(0, amdgpu_bo_unmap_and_free(ibResultHandle, vaHandle, + EXPECT_EQ(0, amdgpu_bo_unmap_and_free(ibResultHandle, vaHandle, ibResultMcAddress, PAGE_SIZE)); - ASSERT_EQ(0, amdgpu_cs_ctx_free(contextHandle)); + EXPECT_EQ(0, amdgpu_cs_ctx_free(contextHandle)); } void KFDEvictTest::ForkChildProcesses(int nprocesses) { @@ -283,7 +283,7 @@ void KFDEvictTest::WaitChildProcesses() { m_ChildPids.erase(m_ChildPids.begin()); } - ASSERT_EQ(childExitOkNum, size); + EXPECT_EQ(childExitOkNum, size); } /* Child process or parent process finished successfully */ @@ -598,7 +598,7 @@ TEST_F(KFDEvictTest, QueueTest) { /* Wait for shader to finish or timeout if shader has vm page fault */ dispatch0.SyncWithStatus(120000); - ASSERT_SUCCESS(pm4Queue.Destroy()); + EXPECT_SUCCESS(pm4Queue.Destroy()); FreeAmdgpuBo(handle); @@ -610,7 +610,7 @@ TEST_F(KFDEvictTest, QueueTest) { /* Check if all wavefronts finished successfully */ for (i = 0; i < wavefront_num; i++) - ASSERT_EQ(0x5678, *(result + i)); + EXPECT_EQ(0x5678, *(result + i)); WaitChildProcesses(); diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index f7e79aaf0f..428e4a64d7 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -144,11 +144,11 @@ TEST_F(KFDExceptionTest, InvalidBadAddress) { waitpid(m_ChildPid, &childStatus, 0); if (is_dgpu()) { - ASSERT_EQ(WIFEXITED(childStatus), true); - ASSERT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); } else { - ASSERT_EQ(WIFSIGNALED(childStatus), true); - ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); } } @@ -204,8 +204,8 @@ TEST_F(KFDExceptionTest, InvalidPPRWriteProtection) { int childStatus; waitpid(m_ChildPid, &childStatus, 0); - ASSERT_EQ(WIFSIGNALED(childStatus), true); - ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); } TEST_END @@ -254,8 +254,8 @@ TEST_F(KFDExceptionTest, InvalidPPRReadProtection) { int childStatus; waitpid(m_ChildPid, &childStatus, 0); - ASSERT_EQ(WIFSIGNALED(childStatus), true); - ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); } TEST_END @@ -298,11 +298,11 @@ TEST_F(KFDExceptionTest, InvalidWriteAddress) { waitpid(m_ChildPid, &childStatus, 0); if (is_dgpu()) { - ASSERT_EQ(WIFEXITED(childStatus), true); - ASSERT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); } else { - ASSERT_EQ(WIFSIGNALED(childStatus), true); - ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); } } @@ -340,11 +340,11 @@ TEST_F(KFDExceptionTest, FaultStorm) { waitpid(m_ChildPid, &childStatus, 0); if (is_dgpu()) { - ASSERT_EQ(WIFEXITED(childStatus), true); - ASSERT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); } else { - ASSERT_EQ(WIFSIGNALED(childStatus), true); - ASSERT_EQ(WTERMSIG(childStatus), SIGSEGV); + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); } } diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp index 51232216e2..8d97885a30 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.cpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -66,17 +66,17 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { void *pCpuMap; ASSERT_EQ(0, amdgpu_bo_cpu_map(handle, &pCpuMap)); memset(pCpuMap, 0xaa, PAGE_SIZE); - ASSERT_EQ(0, amdgpu_bo_cpu_unmap(handle)); + EXPECT_EQ(0, amdgpu_bo_cpu_unmap(handle)); struct amdgpu_bo_metadata meta; meta.flags = 0; meta.tiling_info = 0; meta.size_metadata = metadata_size; memcpy(meta.umd_metadata, metadata, metadata_size); - ASSERT_EQ(0, amdgpu_bo_set_metadata(handle, &meta)); + EXPECT_EQ(0, amdgpu_bo_set_metadata(handle, &meta)); uint32_t dmabufFd; - ASSERT_EQ(0, amdgpu_bo_export(handle, amdgpu_bo_handle_type_dma_buf_fd, &dmabufFd)); + EXPECT_EQ(0, amdgpu_bo_export(handle, amdgpu_bo_handle_type_dma_buf_fd, &dmabufFd)); // Register it with HSA HsaGraphicsResourceInfo info; @@ -86,13 +86,13 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { /* DMA buffer handle and GEM handle are no longer needed, KFD * should have taken a reference to the BO */ - ASSERT_EQ(0, close(dmabufFd)); - ASSERT_EQ(0, amdgpu_bo_free(handle)); + EXPECT_EQ(0, close(dmabufFd)); + EXPECT_EQ(0, amdgpu_bo_free(handle)); // Check that buffer size and metadata match - ASSERT_EQ(info.SizeInBytes, alloc.alloc_size); - ASSERT_EQ(info.MetadataSizeInBytes, metadata_size); - ASSERT_EQ(0, strcmp(metadata, (const char *)info.Metadata)); + EXPECT_EQ(info.SizeInBytes, alloc.alloc_size); + EXPECT_EQ(info.MetadataSizeInBytes, metadata_size); + EXPECT_EQ(0, strcmp(metadata, (const char *)info.Metadata)); // Map the buffer ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(info.MemoryAddress, @@ -113,9 +113,9 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { dispatch.Submit(queue); dispatch.Sync(g_TestTimeOut); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); - ASSERT_EQ(dstBuffer.As()[0], 0xaaaaaaaa); + EXPECT_EQ(dstBuffer.As()[0], 0xaaaaaaaa); // Test QueryMem before the cleanup HsaPointerInfo ptrInfo; @@ -126,8 +126,8 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { EXPECT_EQ(ptrInfo.SizeInBytes, alloc.alloc_size); // Cleanup - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(info.MemoryAddress)); - ASSERT_SUCCESS(hsaKmtDeregisterMemory(info.MemoryAddress)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(info.MemoryAddress)); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(info.MemoryAddress)); TEST_END } @@ -214,11 +214,11 @@ TEST_F(KFDGraphicsInterop, RegisterForeignDeviceMem) { dispatch.Submit(queue); dispatch.Sync(g_TestTimeOut); - ASSERT_SUCCESS(queue.Destroy()); - ASSERT_EQ(dstBuffer.As()[0], 0xAAAAAAAA); + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_EQ(dstBuffer.As()[0], 0xAAAAAAAA); - ASSERT_EQ(0, amdgpu_bo_cpu_unmap(handle)); - ASSERT_EQ(0, amdgpu_bo_free(handle)); + EXPECT_EQ(0, amdgpu_bo_cpu_unmap(handle)); + EXPECT_EQ(0, amdgpu_bo_free(handle)); TEST_END } diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index b68940e2a6..1150076a6b 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -89,16 +89,16 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(tempSysBuffer.As(), sharedLocalBuffer, size)); sdmaQueue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xAAAAAAAA)); + EXPECT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xAAAAAAAA)); /* Fill in the Local Memory with different pattern */ sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sharedLocalBuffer, 0xBBBBBBBB)); sdmaQueue.Wait4PacketConsumption(); /* Clean up */ - ASSERT_SUCCESS(sdmaQueue.Destroy()); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(sharedLocalBuffer)); - ASSERT_SUCCESS(hsaKmtDeregisterMemory(sharedLocalBuffer)); + EXPECT_SUCCESS(sdmaQueue.Destroy()); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(sharedLocalBuffer)); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(sharedLocalBuffer)); } /* Fill a pattern into Local Memory and share with the child process. @@ -133,18 +133,18 @@ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pip /* Wait for the child to finish */ waitpid(cpid, &status, 0); - ASSERT_EQ(WIFEXITED(status), 1); - ASSERT_EQ(WEXITSTATUS(status), 0); + EXPECT_EQ(WIFEXITED(status), 1); + EXPECT_EQ(WEXITSTATUS(status), 0); /* Check for the new pattern filled in by child process */ sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(tempSysBuffer.As(), toShareLocalBuffer.As(), size)); sdmaQueue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xBBBBBBBB)); + EXPECT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xBBBBBBBB)); /* Clean up */ - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(toShareLocalBuffer.As())); - ASSERT_SUCCESS(sdmaQueue.Destroy()); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(toShareLocalBuffer.As())); + EXPECT_SUCCESS(sdmaQueue.Destroy()); } /* Test IPC memory. @@ -620,23 +620,23 @@ TEST_F(KFDIPCTest, CrossMemoryAttachTest) { /* Child Process */ status = CrossMemoryAttachChildProcess(defaultGPUNode, pipeCtoP[1], pipePtoC[0], CMA_READ_TEST); - ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Child: Read Test Fail"; + EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Child: Read Test Fail"; status = CrossMemoryAttachChildProcess(defaultGPUNode, pipeCtoP[1], pipePtoC[0], CMA_WRITE_TEST); - ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Child: Write Test Fail"; + EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Child: Write Test Fail"; } else { int childStatus; status = CrossMemoryAttachParentProcess(defaultGPUNode, m_ChildPid, pipePtoC[1], pipeCtoP[0], CMA_READ_TEST); /* Parent proces */ - ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Read Test Fail"; + EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Read Test Fail"; status = CrossMemoryAttachParentProcess(defaultGPUNode, m_ChildPid, pipePtoC[1], pipeCtoP[0], CMA_WRITE_TEST); - ASSERT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Write Test Fail"; + EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Write Test Fail"; waitpid(m_ChildPid, &childStatus, 0); - ASSERT_EQ(WIFEXITED(childStatus), true); - ASSERT_EQ(WEXITSTATUS(childStatus), 0); + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), 0); } /* Code path executed by both parent and child with respective fds */ @@ -690,10 +690,10 @@ TEST_F(KFDIPCTest, CMABasicTest) { dstRange.MemoryAddress = testLocalBuffer.As(); dstRange.SizeInBytes = size; ASSERT_SUCCESS(hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied)); - ASSERT_EQ(copied, size); + EXPECT_EQ(copied, size); - ASSERT_TRUE(testLocalBuffer.IsPattern(0, PATTERN1, sdmaQueue, tmp)); - ASSERT_TRUE(testLocalBuffer.IsPattern(size - 4, PATTERN2, sdmaQueue, tmp)); + EXPECT_TRUE(testLocalBuffer.IsPattern(0, PATTERN1, sdmaQueue, tmp)); + EXPECT_TRUE(testLocalBuffer.IsPattern(size - 4, PATTERN2, sdmaQueue, tmp)); /* Test2. Test unaligned byte copy. Write 3 bytes to an unaligned destination address */ @@ -712,10 +712,10 @@ TEST_F(KFDIPCTest, CMABasicTest) { dstRange.MemoryAddress = reinterpret_cast(testLocalBuffer.As() + (size / 2) + unaligned_offset); dstRange.SizeInBytes = unaligned_size; ASSERT_SUCCESS(hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied)); - ASSERT_EQ(copied, unaligned_size); + EXPECT_EQ(copied, unaligned_size); expected_pattern = (PATTERN2 & ~unaligned_mask | (PATTERN1 & unaligned_mask)); - ASSERT_TRUE(testLocalBuffer.IsPattern(size/2, expected_pattern, sdmaQueue, tmp)); + EXPECT_TRUE(testLocalBuffer.IsPattern(size/2, expected_pattern, sdmaQueue, tmp)); /* Test3. Test overflow and expect failure */ @@ -727,7 +727,7 @@ TEST_F(KFDIPCTest, CMABasicTest) { EXPECT_NE(status, HSAKMT_STATUS_SUCCESS); EXPECT_LE(copied, (size - 4)); - ASSERT_SUCCESS(sdmaQueue.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); TEST_END } diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index a2928b0c38..c879080b25 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -96,11 +96,11 @@ TEST_F(KFDLocalMemoryTest, BasicTest) { dispatch.Submit(queue); dispatch.Sync(g_TestTimeOut); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(srcLocalBuffer.As())); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(dstLocalBuffer.As())); - ASSERT_EQ(destSysBuffer.As()[0], 0x01010101); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(srcLocalBuffer.As())); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(dstLocalBuffer.As())); + EXPECT_EQ(destSysBuffer.As()[0], 0x01010101); TEST_END } @@ -142,17 +142,17 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { dispatch.Submit(queue); dispatch.Sync(g_TestTimeOut); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As())); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As())); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU)); dispatch.SetArgs(LocalBuffer.As(), SysBufferB.As()); dispatch.Submit(queue); dispatch.Sync(g_TestTimeOut); - ASSERT_SUCCESS(queue.Destroy()); - ASSERT_EQ(SysBufferB.As()[0], 0x01010101); + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_EQ(SysBufferB.As()[0], 0x01010101); if (!is_dgpu()) - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As())); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As())); TEST_END } @@ -378,7 +378,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { delete[] pages[order].pointers; } - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index dfd279ef87..8b25ad7f8f 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -177,8 +177,8 @@ TEST_F(KFDMemoryTest, MMapLarge) { << "GB system memory to gpu" << std::endl; while (i--) { - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(AlternateVAGPU[i]))); - ASSERT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(AlternateVAGPU[i]))); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(AlternateVAGPU[i]))); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(AlternateVAGPU[i]))); } munmap(addr, s); @@ -253,8 +253,8 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { /* Fill src buffer so shader quits */ srcBuffer.Fill(0x5678); WaitOnValue(dstBuffer.As(), 0x5678); - ASSERT_EQ(*dstBuffer.As(), 0x5678); - ASSERT_SUCCESS(pm4Queue.Destroy()); + EXPECT_EQ(*dstBuffer.As(), 0x5678); + EXPECT_SUCCESS(pm4Queue.Destroy()); TEST_END } @@ -273,9 +273,9 @@ TEST_F(KFDMemoryTest , MapMemoryToGPU) { // verify that pDb is not null before it's being used ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(pDb, PAGE_SIZE, NULL)); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); // Release the buffers - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, PAGE_SIZE)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, PAGE_SIZE)); TEST_END } @@ -335,7 +335,7 @@ TEST_F(KFDMemoryTest, AccessPPRMem) { WaitOnValue(destBuf, 0xABCDEF09); WaitOnValue(destBuf + 1, 0x12345678); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); /* This sleep hides the dmesg PPR message storm on Raven, which happens * when the CPU buffer is freed before the excessive PPRs are all @@ -402,7 +402,7 @@ TEST_F(KFDMemoryTest, MemoryRegister) { sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaBuffer.As(), 0x12345678)); sdmaQueue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(&stackData[sdmaOffset], 0x12345678)); + EXPECT_TRUE(WaitOnValue(&stackData[sdmaOffset], 0x12345678)); /* Fork a child process to mark pages as COW */ pid_t pid = fork(); @@ -446,12 +446,12 @@ TEST_F(KFDMemoryTest, MemoryRegister) { sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaBuffer.As(), 0xD0BED0BE)); sdmaQueue.Wait4PacketConsumption(); - ASSERT_SUCCESS(pm4Queue.Destroy()); - ASSERT_SUCCESS(sdmaQueue.Destroy()); + EXPECT_SUCCESS(pm4Queue.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); - ASSERT_EQ(0xD00BED00, globalData); - ASSERT_EQ(0xD00BED00, stackData[dstOffset]); - ASSERT_EQ(0xD0BED0BE, stackData[sdmaOffset]); + EXPECT_EQ(0xD00BED00, globalData); + EXPECT_EQ(0xD00BED00, stackData[dstOffset]); + EXPECT_EQ(0xD0BED0BE, stackData[sdmaOffset]); TEST_END } @@ -557,9 +557,9 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(1), SCRATCH_SLICE_SIZE, NULL)); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(1))); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(2))); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(0))); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(1))); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(2))); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(scratchBuffer.As() + SCRATCH_SLICE_OFFSET(0))); // Map everything for test below ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(scratchBuffer.As(), SCRATCH_SIZE, NULL)); @@ -622,7 +622,7 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { dispatchScratchToDst.Sync(); // Check that the scratch buffer contents were correctly copied over to the system memory buffer - ASSERT_EQ(dstMemBuffer.As()[0], 0x01010101); + EXPECT_EQ(dstMemBuffer.As()[0], 0x01010101); } } @@ -701,12 +701,12 @@ void KFDMemoryTest::BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granular ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode)); if (ret) { - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); highMB = sizeMB; continue; } - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); lowMB = sizeMB; lastTestedSize = sizeMB; @@ -757,12 +757,12 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode)); if (ret) { - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); highMB = sizeMB; continue; } - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); lowMB = sizeMB; lastTestedSize = sizeMB; @@ -828,7 +828,7 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { ret = hsaKmtMapMemoryToGPUNodes(pDb_array[i], block_size, &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode)); if (ret) { - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); break; } } while (++i < ARRAY_ENTRIES); @@ -837,8 +837,8 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { << block_size_mb << "MB" << std::endl; while (i--) { - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[i])); - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[i])); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); } TEST_END @@ -954,7 +954,7 @@ TEST_F(KFDMemoryTest, MMBench) { /* Unmap from GPU */ start = GetSystemTickCountInMicroSec(); for (i = 0; i < nBufs; i++) { - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); INTERLEAVE_SDMA(); } unmap1Time = GetSystemTickCountInMicroSec() - start; @@ -972,7 +972,7 @@ TEST_F(KFDMemoryTest, MMBench) { /* Unmap from all GPUs */ start = GetSystemTickCountInMicroSec(); for (i = 0; i < nBufs; i++) { - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); INTERLEAVE_SDMA(); } unmapAllTime = GetSystemTickCountInMicroSec() - start; @@ -981,7 +981,7 @@ TEST_F(KFDMemoryTest, MMBench) { /* Free */ start = GetSystemTickCountInMicroSec(); for (i = 0; i < nBufs; i++) { - ASSERT_SUCCESS(hsaKmtFreeMemory(bufs[i], bufSize)); + EXPECT_SUCCESS(hsaKmtFreeMemory(bufs[i], bufSize)); INTERLEAVE_SDMA(); } freeTime = GetSystemTickCountInMicroSec() - start; @@ -1182,8 +1182,8 @@ TEST_F(KFDMemoryTest, PtraceAccess) { HSAuint8 *addr = reinterpret_cast(reinterpret_cast(mem[0]) + i) + i; errno = 0; long data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); - ASSERT_EQ(0, errno); - ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, reinterpret_cast(data))); if (mem[1] == NULL) @@ -1192,8 +1192,8 @@ TEST_F(KFDMemoryTest, PtraceAccess) { addr = reinterpret_cast(reinterpret_cast(mem[1]) + i) + i; errno = 0; data = ptrace(PTRACE_PEEKDATA, tracePid, addr, NULL); - ASSERT_EQ(0, errno); - ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, addr + PAGE_SIZE, reinterpret_cast(data))); } } catch (...) { @@ -1321,17 +1321,17 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { /* Peek the memory */ errno = 0; HSAint64 data0 = ptrace(PTRACE_PEEKDATA, tracePid, mem0, NULL); - ASSERT_EQ(0, errno); - ASSERT_EQ(data[0], data0); + EXPECT_EQ(0, errno); + EXPECT_EQ(data[0], data0); HSAint64 data1 = ptrace(PTRACE_PEEKDATA, tracePid, mem1, NULL); - ASSERT_EQ(0, errno); - ASSERT_EQ(data[1], data1); + EXPECT_EQ(0, errno); + EXPECT_EQ(data[1], data1); /* Swap mem0 and mem1 by poking */ - ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, reinterpret_cast(data[1]))); - ASSERT_EQ(0, errno); - ASSERT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, reinterpret_cast(data[0]))); - ASSERT_EQ(0, errno); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem0, reinterpret_cast(data[1]))); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ptrace(PTRACE_POKEDATA, tracePid, mem1, reinterpret_cast(data[0]))); + EXPECT_EQ(0, errno); } catch (...) { err = 1; } @@ -1360,19 +1360,19 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { dispatch0.SetArgs(mem0, dstBuffer.As()); dispatch0.Submit(queue); dispatch0.Sync(); - ASSERT_EQ(data1[0], dstBuffer.As()[0]); + EXPECT_EQ(data1[0], dstBuffer.As()[0]); Dispatch dispatch1(isaBuffer); dispatch1.SetArgs(mem1, dstBuffer.As()); dispatch1.Submit(queue); dispatch1.Sync(); WaitOnValue(dstBuffer.As(), data0[0]); - ASSERT_EQ(data0[0], dstBuffer.As()[0]); + EXPECT_EQ(data0[0], dstBuffer.As()[0]); // Clean up - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(mem)); - ASSERT_SUCCESS(hsaKmtFreeMemory(mem, size)); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(mem)); + EXPECT_SUCCESS(hsaKmtFreeMemory(mem, size)); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -1414,12 +1414,12 @@ TEST_F(KFDMemoryTest, SignalHandling) { ASSERT_SUCCESS(hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, reinterpret_cast(&pDb))); // Verify that pDb is not null before it's being used - ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; + EXPECT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; pid_t childPid = fork(); ASSERT_GE(childPid, 0); if (childPid == 0) { - ASSERT_EQ(0, kill(ParentPid, SIGUSR1)); + EXPECT_EQ(0, kill(ParentPid, SIGUSR1)); exit(0); } else { LOG() << "Start Memory Mapping..." << std::endl; @@ -1428,21 +1428,21 @@ TEST_F(KFDMemoryTest, SignalHandling) { int childStatus; // Parent process, just wait for the child to finish - ASSERT_EQ(childPid, waitpid(childPid, &childStatus, 0)); - ASSERT_NE(0, WIFEXITED(childStatus)); - ASSERT_EQ(0, WEXITSTATUS(childStatus)); + EXPECT_EQ(childPid, waitpid(childPid, &childStatus, 0)); + EXPECT_NE(0, WIFEXITED(childStatus)); + EXPECT_EQ(0, WEXITSTATUS(childStatus)); } pDb[0] = 0x02020202; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(pDb, 0x01010101) ); queue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(pDb, 0x01010101)); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_TRUE(WaitOnValue(pDb, 0x01010101)); + EXPECT_SUCCESS(queue.Destroy()); - ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); // Release the buffers - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); TEST_END } @@ -1497,7 +1497,7 @@ TEST_F(KFDMemoryTest, CheckZeroInitializationSysMem) { EXPECT_EQ(0, pDb[size-1]); pDb[size-1] = size; - ASSERT_SUCCESS(hsaKmtFreeMemory(pDb, sysBufSize)); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, sysBufSize)); } TEST_END @@ -1561,7 +1561,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - ASSERT_NE(tmp, MAP_FAILED); + EXPECT_NE(tmp, MAP_FAILED); memset(tmp, 0, tmpBufferSize); LOG() << "Test (avg. ns)\t memcpyRTime memcpyWTime accessRTime accessWTime" << std::endl; @@ -1616,7 +1616,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { accessRTime = GetSystemTickCountInMicroSec() - start; for (i = 0; i < nBufs; i++) - ASSERT_SUCCESS(hsaKmtFreeMemory(bufs[i], bufSize)); + EXPECT_SUCCESS(hsaKmtFreeMemory(bufs[i], bufSize)); LOG() << std::dec << std::setiosflags(std::ios::right) << std::setw(3) << (bufSize >> 10) << "K-" diff --git a/tests/kfdtest/src/KFDPMTest.cpp b/tests/kfdtest/src/KFDPMTest.cpp index c45e190e22..734b32ba51 100644 --- a/tests/kfdtest/src/KFDPMTest.cpp +++ b/tests/kfdtest/src/KFDPMTest.cpp @@ -64,7 +64,7 @@ void KFDPMTest::TearDown() { TEST_F(KFDPMTest, SuspendWithActiveProcess) { TEST_START(TESTPROFILE_RUNALL) - ASSERT_EQ(true, SuspendAndWakeUp()); + EXPECT_EQ(true, SuspendAndWakeUp()); TEST_END } @@ -78,9 +78,9 @@ TEST_F(KFDPMTest, SuspendWithIdleQueue) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - ASSERT_EQ(true, SuspendAndWakeUp()); + EXPECT_EQ(true, SuspendAndWakeUp()); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -103,18 +103,18 @@ TEST_F(KFDPMTest, SuspendWithIdleQueueAfterWork) { destBuffer.Fill(0); - ASSERT_EQ(true, SuspendAndWakeUp()); + EXPECT_EQ(true, SuspendAndWakeUp()); queue.PlaceAndSubmitPacket(PM4WriteDataPacket(&(destBuffer.As()[2]), 0x3, 0x4)); queue.Wait4PacketConsumption(); - ASSERT_EQ(destBuffer.As()[0], 0); - ASSERT_EQ(destBuffer.As()[1], 0); + EXPECT_EQ(destBuffer.As()[0], 0); + EXPECT_EQ(destBuffer.As()[1], 0); WaitOnValue(&(destBuffer.As()[2]), 0x3); WaitOnValue(&(destBuffer.As()[3]), 0x4); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } diff --git a/tests/kfdtest/src/KFDPNPTest.cpp b/tests/kfdtest/src/KFDPNPTest.cpp index f2a0bb7ad3..fc53f891ab 100644 --- a/tests/kfdtest/src/KFDPNPTest.cpp +++ b/tests/kfdtest/src/KFDPNPTest.cpp @@ -88,12 +88,12 @@ TEST_F(KFDPNPTest, DisableAndCreateQueue) { WaitOnValue(&(destBuffer.As()[0]), 0x1); WaitOnValue(&(destBuffer.As()[1]), 0x2); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); DisableKfd(); EnableKfd(); - ASSERT_NE(HSAKMT_STATUS_SUCCESS, queue.Create(defaultGPUNode)) + EXPECT_NE(HSAKMT_STATUS_SUCCESS, queue.Create(defaultGPUNode)) << "Queue creation should fail after a topology change."; TEST_END diff --git a/tests/kfdtest/src/KFDPerfCounters.cpp b/tests/kfdtest/src/KFDPerfCounters.cpp index 22341458b6..ca0bb2c129 100644 --- a/tests/kfdtest/src/KFDPerfCounters.cpp +++ b/tests/kfdtest/src/KFDPerfCounters.cpp @@ -176,7 +176,7 @@ TEST_F(KFDPerfCountersTest, RegisterTrace) { block->NumConcurrent, block->Counters, &root)); - ASSERT_SUCCESS(hsaKmtPmcUnregisterTrace(defaultGPUNode, root.TraceId)); + EXPECT_SUCCESS(hsaKmtPmcUnregisterTrace(defaultGPUNode, root.TraceId)); TEST_END } @@ -237,7 +237,7 @@ TEST_F(KFDPerfCountersTest, StartStopQueryTrace) { /* Delay between START and STOP tracing */ Delay(START_STOP_DELAY); - /* Stopping th trace */ + /* Stopping the trace */ ASSERT_SUCCESS(hsaKmtPmcStopTrace(root.TraceId)); /* Querying the trace */ @@ -247,9 +247,9 @@ TEST_F(KFDPerfCountersTest, StartStopQueryTrace) { LOG() << "Counter " << std::dec << i << ": " << *buf << std::endl; /* Releasing the trace */ - ASSERT_SUCCESS(hsaKmtPmcReleaseTraceAccess(0, root.TraceId)); + EXPECT_SUCCESS(hsaKmtPmcReleaseTraceAccess(0, root.TraceId)); - ASSERT_SUCCESS(hsaKmtPmcUnregisterTrace(defaultGPUNode, root.TraceId)); + EXPECT_SUCCESS(hsaKmtPmcUnregisterTrace(defaultGPUNode, root.TraceId)); TEST_END } diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 1e3fb65eca..c0d8478e95 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -74,7 +74,7 @@ TEST_F(KFDQMTest, CreateCpQueue) { WaitOnValue(destBuf.As(), 0); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -97,9 +97,9 @@ TEST_F(KFDQMTest, CreateSdmaQueue) { queue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(destBuf.As(), 0x02020202)); + EXPECT_TRUE(WaitOnValue(destBuf.As(), 0x02020202)); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -137,14 +137,14 @@ TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { queues[qidx].Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); + EXPECT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); - ASSERT_SUCCESS(memcmp( + EXPECT_SUCCESS(memcmp( destBuf.As(), srcBuf.As(), bufSize)); } for (unsigned int qidx = 0; qidx < MAX_SDMA_QUEUES; ++qidx) - ASSERT_SUCCESS(queues[qidx].Destroy()); + EXPECT_SUCCESS(queues[qidx].Destroy()); TEST_END } @@ -205,9 +205,9 @@ TEST_F(KFDQMTest, SdmaConcurrentCopies) { queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(srcBuf.As(), 0x02020202)); queue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(srcBuf.As(), 0x02020202)); + EXPECT_TRUE(WaitOnValue(srcBuf.As(), 0x02020202)); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -238,7 +238,7 @@ TEST_F(KFDQMTest, CreateMultipleCpQueues) { } for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) - ASSERT_SUCCESS(queues[qidx].Destroy()); + EXPECT_SUCCESS(queues[qidx].Destroy()); TEST_END } @@ -265,23 +265,23 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) { destBuf.Fill(0xFFFFFFFF); - ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true)); + EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true)); queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 1, 1)); // Don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); - ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) + EXPECT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; - ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); + EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); queue.Wait4PacketConsumption(); WaitOnValue(destBuf.As(), 1); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -306,23 +306,23 @@ TEST_F(KFDQMTest, DisableSdmaQueueByUpdateWithNullAddress) { destBuf.Fill(0xFFFFFFFF); - ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true)); + EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true)); queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0)); // Don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); - ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) + EXPECT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; - ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); + EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); queue.Wait4PacketConsumption(); WaitOnValue(destBuf.As(), 0); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -353,23 +353,23 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) { destBuf.Fill(0xFFFFFFFF); - ASSERT_SUCCESS(queue.Update(0/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false)); + EXPECT_SUCCESS(queue.Update(0/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false)); queue.PlaceAndSubmitPacket(packet2); // Don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); - ASSERT_EQ(destBuf.As()[0], 0xFFFFFFFF) + EXPECT_EQ(destBuf.As()[0], 0xFFFFFFFF) << "Packet executed even though the queue is supposed to be disabled!"; - ASSERT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); + EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); queue.Wait4PacketConsumption(); WaitOnValue(destBuf.As(), 1); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -401,8 +401,8 @@ TEST_F(KFDQMTest, CreateQueueStressSingleThreaded) { ASSERT_SUCCESS(queues[firstToCreate]->Create(defaultGPUNode)); ASSERT_SUCCESS(queues[secondToCreate]->Create(defaultGPUNode)); - ASSERT_SUCCESS(queues[firstToDestroy]->Destroy()); - ASSERT_SUCCESS(queues[secondToDestroy]->Destroy()); + EXPECT_SUCCESS(queues[firstToDestroy]->Destroy()); + EXPECT_SUCCESS(queues[secondToDestroy]->Destroy()); delete queues[0]; delete queues[1]; @@ -453,10 +453,10 @@ TEST_F(KFDQMTest, OverSubscribeCpQueues) { Delay(5000); for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) - ASSERT_TRUE(queues[qidx].AllPacketsSubmitted())<< "QueueId=" << qidx;; + EXPECT_TRUE(queues[qidx].AllPacketsSubmitted())<< "QueueId=" << qidx;; for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) - ASSERT_SUCCESS(queues[qidx].Destroy()); + EXPECT_SUCCESS(queues[qidx].Destroy()); TEST_END } @@ -658,7 +658,7 @@ TEST_F(KFDQMTest, BasicCuMaskingLinear) { LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; - ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); + EXPECT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); } } else { LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl; @@ -729,7 +729,7 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; - ASSERT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); + EXPECT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); } } else { LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl; @@ -772,7 +772,7 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { for (i = 0; i < 2; i++) { syncBuffer[i] = -1; - EXPECT_SUCCESS(queue[i].Create(node)); + ASSERT_SUCCESS(queue[i].Create(node)); queue[i].Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, priority[i], false); pHsaEvent[i] = dispatch[i].GetHsaEvent(); pHsaEvent[i]->EventData.EventData.SyncVar.SyncVar.UserData = &syncBuffer[i]; @@ -899,7 +899,7 @@ void KFDQMTest::SyncDispatch(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, vo dispatch.Submit(queue); dispatch.Sync(); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); } TEST_F(KFDQMTest, EmptyDispatch) { @@ -933,7 +933,7 @@ TEST_F(KFDQMTest, SimpleWriteDispatch) { SyncDispatch(isaBuffer, srcBuffer.As(), destBuffer.As()); - ASSERT_EQ(destBuffer.As()[0], 0x01010101); + EXPECT_EQ(destBuffer.As()[0], 0x01010101); TEST_END } @@ -982,7 +982,7 @@ TEST_F(KFDQMTest, MultipleCpQueuesStressDispatch) { } for (i = 0; i < MAX_CP_QUEUES; ++i) { dispatch[i]->Sync(); - ASSERT_EQ(dst[i], src[i]); + EXPECT_EQ(dst[i], src[i]); delete dispatch[i]; } ++numIter; @@ -993,7 +993,7 @@ TEST_F(KFDQMTest, MultipleCpQueuesStressDispatch) { LOG() << "Total iterated : " << std::dec << numIter << std::endl; for (i = 0; i < MAX_CP_QUEUES; ++i) - ASSERT_SUCCESS(queues[i].Destroy()); + EXPECT_SUCCESS(queues[i].Destroy()); TEST_END } @@ -1051,7 +1051,7 @@ TEST_F(KFDQMTest, CreateAqlCpQueue) { ASSERT_SUCCESS(queue.Create(defaultGPUNode, PAGE_SIZE, pointers.As())); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -1126,7 +1126,7 @@ TEST_F(KFDQMTest, QueueLatency) { do { HSAint64 queue_latency = qts[i] - ts[i].GPUClockCounter; - ASSERT_GE(queue_latency, 0); + EXPECT_GE(queue_latency, 0); queue_latency_arr[i] = queue_latency; if (i >= skip) @@ -1155,7 +1155,7 @@ TEST_F(KFDQMTest, QueueLatency) { workload = qts[slots - 1] - qts[skip]; workload /= (slots - 1 - skip); - ASSERT_GE(workload, 0); + EXPECT_GE(workload, 0); i = 0; do { @@ -1207,7 +1207,7 @@ TEST_F(KFDQMTest, CpQueueWraparound) { WaitOnValue(destBuf.As(), pktIdx); } - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -1235,9 +1235,9 @@ TEST_F(KFDQMTest, SdmaQueueWraparound) { SDMAWriteDataPacket(destBuf.As() + bufSize/4, 0x02020202)); queue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); + EXPECT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); - ASSERT_SUCCESS(memcmp( + EXPECT_SUCCESS(memcmp( destBuf.As(), srcBuf.As(), bufSize)); } @@ -1247,7 +1247,7 @@ TEST_F(KFDQMTest, SdmaQueueWraparound) { WaitOnValue(destBuf.As(), pktIdx); } - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -1355,7 +1355,7 @@ TEST_F(KFDQMTest, mGPUShareBO) { m_pIsaGen->GetCopyDwordIsa(isaBufferDst); SyncDispatch(isaBufferDst, shared_addr.As(), dstNodeMem.As(), dst_node); - ASSERT_EQ(dstNodeMem.As()[0], 0x05050505); + EXPECT_EQ(dstNodeMem.As()[0], 0x05050505); EXPECT_SUCCESS(shared_addr.UnmapMemToNodes(&dst_node, 1)); @@ -1370,7 +1370,7 @@ static void sdma_copy(HSAint32 node, void *src, void *const dst[], int n, unsign ASSERT_SUCCESS(sdmaQueue.Create(node)); sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(dst, src, n, size)); sdmaQueue.Wait4PacketConsumption(); - ASSERT_SUCCESS(sdmaQueue.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); ROUTINE_END; } @@ -1382,7 +1382,7 @@ static void sdma_fill(HSAint32 node, void *dst, unsigned int data, unsigned int ASSERT_SUCCESS(sdmaQueue.Create(node)); sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(dst, data, size)); sdmaQueue.Wait4PacketConsumption(); - ASSERT_SUCCESS(sdmaQueue.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); ROUTINE_END; } @@ -1485,8 +1485,8 @@ TEST_F(KFDQMTest, P2PTest) { sdma_copy(cur, src, dst_array, n, size); /* Verify the data*/ - ASSERT_EQ(sysBuf[0], MAGIC_NUM); - ASSERT_EQ(sysBuf[end], MAGIC_NUM); + EXPECT_EQ(sysBuf[0], MAGIC_NUM); + EXPECT_EQ(sysBuf[end], MAGIC_NUM); LOG() << "PASS " << cur << " -> " << next << std::endl; @@ -1522,11 +1522,11 @@ TEST_F(KFDQMTest, SdmaEventInterrupt) { queue.Wait4PacketConsumption(); - ASSERT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); hsaKmtDestroyEvent(event); - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); TEST_END } @@ -1610,11 +1610,11 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { } /* Check that the PM4 packet has been executed */ - ASSERT_TRUE(WaitOnValue(destBuf.As(), 0x12345678)); - ASSERT_TRUE(WaitOnValue(destBuf.As()+1, 0x87654321)); + EXPECT_TRUE(WaitOnValue(destBuf.As(), 0x12345678)); + EXPECT_TRUE(WaitOnValue(destBuf.As()+1, 0x87654321)); - ASSERT_SUCCESS(pm4Queue.Destroy()); - ASSERT_SUCCESS(otherQueue.Destroy()); + EXPECT_SUCCESS(pm4Queue.Destroy()); + EXPECT_SUCCESS(otherQueue.Destroy()); TEST_END } diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index c50dcb811c..4c80a04ae6 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -233,7 +233,7 @@ void HsaMemoryBuffer::Fill(HSAuint32 value, BaseQueue& baseQueue, HSAuint64 offs baseQueue.PlacePacket(SDMAFillDataPacket((reinterpret_cast(this->As() + offset)), value, size)); baseQueue.PlacePacket(SDMAFencePacket(reinterpret_cast(event->EventData.HWData2), event->EventId)); baseQueue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); - ASSERT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); hsaKmtDestroyEvent(event); } diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index 1bd70bebd1..95b7a9297b 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -254,7 +254,7 @@ void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned i */ m_pPacketData = reinterpret_cast(malloc(m_packetSize)); - ASSERT_NOTNULL(m_pPacketData); + EXPECT_NOTNULL(m_pPacketData); m_packetDataAllocated = true; diff --git a/tests/kfdtest/src/RDMATest.cpp b/tests/kfdtest/src/RDMATest.cpp index 59a25f08f6..973dbdcdc8 100644 --- a/tests/kfdtest/src/RDMATest.cpp +++ b/tests/kfdtest/src/RDMATest.cpp @@ -89,7 +89,7 @@ TEST_F(RDMATest, GPUDirect) { dispatch.Submit(queue); dispatch.Sync(g_TestTimeOut); // GPU executed the command - ASSERT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue.Destroy()); LocalMemoryAccess Rdma; @@ -109,7 +109,7 @@ TEST_F(RDMATest, GPUDirect) { /* Read the memory to confirm that application can read the local memory * correctly from the mapped address. */ - ASSERT_EQ(memcmp(gpuAddr, srcSysBuffer.As(), 4), 0); + EXPECT_EQ(memcmp(gpuAddr, srcSysBuffer.As(), 4), 0); Rdma.UnMap(gpuAddr, PAGE_SIZE); Rdma.Close(); From 3437a356c7330b81afc969740b58a5e2b8142cd3 Mon Sep 17 00:00:00 2001 From: Mike Li Date: Tue, 14 Aug 2018 13:39:22 -0400 Subject: [PATCH 0397/1247] Decouple user NodeID and sysfs NodeID Currently, all HSA nodes are exposed to user. So the existing implementation assumes a one to one mapping between user NodeId and sysfs nodeId. GPU Resource Management will provide control over the exposed HSA nodes. This means not all HSA nodes will be exposed to the user. Decouple it. The mapping from user NodeId to sysfs NodeId will be local to topology.c and topology helper functions. For others NodeId should be sequential from 0 to Number of Nodes exposed to user. v1: initial implementation v2: map node id within the topology_* functions v3: remove two static globals v4: add bounds check got node id Change-Id: Id12147ece41d682430f398944bbb339ca906eb1b Signed-off-by: Mike Li --- src/fmm.c | 2 +- src/fmm.h | 2 + src/libhsakmt.h | 1 - src/topology.c | 212 ++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 190 insertions(+), 27 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 12c0be415d..c4abad2081 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1659,7 +1659,7 @@ static HSAKMT_STATUS get_process_apertures( #define DRM_LAST_RENDER_NODE 255 static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE]; -static int open_drm_render_device(int minor) +int open_drm_render_device(int minor) { char path[128]; int index, fd; diff --git a/src/fmm.h b/src/fmm.h index 2154f5250b..1faecdd207 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -85,4 +85,6 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address); + +int open_drm_render_device(int minor); #endif /* FMM_H_ */ diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 3186787c26..9646190609 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -115,7 +115,6 @@ int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); -HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, struct pci_access* pacc); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); diff --git a/src/topology.c b/src/topology.c index 53ea9b5b0b..0b04ad30bd 100644 --- a/src/topology.c +++ b/src/topology.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "libhsakmt.h" #include "fmm.h" @@ -59,6 +60,12 @@ static HsaSystemProperties *_system = NULL; static node_props_t *g_props; static int is_valgrind; +/* This array caches sysfs based node IDs of CPU nodes + all supported GPU nodes. + * It will be used to map user-node IDs to sysfs-node IDs. + */ +static uint32_t *map_user_to_sysfs_node_id; +static uint32_t map_user_to_sysfs_node_id_size; + static int processor_vendor; /* Supported System Vendors */ enum SUPPORTED_PROCESSOR_VENDORS { @@ -473,6 +480,113 @@ err: return ret; } +static HSAKMT_STATUS topology_sysfs_map_node_id(uint32_t node_id, uint32_t *sys_node_id) +{ + if ((!map_user_to_sysfs_node_id) || (node_id >= map_user_to_sysfs_node_id_size)) + return HSAKMT_STATUS_NOT_SUPPORTED; + + *sys_node_id = map_user_to_sysfs_node_id[node_id]; + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t sysfs_node_id, uint32_t *gpu_id) +{ + FILE *fd; + char path[256]; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + assert(gpu_id); + snprintf(path, 256, "%s/%d/gpu_id", KFD_SYSFS_PATH_NODES, sysfs_node_id); + fd = fopen(path, "r"); + if (!fd) + return HSAKMT_STATUS_ERROR; + if (fscanf(fd, "%ul", gpu_id) != 1) + ret = HSAKMT_STATUS_ERROR; + fclose(fd); + + return ret; +} + +/* Check if the sysfs node is supported. This function will be passed with sysfs node id. + * This function can not use topology_* help functions, because those functions are + * using user node id. A sysfs node is supported only if corresponding drm render node is available. + */ +static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id, bool *is_node_supported) +{ + uint32_t gpu_id; + FILE *fd; + char *read_buf, *p; + int read_size; + char prop_name[256]; + char path[256]; + unsigned long long prop_val; + uint32_t prog; + uint32_t drm_render_minor = 0; + int ret_value; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + *is_node_supported = false; + + /* Retrieve the GPU ID */ + ret = topology_sysfs_get_gpu_id(sysfs_node_id, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + if (gpu_id == 0) { + *is_node_supported = true; + return HSAKMT_STATUS_SUCCESS; + } + + read_buf = malloc(PAGE_SIZE); + if (!read_buf) + return HSAKMT_STATUS_NO_MEMORY; + + /* Retrieve the node properties */ + snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, sysfs_node_id); + fd = fopen(path, "r"); + if (!fd) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + + read_size = fread(read_buf, 1, PAGE_SIZE, fd); + if (read_size <= 0) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + + /* Since we're using the buffer as a string, we make sure the string terminates */ + if (read_size >= PAGE_SIZE) + read_size = PAGE_SIZE - 1; + read_buf[read_size] = 0; + + /* Read the node properties */ + prog = 0; + p = read_buf; + while (sscanf(p += prog, "%s %llu\n%n", prop_name, &prop_val, &prog) == 2) { + if (strcmp(prop_name, "drm_render_minor") == 0) { + drm_render_minor = (int32_t)prop_val; + break; + } + } + if (!drm_render_minor) { + ret = HSAKMT_STATUS_ERROR; + goto err; + } + + /* Open DRM Render device */ + ret_value = open_drm_render_device(drm_render_minor); + if (ret_value > 0) + *is_node_supported = true; + else if (ret_value != -ENOENT) + ret = HSAKMT_STATUS_ERROR; + +err: + free(read_buf); + fclose(fd); + return ret; +} + HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) { FILE *fd; @@ -482,6 +596,9 @@ HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t num_sysfs_nodes = 0; + bool is_node_supported = true; + uint32_t num_supported_nodes = 0; assert(props); @@ -519,12 +636,48 @@ HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) } /* - * Discover the number of nodes: + * Discover the number of sysfs nodes: * Assuming that inside nodes folder there are only folders * which represent the node numbers */ - props->NumNodes = num_subdirs(KFD_SYSFS_PATH_NODES, ""); + num_sysfs_nodes = num_subdirs(KFD_SYSFS_PATH_NODES, ""); + if (map_user_to_sysfs_node_id == NULL) { + /* Trade off - num_sysfs_nodes includes all CPU and GPU nodes. + * Slightly more memory is allocated than necessary. + */ + map_user_to_sysfs_node_id = calloc(num_sysfs_nodes, sizeof(uint32_t)); + if (map_user_to_sysfs_node_id == NULL) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err2; + } + map_user_to_sysfs_node_id_size = num_sysfs_nodes; + } else if (num_sysfs_nodes > map_user_to_sysfs_node_id_size) { + free(map_user_to_sysfs_node_id); + map_user_to_sysfs_node_id = calloc(num_sysfs_nodes, sizeof(uint32_t)); + if (map_user_to_sysfs_node_id == NULL) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err2; + } + map_user_to_sysfs_node_id_size = num_sysfs_nodes; + } + + for (uint32_t i = 0; i < num_sysfs_nodes; i++) { + ret = topology_sysfs_check_node_supported(i, &is_node_supported); + if (ret != HSAKMT_STATUS_SUCCESS) + goto sysfs_parse_failed; + if (is_node_supported) + map_user_to_sysfs_node_id[num_supported_nodes++] = i; + } + props->NumNodes = num_supported_nodes; + + free(read_buf); + fclose(fd); + return ret; + +sysfs_parse_failed: + free(map_user_to_sysfs_node_id); + map_user_to_sysfs_node_id = NULL; err2: free(read_buf); err1: @@ -532,24 +685,6 @@ err1: return ret; } -HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t node_id, uint32_t *gpu_id) -{ - FILE *fd; - char path[256]; - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - - assert(gpu_id); - snprintf(path, 256, "%s/%d/gpu_id", KFD_SYSFS_PATH_NODES, node_id); - fd = fopen(path, "r"); - if (!fd) - return HSAKMT_STATUS_ERROR; - if (fscanf(fd, "%ul", gpu_id) != 1) - ret = HSAKMT_STATUS_ERROR; - fclose(fd); - - return ret; -} - static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id) { uint32_t i, table_size; @@ -737,20 +872,25 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, const struct hsa_gfxip_table *hsa_gfxip; char namebuf[HSA_PUBLIC_NAME_SIZE]; const char *name; + uint32_t sys_node_id; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); assert(gpu_id); + ret = topology_sysfs_map_node_id(node_id, &sys_node_id); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + /* Retrieve the GPU ID */ - ret = topology_sysfs_get_gpu_id(node_id, gpu_id); + ret = topology_sysfs_get_gpu_id(sys_node_id, gpu_id); read_buf = malloc(PAGE_SIZE); if (!read_buf) return HSAKMT_STATUS_NO_MEMORY; /* Retrieve the node properties */ - snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, node_id); + snprintf(path, 256, "%s/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id); fd = fopen(path, "r"); if (!fd) { free(read_buf); @@ -907,9 +1047,14 @@ static HSAKMT_STATUS topology_sysfs_get_mem_props(uint32_t node_id, uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t sys_node_id; assert(props); - snprintf(path, 256, "%s/%d/mem_banks/%d/properties", KFD_SYSFS_PATH_NODES, node_id, mem_id); + ret = topology_sysfs_map_node_id(node_id, &sys_node_id); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + snprintf(path, 256, "%s/%d/mem_banks/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, mem_id); fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -1160,9 +1305,14 @@ static HSAKMT_STATUS topology_sysfs_get_cache_props(uint32_t node_id, uint32_t i, prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t sys_node_id; assert(props); - snprintf(path, 256, "%s/%d/caches/%d/properties", KFD_SYSFS_PATH_NODES, node_id, cache_id); + ret = topology_sysfs_map_node_id(node_id, &sys_node_id); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + snprintf(path, 256, "%s/%d/caches/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, cache_id); fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -1233,9 +1383,14 @@ static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + uint32_t sys_node_id; assert(props); - snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, node_id, iolink_id); + ret = topology_sysfs_map_node_id(node_id, &sys_node_id); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; + + snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, iolink_id); fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -1659,6 +1814,13 @@ HSAKMT_STATUS topology_drop_snapshot(void) free(_system); _system = NULL; + + if (map_user_to_sysfs_node_id) { + free(map_user_to_sysfs_node_id); + map_user_to_sysfs_node_id = NULL; + map_user_to_sysfs_node_id_size = 0; + } + err = HSAKMT_STATUS_SUCCESS; out: From 1076075a1c66991a9e9ca0df5a8c1af2c136878e Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Wed, 22 Aug 2018 15:54:54 +0800 Subject: [PATCH 0398/1247] kfdtest: Add some asserts in BigBufferStressTest It should have PASS/FAIL report for the vram allocated size. Change-Id: I546c02c2ed02f1cfb5278e0dfd7b18ade39faafb Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDMemoryTest.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 8b25ad7f8f..878c487afa 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -772,6 +772,12 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, << lastTestedSize << "MB" << std::endl; if (lastSize) *lastSize = lastTestedSize * 1024 * 1024; + + /* Make sure 3/4 vram can be allocated.*/ + EXPECT_GE(lastTestedSize * 4, vramSizeMB * 3); + if (lastTestedSize * 16 < vramSizeMB * 15) + WARN() << "The biggest allocated VRAM buffer size is smaller than the expected " + << vramSizeMB * 15 / 16 << "MB" << std::endl; } /* BigBufferStressTest allocs, maps/unmaps, and frees the biggest possible system From bdb1f8a066bf9f88c29705ea130f3537a3263a5b Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Fri, 24 Aug 2018 11:27:34 +0800 Subject: [PATCH 0399/1247] kfdtest: Let SdmaEventInterrupt test more meaningful Simulate some workload there to verify the sDMA event interrupt. Change-Id: Ib5ad0c238cc66898f7835e765df50427ef106b04 Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDQMTest.cpp | 40 +++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index c0d8478e95..e919acc398 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1509,24 +1509,40 @@ TEST_F(KFDQMTest, SdmaEventInterrupt) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HsaEvent* event = NULL; - ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_NodeInfo.HsaDefaultGPUNode(), &event)); + const HSAuint64 bufSize = 4 << 20; + HsaMemoryBuffer srcBuf(bufSize, 0); // System memory. + HsaMemoryBuffer destBuf(bufSize, defaultGPUNode, true, true); // local vram. - SDMAQueue queue; + /* Two sdma queues will be running at same time.*/ + const int numSDMAQueue = 2; + HsaEvent *event[numSDMAQueue]; + SDMAQueue queue[numSDMAQueue]; - ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + /* A simple loop here to give more pressure.*/ + for (int test_count = 0; test_count < 4096; test_count++) { + for (int i = 0; i < numSDMAQueue; i++) { + ASSERT_SUCCESS(queue[i].Create(defaultGPUNode)); + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event[i])); - queue.PlaceAndSubmitPacket(SDMAFencePacket(reinterpret_cast(event->EventData.HWData2), event->EventId)); + /* Let sDMA have some workload first.*/ + queue[i].PlacePacket( + SDMACopyDataPacket(destBuf.As(), srcBuf.As(), bufSize)); + queue[i].PlacePacket( + SDMAFencePacket(reinterpret_cast(event[i]->EventData.HWData2), event[i]->EventId)); + queue[i].PlacePacket(SDMATrapPacket(event[i]->EventId)); + } - queue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); + for (int i = 0; i < numSDMAQueue; i++) + queue[i].SubmitPacket(); - queue.Wait4PacketConsumption(); + for (int i = 0; i < numSDMAQueue; i++) + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event[i], g_TestTimeOut)); - EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); - - hsaKmtDestroyEvent(event); - - EXPECT_SUCCESS(queue.Destroy()); + for (int i = 0; i < numSDMAQueue; i++) { + EXPECT_SUCCESS(queue[i].Destroy()); + EXPECT_SUCCESS(hsaKmtDestroyEvent(event[i])); + } + } TEST_END } From 3e527bc7e8d7417c13f813e8d1af32722be60e6d Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Thu, 23 Aug 2018 13:27:34 +0800 Subject: [PATCH 0400/1247] kfdtest: add PM4EventInterrupt test Similar with SdmaEventInterrupt, verify event interrupt on pm4 queue. Change-Id: I0e43f26fd0d965126985820704215d2ef5e52c1a Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDQMTest.cpp | 63 +++++++++++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.cpp | 14 +++++++ tests/kfdtest/src/KFDTestUtil.hpp | 2 + 3 files changed, 79 insertions(+) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index e919acc398..bb0196ccf4 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1503,6 +1503,69 @@ TEST_F(KFDQMTest, P2PTest) { TEST_END } +TEST_F(KFDQMTest, PM4EventInterrupt) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + const HSAuint64 bufSize = PAGE_SIZE; + const int packetCount = bufSize / sizeof(unsigned int); + const int totalPacketSize = packetCount * PM4WriteDataPacket(0, 0).SizeInBytes() + + PM4ReleaseMemoryPacket(0, 0, 0).SizeInBytes(); + const int queueSize = RoundToPowerOf2(totalPacketSize); + + /* 4 PM4 queues will be running at same time.*/ + const int numPM4Queue = 4; + HsaEvent *event[numPM4Queue]; + PM4Queue queue[numPM4Queue]; + HsaMemoryBuffer *destBuf[numPM4Queue]; + unsigned int *buf[numPM4Queue]; + + for (int i = 0; i < numPM4Queue; i++) { + destBuf[i] = new HsaMemoryBuffer(bufSize, defaultGPUNode, true, false); // System memory + buf[i] = destBuf[i]->As(); + } + + /* A simple loop here to give more pressure.*/ + for (int test_count = 0; test_count < 1024; test_count++) { + for (int i = 0; i < numPM4Queue; i++) { + ASSERT_SUCCESS(queue[i].Create(defaultGPUNode, queueSize)); + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event[i])); + + /* Let CP have some workload first.*/ + for(int index = 0; index < packetCount; index++) + queue[i].PlacePacket(PM4WriteDataPacket(buf[i] + index, 0xdeadbeaf)); + + /* releaseMemory packet makes sure all previous written data is visible.*/ + queue[i].PlacePacket(PM4ReleaseMemoryPacket(0, + reinterpret_cast(event[i]->EventData.HWData2), + event[i]->EventId, + true)); + } + + for (int i = 0; i < numPM4Queue; i++) + queue[i].SubmitPacket(); + + for (int i = 0; i < numPM4Queue; i++) { + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event[i], g_TestTimeOut)); + EXPECT_EQ(buf[i][0], 0xdeadbeaf); + EXPECT_EQ(buf[i][packetCount - 1], 0xdeadbeaf); + memset(buf[i], 0, bufSize); + } + + for (int i = 0; i < numPM4Queue; i++) { + EXPECT_SUCCESS(queue[i].Destroy()); + EXPECT_SUCCESS(hsaKmtDestroyEvent(event[i])); + } + } + + for (int i = 0; i < numPM4Queue; i++) + delete destBuf[i]; + + TEST_END +} + TEST_F(KFDQMTest, SdmaEventInterrupt) { TEST_START(TESTPROFILE_RUNALL) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 4c80a04ae6..2195d6f37e 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -30,6 +30,20 @@ #include "Dispatch.hpp" #include "SDMAPacket.hpp" +uint64_t RoundToPowerOf2(uint64_t val) { + int bytes = sizeof(uint64_t); + + val--; + + for (int i = 0; i < bytes; i++) { + val |= val >> (1 << i); + } + + val++; + + return val; +} + bool WaitOnValue(const volatile unsigned int *buf, unsigned int value) { unsigned int timeOut = g_TestTimeOut; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 853153ef9b..014b31d13f 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -33,6 +33,8 @@ class BaseQueue; #define ARRAY_SIZE(_x) (sizeof(_x)/sizeof(_x[0])) +uint64_t RoundToPowerOf2(uint64_t val); + // @brief: waits until the value is written to the buffer or until time out if received through args bool WaitOnValue(const volatile unsigned int *buf, unsigned int value); From f9faf05fd9f2a133e238cb3729f76172c5a4d2f4 Mon Sep 17 00:00:00 2001 From: Shaoyun Liu Date: Thu, 9 Aug 2018 17:21:54 -0400 Subject: [PATCH 0401/1247] Thunk: Add xgmi thunk interface definition Add XGMI related defines in thunk according to the document HSAKMT library interface specification v1.16 Change-Id: Ib25ff0ddf7380c97d06bd76fb730915e7c634270 Signed-off-by: Shaoyun Liu --- include/hsakmttypes.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index c43e48911e..a5272151c7 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -273,7 +273,10 @@ typedef struct _HsaNodeProperties // Unicode string HSAuint8 AMDName[HSA_PUBLIC_NAME_SIZE]; //CAL Name of the "device", ASCII HSA_ENGINE_VERSION uCodeEngineVersions; - HSAuint8 Reserved[60]; + HSAuint64 DebugProperties; + HSAuint64 HiveID; // XGMI Hive the GPU node belongs to in the system. It is an opaque and static + // number hash created by the PSP + HSAuint8 Reserved[44]; } HsaNodeProperties; @@ -398,7 +401,12 @@ typedef enum _HSA_IOLINKTYPE { HSA_IOLINK_TYPE_RAPID_IO = 8, HSA_IOLINK_TYPE_INFINIBAND = 9, HSA_IOLINK_TYPE_RESERVED3 = 10, - HSA_IOLINKTYPE_OTHER = 11, + HSA_IOLINK_TYPE_XGMI = 11, + HSA_IOLINK_TYPE_XGOP = 12, + HSA_IOLINK_TYPE_GZ = 13, + HSA_IOLINK_TYPE_ETHERNET_RDMA = 14, + HSA_IOLINK_TYPE_RDMA_OTHER = 15, + HSA_IOLINK_TYPE_OTHER = 16, HSA_IOLINKTYPE_NUMIOLINKTYPES, HSA_IOLINKTYPE_SIZE = 0xFFFFFFFF } HSA_IOLINKTYPE; From a040a2424316a7e727d07e69617b0eac5d18d7bc Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Fri, 24 Aug 2018 11:57:04 +0800 Subject: [PATCH 0402/1247] kfdtest: Let BigBufferStressTest detect memory leak As it will alloc as much as small system memory to reach the allocation limit. We can try to alloc memory several times to see if any allocation in the previous step cause memory leak. Also we test if GPU can access these memory correctly or not. Change-Id: I309f9821b6bc99c212a6bfbc21fe3086ab589fd3 Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDMemoryTest.cpp | 53 +++++++++++++++++++---------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 878c487afa..8617b0893c 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -819,33 +819,50 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { */ #define ARRAY_ENTRIES 2048 - int i = 0; + int i = 0, allocationCount = 0; unsigned int* pDb_array[ARRAY_ENTRIES]; HSAuint64 block_size_mb = 128; HSAuint64 block_size = block_size_mb * 1024 * 1024; + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - do { - ret = hsaKmtAllocMemory(0 /* system */, block_size, m_MemoryFlags, - reinterpret_cast(&pDb_array[i])); - if (ret) { - break; + /* Test 4 times to see if there is any memory leak.*/ + for (int repeat = 1; repeat < 5; repeat++) { + for (i = 0; i < ARRAY_ENTRIES; i++) { + ret = hsaKmtAllocMemory(0 /* system */, block_size, m_MemoryFlags, + reinterpret_cast(&pDb_array[i])); + if (ret) + break; + + ret = hsaKmtMapMemoryToGPUNodes(pDb_array[i], block_size, + &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode)); + if (ret) { + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); + break; + } } - ret = hsaKmtMapMemoryToGPUNodes(pDb_array[i], block_size, - &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode)); - if (ret) { + LOG() << "Allocated system buffers time " << std::dec << repeat << ": " << i << "x" + << block_size_mb << "MB" << std::endl; + + if (allocationCount == 0) + allocationCount = i; + EXPECT_GE(i, allocationCount) << "There might be memory leak!" << std::endl; + + while (i--) { + /* To see if GPU can access the memory correctly*/ + unsigned int *begin = pDb_array[i]; + *begin = 0; + queue.PlaceAndSubmitPacket( + PM4WriteDataPacket(begin, 0xdeadbeaf)); + queue.Wait4PacketConsumption(); + EXPECT_TRUE(WaitOnValue(begin, 0xdeadbeaf)); + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[i])); EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); - break; } - } while (++i < ARRAY_ENTRIES); - - LOG() << "Allocated system buffers: " << std::dec << i << "x" - << block_size_mb << "MB" << std::endl; - - while (i--) { - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[i])); - EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); } + EXPECT_SUCCESS(queue.Destroy()); TEST_END } From 7796994f4691fde8cf002226f4dfc75120f140ad Mon Sep 17 00:00:00 2001 From: Shaoyun Liu Date: Mon, 13 Aug 2018 16:29:16 -0400 Subject: [PATCH 0403/1247] Thunk: Avoid add indirect link for the GPUS with xGMI link Change-Id: I06f511c55e28919512fda79b504566818dc2a5ab Signed-off-by: Shaoyun Liu --- src/topology.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/topology.c b/src/topology.c index 0b04ad30bd..6afb03ab98 100644 --- a/src/topology.c +++ b/src/topology.c @@ -964,6 +964,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->DrmRenderMinor = (int32_t)prop_val; else if (strcmp(prop_name, "sdma_fw_version") == 0) props->uCodeEngineVersions.Value = (uint32_t)prop_val & 0x3ff; + else if (strcmp(prop_name, "hive_id") == 0) + props->HiveID = prop_val; } hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); @@ -1573,6 +1575,11 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, if (!node_props[node1].gpu_id && !node_props[node2].gpu_id) return HSAKMT_STATUS_INVALID_NODE_UNIT; + if (node_props[node1].node.HiveID && + node_props[node2].node.HiveID && + node_props[node1].node.HiveID == node_props[node2].node.HiveID) + return HSAKMT_STATUS_INVALID_PARAMETER; + if (node_props[node1].gpu_id) dir_cpu1 = gpu_get_direct_link_cpu(node1, node_props); if (node_props[node2].gpu_id) From 30a4ab39f39392129c785b3e9b7c2e0c56bafbb4 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Wed, 29 Aug 2018 13:16:38 -0400 Subject: [PATCH 0404/1247] thunk: Avoid create PCIe indirect link on none large bar target PCIe P2P (indirect) IOLinks should only be created if the remote GPU is large-BAR Change-Id: I55cbb5e37c5d41267583e07aca6bdcc708403029 Signed-off-by: shaoyunl --- src/topology.c | 54 +++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/topology.c b/src/topology.c index 6afb03ab98..64872c0b2a 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1479,37 +1479,28 @@ static HsaIoLinkProperties *topology_get_free_io_link_slot_for_node(uint32_t nod } /* topology_add_io_link_for_node - If a free slot is available, - * add io_link for the given Node. If bi_directional is true, set up two - * links for both directions. + * add io_link for the given Node. * TODO: Add other members of HsaIoLinkProperties */ -static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_id, +static HSAKMT_STATUS topology_add_io_link_for_node(uint32_t node_from, const HsaSystemProperties *sys_props, node_props_t *node_props, HSA_IOLINKTYPE IoLinkType, - uint32_t NodeTo, - uint32_t Weight, bool bi_dir) + uint32_t node_to, + uint32_t Weight) { HsaIoLinkProperties *props; - /* If bi-directional is set true, it's two links to add. */ - uint32_t i, num_links = (bi_dir == true) ? 2 : 1; - uint32_t node_from = node_id, node_to = NodeTo; - for (i = 0; i < num_links; i++) { - props = topology_get_free_io_link_slot_for_node(node_from, + props = topology_get_free_io_link_slot_for_node(node_from, sys_props, node_props); - if (!props) - return HSAKMT_STATUS_NO_MEMORY; + if (!props) + return HSAKMT_STATUS_NO_MEMORY; - props->IoLinkType = IoLinkType; - props->NodeFrom = node_from; - props->NodeTo = node_to; - props->Weight = Weight; - node_props[node_from].node.NumIOLinks++; - /* switch direction on the 2nd link when num_links=2 */ - node_from = NodeTo; - node_to = node_id; - } + props->IoLinkType = IoLinkType; + props->NodeFrom = node_from; + props->NodeTo = node_to; + props->Weight = Weight; + node_props[node_from].node.NumIOLinks++; return HSAKMT_STATUS_SUCCESS; } @@ -1564,6 +1555,7 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, int32_t dir_cpu1 = -1, dir_cpu2 = -1; HSAuint32 weight1 = 0, weight2 = 0, weight3 = 0; HSAKMT_STATUS ret; + uint32_t i; *weight = 0; *type = HSA_IOLINKTYPE_UNDEFINED; @@ -1588,6 +1580,15 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, if (dir_cpu1 < 0 && dir_cpu2 < 0) return HSAKMT_STATUS_ERROR; + /* if the node2(dst) is GPU , it need to be large bar for host access*/ + if (node_props[node2].gpu_id) { + for (i = 0; i < node_props[node2].node.NumMemoryBanks; ++i) + if (node_props[node2].mem[i].HeapType == + HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC) + break; + if (i >= node_props[node2].node.NumMemoryBanks) + return HSAKMT_STATUS_ERROR; + } /* Possible topology: * GPU --(weight1) -- CPU -- (weight2) -- GPU * GPU --(weight1) -- CPU -- (weight2) -- CPU -- (weight3) -- GPU @@ -1659,10 +1660,17 @@ static void topology_create_indirect_gpu_links(const HsaSystemProperties *sys_pr for (j = i + 1; j < sys_props->NumNodes; j++) { get_indirect_iolink_info(i, j, node_props, &weight, &type); if (!weight) - continue; + goto try_alt_dir; if (topology_add_io_link_for_node(i, sys_props, node_props, - type, j, weight, true) != HSAKMT_STATUS_SUCCESS) + type, j, weight) != HSAKMT_STATUS_SUCCESS) pr_err("Fail to add IO link %d->%d\n", i, j); +try_alt_dir: + get_indirect_iolink_info(j, i, node_props, &weight, &type); + if (!weight) + continue; + if (topology_add_io_link_for_node(j, sys_props, node_props, + type, i, weight) != HSAKMT_STATUS_SUCCESS) + pr_err("Fail to add IO link %d->%d\n", j, i); } } } From 855f1a32a9bd8599fd728532c0bcf0d8b2165e0e Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 30 Aug 2018 23:44:33 -0400 Subject: [PATCH 0405/1247] libhsakmt: Fix and deduplicate object lookup code Added a helper vm_find_object that can be used everywhere we need to lookup objects by their address and optionally size. This unifies all subtly different, partially incomplete, or broken ways of doing this in various functions: * map * unmap * register * deregister * free * get_mem_info * set_mem_user_data At the same time fix some subtle problems for userptr lookup that got a bit more complex when the userptr address can match the GPU address. Change-Id: I98572d1734fc7688a1d68f6a784e02c8dee90af5 Signed-off-by: Felix Kuehling --- src/fmm.c | 626 ++++++++++++++++++++---------------------------------- 1 file changed, 227 insertions(+), 399 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index c4abad2081..22c572b43e 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1036,6 +1036,115 @@ void fmm_print(uint32_t gpu_id) } #endif +/* vm_find_object - Find a VM object in any aperture + * + * @addr: VM address of the object + * @size: size of the object, 0 means "don't care", + * UINT64_MAX means addr can match any address within the object + * @out_aper: Aperture where the object was found + * + * Returns a pointer to the object if found, NULL otherwise. If an + * object is found, this function returns with the + * (*out_aper)->fmm_mutex locked. + */ +static vm_object_t *vm_find_object(const void *addr, uint64_t size, + manageable_aperture_t **out_aper) +{ + manageable_aperture_t *aper = NULL; + bool range = (size == UINT64_MAX); + bool userptr = false; + vm_object_t *obj = NULL; + uint32_t i; + + for (i = 0; i < gpu_mem_count; i++) + if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && + addr >= gpu_mem[i].gpuvm_aperture.base && + addr <= gpu_mem[i].gpuvm_aperture.limit) { + aper = &gpu_mem[i].gpuvm_aperture; + break; + } + + if (!aper) { + if ((addr >= svm.dgpu_aperture->base) && + (addr <= svm.dgpu_aperture->limit)) + aper = svm.dgpu_aperture; + else if ((addr >= svm.dgpu_alt_aperture->base) && + (addr <= svm.dgpu_alt_aperture->limit)) + aper = svm.dgpu_alt_aperture; + else { + aper = svm.dgpu_aperture; + userptr = true; + } + } + + pthread_mutex_lock(&aper->fmm_mutex); + if (range) { + /* mmap_apertures can have userptrs in them. Try to + * look up addresses as userptrs first to sort out any + * ambiguity of multiple overlapping mappings at + * different GPU addresses. + */ + if (userptr || aper->ops == &mmap_aperture_ops) + obj = vm_find_object_by_userptr_range(aper, addr); + if (!obj && !userptr) + obj = vm_find_object_by_address_range(aper, addr); + } else { + if (userptr || aper->ops == &mmap_aperture_ops) + obj = vm_find_object_by_userptr(aper, addr, size); + if (!obj && !userptr) { + long page_offset = (long)addr & (PAGE_SIZE-1); + const void *page_addr = (const uint8_t *)addr - page_offset; + + obj = vm_find_object_by_address(aper, page_addr, 0); + /* If we find a userptr here, it's a match on + * the aligned GPU address. Make sure that the + * page offset and size match too. + */ + if (obj && obj->userptr && + (((long)obj->userptr & (PAGE_SIZE - 1)) != page_offset || + (size && size != obj->userptr_size))) + obj = NULL; + } + } + + if (!obj && !is_dgpu) { + /* On APUs try finding it in the CPUVM aperture */ + pthread_mutex_unlock(&aper->fmm_mutex); + + aper = &cpuvm_aperture; + + pthread_mutex_lock(&aper->fmm_mutex); + if (range) + obj = vm_find_object_by_address_range(aper, addr); + else + obj = vm_find_object_by_address(aper, addr, 0); + } + + if (obj) { + *out_aper = aper; + return obj; + } + + pthread_mutex_unlock(&aper->fmm_mutex); + return NULL; +} + +static HSAuint8 fmm_check_user_memory(const void *addr, HSAuint64 size) +{ + volatile const HSAuint8 *ptr = addr; + volatile const HSAuint8 *end = ptr + size; + HSAuint8 sum = 0; + + /* Access every page in the buffer to make sure the mapping is + * valid. If it's not, it will die with a segfault that's easy + * to debug. + */ + for (; ptr < end; ptr = (void *)PAGE_ALIGN_UP(ptr + 1)) + sum += *ptr; + + return sum; +} + static void fmm_release_scratch(uint32_t gpu_id) { int32_t gpu_mem_id; @@ -1530,66 +1639,41 @@ static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) HSAKMT_STATUS fmm_release(void *address) { - uint32_t i; - vm_object_t *object = NULL; manageable_aperture_t *aperture = NULL; + vm_object_t *object = NULL; + uint32_t i; - for (i = 0; i < gpu_mem_count; i++) { - if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) - continue; - if (address >= gpu_mem[i].scratch_physical.base && - address <= gpu_mem[i].scratch_physical.limit) { + /* Special handling for scratch memory */ + for (i = 0; i < gpu_mem_count; i++) + if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && + address >= gpu_mem[i].scratch_physical.base && + address <= gpu_mem[i].scratch_physical.limit) { fmm_release_scratch(gpu_mem[i].gpu_id); return HSAKMT_STATUS_SUCCESS; } - if (address >= gpu_mem[i].gpuvm_aperture.base && - address <= gpu_mem[i].gpuvm_aperture.limit) { - aperture = &gpu_mem[i].gpuvm_aperture; - break; - } - } + object = vm_find_object(address, 0, &aperture); - if (!aperture) { - if (address >= svm.dgpu_aperture->base && - address <= svm.dgpu_aperture->limit) { - aperture = svm.dgpu_aperture; - } else if (address >= svm.dgpu_alt_aperture->base && - address <= svm.dgpu_alt_aperture->limit) { - aperture = svm.dgpu_alt_aperture; - } - } + if (!object) + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; - if (aperture) { - pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_address(aperture, address, 0); - pthread_mutex_unlock(&aperture->fmm_mutex); - if (object) - __fmm_release(object, aperture); - if (i < gpu_mem_count) - fmm_print(gpu_mem[i].gpu_id); - } else { - /* - * If memory address isn't inside of any defined GPU aperture - it - * refers to the system memory - */ + if (aperture == &cpuvm_aperture) { + /* APU system memory */ uint64_t size = 0; - /* Release the vm object in CPUVM */ - pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); - object = vm_find_object_by_address(&cpuvm_aperture, address, 0); - if (object) { - size = object->size; - vm_remove_object(&cpuvm_aperture, object); - } - pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); - /* Free the memory from the system */ - if (size) - munmap(address, size); + + size = object->size; + vm_remove_object(&cpuvm_aperture, object); + pthread_mutex_unlock(&aperture->fmm_mutex); + munmap(address, size); + } else { + pthread_mutex_unlock(&aperture->fmm_mutex); + + __fmm_release(object, aperture); + if (!aperture->is_cpu_accessible) + fmm_print(gpu_mem[i].gpu_id); } - return object ? - HSAKMT_STATUS_SUCCESS : - HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + return HSAKMT_STATUS_SUCCESS; } static int fmm_set_memory_policy(uint32_t gpu_id, int default_policy, int alt_policy, @@ -2420,46 +2504,10 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert return ret; } -static int _fmm_map_to_apu_local(uint32_t gpu_id, - manageable_aperture_t *aperture, - void *address, uint64_t size, - uint64_t *gpuvm_address) -{ - vm_object_t *object; - - if (gpuvm_address) - *gpuvm_address = 0; - /* Check that address space was previously reserved */ - if (!vm_find(aperture, address)) - return -1; - - pthread_mutex_lock(&aperture->fmm_mutex); - - /* Find the object to retrieve the handle */ - object = vm_find_object_by_address(aperture, address, 0); - if (!object) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; - } - pthread_mutex_unlock(&aperture->fmm_mutex); - - if (_fmm_map_to_gpu(aperture, address, size, object, NULL, 0)) - return -1; - - if (gpuvm_address) { - *gpuvm_address = (uint64_t)object->start; - if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) - *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); - } - - return 0; -} - static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, uint64_t *gpuvm_addr, vm_object_t *object) { manageable_aperture_t *aperture; - vm_object_t *obj; void *svm_addr; HSAuint64 svm_size; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); @@ -2467,31 +2515,16 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, aperture = svm.dgpu_aperture; - /* Find the start address in SVM space for GPU mapping */ - if (!object) - pthread_mutex_lock(&aperture->fmm_mutex); - - obj = object; - if (!obj) { - obj = vm_find_object_by_userptr(aperture, addr, size); - if (!obj) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_ERROR; - } - } - svm_addr = obj->start; - svm_size = obj->size; + svm_addr = object->start; + svm_size = object->size; /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, obj, NULL, 0); + ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, object, NULL, 0); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; - if (!object) - pthread_mutex_unlock(&aperture->fmm_mutex); - return ret; } @@ -2499,83 +2532,47 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) { manageable_aperture_t *aperture; vm_object_t *object; - bool userptr = false; uint32_t i; - uint64_t pi; int ret; - /* Find an aperture the requested address belongs to */ - for (i = 0; i < gpu_mem_count; i++) { - if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) - continue; - - if ((address >= gpu_mem[i].scratch_physical.base) && - (address <= gpu_mem[i].scratch_physical.limit)) + /* Special handling for scratch memory */ + for (i = 0; i < gpu_mem_count; i++) + if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && + address >= gpu_mem[i].scratch_physical.base && + address <= gpu_mem[i].scratch_physical.limit) return _fmm_map_to_gpu_scratch(gpu_mem[i].gpu_id, &gpu_mem[i].scratch_physical, address, size); - if ((address >= gpu_mem[i].gpuvm_aperture.base) && - (address <= gpu_mem[i].gpuvm_aperture.limit)) - /* map it */ - return _fmm_map_to_apu_local(gpu_mem[i].gpu_id, - &gpu_mem[i].gpuvm_aperture, - address, size, gpuvm_address); - } - - if ((address >= svm.dgpu_aperture->base) && - (address <= svm.dgpu_aperture->limit)) - aperture = svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture->base) && - (address <= svm.dgpu_alt_aperture->limit)) - aperture = svm.dgpu_alt_aperture; - else { - aperture = svm.dgpu_aperture; - userptr = true; - } - - pthread_mutex_lock(&aperture->fmm_mutex); - if (userptr && is_dgpu) - object = vm_find_object_by_userptr(aperture, address, size); - else { - object = vm_find_object_by_address(aperture, address, 0); - /* If the object wasn't found in an unreserved - * aperture, it may be a userptr - */ - if (!object && aperture->ops == &mmap_aperture_ops) { - object = vm_find_object_by_userptr(aperture, address, - size); - userptr = true; - } else if (object && object->userptr == address) { - /* We found a userptr, but make sure we get - * the right one - */ - object = vm_find_object_by_userptr(aperture, address, - size); - userptr = true; + object = vm_find_object(address, size, &aperture); + if (!object) { + if (!is_dgpu) { + /* Prefetch memory on APUs with dummy-reads */ + fmm_check_user_memory(address, size); + return 0; } + pr_err("Object not found at %p\n", address); + return -EINVAL; } + /* Successful vm_find_object returns with the aperture locked */ - if (object) { - if (userptr && is_dgpu) - ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); - else - ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0); - - pthread_mutex_unlock(&aperture->fmm_mutex); - return ret; + if (aperture == &cpuvm_aperture) { + /* Prefetch memory on APUs with dummy-reads */ + fmm_check_user_memory(address, size); + ret = 0; + } else if (object->userptr) { + ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); + } else { + ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0); + /* Update alternate GPUVM address only for + * CPU-invisible apertures on old APUs + */ + if (!ret && gpuvm_address && !aperture->is_cpu_accessible) + *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); } pthread_mutex_unlock(&aperture->fmm_mutex); - - /* - * On an APU a system memory address is accessed through - * IOMMU. Thus we "prefetch" it. - */ - for (pi = 0; pi < size / PAGE_SIZE; pi++) - ((char *) address)[pi * PAGE_SIZE] = 0; - - return 0; + return ret; } static void print_device_id_array(uint32_t *device_id_array, uint32_t device_id_array_size) @@ -2719,78 +2716,37 @@ err: return ret; } -static int _fmm_unmap_from_gpu_userptr(void *addr) -{ - manageable_aperture_t *aperture; - vm_object_t *obj; - void *svm_addr; - - aperture = svm.dgpu_aperture; - - /* Find the start address in SVM space for GPU unmapping */ - pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_userptr(aperture, addr, 0); - if (!obj) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_ERROR; - } - svm_addr = obj->start; - pthread_mutex_unlock(&aperture->fmm_mutex); - - /* Unmap */ - return _fmm_unmap_from_gpu(aperture, svm_addr, NULL, 0, NULL); -} - int fmm_unmap_from_gpu(void *address) { - manageable_aperture_t *aperture = NULL; + manageable_aperture_t *aperture; + vm_object_t *object; uint32_t i; int ret; - /* Find the aperture the requested address belongs to */ - for (i = 0; i < gpu_mem_count; i++) { - if (gpu_mem[i].gpu_id == NON_VALID_GPU_ID) - continue; - - if ((address >= gpu_mem[i].scratch_physical.base) && - (address <= gpu_mem[i].scratch_physical.limit)) + /* Special handling for scratch memory */ + for (i = 0; i < gpu_mem_count; i++) + if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && + address >= gpu_mem[i].scratch_physical.base && + address <= gpu_mem[i].scratch_physical.limit) return _fmm_unmap_from_gpu_scratch(gpu_mem[i].gpu_id, &gpu_mem[i].scratch_physical, address); - if ((address >= gpu_mem[i].gpuvm_aperture.base) && - (address <= gpu_mem[i].gpuvm_aperture.limit)) - /* unmap it */ - return _fmm_unmap_from_gpu(&gpu_mem[i].gpuvm_aperture, - address, NULL, 0, NULL); - } + object = vm_find_object(address, 0, &aperture); + if (!object) + /* On APUs GPU unmapping of system memory is a no-op */ + return is_dgpu ? -EINVAL : 0; + /* Successful vm_find_object returns with the aperture locked */ - if ((address >= svm.dgpu_aperture->base) && - (address <= svm.dgpu_aperture->limit)) - aperture = svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture->base) && - (address <= svm.dgpu_alt_aperture->limit)) - aperture = svm.dgpu_alt_aperture; + if (aperture == &cpuvm_aperture) + /* On APUs GPU unmapping of system memory is a no-op */ + ret = 0; + else + ret = _fmm_unmap_from_gpu(aperture, address, NULL, 0, object); - if (aperture) { - ret = _fmm_unmap_from_gpu(aperture, address, NULL, 0, NULL); - /* If unmap failed for an address in a reserved - * aperture, it can't be a userptr address - */ - if (!ret || aperture->ops == &reserved_aperture_ops) - return ret; - /* fall through: try userptr */ - } + pthread_mutex_unlock(&aperture->fmm_mutex); - /* - * If address isn't an SVM address, we assume that this is - * system memory address. - */ - if (is_dgpu) - /* TODO: support mixed APU and dGPU configurations */ - return _fmm_unmap_from_gpu_userptr(address); - - return 0; + return ret; } bool fmm_get_handle(void *address, uint64_t *handle) @@ -2841,31 +2797,15 @@ bool fmm_get_handle(void *address, uint64_t *handle) return found; } -static HSAuint8 fmm_check_user_memory(const void *addr, HSAuint64 size) -{ - volatile const HSAuint8 *ptr = addr; - volatile const HSAuint8 *end = ptr + size; - HSAuint8 sum = 0; - - /* Access every page in the buffer to make sure the mapping is - * valid. If it's not, it will die with a segfault that's easy - * to debug. - */ - for (; ptr < end; ptr = (void *)PAGE_ALIGN_UP(ptr + 1)) - sum += *ptr; - - return sum; -} - static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_object_t **obj_ret) { - HSAuint32 gpu_id; - manageable_aperture_t *aperture; - void *svm_addr = NULL; - vm_object_t *obj; + manageable_aperture_t *aperture = svm.dgpu_aperture; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); HSAuint64 aligned_addr = (HSAuint64)addr - page_offset; HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); + void *svm_addr; + HSAuint32 gpu_id; + vm_object_t *obj; /* Find first GPU for creating the userptr BO */ if (!g_first_gpu_mem) @@ -2873,19 +2813,6 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj gpu_id = g_first_gpu_mem->gpu_id; - aperture = svm.dgpu_aperture; - - /* Check if this address was already registered */ - pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_userptr(aperture, addr, size); - if (obj) { - ++obj->registration_count; - pthread_mutex_unlock(&aperture->fmm_mutex); - *obj_ret = obj; - return HSAKMT_STATUS_SUCCESS; - } - pthread_mutex_unlock(&aperture->fmm_mutex); - /* Optionally check that the CPU mapping is valid */ if (svm.check_userptr) fmm_check_user_memory(addr, size); @@ -2926,31 +2853,13 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; - if ((address >= svm.dgpu_aperture->base) && - (address <= svm.dgpu_aperture->limit)) - aperture = svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture->base) && - (address <= svm.dgpu_alt_aperture->limit)) - aperture = svm.dgpu_alt_aperture; - /* else it's probably a userptr, handled later */ - - if (aperture) { - pthread_mutex_lock(&aperture->fmm_mutex); - object = vm_find_object_by_address(aperture, address, 0); - /* If the SVM aperture is reserved, it can't be a userptr */ - if (!object && aperture->ops == &reserved_aperture_ops) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_NOT_SUPPORTED; - } - } - + object = vm_find_object(address, size_in_bytes, &aperture); if (!object) { - if (aperture) - pthread_mutex_unlock(&aperture->fmm_mutex); - /* - * If address isn't SVM address, we assume that this - * is system memory address. - */ + if (!is_dgpu) + /* System memory registration on APUs is a no-op */ + return HSAKMT_STATUS_SUCCESS; + + /* Register a new user ptr */ ret = fmm_register_user_memory(address, size_in_bytes, &object); if (ret != HSAKMT_STATUS_SUCCESS) return ret; @@ -2959,7 +2868,11 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, aperture = svm.dgpu_aperture; pthread_mutex_lock(&aperture->fmm_mutex); /* fall through for registered device ID array setup */ + } else if (object->userptr) { + /* Update an existing userptr */ + ++object->registration_count; } + /* Successful vm_find_object returns with aperture locked */ if (object->registered_device_id_array_size > 0) { /* Multiple registration is allowed, but not changing nodes */ @@ -3246,74 +3159,27 @@ err_import: return err; } -static HSAKMT_STATUS fmm_deregister_user_memory(void *addr) -{ - manageable_aperture_t *aperture; - vm_object_t *obj; - - aperture = svm.dgpu_aperture; - - /* Find the size and start address in SVM space */ - pthread_mutex_lock(&aperture->fmm_mutex); - obj = vm_find_object_by_userptr(aperture, addr, 0); - if (!obj || obj->registration_count > 1) { - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_ERROR; - } - pthread_mutex_unlock(&aperture->fmm_mutex); - - /* Destroy BO */ - __fmm_release(obj, aperture); - - return HSAKMT_STATUS_SUCCESS; -} - HSAKMT_STATUS fmm_deregister_memory(void *address) { - manageable_aperture_t *aperture = NULL; - vm_object_t *object = NULL; - unsigned int i; - HSAuint32 page_offset = (HSAint64)address & (PAGE_SIZE - 1); + manageable_aperture_t *aperture; + vm_object_t *object; - if ((address >= svm.dgpu_aperture->base) && - (address <= svm.dgpu_aperture->limit)) - aperture = svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture->base) && - (address <= svm.dgpu_alt_aperture->limit)) - aperture = svm.dgpu_alt_aperture; - else - for (i = 0; i < gpu_mem_count; i++) { - if (gpu_mem[i].gpu_id != NON_VALID_GPU_ID && - address >= gpu_mem[i].gpuvm_aperture.base && - address <= gpu_mem[i].gpuvm_aperture.limit) { - aperture = &gpu_mem[i].gpuvm_aperture; - break; - } - } - - if (!aperture) { - /* If address isn't found in any aperture, we assume - * that this is system memory address. On APUs, there - * is nothing to do (for now). + object = vm_find_object(address, 0, &aperture); + if (!object) + /* On APUs we assume it's a random system memory address + * where registration and dergistration is a no-op */ - if (!is_dgpu) - return HSAKMT_STATUS_SUCCESS; - /* If the userptr object had a - * registered_device_id_array, it will be freed by - * __fmm_release. Also the object will be - * removed. Therefore we can short-circuit the rest of - * the function below. + return is_dgpu ? + HSAKMT_STATUS_MEMORY_NOT_REGISTERED : + HSAKMT_STATUS_SUCCESS; + /* Successful vm_find_object returns with aperture locked */ + + if (aperture == &cpuvm_aperture) { + /* API-allocated system memory on APUs, deregistration + * is a no-op */ - return fmm_deregister_user_memory(address); - } - - pthread_mutex_lock(&aperture->fmm_mutex); - - object = vm_find_object_by_address(aperture, - VOID_PTR_SUB(address, page_offset), 0); - if (!object) { pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + return HSAKMT_STATUS_SUCCESS; } if (object->registration_count > 1) { @@ -3363,9 +3229,8 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint64_t *gpuvm_address) { manageable_aperture_t *aperture; - vm_object_t *object = NULL; + vm_object_t *object; uint32_t i; - bool userptr = false; uint32_t *registered_node_id_array, registered_node_id_array_size; HSAKMT_STATUS ret = HSAKMT_STATUS_ERROR; int retcode = 0; @@ -3373,41 +3238,13 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, if (!num_of_nodes || !nodes_to_map || !address) return HSAKMT_STATUS_INVALID_PARAMETER; - /* Find object by address */ - if ((address >= svm.dgpu_aperture->base) && - (address <= svm.dgpu_aperture->limit)) - aperture = svm.dgpu_aperture; - else if ((address >= svm.dgpu_alt_aperture->base) && - (address <= svm.dgpu_alt_aperture->limit)) - aperture = svm.dgpu_alt_aperture; - else { - aperture = svm.dgpu_aperture; - userptr = true; - } + object = vm_find_object(address, size, &aperture); + if (!object) + return HSAKMT_STATUS_ERROR; + /* Successful vm_find_object returns with aperture locked */ - pthread_mutex_lock(&aperture->fmm_mutex); - if (userptr && is_dgpu) - object = vm_find_object_by_userptr(aperture, address, size); - else { - object = vm_find_object_by_address(aperture, address, 0); - /* If the object wasn't found in an unreserved - * aperture, it may be a userptr - */ - if (!object && aperture->ops == &mmap_aperture_ops) { - object = vm_find_object_by_userptr(aperture, address, - size); - userptr = true; - } else if (object && object->userptr == address) { - /* We found a userptr, but make sure we get - * the right one - */ - object = vm_find_object_by_userptr(aperture, address, - size); - userptr = true; - } - } - - if (!object) { + /* APU memory is not supported by this function */ + if (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } @@ -3416,11 +3253,11 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, * This is to simply the implementation of allowing the same memory * region to be registered multiple times. */ - if (userptr && is_dgpu) { + if (object->userptr) { retcode = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); pthread_mutex_unlock(&aperture->fmm_mutex); - return retcode; + return retcode ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } /* Verify that all nodes to map are registered already */ @@ -3501,18 +3338,12 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) memset(info, 0, sizeof(HsaPointerInfo)); - aperture = fmm_find_aperture(address, NULL); - - pthread_mutex_lock(&aperture->fmm_mutex); - vm_obj = vm_find_object_by_address_range(aperture, address); - if (!vm_obj) - vm_obj = vm_find_object_by_userptr_range(aperture, address); - + vm_obj = vm_find_object(address, UINT64_MAX, &aperture); if (!vm_obj) { info->Type = HSA_POINTER_UNKNOWN; - ret = HSAKMT_STATUS_ERROR; - goto exit; + return HSAKMT_STATUS_ERROR; } + /* Successful vm_find_object returns with the aperture locked */ if (vm_obj->metadata) info->Type = HSA_POINTER_REGISTERED_GRAPHICS; @@ -3565,7 +3396,6 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) info->CPUAddress = vm_obj->start; } -exit: pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } @@ -3575,15 +3405,13 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) manageable_aperture_t *aperture; vm_object_t *vm_obj; - aperture = fmm_find_aperture(mem, NULL); - - vm_obj = vm_find_object_by_address(aperture, mem, 0); - if (!vm_obj) - vm_obj = vm_find_object_by_userptr(aperture, mem, 0); + vm_obj = vm_find_object(mem, 0, &aperture); if (!vm_obj) return HSAKMT_STATUS_ERROR; vm_obj->user_data = usr_data; + + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } From 608dddbe9dafcef02a376147835e19bd94119079 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 31 Aug 2018 00:49:06 -0400 Subject: [PATCH 0406/1247] kfdtest: Fix gfx902 blacklist Removed some tests from the blacklist that are now passing. Added two new tests that hang the GPU. Change-Id: I09e729590e5181311375058be492d387342ba2fe Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/kfdtest.exclude | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 9fdd2e19f9..ba80c275d8 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -124,10 +124,8 @@ VEGA20_TESTS_BLACKLIST=\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDMemoryTest.MemoryRegister" - RAVEN_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "KFDDBGTest.BasicAddressWatch:"\ -"KFDQMTest.CreateQueueStressSingleThreaded:"\ -"KFDQMTest.GPUDoorbellWrite:"\ -"KFDMemoryTest.MemoryRegister" +"KFDQMTest.QueueLatency:"\ +"KFDQMTest.SdmaEventInterrupt" From a9bd6e6f8bd517e4282da90a6336b90b4949a849 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 31 Aug 2018 14:36:46 -0400 Subject: [PATCH 0407/1247] Revert "libhsakmt: Try to use CPU addr as GPU addr for userptrs" This reverts commit ab181c46c0af4aaceaa981919ba37fd52c44a479. This fixes ambiguity when looking up GPU addresses with hsaKmtQueryPointerInfo. hsa_amd_agents_allow_access uses hsaKmtQueryPointerInfo, and depends on finding the correct object from a GPU address. Finding the wrong userptr object based on its CPU address leads to incorrect GPU mappings and results in VM faults. Change-Id: I7c5f571ee6e1f9d32687aa3eab6d96944ad032be Signed-off-by: Felix Kuehling --- src/fmm.c | 69 +++++-------------------------------------------- src/libhsakmt.h | 4 +-- 2 files changed, 7 insertions(+), 66 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 22c572b43e..24ddddcd5e 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -502,38 +502,6 @@ loop: return cur; /* NULL if not found */ } -/* Returns true if there is any object in GPU VM address space - * overlapping the specified address range - */ -static bool vm_exists_overlapping_object(manageable_aperture_t *aper, - const void *addr, uint64_t size) -{ - unsigned long start_addr = (unsigned long)addr; - unsigned long end_addr = start_addr + size; - rbtree_t *tree = vm_object_tree(aper, 0); - rbtree_key_t start_key, end_key; - rbtree_node_t *rn_start, *ln_end; - vm_object_t *cur; - - start_key = rbtree_key(start_addr, 0); - rn_start = rbtree_lookup_nearest(tree, &start_key, LKP_ALL, RIGHT); - if (rn_start) { - cur = vm_object_entry(rn_start, 0); - if ((unsigned long)cur->start < end_addr) - return true; - } - - end_key = rbtree_key(end_addr, 0); - ln_end = rbtree_lookup_nearest(tree, &end_key, LKP_ALL, LEFT); - if (ln_end) { - cur = vm_object_entry(ln_end, 0); - if ((unsigned long)cur->start + cur->size > start_addr) - return true; - } - - return false; -} - static vm_object_t *vm_find_object_by_address(manageable_aperture_t *app, const void *address, uint64_t size) { @@ -1277,7 +1245,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t *mmap_offset, uint32_t flags, vm_object_t **vm_obj) { - void *mem = NULL, *userpage = NULL; + void *mem = NULL; vm_object_t *obj; /* Check that aperture is properly initialized/supported */ @@ -1286,27 +1254,9 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); - /* If it's a userptr within the GPU-addressable address space, - * and no object is registered yet in the same address range, - * then use the CPU address as GPU address - */ - if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) && mmap_offset) { - userpage = (void *)*mmap_offset; - if (userpage >= aperture->base && - VOID_PTR_ADD(userpage, MemorySizeInBytes - 1) <= aperture->limit && - !vm_exists_overlapping_object(aperture, userpage, MemorySizeInBytes)) - mem = userpage; - } - if (!mem) - mem = aperture_allocate_area(aperture, MemorySizeInBytes); + mem = aperture_allocate_area(aperture, MemorySizeInBytes); pthread_mutex_unlock(&aperture->fmm_mutex); - if (!mem) { - pr_err("Failed to allocate %ld bytes virtual address space\n", - MemorySizeInBytes); - return NULL; - } - /* * Now that we have the area reserved, allocate memory in the device * itself @@ -1318,11 +1268,9 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, * allocation of memory in device failed. * Release region in aperture */ - if (mem != userpage) { - pthread_mutex_lock(&aperture->fmm_mutex); - aperture_release_area(aperture, mem, MemorySizeInBytes); - pthread_mutex_unlock(&aperture->fmm_mutex); - } + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, MemorySizeInBytes); + pthread_mutex_unlock(&aperture->fmm_mutex); /* Assign NULL to mem to indicate failure to calling function */ mem = NULL; @@ -1626,12 +1574,7 @@ static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) args.handle = object->handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); - /* Userptrs with CPU addr == GPU addr don't have aperture - * space allocated to them - */ - if (!object->userptr || - (void *)PAGE_ALIGN_DOWN(object->userptr) != object->start) - aperture_release_area(aperture, object->start, object->size); + aperture_release_area(aperture, object->start, object->size); vm_remove_object(aperture, object); pthread_mutex_unlock(&aperture->fmm_mutex); diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 9646190609..8ed894bf4e 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -65,10 +65,8 @@ extern int PAGE_SHIFT; do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) -#define ALIGN_DOWN(x,align) ((uint64_t)(x) & ~(uint64_t)((align)-1)) #define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1)) -#define PAGE_ALIGN_UP(x) ALIGN_UP((x),PAGE_SIZE) -#define PAGE_ALIGN_DOWN(x) ALIGN_DOWN((x),PAGE_SIZE) +#define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) #define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) From 9c7cfc0df2ccbd0c5a99f4460ba5750b775623fc Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Mon, 27 Aug 2018 14:56:48 +0800 Subject: [PATCH 0408/1247] kfdtest: Add event-based synchronization mechanism to queues Wait4PacketConsumption now can accept an event to wait all packets subbmitted to be processed. Change-Id: I1497b7704e892b04d05811b8d3e4742237c1be57 Signed-off-by: xinhui pan --- tests/kfdtest/src/BaseQueue.cpp | 3 ++- tests/kfdtest/src/BaseQueue.hpp | 2 +- tests/kfdtest/src/PM4Queue.cpp | 12 ++++++++++++ tests/kfdtest/src/PM4Queue.hpp | 8 ++++++++ tests/kfdtest/src/SDMAQueue.cpp | 12 ++++++++++++ tests/kfdtest/src/SDMAQueue.hpp | 9 +++++++++ 6 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 50ca76f778..e8a65ff10a 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -116,7 +116,8 @@ void BaseQueue::PlaceAndSubmitPacket(const BasePacket &packet) { SubmitPacket(); } -void BaseQueue::Wait4PacketConsumption() { +void BaseQueue::Wait4PacketConsumption(HsaEvent *event) { + ASSERT_TRUE(!event) << "Not supported!" << std::endl; ASSERT_TRUE(WaitOnValue(m_Resources.Queue_read_ptr, RptrWhenConsumed())); } diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index a45c44d8b9..ba0464a6e0 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -63,7 +63,7 @@ class BaseQueue { /** Wait for all the packets submitted to the queue to be consumed. (i.e. wait until RPTR=WPTR). * Note that all packets being consumed is not the same as all packets being processed. */ - virtual void Wait4PacketConsumption(); + virtual void Wait4PacketConsumption(HsaEvent *event = NULL); /** @brief Place packet and submit it in one function */ virtual void PlaceAndSubmitPacket(const BasePacket &packet); diff --git a/tests/kfdtest/src/PM4Queue.cpp b/tests/kfdtest/src/PM4Queue.cpp index d1152c0b34..99707fb9ad 100644 --- a/tests/kfdtest/src/PM4Queue.cpp +++ b/tests/kfdtest/src/PM4Queue.cpp @@ -71,3 +71,15 @@ void PM4Queue::SubmitPacket() { } } +void PM4Queue::Wait4PacketConsumption(HsaEvent *event) { + if (event) { + PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(0, + event->EventData.HWData2, + event->EventId, + true)); + + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + } else { + BaseQueue::Wait4PacketConsumption(); + } +} diff --git a/tests/kfdtest/src/PM4Queue.hpp b/tests/kfdtest/src/PM4Queue.hpp index dd5d35b85f..64e8f4ec96 100644 --- a/tests/kfdtest/src/PM4Queue.hpp +++ b/tests/kfdtest/src/PM4Queue.hpp @@ -41,6 +41,14 @@ class PM4Queue : public BaseQueue { virtual unsigned int Wptr(); // @ return expected m_Resources.Queue_read_ptr when all packets consumed virtual unsigned int RptrWhenConsumed(); + /** Wait for all the packets submitted to the queue to be consumed. (i.e. wait until RPTR=WPTR). + * Note that all packets being consumed is not the same as all packets being processed. + * If event is set, wait all packets being processed. + * And we can benefit from that as it has + * 1) Less CPU usage (process can sleep, waiting for interrupt). + * 2) Lower latency (GPU only updates RPTR in memory periodically). + */ + virtual void Wait4PacketConsumption(HsaEvent *event = NULL); protected: virtual PACKETTYPE PacketTypeSupported() { return PACKETTYPE_PM4; } diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/tests/kfdtest/src/SDMAQueue.cpp index 6f352b2cef..047b9eae74 100644 --- a/tests/kfdtest/src/SDMAQueue.cpp +++ b/tests/kfdtest/src/SDMAQueue.cpp @@ -22,6 +22,7 @@ */ #include "SDMAQueue.hpp" +#include "SDMAPacket.hpp" SDMAQueue::SDMAQueue(void) { CMD_NOP = 0; @@ -79,3 +80,14 @@ void SDMAQueue::SubmitPacket() { } } +void SDMAQueue::Wait4PacketConsumption(HsaEvent *event) { + if (event) { + PlacePacket(SDMAFencePacket((void*)event->EventData.HWData2, event->EventId)); + + PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); + + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + } else { + BaseQueue::Wait4PacketConsumption(); + } +} diff --git a/tests/kfdtest/src/SDMAQueue.hpp b/tests/kfdtest/src/SDMAQueue.hpp index fa762e3742..75a0f2a250 100644 --- a/tests/kfdtest/src/SDMAQueue.hpp +++ b/tests/kfdtest/src/SDMAQueue.hpp @@ -34,6 +34,15 @@ class SDMAQueue : public BaseQueue { // @brief Update queue write pointer and set the queue doorbell to the queue write pointer virtual void SubmitPacket(); + /** Wait for all the packets submitted to the queue to be consumed. (i.e. wait until RPTR=WPTR). + * Note that all packets being consumed is not the same as all packets being processed. + * If event is set, wait all packets being processed. + * And we can benefit from that as it has + * 1) Less CPU usage (process can sleep, waiting for interrupt). + * 2) Lower latency (GPU only updates RPTR in memory periodically). + */ + virtual void Wait4PacketConsumption(HsaEvent *event = NULL); + protected: // @ return Write pointer modulo queue size in dwords virtual unsigned int Wptr(); From f84a99e9533b01a333746dbd7ade8a4d42eb59b7 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 21 Aug 2018 10:44:20 -0400 Subject: [PATCH 0409/1247] Replace global variable _system with g_system Change-Id: I452090473a5b46b32204f7f916bdcfdd3e8a47bd Signed-off-by: Harish Kasiviswanathan --- src/topology.c | 56 +++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/topology.c b/src/topology.c index 64872c0b2a..ab41c2f5cc 100644 --- a/src/topology.c +++ b/src/topology.c @@ -56,7 +56,7 @@ typedef struct { HsaIoLinkProperties *link; } node_props_t; -static HsaSystemProperties *_system = NULL; +static HsaSystemProperties *g_system; static node_props_t *g_props; static int is_valgrind; @@ -1793,15 +1793,15 @@ retry: goto retry; } - if (!_system) { - _system = malloc(sizeof(HsaSystemProperties)); - if (!_system) { + if (!g_system) { + g_system = malloc(sizeof(HsaSystemProperties)); + if (!g_system) { free_properties(temp_props, sys_props.NumNodes); return HSAKMT_STATUS_NO_MEMORY; } } - *_system = sys_props; + *g_system = sys_props; if (g_props) free(g_props); g_props = temp_props; @@ -1815,7 +1815,7 @@ HSAKMT_STATUS topology_drop_snapshot(void) { HSAKMT_STATUS err; - if (!!_system != !!g_props) { + if (!!g_system != !!g_props) { pr_warn("Probably inconsistency?\n"); err = HSAKMT_STATUS_SUCCESS; goto out; @@ -1823,12 +1823,12 @@ HSAKMT_STATUS topology_drop_snapshot(void) if (g_props) { /* Remove state */ - free_properties(g_props, _system->NumNodes); + free_properties(g_props, g_system->NumNodes); g_props = NULL; } - free(_system); - _system = NULL; + free(g_system); + g_system = NULL; if (map_user_to_sysfs_node_id) { free(map_user_to_sysfs_node_id); @@ -1844,7 +1844,7 @@ out: HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) { - if (!g_props || !_system || _system->NumNodes <= nodeid) + if (!g_props || !g_system || g_system->NumNodes <= nodeid) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (gpu_id) *gpu_id = g_props[nodeid].gpu_id; @@ -1856,7 +1856,7 @@ HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id) { uint64_t node_idx; - for (node_idx = 0; node_idx < _system->NumNodes; node_idx++) { + for (node_idx = 0; node_idx < g_system->NumNodes; node_idx++) { if (g_props[node_idx].gpu_id == gpu_id) { *node_id = node_idx; return HSAKMT_STATUS_SUCCESS; @@ -1882,9 +1882,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties(HsaSystemProperties *Syste if (err != HSAKMT_STATUS_SUCCESS) goto out; - assert(_system); + assert(g_system); - *SystemProperties = *_system; + *SystemProperties = *g_system; err = HSAKMT_STATUS_SUCCESS; out: @@ -1920,13 +1920,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (!_system) { + if (!g_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(_system); + assert(g_system); goto out; } - if (NodeId >= _system->NumNodes) { + if (NodeId >= g_system->NumNodes) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -1965,14 +1965,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (!_system) { + if (!g_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(_system); + assert(g_system); goto out; } /* Check still necessary */ - if (NodeId >= _system->NumNodes) { + if (NodeId >= g_system->NumNodes) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -2052,13 +2052,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeCacheProperties(HSAuint32 NodeId, pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (!_system) { + if (!g_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(_system); + assert(g_system); goto out; } - if (NodeId >= _system->NumNodes || NumCaches > g_props[NodeId].node.NumCaches) { + if (NodeId >= g_system->NumNodes || NumCaches > g_props[NodeId].node.NumCaches) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -2090,13 +2090,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (!_system) { + if (!g_system) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(_system); + assert(g_system); goto out; } - if (NodeId >= _system->NumNodes || NumIoLinks > g_props[NodeId].node.NumIOLinks) { + if (NodeId >= g_system->NumNodes || NumIoLinks > g_props[NodeId].node.NumIOLinks) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -2115,7 +2115,7 @@ out: uint16_t get_device_id_by_node_id(HSAuint32 node_id) { - if (!g_props || !_system || _system->NumNodes <= node_id) + if (!g_props || !g_system || g_system->NumNodes <= node_id) return 0; return g_props[node_id].node.DeviceId; @@ -2138,10 +2138,10 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; - if (!g_props || !_system) + if (!g_props || !g_system) return 0; - for (i = 0; i < _system->NumNodes; i++) { + for (i = 0; i < g_system->NumNodes; i++) { if (g_props[i].gpu_id == gpu_id) return g_props[i].node.DeviceId; } From 866ef2005415e2f4a17ad8e92bfbfaf7dd53fb2f Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 21 Aug 2018 12:39:58 -0400 Subject: [PATCH 0410/1247] iolinks: Handle GPU resource management With GPU resource management, some nodes are unavailable based on the cgroup hierarchy of the task. Kernel via sysfs specifies all the iolinks. Skip the links which are not accessible. Also iolinks specified by the kernel refer to sysfs Node IDs. Map it to relevant user Node IDs v2: NodeFrom mapped from sysfs Node to User Node Change-Id: I95312ee6ca51b89fe9e6ca2a9185c2ea1e94afc4 Signed-off-by: Harish Kasiviswanathan --- src/topology.c | 68 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/src/topology.c b/src/topology.c index ab41c2f5cc..3a1ed47a49 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1373,6 +1373,25 @@ err1: return ret; } +static HSAKMT_STATUS topology_map_sysfs_to_user_node_id(uint32_t sys_node_id, uint32_t *user_node_id) +{ + uint32_t node_id; + + for (node_id = 0; node_id < map_user_to_sysfs_node_id_size; node_id++) + if (map_user_to_sysfs_node_id[node_id] == sys_node_id) { + *user_node_id = node_id; + return HSAKMT_STATUS_SUCCESS; + } + return HSAKMT_STATUS_INVALID_NODE_UNIT; +} + + +/* For a give Node @node_id the function gets @iolink_id information i.e. parses sysfs the following sysfs entry + * ./nodes/@node_id/io_links/@iolink_id/properties. @node_id has to be valid accessible node. + * + * If node_to specified by the @iolink_id is not accessible the function returns HSAKMT_STATUS_NOT_SUPPORTED. + * If node_to is accessible, then node_to is mapped from sysfs_node to user_node and returns HSAKMT_STATUS_SUCCESS. + */ static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, uint32_t iolink_id, HsaIoLinkProperties *props) @@ -1422,11 +1441,27 @@ static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, props->VersionMajor = (uint32_t)prop_val; else if (strcmp(prop_name, "version_minor") == 0) props->VersionMinor = (uint32_t)prop_val; - else if (strcmp(prop_name, "node_from") == 0) - props->NodeFrom = (uint32_t)prop_val; - else if (strcmp(prop_name, "node_to") == 0) - props->NodeTo = (uint32_t)prop_val; - else if (strcmp(prop_name, "weight") == 0) + else if (strcmp(prop_name, "node_from") == 0) { + if (sys_node_id != (uint32_t)prop_val) { + ret = HSAKMT_STATUS_INVALID_NODE_UNIT; + goto err2; + } + props->NodeFrom = node_id; + } else if (strcmp(prop_name, "node_to") == 0) { + bool is_node_supported; + uint32_t sysfs_node_id; + + sysfs_node_id = (uint32_t)prop_val; + ret = topology_sysfs_check_node_supported(sysfs_node_id, &is_node_supported); + if (!is_node_supported) { + ret = HSAKMT_STATUS_NOT_SUPPORTED; + memset(props, 0, sizeof(*props)); + goto err2; + } + ret = topology_map_sysfs_to_user_node_id(sysfs_node_id, &props->NodeTo); + if (ret != HSAKMT_STATUS_SUCCESS) + goto err2; + } else if (strcmp(prop_name, "weight") == 0) props->Weight = (uint32_t)prop_val; else if (strcmp(prop_name, "min_latency") == 0) props->MinimumLatency = (uint32_t)prop_val; @@ -1677,7 +1712,7 @@ try_alt_dir: HSAKMT_STATUS topology_take_snapshot(void) { - uint32_t gen_start, gen_end, i, mem_id, cache_id, link_id; + uint32_t gen_start, gen_end, i, mem_id, cache_id; HsaSystemProperties sys_props; node_props_t *temp_props = 0; void *cpu_ci_list = NULL; @@ -1764,13 +1799,26 @@ retry: } if (temp_props[i].node.NumIOLinks) { - for (link_id = 0; link_id < temp_props[i].node.NumIOLinks; link_id++) { - ret = topology_sysfs_get_iolink_props(i, link_id, &temp_props[i].link[link_id]); - if (ret != HSAKMT_STATUS_SUCCESS) { - free_properties(temp_props, i+1); + uint32_t sys_link_id = 0, link_id = 0; + + /* Parse all the sysfs specified io links. Skip the ones where the + * remote node (node_to) is not accessible + */ + while (sys_link_id < temp_props[i].node.NumIOLinks && + link_id < sys_props.NumNodes - 1) { + ret = topology_sysfs_get_iolink_props(i, sys_link_id++, + &temp_props[i].link[link_id]); + if (ret == HSAKMT_STATUS_NOT_SUPPORTED) { + ret = HSAKMT_STATUS_SUCCESS; + continue; + } else if (ret != HSAKMT_STATUS_SUCCESS) { + free_properties(temp_props, i + 1); goto err; } + link_id++; } + /* sysfs specifies all the io links. Limit the number to valid ones */ + temp_props[i].node.NumIOLinks = link_id; } } pci_cleanup(pacc); From 7876bb70a9c49e44e33742b6f16b68a25d94e43b Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 21 Aug 2018 13:59:48 -0400 Subject: [PATCH 0411/1247] Add cgroup support Some nodes are unavailable based on the task's cgroup hierarchy. Handle this situation by ignoring those nodes Change-Id: I72f9e822d2ec8cf15732df95e427d5549a75b55d Signed-off-by: Harish Kasiviswanathan --- src/topology.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/topology.c b/src/topology.c index 3a1ed47a49..12e58b8ba7 100644 --- a/src/topology.c +++ b/src/topology.c @@ -501,15 +501,19 @@ static HSAKMT_STATUS topology_sysfs_get_gpu_id(uint32_t sysfs_node_id, uint32_t if (!fd) return HSAKMT_STATUS_ERROR; if (fscanf(fd, "%ul", gpu_id) != 1) - ret = HSAKMT_STATUS_ERROR; + ret = (errno == EPERM) ? HSAKMT_STATUS_NOT_SUPPORTED : + HSAKMT_STATUS_ERROR; fclose(fd); return ret; } -/* Check if the sysfs node is supported. This function will be passed with sysfs node id. +/* Check if the @sysfs_node_id is supported. This function will be passed with sysfs node id. * This function can not use topology_* help functions, because those functions are - * using user node id. A sysfs node is supported only if corresponding drm render node is available. + * using user node id. + * A sysfs node is not supported + * - if corresponding drm render node is not available. + * - if node information is not accessible (EPERM) */ static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id, bool *is_node_supported) { @@ -529,6 +533,8 @@ static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id, /* Retrieve the GPU ID */ ret = topology_sysfs_get_gpu_id(sysfs_node_id, &gpu_id); + if (ret == HSAKMT_STATUS_NOT_SUPPORTED) + return HSAKMT_STATUS_SUCCESS; if (ret != HSAKMT_STATUS_SUCCESS) return ret; From 1fda42972615d17a08609f2166cd17e935ab09fc Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 6 Sep 2018 11:38:53 -0400 Subject: [PATCH 0412/1247] kfdtest: GetNodeIoLinkProperties: Display NodeFrom Use the NodeFrom returned by hsaKmtGetNodeIoLinkProperties() to check its correctness. Change-Id: I6ce436dc7c5d5b192bee21156292bd3eff77f916 Signed-off-by: Harish Kasiviswanathan --- tests/kfdtest/src/KFDTopologyTest.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index 16fe8cec99..ac9960bffb 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -203,7 +203,9 @@ TEST_F(KFDTopologyTest, GetNodeIoLinkProperties) { for (linkId = 0; linkId < pNodeProperties->NumIOLinks; linkId++) { if (linkId == 0) { // First io_link. Print Parent Node and io_link Node - LOG() << "[" << node << "]--(" << IolinkProperties[linkId].Weight << ")-->" << + EXPECT_EQ(node, IolinkProperties[linkId].NodeFrom); + LOG() << "[" << IolinkProperties[linkId].NodeFrom << "]--(" << + IolinkProperties[linkId].Weight << ")-->" << "[" << IolinkProperties[linkId].NodeTo << "]" << std::endl; continue; } From be574169c1f5f428b9657b3df3313016a5e73066 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 7 Sep 2018 17:21:37 -0400 Subject: [PATCH 0413/1247] libhsakmt: Fix segfault on gfx801 Handle the case that svm.dgpu_aperture does not exist in vm_find_object. Change-Id: Ic0983d4f321f1b6248514f2fa25162976e90bd75 Signed-off-by: Felix Kuehling --- src/fmm.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 24ddddcd5e..223ff364af 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1033,6 +1033,9 @@ static vm_object_t *vm_find_object(const void *addr, uint64_t size, } if (!aper) { + if (!svm.dgpu_aperture) + goto no_svm; + if ((addr >= svm.dgpu_aperture->base) && (addr <= svm.dgpu_aperture->limit)) aper = svm.dgpu_aperture; @@ -1075,9 +1078,11 @@ static vm_object_t *vm_find_object(const void *addr, uint64_t size, } } +no_svm: if (!obj && !is_dgpu) { /* On APUs try finding it in the CPUVM aperture */ - pthread_mutex_unlock(&aper->fmm_mutex); + if (aper) + pthread_mutex_unlock(&aper->fmm_mutex); aper = &cpuvm_aperture; @@ -1093,7 +1098,8 @@ static vm_object_t *vm_find_object(const void *addr, uint64_t size, return obj; } - pthread_mutex_unlock(&aper->fmm_mutex); + if (aper) + pthread_mutex_unlock(&aper->fmm_mutex); return NULL; } From 07bd97a8647c3e55d5d418c3ac5597521e1a2980 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Mon, 10 Sep 2018 14:45:48 +0800 Subject: [PATCH 0414/1247] kfdtest: Fix queuelatency fail issue The timestamp written by releaseMemory packet might still not be visible when we fetch it. To fix this bug, use event-based wait. Change-Id: If2324eb3b3a632c711ee4dff4d03a93d5306c289 Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDQMTest.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index bb0196ccf4..cc8671530f 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1074,8 +1074,9 @@ TEST_F(KFDQMTest, QueueLatency) { * |E|E|E|E|E|E|E|rptr...wptr|E|E|E|E|E| ---> |P|P|P|P|P|P|E|rptr...wptr|N|N|N|N|N| * So to respect that, we reserve packetSize space for these additional NOPs. * Also we reserve the remainder of the division by packetSize explicitly. + * Reserve another packetSize for event-based wait which uses a releseMemory packet. */ - const int reservedSpace = packetSize + queueSize % packetSize; + const int reservedSpace = packetSize + queueSize % packetSize + packetSize; const int slots = (queueSize - reservedSpace) / packetSize; HSAint64 queue_latency_avg = 0, queue_latency_min, queue_latency_max, queue_latency_med; HSAint64 overhead, workload; @@ -1101,6 +1102,9 @@ TEST_F(KFDQMTest, QueueLatency) { HsaMemoryBuffer qbuf(ALIGN_UP(slots * sizeof(HSAuint64), PAGE_SIZE), 0); qts = qbuf.As(); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); + /* GpuCounter overhead*/ do { hsaKmtGetClockCounters(defaultGPUNode, &ts[i]); @@ -1118,7 +1122,7 @@ TEST_F(KFDQMTest, QueueLatency) { 1)); hsaKmtGetClockCounters(defaultGPUNode, &ts[i]); queue.SubmitPacket(); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); } while (++i < slots); /* Calculate timing which includes workload and overhead*/ @@ -1145,8 +1149,9 @@ TEST_F(KFDQMTest, QueueLatency) { 1)); } while (++i < slots); queue.SubmitPacket(); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); + hsaKmtDestroyEvent(event); /* qts[i] records the timestamp of the end of packet[i] which is * approximate that of the beginging of packet[i+1]. * The workload total is [0, skip], [skip+1, slots-1]. From a6287ba919dcf7c9aae165183dcace1ce03b62aa Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Tue, 11 Sep 2018 10:19:56 +0800 Subject: [PATCH 0415/1247] kfdtest: Do not set GTEST_FLAG throw_on_failure This change is from commit 62f7dc2a("kfdtest: Do not set GTEST_FLAG throw_on_failure"). But it is unexpected to reverted by commit 414042ab("kfdtest: Clean up comments"). So add this change back. Fix: 414042ab Change-Id: Ia9e99c9ca17b99aab62b4db55017018ddae43dfb Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDTestMain.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index 98d186d9ee..7715bd06a1 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -61,8 +61,6 @@ GTEST_API_ int main(int argc, char **argv) { g_TestENVCaps = ENVCAPS_NOADDEDCAPS | ENVCAPS_64BITLINUX; g_TestTimeOut = KFD_TEST_DEFAULT_TIMEOUT; - // Every fatal fail ( = assert that failed ) will throw an exception - testing::GTEST_FLAG(throw_on_failure) = true; testing::InitGoogleTest(&argc, argv); CommandLineArguments args; From 247fa9f1e0f51708d98aa5f9b17bf95c57bca7d3 Mon Sep 17 00:00:00 2001 From: Xiaojie Yuan Date: Wed, 12 Sep 2018 17:30:42 +0800 Subject: [PATCH 0416/1247] Use 'RecordProperty' to record performance scores For following test cases: - KFDQMTest.QueueLatency - KFDQMTest.BasicCuMaskingLinear - KFDQMTest.BasicCuMaskingEven - KFDMemoryTest.MMBandWidth - KFDMemoryTest.MMapLarge - KFDMemoryTest.MMBench v2: xml element cannot start with a number, so change the key name of MMBandWidth and MMBench accordingly xml element cannot contain whitespaces, so trim whitespaces in "VRAM " v3: introduce KFDLog-like way to use KFDRecord Change-Id: Ifc3ed5657621252a7b39dccf1ef4f50a92593f77 Signed-off-by: Xiaojie Yuan --- tests/kfdtest/src/GoogleTestExtension.hpp | 21 +++++++++++++++++ tests/kfdtest/src/KFDMemoryTest.cpp | 28 +++++++++++++++++++---- tests/kfdtest/src/KFDQMTest.cpp | 11 +++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/GoogleTestExtension.hpp b/tests/kfdtest/src/GoogleTestExtension.hpp index b53ce6bf84..196b7242c4 100644 --- a/tests/kfdtest/src/GoogleTestExtension.hpp +++ b/tests/kfdtest/src/GoogleTestExtension.hpp @@ -42,6 +42,27 @@ std::ostream& operator << (KFDLog log, LOGTYPE level); #define LOG() KFDLog() << LOGTYPE_INFO #define WARN() KFDLog() << LOGTYPE_WARNING +class KFDRecord: public testing::Test { +public: + KFDRecord(const char *val): m_val(val) {} + KFDRecord(std::string &val): m_val(val) {} + KFDRecord(HSAint64 val): m_val(std::to_string(val)) {} + KFDRecord(HSAuint64 val): m_val(std::to_string(val)) {} + KFDRecord(double val): m_val(std::to_string(val)) {} + ~KFDRecord() { + RecordProperty(m_key.str().c_str(), m_val.c_str()); + } + std::stringstream &get_key_stream() { + return m_key; + } + virtual void TestBody() {}; +private: + std::string m_val; + std::stringstream m_key; +}; + +#define RECORD(val) (KFDRecord(val).get_key_stream()) + // All tests MUST be in a try catch since the gtest flag to throw an exception on any fatal failure is enabled #define TEST_START(testProfile) if (Ok2Run(testProfile)) try { #define TEST_END } catch (...) {} diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 8617b0893c..8cc641c375 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -176,6 +176,8 @@ TEST_F(KFDMemoryTest, MMapLarge) { LOG() << "Successfully registered and mapped " << (i * s >> 30) << "GB system memory to gpu" << std::endl; + RECORD(i * s >> 30) << "Mmap-SysMem-Size"; + while (i--) { EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(AlternateVAGPU[i]))); EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(AlternateVAGPU[i]))); @@ -1020,6 +1022,16 @@ TEST_F(KFDMemoryTest, MMBench) { << std::setw(8) << mapAllTime << std::setw(8) << unmapAllTime << std::setw(8) << freeTime << std::endl; + +#define MMBENCH_KEY_PREFIX memTypeStrings[memType] << "-" \ + << (interleaveSDMA ? "SDMA" : "noSDMA") << "-" \ + << (bufSize >> 10) << "K-" + RECORD(allocTime) << MMBENCH_KEY_PREFIX << "alloc"; + RECORD(map1Time) << MMBENCH_KEY_PREFIX << "mapOne"; + RECORD(unmap1Time) << MMBENCH_KEY_PREFIX << "unmapOne"; + RECORD(mapAllTime) << MMBENCH_KEY_PREFIX << "mapAll"; + RECORD(unmapAllTime) << MMBENCH_KEY_PREFIX << "unmapAll"; + RECORD(freeTime) << MMBENCH_KEY_PREFIX << "free"; } TEST_END @@ -1551,7 +1563,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { const unsigned nBufs = 1000; /* measure us, report ns */ unsigned testIndex, sizeIndex, memType; const unsigned nMemTypes = 2; - const char *memTypeStrings[nMemTypes] = {"SysMem", "VRAM "}; + const char *memTypeStrings[nMemTypes] = {"SysMem", "VRAM"}; const unsigned nSizes = 4; const unsigned bufSizes[nSizes] = {PAGE_SIZE, PAGE_SIZE*4, PAGE_SIZE*16, PAGE_SIZE*64}; const unsigned nTests = nSizes * nMemTypes; @@ -1641,14 +1653,22 @@ TEST_F(KFDMemoryTest, MMBandWidth) { for (i = 0; i < nBufs; i++) EXPECT_SUCCESS(hsaKmtFreeMemory(bufs[i], bufSize)); - LOG() << std::dec << std::setiosflags(std::ios::right) - << std::setw(3) << (bufSize >> 10) << "K-" - << memTypeStrings[memType] << "\t" + LOG() << std::dec + << std::right << std::setw(3) << (bufSize >> 10) << "K-" + << std::left << std::setw(14) << memTypeStrings[memType] + << std::right << std::setw(12) << mcpRTime << std::setw(12) << mcpWTime << std::setw(12) << accessRTime << std::setw(12) << accessWTime << std::endl; + +#define MMBANDWIDTH_KEY_PREFIX memTypeStrings[memType] << "-" \ + << (bufSize >> 10) << "K" << "-" + RECORD(mcpRTime) << MMBANDWIDTH_KEY_PREFIX << "mcpRTime"; + RECORD(mcpWTime) << MMBANDWIDTH_KEY_PREFIX << "mcpWTime"; + RECORD(accessRTime) << MMBANDWIDTH_KEY_PREFIX << "accessRTime"; + RECORD(accessWTime) << MMBANDWIDTH_KEY_PREFIX << "accessWTime"; } munmap(tmp, tmpBufferSize); diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index cc8671530f..27e84fef5e 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -659,6 +659,8 @@ TEST_F(KFDQMTest, BasicCuMaskingLinear) { << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; EXPECT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); + + RECORD(ratio) << "Ratio-" << nCUs << "-CUs"; } } else { LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl; @@ -730,6 +732,8 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; EXPECT_TRUE((ratio >= CuNegVariance) && (ratio <= CuPosVariance)); + + RECORD(ratio) << "Ratio-" << nCUs << "-CUs"; } } else { LOG() << "Skipping test: Test not supported for family ID 0x" << m_FamilyId << "." << std::endl; @@ -1184,6 +1188,13 @@ TEST_F(KFDQMTest, QueueLatency) { LOG() << "Queue Packet Workload: \t" << CounterToNanoSec(workload) << std::endl; LOG() << "Get GpuCounter Overhead: \t" << CounterToNanoSec(overhead) << std::endl; + RECORD(CounterToNanoSec(queue_latency_avg)) << "Queue-Latency-Avg"; + RECORD(CounterToNanoSec(queue_latency_min)) << "Queue-Latency-Min"; + RECORD(CounterToNanoSec(queue_latency_med)) << "Queue-Latency-Med"; + RECORD(CounterToNanoSec(queue_latency_max)) << "Queue-Latency-Max"; + RECORD(CounterToNanoSec(workload)) << "Queue-Packet-Workload"; + RECORD(CounterToNanoSec(overhead)) << "GpuCounter-Overhead"; + TEST_END } From f618b3f075d5e02345f6543b5bd1250a0b9570b8 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Mon, 27 Aug 2018 15:01:21 +0800 Subject: [PATCH 0417/1247] kfdtest: add KFDTestUtilQueue Some infrastructures below, Implement SdmaTimePacket which records the global GPU timestamp. Introduce class AsyncMPSQ and AsyncMPMQ. AsyncMPSQ is aka async multiple packet single queue. It takes a set of packet when create and submits them to a GPU to run. While AsyncMPMQ is aka async multiple packet multiple queue. It manages a set of AsyncMPSQ, and use a forloop to do operations of AsyncMPSQ. Implement sdma_multicopy helper functions. Change-Id: I47e1d2ca9630113b2a1d85a0055f3f8ee629fb5f Signed-off-by: xinhui pan --- tests/kfdtest/include/sdma_pkt_struct.h | 38 ++ tests/kfdtest/src/KFDTestUtil.hpp | 2 + tests/kfdtest/src/KFDTestUtilQueue.cpp | 512 ++++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtilQueue.hpp | 70 ++++ tests/kfdtest/src/SDMAPacket.cpp | 21 + tests/kfdtest/src/SDMAPacket.hpp | 19 + 6 files changed, 662 insertions(+) create mode 100644 tests/kfdtest/src/KFDTestUtilQueue.cpp create mode 100644 tests/kfdtest/src/KFDTestUtilQueue.hpp diff --git a/tests/kfdtest/include/sdma_pkt_struct.h b/tests/kfdtest/include/sdma_pkt_struct.h index bc37b5189f..127c44f43d 100644 --- a/tests/kfdtest/include/sdma_pkt_struct.h +++ b/tests/kfdtest/include/sdma_pkt_struct.h @@ -30,6 +30,7 @@ const unsigned int SDMA_OP_WRITE = 2; const unsigned int SDMA_OP_FENCE = 5; const unsigned int SDMA_OP_TRAP = 6; +const unsigned int SDMA_OP_TIMESTAMP = 13; const unsigned int SDMA_OP_CONST_FILL = 11; @@ -316,4 +317,41 @@ typedef struct SDMA_PKT_TRAP_TAG } SDMA_PKT_TRAP, *PSDMA_PKT_TRAP; +/* +** Definitions for SDMA_PKT_TIMESTAMP packet +*/ + +typedef struct SDMA_PKT_TIMESTAMP_TAG +{ + + union + { + struct + { + unsigned int op:8; + unsigned int sub_op:8; + unsigned int reserved_0:16; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union + { + struct + { + unsigned int addr_31_0:32; + }; + unsigned int DW_1_DATA; + } ADDR_LO_UNION; + + union + { + struct + { + unsigned int addr_63_32:32; + }; + unsigned int DW_2_DATA; + } ADDR_HI_UNION; +} SDMA_PKT_TIMESTAMP, *PSDMA_PKT_TIMESTAMP; + #endif // __SDMA_PKT_STRUCT_H__ diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 014b31d13f..fb4937e7dc 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -32,6 +32,8 @@ class BaseQueue; #define ARRAY_SIZE(_x) (sizeof(_x)/sizeof(_x[0])) +#define ALIGN_UP(x, align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) +#define CounterToNanoSec(x) ((x) * 1000 / (is_dgpu() ? 27 : 100)) uint64_t RoundToPowerOf2(uint64_t val); diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/tests/kfdtest/src/KFDTestUtilQueue.cpp new file mode 100644 index 0000000000..b6604287cf --- /dev/null +++ b/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -0,0 +1,512 @@ +/* + * Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include +#include +#include +#include "SDMAQueue.hpp" +#include "PM4Queue.hpp" +#include "SDMAPacket.hpp" +#include "PM4Packet.hpp" +#include "KFDTestUtil.hpp" +#include "KFDTestUtilQueue.hpp" + +#define MB_PER_SEC(size, time) ((((size) * 1ULL) >> 20) * 1000ULL * 1000ULL * 1000ULL / (time)) + +class AsyncMPSQ; +class AsyncMPMQ; + +typedef std::shared_ptr sharedAsyncMPSQ; +typedef std::list AsyncMPSQList; + +typedef std::shared_ptr sharedPacket; +typedef std::list PacketList; + +/* AsyncMPSQ is short for Async multiple packet single queue. + * It is allowed to place a list of packets to run on one queue of the specified GPU node. + */ +class AsyncMPSQ { + public: + AsyncMPSQ() : m_queue(NULL), m_buf(NULL), m_event(NULL) { /*do nothing*/} + + virtual ~AsyncMPSQ(void) { Destroy(); } + + /* It is the main function to deal with the packet and queue.*/ + void PlacePacketOnNode(PacketList &packetList, int node, TSPattern tsp); + + /* Run the packets placed on nodes and return immediately.*/ + void Submit(void) { ASSERT_NE((HSAuint64)m_queue, NULL); m_queue->SubmitPacket(); } + + /* Return only when all packets are consumed. + * If there is any packet issues some IO operations, wait these IO to complete too. + */ + void Wait(void) { ASSERT_NE((HSAuint64)m_queue, NULL); m_queue->Wait4PacketConsumption(m_event); } + + /* Report the time used between packet [begin, end) in Global Counter on success. + * Return 0 on failure. + */ + HSAuint64 Report(int indexOfPacketBegin = 0, int indexOfPacketEnd = 0); + /* Report the timestamp around the packet. + * Return the time used on success. + * Return 0 on failure. + */ + HSAuint64 Report(int indexOfPacket, HSAuint64 &tsBegin, HSAuint64 &tsEnd); + + private: + BaseQueue *m_queue; + HSA_QUEUE_TYPE m_queueType; + HsaEvent *m_event; + /* m_ts points to m_buf's memory.*/ + HsaMemoryBuffer *m_buf; + TimeStamp *m_ts; + unsigned m_ts_count; + TSPattern m_ts_pattern; + + void AllocTimeStampBuf(int packetCount); + void Destroy(); + + /* It determines which queue will be created.*/ + void InitQueueType(PACKETTYPE packetType) { + if (packetType == PACKETTYPE_SDMA) + m_queueType = HSA_QUEUE_SDMA; + else if (packetType == PACKETTYPE_PM4) + m_queueType = HSA_QUEUE_COMPUTE; + else + WARN() << "Unsupported queue type!" << std::endl; + } + + unsigned int TimePacketSize(void) { + if (m_queueType == HSA_QUEUE_SDMA) + return SDMATimePacket(0).SizeInBytes(); + else if (m_queueType == HSA_QUEUE_COMPUTE) + return PM4ReleaseMemoryPacket(0, 0, 0, 0, 0).SizeInBytes(); + return 0; + } + + void CreateNewQueue(int node, unsigned int queueSize) { + if (m_queueType == HSA_QUEUE_SDMA) + m_queue = new SDMAQueue(); + else if (m_queueType == HSA_QUEUE_COMPUTE) + m_queue = new PM4Queue(); + else { + m_queue = NULL; + WARN() << "Unsupported queue type!" << std::endl; + } + + if (m_queue) + ASSERT_SUCCESS(m_queue->Create(node, queueSize)); + } + + void PlaceTimestampPacket(void *addr) { + if (m_queueType == HSA_QUEUE_SDMA) + PlacePacket(SDMATimePacket(addr)); + else if (m_queueType == HSA_QUEUE_COMPUTE) + PlacePacket( + PM4ReleaseMemoryPacket(true, (HSAuint64)addr, 0, true, true)); + else + WARN() << "Unsupported queue type!" << std::endl; + } + + void PlacePacket(const BasePacket &packet) { + m_queue->PlacePacket(packet); + } +}; + +void AsyncMPSQ::Destroy(void) { + /* Delete queue first.*/ + if (m_queue) { + delete m_queue; + } + + if (m_buf) + delete m_buf; + + if (m_event) + hsaKmtDestroyEvent(m_event); +} + +void AsyncMPSQ::AllocTimeStampBuf(int packetCount) { + if (m_ts_pattern == NOTS) { + m_buf = NULL; + m_ts = NULL; + m_ts_count = 0; + return; + } + + if (m_ts_pattern == ALLTS) + /* One extra timestamp packet.*/ + m_ts_count = packetCount + 1; + else + m_ts_count = 2; + + /* One more timestamp space to fit with alignment.*/ + HSAuint64 size = ALIGN_UP(sizeof(TimeStamp) * (m_ts_count + 1), PAGE_SIZE); + + m_buf = new HsaMemoryBuffer(size, 0, true, false); + + TimeStamp *array = m_buf->As(); + + /* SDMATimePacket need 32bytes aligned boundary dst address*/ + m_ts = reinterpret_castALIGN_UP(array, sizeof(TimeStamp)); +} + +void AsyncMPSQ::PlacePacketOnNode(PacketList &packets, int node, TSPattern tsp = ALLTS) { + int nPacket = packets.size(); + + if (nPacket == 0) { + WARN() << "Empty packetList!" << std::endl; + return; + } + + /*1: All resources should be freed.*/ + Destroy(); + + /*2: Must initialize queueType first.*/ + InitQueueType(packets.front()->PacketType()); + /*3: Initialize timestamp buf second with the pattern.*/ + m_ts_pattern = tsp; + AllocTimeStampBuf(nPacket); + /*4: Create a event for Wait().*/ + CreateQueueTypeEvent(false, false, node, &m_event); + + int i = -1; + int packetSize = 0; + /* Calculate the space to put all timestamp packet.*/ + int timePacketSize = TimePacketSize() * m_ts_count; + /* Another one page space to put fence, trap, etc*/ + int extraPacketSize = PAGE_SIZE + timePacketSize; + + /* To calculate the total packet size we will need to create the queue. + * As the packet in the vector might be different with each other, + * we have no other way to calculate the queuesize. + */ + for (auto &packet : packets) + packetSize += packet->SizeInBytes(); + + /* queueSize need be power of 2.*/ + const int queueSize = RoundToPowerOf2(packetSize + extraPacketSize); + + /*5: Create a new queue on node for the packets.*/ + CreateNewQueue(node, queueSize); + + if (tsp != NOTS) { + i++; + PlaceTimestampPacket(m_ts + i); + } + + for (auto &packet : packets) { + PlacePacket(*packet); + if (tsp == ALLTS) { + i++; + PlaceTimestampPacket(m_ts + i); + } + } + + if (tsp == HEAD_TAIL) { + i++; + PlaceTimestampPacket(m_ts + i); + } + + ASSERT_EQ(i + 1, m_ts_count); +} + +HSAuint64 AsyncMPSQ::Report(int indexOfPacket, HSAuint64 &begin, HSAuint64 &end) { + /* Should not get any timestamp if NOTS is specified.*/ + int error = 0; + EXPECT_NE(m_ts_pattern, NOTS) + << " Error " << ++error << ": No timestamp would be reported!" << std::endl; + + if (m_ts_pattern == HEAD_TAIL) + indexOfPacket = 0; + + EXPECT_NE((HSAuint64)m_ts, NULL) + << " Error " << ++error << ": No timestamp buf!" << std::endl; + /* m_ts_count is equal to packets count + 1, see PlacePacketOnNode(). + * So the max index of a packet is m_ts_count - 2. + * make it unsigned to defend any minus values. + */ + EXPECT_GE(m_ts_count - 2, (unsigned)indexOfPacket) + << " Error " << ++error << ": Index overflow!" << std::endl; + + if (error) + return 0; + + begin = m_ts[indexOfPacket].timestamp; + end = m_ts[indexOfPacket + 1].timestamp; + return end - begin; +} + +HSAuint64 AsyncMPSQ::Report(int indexOfPacketBegin, int indexOfPacketEnd) { + HSAuint64 ts[4]; + int error = 0; + + if (indexOfPacketEnd == 0) + indexOfPacketEnd = m_ts_count - 1; + + EXPECT_GT((unsigned)indexOfPacketEnd, (unsigned)indexOfPacketBegin) + << " Error " << ++error << ": Index inverted!" << std::endl; + + if (error) + return 0; + /* Get the timestamps around the two packets.*/ + if (!Report(indexOfPacketBegin, ts[0], ts[1])) + return 0; + /* [begin, end)*/ + if (!Report(indexOfPacketEnd - 1, ts[2], ts[3])) + return 0; + + EXPECT_GT(ts[3], ts[0]) + << " Waring: Might be wrong timestamp values!" << std::endl; + + return ts[3] - ts[0]; +} + +/* AsyncMPMQ is short for Async multiple packet multiple queue. + * AsyncMPMQ manages a list of AsyncMPSQ. + * So the packet can be running on multiple GPU nodes at same time. + */ + +class AsyncMPMQ { + public: + AsyncMPMQ(void) { /* do nothing*/} + + virtual ~AsyncMPMQ(void) { /*do nothing*/} + + sharedAsyncMPSQ PlacePacketOnNode(PacketList &packetList, int node, TSPattern tsp = ALLTS) { + /* Create a sharedAsyncMPSQ object and push it into the AsyncMPSQList. + * As we might submit packet to same GPU nodes several times, AsyncMPSQ * + * is returned to stand for the AsyncMPSQ it is created with + */ + sharedAsyncMPSQ mpsq_ptr(new AsyncMPSQ); + mpsq_ptr->PlacePacketOnNode(packetList, node, tsp); + m_mpsqList.push_back(mpsq_ptr); + return mpsq_ptr; + } + + void Submit(void) { + for (auto &mpsq : m_mpsqList) + mpsq->Submit(); + } + + void Wait(void) { + for (auto &mpsq : m_mpsqList) + mpsq->Wait(); + } + + private: + AsyncMPSQList m_mpsqList; +}; + + +/* + * SDMA queue helper functions. + */ + +bool sort_SDMACopyParams(SDMACopyParams &a1, SDMACopyParams &a2) { + if (a1.node != a2.node) + return a1.node < a2.node; + return a1.group < a2.group; +} + +/* + * Copy from src to dst with corresponding sDMA. + * It will try to merge copy on same node into one queue unless + * caller forbid it by setting mashup to 0 and SDMACopyParams::group to different values. + * On condition of mashup is 1, it will re-sort array into mergeable state. + * All mergeable copy will be placed together. + * On condition os mashup is 0, it keeps array in original order. + * It will merge nearby copy if they have same group and node anyway. + */ +void sdma_multicopy(std::vector &array, int mashup, TSPattern tsp) { + int i, packet_index = 0, queue_index = 0; + PacketList packetList; + AsyncMPMQ obj; + std::vector handle; + + /* Sort it and then reduce the amount of queues if caller permits. + * We might change the order of array only here. + */ + if (mashup) + std::sort(array.begin(), array.end(), sort_SDMACopyParams); + + for (i = 0; i < array.size(); i++) { + sharedPacket packet(new + SDMACopyDataPacket(array[i].dst, array[i].src, array[i].size)); + packetList.push_back(packet); + + /* We put the real queue_id in local handle[] to reduce some assignment.*/ + array[i].queue_id = queue_index; + /* Every queue has its packets with the index starts from 0.*/ + array[i].packet_id = packet_index++; + + /* If next copy is on same node and group, try to merge it into same queue.*/ + if (i + 1 < array.size() && array[i].node == array[i + 1].node + && array[i].group == array[i + 1].group) + continue; + + /* Now we have prepare one packetList, place packet into the queue on GPU node.*/ + queue_index++; + handle.push_back(obj.PlacePacketOnNode(packetList, array[i].node, tsp)); + + /* Prepare a new(empty) packetList.*/ + packetList.clear(); + + /* Prepare a new(zero) packet index for the packets in the new queue.*/ + packet_index = 0; + } + + obj.Submit(); + obj.Wait(); + + if (tsp == NOTS) + return; + + /* Get the time used by packet.*/ + for (i = 0; i < array.size(); i++) + array[i].timeConsumption = (handle[array[i].queue_id])->Report( + array[i].packet_id, array[i].timeBegin, array[i].timeEnd); +} + +static +void sdma_multicopy_report(std::vector &array, HSAuint64 countPerGroup, std::stringstream *msg, + HSAuint64 &timeConsumptionMin, HSAuint64 &timeConsumptionMax, + HSAuint64 &totalSizeMin, HSAuint64 &totalSizeMax) { + HSAuint64 begin, end; + /* There can be different count of copies in different groups in the future. + * But assume they are same now. + */ + HSAuint64 group = array.size() / countPerGroup; + HSAuint64 interval = -1; + timeConsumptionMin = -1; + timeConsumptionMax = 0; + totalSizeMin = totalSizeMax = 0; + + /* Try to find out + * 1) The max/min timeConsumption of one copy in all copies. + * 2) The minimal average of timeConsumption of one packet in all copies. + * And one char # or - stands for one interval, aka minimal average. + * Say, one copy use 10ns with 10 copy packets. the other copy use 20ns + * with 10 copy packets. So the interval is 1ns, the timeConsumption is 20ns. + * So the ouput msg will be like + * ########## //copy1 10ns + * #---##----####### //copy2 20ns + */ + for (int i = 0; i < group; i++) { + HSAuint64 begin, end, base = i * countPerGroup; + + begin = array[base].timeBegin; + end = array[base + countPerGroup - 1].timeEnd; + + if (timeConsumptionMax < end - begin) + timeConsumptionMax = end - begin; + + if (timeConsumptionMin > end - begin) + timeConsumptionMin = end - begin; + } + + interval = timeConsumptionMin / countPerGroup; + + /* Draw the timestamp event for each copy list. + * - means still doing copy. + * # means just finish one copy. + */ + if (msg) + for (int i = 0; i < group; i++) { + HSAuint64 base = i * countPerGroup; + HSAuint64 last = array[base].timeBegin; + HSAuint64 timeConsumption; + + *msg << "[" << array[base].node << " : " << array[base].group << "] "; + + for (int j = 0; j < countPerGroup; j++) { + timeConsumption = array[base + j].timeEnd - last; + + while (timeConsumption >= interval) { + timeConsumption -= interval; + last += interval; + + if (timeConsumption >= interval) + *msg << "-"; + else + *msg << "#"; + }; + } + + *msg << std::endl; + } + + /* Try to find out + * 1) The size of all copies in all queues. + * 2) The size of the copies running within the same period in all queues. + * We assume all packets begin to run at same time. + */ + for (int i = 0; i < group; i++) { + HSAuint64 base = i * countPerGroup; + HSAuint64 time = 0; + + for (int j = 0; j < countPerGroup; j++) { + totalSizeMax += array[base + j].size; + + if (time < timeConsumptionMin) { + time += array[base + j].timeConsumption; + totalSizeMin += array[base + j].size; + } + } + } +} + +/* + * Do copy with corresponding sDMA. + */ +void +sdma_multicopy(SDMACopyParams *copyArray, int arrayCount, + HSAuint64 *minSpeed, HSAuint64 *maxSpeed, std::stringstream *msg) { + const HSAuint64 countPerGroup = minSpeed || maxSpeed ? 100 : 1; + std::vector array; + HSAuint64 totalSizeMin, totalSizeMax, timeConsumptionMin, timeConsumptionMax; + + for (int i = 0; i < arrayCount; i++) { + /* Each copy has its own queue.*/ + copyArray[i].group = i; + for (int j = 0; j < countPerGroup; j++) + array.push_back(copyArray[i]); + } + + sdma_multicopy(array, 0, ALLTS); + + sdma_multicopy_report(array, countPerGroup, msg, + timeConsumptionMin, timeConsumptionMax, + totalSizeMin, totalSizeMax); + + if (minSpeed) + *minSpeed = MB_PER_SEC(totalSizeMin, CounterToNanoSec(timeConsumptionMin)); + + if (maxSpeed) + *maxSpeed = MB_PER_SEC(totalSizeMax, CounterToNanoSec(timeConsumptionMax)); +} + +/* + * PM4 queue helper functions. + */ +// TODO diff --git a/tests/kfdtest/src/KFDTestUtilQueue.hpp b/tests/kfdtest/src/KFDTestUtilQueue.hpp new file mode 100644 index 0000000000..dac222b623 --- /dev/null +++ b/tests/kfdtest/src/KFDTestUtilQueue.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD__TEST__UTIL__QUEUE__H__ +#define __KFD__TEST__UTIL__QUEUE__H__ + +#include "hsakmt.h" +#include + +typedef struct { + HSAuint64 timestamp; + HSAuint64 timeConsumption; + HSAuint64 timeBegin; + HSAuint64 timeEnd; +} TimeStamp; + +/* We have three pattern to put timestamp packet, + * NOTS: No timestamp packet insert. + * ALLTS: Put timestamp packet around every packet. This is the default behavoir. + * It will look like |timestamp|packet|timestamp|...|packet|timestamp| + * HEAD_TAIL: Put timestmap packet at head and tail to measure the overhead of a bunch of packet. + * It will look like |timestamp|packet|...|packet|timestamp| + */ +typedef enum { + NOTS = 0, + ALLTS = 1, + HEAD_TAIL = 2, +} TSPattern; + +typedef struct { + /* input values*/ + HSAuint32 node; + void *src; + void *dst; + HSAuint64 size; + /* input value for internal use.*/ + HSAuint64 group; + /* output value*/ + HSAuint64 timeConsumption; + HSAuint64 timeBegin; + HSAuint64 timeEnd; + /* private: Output values for internal use.*/ + HSAuint64 queue_id; + HSAuint64 packet_id; +} SDMACopyParams; + +void sdma_multicopy(SDMACopyParams *array, int n, + HSAuint64 *speedSmall = 0, HSAuint64 *speedLarge = 0, std::stringstream *s = 0); +void sdma_multicopy(std::vector &array, int mashup = 1, TSPattern tsp = ALLTS); +#endif //__KFD__TEST__UTIL__QUEUE__H__ diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index 2dbb449d9f..81ed3ba73d 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -207,3 +207,24 @@ void SDMATrapPacket::InitPacket(unsigned int eventID) { packetData.HEADER_UNION.op = SDMA_OP_TRAP; packetData.INT_CONTEXT_UNION.int_context = eventID; } + +SDMATimePacket::SDMATimePacket(void *destaddr) { + InitPacket(destaddr); +} + +SDMATimePacket::~SDMATimePacket(void) { +} + +void SDMATimePacket::InitPacket(void *destaddr) { + memset(&packetData, 0, SizeInBytes()); + + packetData.HEADER_UNION.op = SDMA_OP_TIMESTAMP; + packetData.HEADER_UNION.sub_op = 1 << 1; /* Get Global GPU Timestamp*/ + + if (reinterpret_cast(destaddr) & 0x1f) + WARN() << "SDMATimePacket dst address must aligned to 32bytes boundary" << std::endl; + + SplitU64(reinterpret_cast(destaddr), + packetData.ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ + packetData.ADDR_HI_UNION.DW_2_DATA); /*dst_addr_63_32*/ +} diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index 074c52976e..b1afb374ce 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -141,4 +141,23 @@ class SDMATrapPacket : public SDMAPacket { SDMA_PKT_TRAP packetData; }; +class SDMATimePacket : public SDMAPacket { + public: + // Empty constructor, before using the packet call the init func + SDMATimePacket(void*); + + virtual ~SDMATimePacket(void); + + // @returns Pointer to the packet + virtual const void *GetPacket() const { return &packetData; } + // @brief Initialise the packet + void InitPacket(void*); + // @returns Packet size in bytes + virtual unsigned int SizeInBytes() const { return sizeof(SDMA_PKT_TIMESTAMP); } + + protected: + SDMA_PKT_TIMESTAMP packetData; +}; + + #endif // __KFD_SDMA_PACKET__H__ From e5a541eaf258351af9f00a6bfa22bb32a45c1cc3 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Mon, 27 Aug 2018 15:08:43 +0800 Subject: [PATCH 0418/1247] kfdtest: Add P2P bandwidth test The test measures the bandwidth between GPUs. Currently we do not care numa topology as some products really support across PCI-e root complex p2p. test result on two gfx900 system. [ RUN ] KFDPerformanceTest.P2PBandWidthTest [ ] Copy from node to node by [push, NONE] [ ] [1 -> 0] 6.13477 - 6.12695 GB/s [ ] [1 -> 2] 3.77734 - 3.76855 GB/s [ ] [2 -> 0] 6.67676 - 6.6543 GB/s [ ] [2 -> 1] 6.14453 - 6.12793 GB/s [ ] Copy from node to node by [pull, NONE] [ ] [1 -> 0] 6.10547 - 6.08105 GB/s [ ] [1 -> 2] 9.65527 - 9.65039 GB/s [ ] [2 -> 0] 6.49805 - 6.4873 GB/s [ ] [2 -> 1] 8.95508 - 8.85254 GB/s [ ] Full duplex copy from node to node by [push|pull, NONE] [ ] [1 -> 0] 11.0986 - 11.0986 GB/s [ ] [1 -> 2] 7.54297 - 7.54297 GB/s [ ] [2 -> 0] 12.0264 - 11.9639 GB/s [ ] [2 -> 1] 12.0469 - 12.0371 GB/s [ ] Full duplex copy from node to node by [push, push] [ ] [1 <-> 2] 11.7324 - 11.4541 GB/s [ ] Full duplex copy from node to node by [pull, pull] [ ] [1 <-> 2] 11.4824 - 11.0508 GB/s [ ] Copy from node to multiple nodes by [push, NONE] [ ] [1 -> [0...2]] 5.625 - 5.73633 GB/s [ ] [2 -> [0...2]] 6.45801 - 6.4707 GB/s [ ] Copy from multiple nodes to node by [push, NONE] [ ] [[1...2] -> 0] 12.8379 - 12.2578 GB/s Now we can get more timestamp info like below. Copy from node to node by [push, NONE] [1 -> 0] [1 : 0] #-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-##-#-#-#-#-############################### [1 : 1] #################################################################################################### [1 -> 2] [1 : 0] #--#-#-#-#-#--#-#-#-#-#--#-#-#-#-#--#-#-#-#-#-#--#-#-#-#-#--#-#-#-#-#--#-#-#-#-#-#--#-#-#-#-#--#-#-###################################### [1 : 1] ##################################################################################################-# [2 -> 0] [2 : 0] ##-###-##-###-###-##-###-##-###-###-##-###-###-##-###-###-##-###-##-###-###-##-###-###-##-###-###-################# [2 : 1] ###############################################################################-#############-###-## [2 -> 1] [2 : 0] ##-##-##-##-##-###-##-##-##-##-##-###-##-##-##-##-###-##-##-##-##-##-###-##-##-##-##-###-##-##-##-#################### [2 : 1] ################################################################################-###-############-## [snip] Full duplex copy from node to node by [push, push] [1 <-> 2] [1 : 0] #-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-#-#-#-#################################### [1 : 1] ################-###################################################-############-####-############# [2 : 2] #-##-##-##-#-##-##-##-##-#-##-##-##-##-#-##-##-##-##-#-##-##-##-##-#-##-##-##-##-##-#-##-##-##-##-#-##-##-##-##-##-#-################## [2 : 3] #####-######-#####-######-#####-######-#####-######-#####-######-#####-######-#####-######-#####-######-#####-#####-## Full duplex copy from node to node by [pull, pull] [1 <-> 2] [1 : 0] ######################################################################-##-#-###############-####-### [1 : 1] #-#-#-##-#-#-##-#-#-##-#-#-##-#-#-##-#-#-##-#-#-##-#-#-#-##-#-#-##-#-#-##-#-#-##-#-#-##-#-#-##-#-#-############################ [2 : 2] ##-##-##-##-###-##-##-##-##-###-##-##-##-###-##-##-##-##-###-##-##-##-###-##-##-##-##-###-##-##-##-##-###-##-##-##-###-##-##-############ [2 : 3] #-#-#-#-#-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#########-############# Copy from node to multiple nodes by [push, NONE] [1 -> [0...2]] [1 : 0] #-#--#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-#-############################### [1 : 1] ########################################################################################-###-###-### [2 -> [0...2]] [2 : 0] ##-##-##-###-##-###-##-##-###-##-###-##-##-###-##-###-##-###-##-##-###-##-###-##-##-###-##-###-##-################## [2 : 1] -################################################################################################-## Copy from multiple nodes to node by [push, NONE] [[1...2] -> 0] [1 : 0] #-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-##-#-#-#-#-#-#-#-#-##-#-#-#-############################### [1 : 1] ################################################################################################-#-# [2 : 2] ##-##-##-###-##-##-###-##-##-##-###-##-##-###-##-##-###-##-##-###-##-##-##-###-##-##-###-##-##-###-##-################## [2 : 3] #########################-#########################-#########################-######################### [ OK ] KFDPerformanceTest.P2PBandWidthTest (15982 ms) Change-Id: Ia90044191d51650ccb220476d31fb317aa3ad6ce Signed-off-by: xinhui pan --- tests/kfdtest/scripts/kfdtest.exclude | 3 +- tests/kfdtest/src/KFDPerformanceTest.cpp | 265 +++++++++++++++++++++++ tests/kfdtest/src/KFDQMTest.cpp | 30 ++- 3 files changed, 280 insertions(+), 18 deletions(-) create mode 100644 tests/kfdtest/src/KFDPerformanceTest.cpp diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index ba80c275d8..6607cacdb8 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -43,7 +43,8 @@ SDMA_BLACKLIST=\ "KFDIPCTest.*:"\ "KFDMemoryTest.MMBench:"\ "KFDQMTest.*Sdma*:"\ -"KFDQMTest.P2PTest" +"KFDQMTest.P2PTest:"\ +"KFDPerformanceTest.P2PBandWidthTest" # Anything involving CP queue creation is failing on Kaveri. Separate them here for convenience (KFD-336) KV_QUEUE_BLACKLIST=\ diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp new file mode 100644 index 0000000000..8b86d98555 --- /dev/null +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include +#include +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "AqlQueue.hpp" +#include "KFDTestUtilQueue.hpp" +#include +#include +#include "KFDBaseComponentTest.hpp" + +class KFDPerformanceTest: public KFDBaseComponentTest { + protected: + virtual void SetUp(); + virtual void TearDown(); +}; + +void KFDPerformanceTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + ROUTINE_END +} + +void KFDPerformanceTest::TearDown() { + ROUTINE_START + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +enum P2PDirection { + IN = 1, + OUT = 2, + IN_OUT = 3, + NONE = 4, +}; + +/* + * Do the copy of one GPU from & to multiple GPUs. + */ +static void +testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirection n1Direction, + P2PDirection n2Direction, HSAuint64 size, HSAuint64 &speed, HSAuint64 &speed2, std::stringstream &msg) { + ASSERT_GT(16, unsigned(n - 1)); + HSAuint32 n2[n]; + void *n1Mem, *n2Mem[n]; + HsaMemFlags memFlags = {0}; + memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + memFlags.ui32.HostAccess = 1; + memFlags.ui32.NonPaged = 1; + SDMACopyParams array[n * 4]; + int array_count = 0; + int i; + + ASSERT_SUCCESS(hsaKmtAllocMemory(n1, size, memFlags, &n1Mem)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n1Mem, size, NULL)); + + for (i = 0; i < n; i++) { + n2[i] = n2Array[i]; + ASSERT_SUCCESS(hsaKmtAllocMemory(n2[i], size, memFlags, &n2Mem[i])); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n2Mem[i], size, NULL)); + } + + for (i = 0; i < n; i++) { + if (n1Direction != NONE) + ASSERT_NE(n1, 0); + if (n2Direction != NONE) + ASSERT_NE(n2[i], 0); + + do { + if (n1Direction == IN || n1Direction == IN_OUT) + /* n2Mem -> n1Mem*/ + array[array_count++] = {n1, n2Mem[i], n1Mem, size}; + if (n1Direction == OUT || n1Direction == IN_OUT) + /* n1Mem -> n2Mem*/ + array[array_count++] = {n1, n1Mem, n2Mem[i], size}; + /* Issue two copies to make full use of sdma.*/ + } while (n1Direction < IN_OUT && n == 1 && array_count % 2); + /* Do nothing if no IN or OUT specified.*/ + + do { + if (n2Direction == IN || n2Direction == IN_OUT) + /* n1Mem -> n2Mem*/ + array[array_count++] = {n2[i], n1Mem, n2Mem[i], size}; + if (n2Direction == OUT || n2Direction == IN_OUT) + /* n2Mem -> n1Mem*/ + array[array_count++] = {n2[i], n2Mem[i], n1Mem, size}; + } while (n2Direction < IN_OUT && array_count % 2); + } + + sdma_multicopy(array, array_count, &speed, &speed2, &msg); + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(n1Mem)); + EXPECT_SUCCESS(hsaKmtFreeMemory(n1Mem, size)); + + for (i = 0; i < n; i++) { + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(n2Mem[i])); + EXPECT_SUCCESS(hsaKmtFreeMemory(n2Mem[i], size)); + } +} + +TEST_F(KFDPerformanceTest, P2PBandWidthTest) { + TEST_START(TESTPROFILE_RUNALL); + if (!is_dgpu()) { + LOG() << "Skipping test: Can't have 2 APUs on the same system." << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + std::vector nodes; + const bool isSpecified = g_TestDstNodeId != -1 && g_TestNodeId != -1; + + for (unsigned i = 0; i < gpuNodes.size(); i++) + if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && + /* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */ + (!isSpecified || gpuNodes.at(i) == g_TestDstNodeId || gpuNodes.at(i) == g_TestNodeId)) + nodes.push_back(gpuNodes.at(i)); + + if (nodes.size() < 2) { + LOG() << "Skipping test: Need at least two large bar GPU." << std::endl; + return; + } + + std::vector sysNodes(nodes); // include sysMem node 0... + sysNodes.insert(sysNodes.begin(),0); + + const int total_tests = 7; + const char *test_suits_string[total_tests] = { + "Copy from node to node by [push, NONE]", + "Copy from node to node by [pull, NONE]", + "Full duplex copy from node to node by [push|pull, NONE]", + "Full duplex copy from node to node by [push, push]", + "Full duplex copy from node to node by [pull, pull]", + "Copy from node to multiple nodes by [push, NONE]", + "Copy from multiple nodes to node by [push, NONE]", + }; + const P2PDirection test_suits[total_tests][2] = { + /* One node used.*/ + {OUT, NONE}, + {IN, NONE}, + {IN_OUT,NONE}, + /* two nodes used.*/ + {OUT, OUT}, + {IN, IN}, + /* Multi nodes used*/ + {OUT, NONE}, + {NONE, OUT}, + }; + const int twoNodesIdx = 3; + const int multiNodesIdx = 5; + const HSAuint32 size = 32ULL << 20; + int s = 0; //test index; + std::stringstream msg; + char str[64]; + + for (; s < twoNodesIdx; s++) { + LOG() << test_suits_string[s] << std::endl; + msg << test_suits_string[s] << std::endl; + + for (unsigned i = 0; i < nodes.size(); i++) { + /* Src node is a GPU.*/ + HSAuint32 n1 = nodes[i]; + HSAuint64 speed, speed2; + + /* Pick up dst node which can be sysMem.*/ + for (unsigned j = 0; j < sysNodes.size(); j++) { + HSAuint32 n2 = sysNodes[j]; + if (n1 == n2) + continue; + + snprintf(str, sizeof(str), "[%d -> %d] ", n1, n2); + msg << str << std::endl; + testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + + LOG() << std::dec << str << (float)speed / 1024 << " - " << + (float)speed2 / 1024 << " GB/s" << std::endl; + } + } + } + + for (; s < multiNodesIdx; s++) { + LOG() << test_suits_string[s] << std::endl; + msg << test_suits_string[s] << std::endl; + + for (unsigned i = 0; i < nodes.size(); i++) { + HSAuint32 n1 = nodes[i]; + HSAuint64 speed, speed2; + + for (unsigned j = i + 1; j < nodes.size(); j++) { + HSAuint32 n2 = nodes[j]; + + snprintf(str, sizeof(str), "[%d <-> %d] ", n1, n2); + msg << str << std::endl; + testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + + LOG() << std::dec << str << (float)speed / 1024 << " - " << + (float)speed2 / 1024 << " GB/s" << std::endl; + } + } + } + + for (; s < total_tests && !isSpecified; s++) { + LOG() << test_suits_string[s] << std::endl; + msg << test_suits_string[s] << std::endl; + /* Just use GPU nodes to do copy.*/ + std::vector &src = test_suits[s][0] != NONE ? nodes : sysNodes; + std::vector &dst = test_suits[s][1] != NONE ? nodes : sysNodes; + + for (unsigned i = 0; i < src.size(); i++) { + HSAuint32 n1 = src[i]; + HSAuint64 speed, speed2; + HSAuint32 n2[dst.size()]; + int n = 0; + char str[64]; + + for (unsigned j = 0; j < dst.size(); j++) + if (dst[j] != n1) + n2[n++] = dst[j]; + /* At least 2 dst GPUs.*/ + if (n < 2) + continue; + + if (test_suits[s][1] == OUT) + snprintf(str, sizeof(str), "[[%d...%d] -> %d] ", dst.front(), dst.back(), n1); + else + snprintf(str, sizeof(str), "[%d -> [%d...%d]] ", n1, dst.front(), dst.back()); + msg << str << std::endl; + testNodeToNodes(n1, n2, n, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + + LOG() << std::dec << str << (float)speed / 1024 << " - " << + (float)speed2 / 1024 << " GB/s" << std::endl; + } + } + + /* New line.*/ + LOG() << std::endl << msg.str() << std::endl; + + TEST_END +} diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 27e84fef5e..dba670f4c1 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -29,6 +29,7 @@ #include "SDMAPacket.hpp" #include "SDMAQueue.hpp" #include "AqlQueue.hpp" +#include #include "Dispatch.hpp" @@ -1060,10 +1061,6 @@ TEST_F(KFDQMTest, CreateAqlCpQueue) { TEST_END } -#define ALIGN_UP(x, align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) -#define CounterToNanoSec(x) ((x) * 1000 / (is_dgpu() ? 27 : 100)) - -#include TEST_F(KFDQMTest, QueueLatency) { TEST_START(TESTPROFILE_RUNALL); @@ -1378,29 +1375,28 @@ TEST_F(KFDQMTest, mGPUShareBO) { TEST_END } - -static void sdma_copy(HSAint32 node, void *src, void *const dst[], int n, unsigned int size) { - ROUTINE_START; - +static void +sdma_copy(HSAuint32 node, void *src, void *const dst[], int n, HSAuint64 size) { SDMAQueue sdmaQueue; + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, node, &event)); ASSERT_SUCCESS(sdmaQueue.Create(node)); sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(dst, src, n, size)); - sdmaQueue.Wait4PacketConsumption(); + sdmaQueue.Wait4PacketConsumption(event); EXPECT_SUCCESS(sdmaQueue.Destroy()); - - ROUTINE_END; + hsaKmtDestroyEvent(event); } -static void sdma_fill(HSAint32 node, void *dst, unsigned int data, unsigned int size) { - ROUTINE_START; - +static void +sdma_fill(HSAint32 node, void *dst, unsigned int data, HSAuint64 size) { SDMAQueue sdmaQueue; + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, node, &event)); ASSERT_SUCCESS(sdmaQueue.Create(node)); sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(dst, data, size)); - sdmaQueue.Wait4PacketConsumption(); + sdmaQueue.Wait4PacketConsumption(event); EXPECT_SUCCESS(sdmaQueue.Destroy()); - - ROUTINE_END; + hsaKmtDestroyEvent(event); } TEST_F(KFDQMTest, P2PTest) { From fb79a0efe2c8d4b48fe9d8ce251c67f1ac8d9bfd Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Tue, 18 Sep 2018 11:07:54 -0400 Subject: [PATCH 0419/1247] Topology: Use processors available to the process The existing call sysconf (_SC_NPROCESSORS_ONLN) provides the number of processors available to the scheduler. When a KFD process is run under a container environment, only a subset (cpuset) of processors are available to the current process. For getting CPU cache information use sched_getaffinity() to get the number of processors available to the current process. Change-Id: Ieac02f1f61c17e24ac34ba502968c69d3bc631cb Signed-off-by: Harish Kasiviswanathan --- src/topology.c | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/src/topology.c b/src/topology.c index 12e58b8ba7..a7aca6aaf8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1141,7 +1141,7 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list void *p_temp_cpu_ci_list; int procs_online; cpu_set_t orig_cpuset; - int i; + int i, j = 0; uint32_t cpuid_op_cache; uint32_t eax, ebx, ecx = 0, edx; /* cpuid registers */ cpu_cacheinfo_t *cpu_ci_list, *this_cpu; @@ -1152,8 +1152,21 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list } *temp_cpu_ci_list = NULL; - procs_online = (int)sysconf(_SC_NPROCESSORS_ONLN); + /* Find the number of processors available to the current process, based + * on a system call that returns the "affinity" between the current process + * and each CPU. For getting each CPU cache information the process affinity + * is modified. Save the original affinity mask so that it can be restored + * after finding the cache information. + */ + CPU_ZERO(&orig_cpuset); + if (sched_getaffinity(0, sizeof(cpu_set_t), &orig_cpuset)) { + pr_warn("Failed to get CPU affinity %s\n", strerror(errno)); + ret = HSAKMT_STATUS_ERROR; + goto exit; + } + procs_online = CPU_COUNT(&orig_cpuset); if (procs_online <= 0) { + pr_warn("Invalid number of online processors %d\n", procs_online); ret = HSAKMT_STATUS_ERROR; goto exit; } @@ -1172,20 +1185,19 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list else cpuid_op_cache = 0x4; - /* lock_to_processor() changes the affinity. Save the current affinity - * so we can restore it after cpuid is done. - */ - CPU_ZERO(&orig_cpuset); - if (sched_getaffinity(0, sizeof(cpu_set_t), &orig_cpuset) != 0) { - pr_err("Failed to get CPU affinity\n"); - free(p_temp_cpu_ci_list); - ret = HSAKMT_STATUS_ERROR; - goto exit; - } + for (i = 0; i < CPU_SETSIZE && j < procs_online; i++) { + /* Ignore the CPUs that is not available for the current process */ + if (!CPU_ISSET(i, &orig_cpuset)) + continue; - for (i = 0; i < procs_online; i++) { - this_cpu = cpu_ci_list + i; - lock_to_processor(i); /* so cpuid is executed in correct cpu */ + this_cpu = cpu_ci_list + j++; + /* so cpuid is executed in correct cpu */ + if (lock_to_processor(i)) { + pr_err("Failed to set CPU affinity %s\n", strerror(errno)); + free(p_temp_cpu_ci_list); + ret = HSAKMT_STATUS_ERROR; + goto exit; + } eax = 0x1; cpuid(&eax, &ebx, &ecx, &edx); @@ -1196,12 +1208,18 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list this_cpu->cache_info = calloc( sizeof(cacheinfo_t) * this_cpu->num_caches, 1); if (!this_cpu->cache_info) { + free(p_temp_cpu_ci_list); ret = HSAKMT_STATUS_NO_MEMORY; goto err; } cpuid_get_cpu_cache_info(cpuid_op_cache, this_cpu); } + if (j < procs_online) { + pr_warn("CPU cache information missing\n"); + cpu_ci_list->len = j; + } + find_cpu_cache_siblings(cpu_ci_list); *temp_cpu_ci_list = p_temp_cpu_ci_list; From 918a45a43053bc21957ed71dd3f01950b7ce399f Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Wed, 19 Sep 2018 15:12:04 +0800 Subject: [PATCH 0420/1247] kfdtest: add P2POverheadTest This is to measure the laterncy + overhead of sdma packet consumption on p2p. It is Similar with QueueLatency test. What's more, the queue's overhead with different workload show more details. test result on two gfx900. [ RUN ] KFDPerformanceTest.P2POverheadTest [ ] Test (avg. ns) | Size 4 8 16 64 256 1024 [ ] ----------------------------------------------------------------------- [ ] [push] [1 -> 0] 333 148 185 111 148 148 [ ] [push] [1 -> 1] 370 222 333 74 148 111 [ ] [push] [1 -> 2] 333 148 148 148 148 148 [ ] [push] [2 -> 0] 111 333 259 148 148 148 [ ] [push] [2 -> 1] 222 148 185 148 148 148 [ ] [push] [2 -> 2] 222 111 370 111 74 148 [ ] [pull] [1 -> 0] 370 296 296 148 185 148 [ ] [pull] [1 -> 1] 185 333 222 148 222 148 [ ] [pull] [1 -> 2] 222 444 259 148 185 111 [ ] [pull] [2 -> 0] 148 148 148 148 148 148 [ ] [pull] [2 -> 1] 148 148 148 148 148 148 [ ] [pull] [2 -> 2] 185 148 148 74 222 296 [ ] [push|pull][1 -> 0] 1259 1222 1259 1074 1037 962 [ ] [push|pull][1 -> 1] 1037 1037 1037 740 740 1000 [ ] [push|pull][1 -> 2] 1259 1259 1296 1037 1000 1074 [ ] [push|pull][2 -> 0] 1037 1037 1037 1074 1037 1148 [ ] [push|pull][2 -> 1] 1037 1037 1037 1037 925 1074 [ ] [push|pull][2 -> 2] 666 666 740 740 703 925 [ OK ] KFDPerformanceTest.P2POverheadTest (459 ms) Change-Id: I422263cb52f7ce184f6f1ff4466d04c239fbe9c9 Signed-off-by: xinhui pan --- tests/kfdtest/scripts/kfdtest.exclude | 3 +- tests/kfdtest/src/KFDPerformanceTest.cpp | 107 +++++++++++++++++++---- 2 files changed, 94 insertions(+), 16 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 6607cacdb8..fcdaeb48fe 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -44,7 +44,8 @@ SDMA_BLACKLIST=\ "KFDMemoryTest.MMBench:"\ "KFDQMTest.*Sdma*:"\ "KFDQMTest.P2PTest:"\ -"KFDPerformanceTest.P2PBandWidthTest" +"KFDPerformanceTest.P2PBandWidthTest:"\ +"KFDPerformanceTest.P2POverheadTest" # Anything involving CP queue creation is failing on Kaveri. Separate them here for convenience (KFD-336) KV_QUEUE_BLACKLIST=\ diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index 8b86d98555..4d49f392cc 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -66,7 +66,8 @@ enum P2PDirection { */ static void testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirection n1Direction, - P2PDirection n2Direction, HSAuint64 size, HSAuint64 &speed, HSAuint64 &speed2, std::stringstream &msg) { + P2PDirection n2Direction, HSAuint64 size, HSAuint64 *speed, HSAuint64 *speed2, std::stringstream *msg, + bool isTestOverhead = false, HSAuint64 *time = 0) { ASSERT_GT(16, unsigned(n - 1)); HSAuint32 n2[n]; void *n1Mem, *n2Mem[n]; @@ -76,15 +77,17 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio memFlags.ui32.NonPaged = 1; SDMACopyParams array[n * 4]; int array_count = 0; + HSAuint64 alloc_size = ALIGN_UP(size, PAGE_SIZE); + std::vector copyArray; int i; - ASSERT_SUCCESS(hsaKmtAllocMemory(n1, size, memFlags, &n1Mem)); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n1Mem, size, NULL)); + ASSERT_SUCCESS(hsaKmtAllocMemory(n1, alloc_size, memFlags, &n1Mem)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n1Mem, alloc_size, NULL)); for (i = 0; i < n; i++) { n2[i] = n2Array[i]; - ASSERT_SUCCESS(hsaKmtAllocMemory(n2[i], size, memFlags, &n2Mem[i])); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n2Mem[i], size, NULL)); + ASSERT_SUCCESS(hsaKmtAllocMemory(n2[i], alloc_size, memFlags, &n2Mem[i])); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(n2Mem[i], alloc_size, NULL)); } for (i = 0; i < n; i++) { @@ -96,10 +99,10 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio do { if (n1Direction == IN || n1Direction == IN_OUT) /* n2Mem -> n1Mem*/ - array[array_count++] = {n1, n2Mem[i], n1Mem, size}; + array[array_count++] = {n1, n2Mem[i], n1Mem, size, n1/*group id, just a hint*/}; if (n1Direction == OUT || n1Direction == IN_OUT) /* n1Mem -> n2Mem*/ - array[array_count++] = {n1, n1Mem, n2Mem[i], size}; + array[array_count++] = {n1, n1Mem, n2Mem[i], size, n1}; /* Issue two copies to make full use of sdma.*/ } while (n1Direction < IN_OUT && n == 1 && array_count % 2); /* Do nothing if no IN or OUT specified.*/ @@ -107,21 +110,30 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio do { if (n2Direction == IN || n2Direction == IN_OUT) /* n1Mem -> n2Mem*/ - array[array_count++] = {n2[i], n1Mem, n2Mem[i], size}; + array[array_count++] = {n2[i], n1Mem, n2Mem[i], size, n2[i]}; if (n2Direction == OUT || n2Direction == IN_OUT) /* n2Mem -> n1Mem*/ - array[array_count++] = {n2[i], n2Mem[i], n1Mem, size}; + array[array_count++] = {n2[i], n2Mem[i], n1Mem, size, n2[i]}; } while (n2Direction < IN_OUT && array_count % 2); } - sdma_multicopy(array, array_count, &speed, &speed2, &msg); + /* We measure a bunch of packets.*/ + if (isTestOverhead) { + for (i = 0; i < 1000; i++) + for (int j = 0; j < array_count; j++) + copyArray.push_back(array[j]); + sdma_multicopy(copyArray, 1, HEAD_TAIL); + *time = CounterToNanoSec(copyArray[0].timeConsumption / (1000 * array_count)); + } else + /* It did not respect the group id we set above.*/ + sdma_multicopy(array, array_count, speed, speed2, msg); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(n1Mem)); - EXPECT_SUCCESS(hsaKmtFreeMemory(n1Mem, size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(n1Mem, alloc_size)); for (i = 0; i < n; i++) { EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(n2Mem[i])); - EXPECT_SUCCESS(hsaKmtFreeMemory(n2Mem[i], size)); + EXPECT_SUCCESS(hsaKmtFreeMemory(n2Mem[i], alloc_size)); } } @@ -196,7 +208,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { snprintf(str, sizeof(str), "[%d -> %d] ", n1, n2); msg << str << std::endl; - testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); LOG() << std::dec << str << (float)speed / 1024 << " - " << (float)speed2 / 1024 << " GB/s" << std::endl; @@ -217,7 +229,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { snprintf(str, sizeof(str), "[%d <-> %d] ", n1, n2); msg << str << std::endl; - testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); LOG() << std::dec << str << (float)speed / 1024 << " - " << (float)speed2 / 1024 << " GB/s" << std::endl; @@ -251,7 +263,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { else snprintf(str, sizeof(str), "[%d -> [%d...%d]] ", n1, dst.front(), dst.back()); msg << str << std::endl; - testNodeToNodes(n1, n2, n, test_suits[s][0], test_suits[s][1], size, speed, speed2, msg); + testNodeToNodes(n1, n2, n, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); LOG() << std::dec << str << (float)speed / 1024 << " - " << (float)speed2 / 1024 << " GB/s" << std::endl; @@ -263,3 +275,68 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { TEST_END } + +TEST_F(KFDPerformanceTest, P2POverheadTest) { + TEST_START(TESTPROFILE_RUNALL); + if (!is_dgpu()) { + LOG() << "Skipping test: Can't have 2 APUs on the same system." << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + std::vector nodes; + + for (unsigned i = 0; i < gpuNodes.size(); i++) + if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i))) + nodes.push_back(gpuNodes.at(i)); + + if (nodes.size() < 2) { + LOG() << "Skipping test: Need at least two large bar GPU." << std::endl; + return; + } + + std::vector sysNodes(nodes); // include sysMem node 0... + sysNodes.insert(sysNodes.begin(),0); + + /* size should be small.*/ + const HSAuint32 sizeArray[] = {4, 8, 16, 64, 256, 1024}; + const int total_tests = 3; + const char *test_suits_string[total_tests] = { + "[push] ", + "[pull] ", + "[push|pull]", + }; + const P2PDirection test_suits[total_tests] = {OUT, IN, IN_OUT}; + std::stringstream msg; + int s; //test index; + + msg << "Test (avg. ns) | Size"; + for (auto &size : sizeArray) + msg << "\t" << size; + LOG() << msg.str() << std::endl; + LOG() << "-----------------------------------------------------------------------" << std::endl; + + for (s = 0; s < total_tests; s++) { + + for (unsigned i = 0; i < nodes.size(); i++) { + /* Src node is a GPU.*/ + HSAuint32 n1 = nodes[i]; + HSAuint64 time; + + /* Pick up dst node which can be sysMem.*/ + for (unsigned j = 0; j < sysNodes.size(); j++) { + HSAuint32 n2 = sysNodes[j]; + std::stringstream msg; + + msg << test_suits_string[s] << "[" << n1 << " -> " << n2 << "]"; + for (auto &size : sizeArray) { + testNodeToNodes(n1, &n2, 1, test_suits[s], NONE, size, 0, 0, 0, 1, &time); + msg << "\t" << time; + } + LOG() << msg.str() << std::endl; + } + } + } + + TEST_END +} From f9bd9603447ff9864978c2612ae253f5e29cc1ba Mon Sep 17 00:00:00 2001 From: Mike Li Date: Fri, 21 Sep 2018 11:49:44 -0400 Subject: [PATCH 0421/1247] Output a error message only when open_drm_render_device failed unexpectedly. Change-Id: I5b9587a8d5c7a900e9ab8611a25d0c49d34b4cef Signed-off-by: Mike Li --- src/fmm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 223ff364af..c714ed4051 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1711,7 +1711,8 @@ int open_drm_render_device(int minor) sprintf(path, "/dev/dri/renderD%d", minor); fd = open(path, O_RDWR | O_CLOEXEC); if (fd < 0) { - pr_err("Failed to open %s: %s\n", path, strerror(errno)); + if (errno != ENOENT) + pr_err("Failed to open %s: %s\n", path, strerror(errno)); return -errno; } drm_render_fds[index] = fd; From c3b47c0959efdfd94de166eb727b2fb33c636db7 Mon Sep 17 00:00:00 2001 From: Mike Li Date: Fri, 21 Sep 2018 11:53:23 -0400 Subject: [PATCH 0422/1247] kfdtest: Handle GPU resource management Currently the FindDRMRenderNode function will access the sysfs directly to find the render node. It doesn't work with the GPU management changes. Have changed code to call hsaKmtGetNodeProperties instead. Change-Id: I3bb537a323bc1e8c49f38d8aabc60c13e268aecd Signed-off-by: Mike Li --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 2f697ce8aa..44de55b93e 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -126,25 +126,20 @@ HSAuint64 KFDBaseComponentTest::GetVramSize(int defaultGPUNode) { } int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) { - char path[PATH_MAX], buf[PAGE_SIZE]; + HsaNodeProperties *nodeProperties; + _HSAKMT_STATUS status; - snprintf(path, PATH_MAX, "/sys/class/kfd/kfd/topology/nodes/%d/properties", gpuNode); + nodeProperties = new HsaNodeProperties(); - int fd = open(path, O_RDONLY); + status = hsaKmtGetNodeProperties(gpuNode, nodeProperties); + EXPECT_SUCCESS(status) << "Node index: " << gpuNode << "hsaKmtGetNodeProperties returned status " << status; - if (fd < 0) { - LOG() << "Failed to open " << path << std::endl; + if (status != HSAKMT_STATUS_SUCCESS) { + delete nodeProperties; return -EINVAL; } - read(fd, buf, PAGE_SIZE); - - close(fd); - - char *s = strstr(buf, "drm_render_minor"); - - int minor = atoi(s + 17); - + int minor = nodeProperties->DrmRenderMinor; if (minor < 128) { LOG() << "Failed to get minor number " << minor << std::endl; return -EINVAL; From 3144a84b9af35dcc56757e496f5008f0b70c7081 Mon Sep 17 00:00:00 2001 From: Mike Li Date: Fri, 21 Sep 2018 14:55:19 -0400 Subject: [PATCH 0423/1247] all_gpu_id_array: Handle GPU resource management GPU Resource management can disable some of the GPU nodes. The Kernel driver could be not aware of this. Get from Kernel driver information of all the nodes and then filter it. Change-Id: I4eeb126a5efce2192c35f5d2b72be1811e9ded32 Signed-off-by: Mike Li --- src/fmm.c | 41 ++++++++++++++++++++++++++++------------- src/libhsakmt.h | 1 + src/topology.c | 7 ++++++- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index c714ed4051..dc07f8261b 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1964,7 +1964,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t gpu_id; HsaNodeProperties props; struct kfd_process_device_apertures *process_apertures; - uint32_t num_of_nodes; + uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; char *hsaDebug; @@ -2058,39 +2058,49 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * required since Number of nodes is already known. Kernel will fill in * the apertures in kfd_process_device_apertures_ptr */ - num_of_nodes = gpu_mem_count; - process_apertures = calloc(num_of_nodes, sizeof(struct kfd_process_device_apertures)); + num_of_sysfs_nodes = get_num_sysfs_nodes(); + if (num_of_sysfs_nodes < gpu_mem_count) { + ret = HSAKMT_STATUS_ERROR; + goto sysfs_parse_failed; + } + + process_apertures = calloc(num_of_sysfs_nodes, sizeof(struct kfd_process_device_apertures)); if (!process_apertures) { ret = HSAKMT_STATUS_NO_MEMORY; goto sysfs_parse_failed; } - ret = get_process_apertures(process_apertures, &num_of_nodes); + /* GPU Resource management can disable some of the GPU nodes. + * The Kernel driver could be not aware of this. + * Get from Kernel driver information of all the nodes and then filter it. + */ + ret = get_process_apertures(process_apertures, &num_of_sysfs_nodes); if (ret != HSAKMT_STATUS_SUCCESS) goto get_aperture_ioctl_failed; all_gpu_id_array_size = 0; all_gpu_id_array = NULL; - if (num_of_nodes > 0) { - all_gpu_id_array = malloc(sizeof(uint32_t) * num_of_nodes); + if (num_of_sysfs_nodes > 0) { + all_gpu_id_array = malloc(sizeof(uint32_t) * gpu_mem_count); if (!all_gpu_id_array) { ret = HSAKMT_STATUS_NO_MEMORY; goto get_aperture_ioctl_failed; } } - for (i = 0 ; i < num_of_nodes ; i++) { + for (i = 0 ; i < num_of_sysfs_nodes ; i++) { /* Map Kernel process device data node i <--> gpu_mem_id which * indexes into gpu_mem[] based on gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(process_apertures[i].gpu_id); - if (gpu_mem_id < 0) { + if (gpu_mem_id < 0) + continue; + + if (all_gpu_id_array_size == gpu_mem_count) { ret = HSAKMT_STATUS_ERROR; goto invalid_gpu_id; } - - all_gpu_id_array[i] = process_apertures[i].gpu_id; - all_gpu_id_array_size += sizeof(uint32_t); + all_gpu_id_array[all_gpu_id_array_size++] = process_apertures[i].gpu_id; gpu_mem[gpu_mem_id].lds_aperture.base = PORT_UINT64_TO_VPTR(process_apertures[i].lds_base); @@ -2143,6 +2153,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (ret != HSAKMT_STATUS_SUCCESS) goto acquire_vm_failed; } + all_gpu_id_array_size *= sizeof(uint32_t); if (svm_limit) { /* At least one GPU uses GPUVM in canonical address @@ -2153,7 +2164,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (ret != HSAKMT_STATUS_SUCCESS) goto init_svm_failed; - for (i = 0 ; i < num_of_nodes ; i++) { + for (i = 0 ; i < num_of_sysfs_nodes ; i++) { uintptr_t alt_base; uint64_t alt_size; int err; @@ -2175,6 +2186,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) pr_err("Failed to set mem policy for GPU [0x%x]\n", process_apertures[i].gpu_id); ret = HSAKMT_STATUS_ERROR; + goto set_memory_policy_failed; } } } @@ -2187,10 +2199,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) free(process_apertures); return ret; +invalid_gpu_id: init_svm_failed: acquire_vm_failed: +set_memory_policy_failed: + free(all_gpu_id_array); + all_gpu_id_array = NULL; get_aperture_ioctl_failed: -invalid_gpu_id: free(process_apertures); sysfs_parse_failed: fmm_destroy_process_apertures(); diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 8ed894bf4e..8609af1df1 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -153,4 +153,5 @@ extern int kmtIoctl(int fd, unsigned long request, void *arg); void clear_events_page(void); void fmm_clear_all_mem(void); void clear_process_doorbells(void); +uint32_t get_num_sysfs_nodes(void); #endif diff --git a/src/topology.c b/src/topology.c index a7aca6aaf8..c00f116203 100644 --- a/src/topology.c +++ b/src/topology.c @@ -65,6 +65,7 @@ static int is_valgrind; */ static uint32_t *map_user_to_sysfs_node_id; static uint32_t map_user_to_sysfs_node_id_size; +static uint32_t num_sysfs_nodes; static int processor_vendor; /* Supported System Vendors */ @@ -602,7 +603,6 @@ HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) uint32_t prog; int read_size; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - uint32_t num_sysfs_nodes = 0; bool is_node_supported = true; uint32_t num_supported_nodes = 0; @@ -2244,3 +2244,8 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, return ret; } + +inline uint32_t get_num_sysfs_nodes(void) +{ + return num_sysfs_nodes; +} From 52ec7f805e3e13b30d9af99aff619168cfc85c3a Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Wed, 5 Sep 2018 11:40:53 -0400 Subject: [PATCH 0424/1247] drm/amdkfd: Added gfx904 and gfx803 for KFD. Change-Id: I4406dc70c776926feaecca3f2146d65259a80517 Signed-off-by: Gang Ba --- src/libhsakmt.h | 6 ++++-- src/pmc_table.c | 4 +++- src/queues.c | 14 ++++++++++++++ src/topology.c | 18 ++++++++++++++---- tests/kfdtest/scripts/kfdtest.exclude | 9 +++++++++ tests/kfdtest/scripts/run_kfdtest.sh | 16 +++++++++++----- 6 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 8609af1df1..0bc971fc0a 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -96,9 +96,11 @@ enum asic_family_type { CHIP_FIJI, CHIP_POLARIS10, CHIP_POLARIS11, + CHIP_POLARIS12, CHIP_VEGA10, - CHIP_RAVEN, - CHIP_VEGA20 + CHIP_VEGA12, + CHIP_VEGA20, + CHIP_RAVEN }; #define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) diff --git a/src/pmc_table.c b/src/pmc_table.c index b0d1a3f00f..046fec91d4 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -437,7 +437,7 @@ static uint32_t gfx8_sq_counter_ids[] = { 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 298 }; -/* Polaris 10/11 have the same SQ cpunter IDs but different from other gfx8's. */ +/* Polaris 10/11/12 have the same SQ cpunter IDs but different from other gfx8's. */ /* Unused counters - 167 and 275 are *_DUMMY_LAST */ static uint32_t gfx8_pl_sq_counter_ids[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -1742,9 +1742,11 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, break; case CHIP_POLARIS10: case CHIP_POLARIS11: + case CHIP_POLARIS12: *block = polaris_blocks[block_id]; break; case CHIP_VEGA10: + case CHIP_VEGA12: case CHIP_VEGA20: case CHIP_RAVEN: *block = vega_blocks[block_id]; diff --git a/src/queues.c b/src/queues.c index 95259bdb30..28281f5731 100644 --- a/src/queues.c +++ b/src/queues.c @@ -92,12 +92,24 @@ const struct device_info polaris11_device_info = { .doorbell_size = DOORBELL_SIZE_GFX8, }; +const struct device_info polaris12_device_info = { + .asic_family = CHIP_POLARIS12, + .eop_buffer_size = TONGA_PAGE_SIZE, + .doorbell_size = DOORBELL_SIZE_GFX8, +}; + const struct device_info vega10_device_info = { .asic_family = CHIP_VEGA10, .eop_buffer_size = 4096, .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info vega12_device_info = { + .asic_family = CHIP_VEGA12, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + const struct device_info raven_device_info = { .asic_family = CHIP_RAVEN, .eop_buffer_size = 4096, @@ -119,7 +131,9 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_FIJI] = &fiji_device_info, [CHIP_POLARIS10] = &polaris10_device_info, [CHIP_POLARIS11] = &polaris11_device_info, + [CHIP_POLARIS12] = &polaris12_device_info, [CHIP_VEGA10] = &vega10_device_info, + [CHIP_VEGA12] = &vega12_device_info, [CHIP_VEGA20] = &vega20_device_info, [CHIP_RAVEN] = &raven_device_info }; diff --git a/src/topology.c b/src/topology.c index c00f116203..2013e6cac1 100644 --- a/src/topology.c +++ b/src/topology.c @@ -169,6 +169,15 @@ static struct hsa_gfxip_table { { 0x67EB, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67EF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67FF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + /* Polaris12 */ + { 0x6980, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x6981, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x6985, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x6986, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x6987, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x6995, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x6997, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x699F, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, /* Vega10 */ { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6861, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, @@ -180,10 +189,11 @@ static struct hsa_gfxip_table { { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, /* Vega12 */ - { 0x69A0, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, - { 0x69A1, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, - { 0x69A3, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, - { 0x69Af, 9, 0, 4, 1, "Vega12", CHIP_VEGA10 }, + { 0x69A0, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, + { 0x69A1, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, + { 0x69A2, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, + { 0x69A3, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, + { 0x69Af, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, /* Raven */ { 0x15DD, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, /* Vega20 */ diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index fcdaeb48fe..4414dbf43b 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -112,11 +112,20 @@ BAFFIN_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST" +LEXA_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST" + # KFDDBG.BasicAddressWatch is not supported on GFXv9 and later. FIXME: it should disable itself VEGA10_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "KFDDBGTest.BasicAddressWatch" +VEGA12_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ +"KFDDBGTest.BasicAddressWatch" + VEGA20_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "KFDDBGTest.BasicAddressWatch:"\ diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index a36ebc957b..5d692b1afa 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -49,7 +49,7 @@ printUsage() { echo echo "Gtest arguments will be forwarded to the app" echo - echo "Valid platform options: cz, kv, tg, fj, hi, pl/el, plb/bf, vg10, all" + echo "Valid platform options: cz, kv, tg, fj, hi, pl10/el, pl11/bf, pl12/lx, vg10, vg12, vg20, all" echo "'all' option runs all tests" return 0 @@ -66,9 +66,11 @@ getFilter() { kv ) FILTER="--gtest_filter=$KV_TESTS_BLACKLIST" ;; tg ) FILTER="--gtest_filter=$TONGA_TESTS_BLACKLIST" ;; fj ) FILTER="--gtest_filter=$FIJI_TESTS_BLACKLIST" ;; - pl | el ) FILTER="--gtest_filter=$ELLESMERE_TESTS_BLACKLIST" ;; - plb | bf ) FILTER="--gtest_filter=$BAFFIN_TESTS_BLACKLIST" ;; + pl10 | el ) FILTER="--gtest_filter=$ELLESMERE_TESTS_BLACKLIST" ;; + pl11 | bf ) FILTER="--gtest_filter=$BAFFIN_TESTS_BLACKLIST" ;; + pl12 | lx ) FILTER="--gtest_filter=$LEXA_TESTS_BLACKLIST" ;; vg10 ) FILTER="--gtest_filter=$VEGA10_TESTS_BLACKLIST" ;; + vg12 ) FILTER="--gtest_filter=$VEGA12_TESTS_BLACKLIST" ;; vg20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; rv ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; all ) FILTER="" ;; @@ -109,11 +111,15 @@ deviceIdToGpuName() { 7300 | 730f) platformName="fj" ;; 67c0 | 67c1 | 67c2 | 67c4 | 67c7 | 67c8 | 67c9 | 67ca | 67cc | 67cf | 67d0 | 67df ) - platformName="pl" ;; + platformName="pl10" ;; 67e0 | 67e1 | 67e3 | 67e7 | 67e8 | 67e9 | 67eb | 67ef | 67ff ) - platformName="plb" ;; + platformName="pl11" ;; + 6980 | 6981 | 6985 | 6986 | 6987 | 6995 | 6997 | 699f) + platformName="pl12" ;; 6860 | 6861 | 6862 | 6863 | 6864 | 6867 | 6868 | 6869 | 686a | 686b | 686c | 687f) platformName="vg10" ;; + 69a0 | 69a1 | 69a2 | 69a3 | 69af) + platformName="vg12" ;; 66a0 | 66a1 | 66a2 | 66a3 |66a7 | 66af) platformName="vg20" ;; 15dd ) From 77bab8596f95a94357eda28fcba85cfd4f6756fa Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 16 Oct 2018 18:34:26 -0400 Subject: [PATCH 0425/1247] kfdtest: Improve the SignalEvent test Create an extra event so that the event id to test is non zero. That way we can be sure the context id received in kernel ISR is non zero, which is different from the default value 0 when context id is not set at all. Change-Id: I7e261d1bbb783d5afd15558c7ac00493b1218cef Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDEventTest.cpp | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index 3888899dde..16e53613fc 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -86,21 +86,32 @@ TEST_F(KFDEventTest, CreateMaxEvents) { TEST_F(KFDEventTest, SignalEvent) { TEST_START(TESTPROFILE_RUNALL); - ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_NodeInfo.HsaDefaultGPUNode(), &m_pHsaEvent)); - ASSERT_NE(0, m_pHsaEvent->EventData.HWData2); - PM4Queue queue; + HsaEvent *tmp_event; + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &tmp_event)); + + /* Intentionally let event id for m_pHsaEvent be non zero */ + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &m_pHsaEvent)); + ASSERT_NE(0, m_pHsaEvent->EventData.HWData2); + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); + /* From gfx9 onward, m_pHsaEvent->EventId will also be passed to int_ctxid in + * the Release Mem packet, which is used as context id in ISR. + */ + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, + m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); queue.Wait4PacketConsumption(); EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pHsaEvent, g_TestTimeOut)); + EXPECT_SUCCESS(hsaKmtDestroyEvent(tmp_event)); + EXPECT_SUCCESS(queue.Destroy()); TEST_END; From d7e6d4706cc2d5a687b47ce8e10779f80909aa40 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 17 Oct 2018 13:03:41 -0400 Subject: [PATCH 0426/1247] kfdtest: Clean up the indentations in PM4ReleaseMemoryPacket::InitPacket() Change-Id: I7f6b08697f6a68bf8c4a388c9f1cf3c3c8e6c81f Signed-off-by: Yong Zhao --- tests/kfdtest/src/PM4Packet.cpp | 94 ++++++++++++++++----------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index 95b7a9297b..71e919d1b7 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -86,7 +86,7 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, m_packetSize = sizeof(PM4_RELEASE_MEM_CI); pkt = reinterpret_cast(calloc(1, m_packetSize)); m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); + EXPECT_NOTNULL(m_pPacketData); InitPM4Header(pkt->header, IT_RELEASE_MEM); @@ -106,37 +106,37 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, // Possible values: // 0 - memory_controller. // 1 - tc_l2. - if (address) { - pkt->bitfields3.int_sel = (isPolling ? - int_sel_mec_release_mem_SEND_DATA_AFTER_WRITE_CONFIRM_3 : - int_sel_mec_release_mem_SEND_INTERRUPT_AFTER_WRITE_CONFIRM_2); - // Possible values: - // 0 - None (Do not send an interrupt). - // 1 - Send Interrupt Only. Program DATA_SEL 0". - // 2 - Send Interrupt when Write Confirm (WC) is received from the MC. - // 3 - Wait for WC, but dont send interrupt (applicable to 7.3+) [g73_1] - // 4 - Reserved for INTERRUPT packet - if (isTimeStamp && is64bit) - pkt->bitfields3.data_sel = data_sel_mec_release_mem_SEND_GPU_CLOCK_COUNTER_3; - else - pkt->bitfields3.data_sel = is64bit ? - data_sel_mec_release_mem_SEND_64_BIT_DATA_2 : - data_sel_mec_release_mem_SEND_32_BIT_LOW_1; - // Possible values: - // 0 - None, i.e., Discard Data. - // 1 - Send 32-bit Data Low (Discard Data High). - // 2 - Send 64-bit Data. - // 3 - Send current value of the 64 bit global GPU clock counter. - // 4 - Send current value of the 64 bit system clock counter. - // 5 - Store GDS Data to memory. - // 6 - Reserved for use by the CP for Signal Semaphore. - // 7 - Reserved for use by the CP for Wait Semaphore. - } else { - pkt->bitfields3.int_sel = (isPolling ? - int_sel_mec_release_mem_NONE_0 : - int_sel_mec_release_mem_SEND_INTERRUPT_ONLY_1); - pkt->bitfields3.data_sel = data_sel_mec_release_mem_NONE_0; - } + if (address) { + pkt->bitfields3.int_sel = (isPolling ? + int_sel_mec_release_mem_SEND_DATA_AFTER_WRITE_CONFIRM_3 : + int_sel_mec_release_mem_SEND_INTERRUPT_AFTER_WRITE_CONFIRM_2); + // Possible values: + // 0 - None (Do not send an interrupt). + // 1 - Send Interrupt Only. Program DATA_SEL 0". + // 2 - Send Interrupt when Write Confirm (WC) is received from the MC. + // 3 - Wait for WC, but dont send interrupt (applicable to 7.3+) [g73_1] + // 4 - Reserved for INTERRUPT packet + if (isTimeStamp && is64bit) + pkt->bitfields3.data_sel = data_sel_mec_release_mem_SEND_GPU_CLOCK_COUNTER_3; + else + pkt->bitfields3.data_sel = is64bit ? + data_sel_mec_release_mem_SEND_64_BIT_DATA_2 : + data_sel_mec_release_mem_SEND_32_BIT_LOW_1; + // Possible values: + // 0 - None, i.e., Discard Data. + // 1 - Send 32-bit Data Low (Discard Data High). + // 2 - Send 64-bit Data. + // 3 - Send current value of the 64 bit global GPU clock counter. + // 4 - Send current value of the 64 bit system clock counter. + // 5 - Store GDS Data to memory. + // 6 - Reserved for use by the CP for Signal Semaphore. + // 7 - Reserved for use by the CP for Wait Semaphore. + } else { + pkt->bitfields3.int_sel = (isPolling ? + int_sel_mec_release_mem_NONE_0 : + int_sel_mec_release_mem_SEND_INTERRUPT_ONLY_1); + pkt->bitfields3.data_sel = data_sel_mec_release_mem_NONE_0; + } pkt->bitfields4a.address_lo_dword_aligned = static_cast((address&0xffffffff) >> 2); pkt->addr_hi = static_cast(address>>32); @@ -149,7 +149,7 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, m_packetSize = sizeof(PM4MEC_RELEASE_MEM_AI); pkt = reinterpret_cast(calloc(1, m_packetSize)); m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); + EXPECT_NOTNULL(m_pPacketData); InitPM4Header(pkt->header, IT_RELEASE_MEM); @@ -163,21 +163,21 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, if (address) { pkt->bitfields3.int_sel = (isPolling ? - int_sel__mec_release_mem__send_data_after_write_confirm: - int_sel__mec_release_mem__send_interrupt_after_write_confirm); + int_sel__mec_release_mem__send_data_after_write_confirm: + int_sel__mec_release_mem__send_interrupt_after_write_confirm); - if (isTimeStamp && is64bit) - pkt->bitfields3.data_sel = data_sel__mec_release_mem__send_gpu_clock_counter; - else - pkt->bitfields3.data_sel = is64bit ? - data_sel__mec_release_mem__send_64_bit_data : - data_sel__mec_release_mem__send_32_bit_low; - } else { + if (isTimeStamp && is64bit) + pkt->bitfields3.data_sel = data_sel__mec_release_mem__send_gpu_clock_counter; + else + pkt->bitfields3.data_sel = is64bit ? + data_sel__mec_release_mem__send_64_bit_data : + data_sel__mec_release_mem__send_32_bit_low; + } else { pkt->bitfields3.int_sel = (isPolling ? - int_sel__mec_release_mem__none: - int_sel__mec_release_mem__send_interrupt_only); - pkt->bitfields3.data_sel = data_sel__mec_release_mem__none; - } + int_sel__mec_release_mem__none: + int_sel__mec_release_mem__send_interrupt_only); + pkt->bitfields3.data_sel = data_sel__mec_release_mem__none; + } pkt->bitfields4a.address_lo_32b = static_cast((address&0xffffffff) >> 2); pkt->address_hi = static_cast(address>>32); @@ -185,7 +185,7 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, pkt->data_lo = static_cast(data); pkt->data_hi = static_cast(data >> 32); - pkt->int_ctxid = static_cast(data); + pkt->int_ctxid = static_cast(data); } } From ab4610cff753d30bf8e88ae16b635bc724690e9c Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Thu, 11 Oct 2018 18:41:37 +0800 Subject: [PATCH 0427/1247] kfdtest: Add more debug information of sdma event interrupt test We observe this test fails on gfx900+. Looks like the sdma packets are not executed at all after we submit sometimes. Run it with timeout 2s on gfx900. [ RUN ] KFDQMTest.SdmaEventInterrupt [----------] SDMACopyData FAIL! 1485262707170 VS 1485262747814 [----------] Event On Queue 1:0 Timeout, try to resubmit packets! [----------] The timeout event is signaled! [ ] Time Consumption (ns) [ ] 1: 1859427148 [ ] 2: 680148 [ ] 3: 6370 [ ] 4: 5481 /home/pp/code/compute/libhsakmt/tests/kfdtest/src/KFDQMTest.cpp:1670: Failure Value of: (ret) Actual: 31 Expected: HSAKMT_STATUS_SUCCESS Which is: 0 [----------] SDMACopyData FAIL! 1485367669958 VS 1485367750022 [----------] Event On Queue 2:1 Timeout, try to resubmit packets! [----------] The timeout event is signaled! [ ] Time Consumption (ns) [ ] 1: 1881615148 [ ] 2: 673629 [ ] 3: 6074 [ ] 4: 5481 /home/pp/code/compute/libhsakmt/tests/kfdtest/src/KFDQMTest.cpp:1670: Failure Value of: (ret) Actual: 31 Expected: HSAKMT_STATUS_SUCCESS Which is: 0 [----------] SDMACopyData FAIL! 1485427671250 VS 1485427751238 [----------] Event On Queue 2:1 Timeout, try to resubmit packets! [----------] The timeout event is signaled! [ ] Time Consumption (ns) [ ] 1: 1881508777 [ ] 2: 741629 [ ] 3: 6074 [ ] 4: 5481 /home/pp/code/compute/libhsakmt/tests/kfdtest/src/KFDQMTest.cpp:1670: Failure Value of: (ret) Actual: 31 Expected: HSAKMT_STATUS_SUCCESS Which is: 0 [ FAILED ] KFDQMTest.SdmaEventInterrupt (23675 ms) Change-Id: I7c1b752537d89782570df20838bf976578614f75 Signed-off-by: xinhui pan --- tests/kfdtest/src/KFDQMTest.cpp | 114 ++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 27 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index dba670f4c1..53471dedd9 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1578,6 +1578,7 @@ TEST_F(KFDQMTest, PM4EventInterrupt) { TEST_END } +#include "KFDTestUtilQueue.hpp" TEST_F(KFDQMTest, SdmaEventInterrupt) { TEST_START(TESTPROFILE_RUNALL) @@ -1586,39 +1587,98 @@ TEST_F(KFDQMTest, SdmaEventInterrupt) { const HSAuint64 bufSize = 4 << 20; HsaMemoryBuffer srcBuf(bufSize, 0); // System memory. - HsaMemoryBuffer destBuf(bufSize, defaultGPUNode, true, true); // local vram. - /* Two sdma queues will be running at same time.*/ - const int numSDMAQueue = 2; + HSAuint64 *src = srcBuf.As(); + TimeStamp *tsbuf = srcBuf.As(); + tsbuf = reinterpret_castALIGN_UP(tsbuf, sizeof(TimeStamp)); + + /* Have 3 queues created for test.*/ + const int numSDMAQueue = 3; HsaEvent *event[numSDMAQueue]; SDMAQueue queue[numSDMAQueue]; + HsaMemoryBuffer *destBuf[numSDMAQueue]; + HSAuint64 *dst[numSDMAQueue]; - /* A simple loop here to give more pressure.*/ - for (int test_count = 0; test_count < 4096; test_count++) { - for (int i = 0; i < numSDMAQueue; i++) { - ASSERT_SUCCESS(queue[i].Create(defaultGPUNode)); - ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event[i])); - - /* Let sDMA have some workload first.*/ - queue[i].PlacePacket( - SDMACopyDataPacket(destBuf.As(), srcBuf.As(), bufSize)); - queue[i].PlacePacket( - SDMAFencePacket(reinterpret_cast(event[i]->EventData.HWData2), event[i]->EventId)); - queue[i].PlacePacket(SDMATrapPacket(event[i]->EventId)); - } - - for (int i = 0; i < numSDMAQueue; i++) - queue[i].SubmitPacket(); - - for (int i = 0; i < numSDMAQueue; i++) - EXPECT_SUCCESS(hsaKmtWaitOnEvent(event[i], g_TestTimeOut)); - - for (int i = 0; i < numSDMAQueue; i++) { - EXPECT_SUCCESS(queue[i].Destroy()); - EXPECT_SUCCESS(hsaKmtDestroyEvent(event[i])); - } + for (int i = 0; i < numSDMAQueue; i++) { + destBuf[i] = new HsaMemoryBuffer(bufSize, defaultGPUNode, true, false); // System memory + dst[i] = destBuf[i]->As(); } + /* Test 1 queue, 2 queues, 3 queues running at same time one by one.*/ + for (int testSDMAQueue = 1; testSDMAQueue <= numSDMAQueue; testSDMAQueue++) + /* A simple loop here to give more pressure.*/ + for (int test_count = 0; test_count < 2048; test_count++) { + for (int i = 0; i < testSDMAQueue; i++) { + TimeStamp *ts = tsbuf + i * 32; + ASSERT_SUCCESS(queue[i].Create(defaultGPUNode)); + /* FIXME + * We create event every time along with queue. + * However that will significantly enhance the failure of sdma event timeout. + */ + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event[i])); + + /* Get the timestamp directly. The first member of HsaClockCounters and TimeStamp is GPU clock counter.*/ + hsaKmtGetClockCounters(defaultGPUNode, reinterpret_cast(&ts[0])); + /* Let sDMA have some workload first.*/ + queue[i].PlacePacket(SDMATimePacket(&ts[1])); + queue[i].PlacePacket( + SDMACopyDataPacket(dst[i], src, bufSize)); + queue[i].PlacePacket(SDMATimePacket(&ts[2])); + queue[i].PlacePacket( + SDMAFencePacket(reinterpret_cast(event[i]->EventData.HWData2), event[i]->EventId)); + queue[i].PlacePacket(SDMATimePacket(&ts[3])); + queue[i].PlacePacket(SDMATrapPacket(event[i]->EventId)); + queue[i].PlacePacket(SDMATimePacket(&ts[4])); + + /* Will verify the value of srcBuf and destBuf later. Give it a different value each time.*/ + src[0] = ts[0].timestamp; + } + + for (int i = 0; i < testSDMAQueue; i++) + queue[i].SubmitPacket(); + + for (int i = 0; i < testSDMAQueue; i++) { + TimeStamp *ts = tsbuf + i * 32; + HSAKMT_STATUS ret = hsaKmtWaitOnEvent(event[i], g_TestTimeOut); + + if (dst[i][0] != src[0]) + WARN() << "SDMACopyData FAIL! " << std::dec + << dst[i][0] << " VS " << src[0] << std::endl; + + if (ret == HSAKMT_STATUS_SUCCESS) { + for (int i = 1; i <= 4; i++) + /* Is queue latency too big? The workload is really small.*/ + if (CounterToNanoSec(ts[i].timestamp - ts[i - 1].timestamp) > 1000000000) + WARN() << "SDMA queue latency is bigger than 1s!" << std::endl; + } else { + WARN() << "Event On Queue " << testSDMAQueue << ":" << i + << " Timeout, try to resubmit packets!" << std::endl; + + queue[i].SubmitPacket(); + + if (hsaKmtWaitOnEvent(event[i], g_TestTimeOut) == HSAKMT_STATUS_SUCCESS) + WARN() << "The timeout event is signaled!" << std::endl; + else + WARN() << "The timeout event is lost after resubmit!" << std::endl; + + LOG() << "Time Consumption (ns)" << std::endl; + for (int i = 1; i <= 4; i++) + LOG() << std::dec << i << ": " + << CounterToNanoSec(ts[i].timestamp - ts[i - 1].timestamp) << std::endl; + } + + EXPECT_SUCCESS(ret); + } + + for (int i = 0; i < testSDMAQueue; i++) { + EXPECT_SUCCESS(queue[i].Destroy()); + EXPECT_SUCCESS(hsaKmtDestroyEvent(event[i])); + } + } + + for (int i = 0; i < numSDMAQueue; i++) + delete destBuf[i]; + TEST_END } From 7a13bb4d66f745f7574f91f3baa20badcd401363 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Thu, 18 Oct 2018 13:17:01 +0800 Subject: [PATCH 0428/1247] kfdtest: blacklist KFDQMTest.SdmaEventInterrupt On gfx900+, the test sometimes timeout due to cp fw bug. Blacklist it until we address the root cause and have a fix. Change-Id: Iff600a6f6dbd86c56e034f530484205520bced32 Signed-off-by: xinhui pan --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 4414dbf43b..4dc6dc573a 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -31,7 +31,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDPMTest.SuspendWithIdleQueueAfterWork:"\ "KFDPNPTest.DisableAndCreateQueue:"\ "KFDQMTest.GPUDoorbellWrite:"\ -"KFDQMTest.mGPUShareBO" +"KFDQMTest.mGPUShareBO:"\ +"KFDQMTest.SdmaEventInterrupt" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From 105edd4bb49ba6d322311b7dcd3009f671d246b8 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Tue, 23 Oct 2018 13:07:10 -0400 Subject: [PATCH 0429/1247] Fix Debug Thunk spec mismatch Move debug trap support capabilities to their own structure to fix thunk spec vs header mismatch. Change-Id: I6694601bfa36097502c8ab932e082d7a4645d5b2 Signed-off-by: Philip Cox --- include/hsakmttypes.h | 24 +++++++++++++++++++----- src/topology.c | 2 ++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index a5272151c7..116e45ec77 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -199,17 +199,31 @@ typedef union // 1: This node has 1.0 doorbell characteristic // 2,3: reserved for future use unsigned int AQLQueueDoubleMap : 1; // The unit needs a VA “double map” - unsigned int WatchAddrMaskLoBit : 4; // Only bits WatchAddrMaskLoBit..WatchAddrMaskHiBit of the - unsigned int WatchAddrMaskHiBit : 6; // watch address mask are used. 0 is the least significant bit. unsigned int DebugTrapSupported : 1; // Indicates if Debug Trap is supported on the node. - unsigned int TrapDataCount : 2; // Number of 32 bit TrapData registers supported. unsigned int WaveLaunchTrapOverrideSupported: 1; // Indicates if Wave Launch Trap Override is supported on the node. unsigned int WaveLaunchModeSupported: 1; // Indicates if Wave Launch Mode is supported on the node. unsigned int PreciseMemoryOperationsSupported: 1; // Indicates if Precise Memory Operations are supported on the node. - unsigned int Reserved : 1; + unsigned int Reserved : 13; } ui32; } HSA_CAPABILITY; +// Debug Properties and values +// HSA runtime may expose a subset of the capabilities outlined to the applicati +typedef union +{ + HSAuint64 Value; + struct + { + HSAuint64 WatchAddrMaskLoBit: 6; // Only bits + // WatchAddrMaskLoBit..WatchAddrMaskHiBit + // of the + HSAuint64 WatchAddrMaskHiBit: 4; // watch address mask are used. + // 0 is the least significant bit. + HSAuint64 TrapDataCount: 4; // Number of 32 bit TrapData + // registers supported. + HSAuint64 Reserved: 50; // + }; +} HSA_DEBUG_PROPERTIES; // // HSA node properties. This structure is an output parameter of hsaKmtGetNodeProperties() @@ -273,7 +287,7 @@ typedef struct _HsaNodeProperties // Unicode string HSAuint8 AMDName[HSA_PUBLIC_NAME_SIZE]; //CAL Name of the "device", ASCII HSA_ENGINE_VERSION uCodeEngineVersions; - HSAuint64 DebugProperties; + HSA_DEBUG_PROPERTIES DebugProperties; // Debug properties of this node. HSAuint64 HiveID; // XGMI Hive the GPU node belongs to in the system. It is an opaque and static // number hash created by the PSP HSAuint8 Reserved[44]; diff --git a/src/topology.c b/src/topology.c index 2013e6cac1..0580ac7738 100644 --- a/src/topology.c +++ b/src/topology.c @@ -944,6 +944,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->FComputeIdLo = (uint32_t)prop_val; else if (strcmp(prop_name, "capability") == 0) props->Capability.Value = (uint32_t)prop_val; + else if (strcmp(prop_name, "debug_prop") == 0) + props->DebugProperties.Value = (uint64_t)prop_val; else if (strcmp(prop_name, "max_waves_per_simd") == 0) props->MaxWavesPerSIMD = (uint32_t)prop_val; else if (strcmp(prop_name, "lds_size_in_kb") == 0) From 10edccb9121c25828ac241b960a5083a5932c25d Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 24 Oct 2018 13:46:57 -0400 Subject: [PATCH 0430/1247] Specify requirement of NUMA libs for Thunk Add the numa libs to the thunk specs for DEB/RPM, so we can remove the manual installation requirement Change-Id: I5aadcf581b64e9a20aee9c1e1204af4715d1e990 --- DEBIAN/control | 2 +- RPM/libhsakmt.spec | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DEBIAN/control b/DEBIAN/control index 779d712fac..fd01eac6bd 100644 --- a/DEBIAN/control +++ b/DEBIAN/control @@ -1,7 +1,7 @@ Package: hsakmt-rocm-dev Architecture: $arch Maintainer: Advanced Micro Devices Inc. -Depends:libpci3 +Depends:libpci3, libnuma1 Priority: optional Version: $version Description: Thunk library for AMD KFD diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index 9e32e53426..631d65ba40 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -9,6 +9,8 @@ Summary: Thunk libraries for AMD KFD Group: System Environment/Libraries License: Advanced Micro Devices Inc. +Requires: numactl-libs + %description This package includes the libhsakmt (Thunk) libraries From c1994e28f05fab5241e70d6c940d6257a54d5b45 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 18 Oct 2018 14:03:02 -0400 Subject: [PATCH 0431/1247] libhsakmt: Support device controller cgroup Device whiltelist controller cgroup allows to track and enforce open and mknod restrictions on device files. Tasks should works with /dev/dri/renderN devices that are whitelisted for its cgroup. If a certain node is not whitelisted it is not an error condition. Change-Id: I0b997423ccdc00aee98df5b6f04ed6794549604e Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 5 ++++- src/topology.c | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index dc07f8261b..ed6da56493 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1711,8 +1711,11 @@ int open_drm_render_device(int minor) sprintf(path, "/dev/dri/renderD%d", minor); fd = open(path, O_RDWR | O_CLOEXEC); if (fd < 0) { - if (errno != ENOENT) + if (errno != ENOENT) { + if (errno == EPERM) + pr_info("Check a) User is in \"video\" group b) cgroup permissions\n"); pr_err("Failed to open %s: %s\n", path, strerror(errno)); + } return -errno; } drm_render_fds[index] = fd; diff --git a/src/topology.c b/src/topology.c index 0580ac7738..0e90f83640 100644 --- a/src/topology.c +++ b/src/topology.c @@ -595,7 +595,7 @@ static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id, ret_value = open_drm_render_device(drm_render_minor); if (ret_value > 0) *is_node_supported = true; - else if (ret_value != -ENOENT) + else if (ret_value != -ENOENT && ret_value != -EPERM) ret = HSAKMT_STATUS_ERROR; err: From c54c1dbdcb604c2f47d404c0a4a70bca90c9750a Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Tue, 16 Oct 2018 14:26:42 -0400 Subject: [PATCH 0432/1247] Add code to support packet capture and replay in the Thunk This feature only support dgpu for now. Change-Id: Ic766ec06892c955dd605ecc335a776335edc0df2 Signed-off-by: Gang Ba --- include/hsakmt.h | 2 +- include/hsakmttypes.h | 3 +- src/fmm.c | 85 +++++++++++++++++++++++++++++-------------- src/fmm.h | 6 +-- src/memory.c | 14 +++++-- 5 files changed, 73 insertions(+), 37 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index eb5307e84c..c05e34d114 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -360,7 +360,7 @@ hsaKmtAllocMemory( HSAuint32 PreferredNode, //IN HSAuint64 SizeInBytes, //IN (multiple of page size) HsaMemFlags MemFlags, //IN - void** MemoryAddress //OUT (page-aligned) + void** MemoryAddress //IN/OUT (page-aligned) ); /** diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 116e45ec77..c4d39271b9 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -518,7 +518,8 @@ typedef struct _HsaMemFlags unsigned int AQLQueueMemory: 1; // default = 0; If 1: The caller indicates that the memory will be used as AQL queue memory. // The KFD will ensure that the memory returned is allocated in the optimal memory location // and optimal alignment requirements - unsigned int Reserved : 17; + unsigned int FixedAddress : 1; // Allocate memory at specified virtual address. Fail if address is not free. + unsigned int Reserved : 16; } ui32; HSAuint32 Value; diff --git a/src/fmm.c b/src/fmm.c index ed6da56493..1b70c9e5f0 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -122,7 +122,7 @@ typedef struct manageable_aperture manageable_aperture_t; * schemes. */ typedef struct { - void *(*allocate_area_aligned)(manageable_aperture_t *aper, + void *(*allocate_area_aligned)(manageable_aperture_t *aper, void *addr, uint64_t size, uint64_t align); void (*release_area)(manageable_aperture_t *aper, void *addr, uint64_t size); @@ -130,6 +130,7 @@ typedef struct { /* Reserved aperture type managed by its own address allocator */ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *aper, + void *addr, uint64_t size, uint64_t align); static void reserved_aperture_release(manageable_aperture_t *aper, void *addr, uint64_t size); @@ -140,6 +141,7 @@ static const manageable_aperture_ops_t reserved_aperture_ops = { /* Unreserved aperture type using mmap to allocate virtual address space */ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, + void *addr, uint64_t size, uint64_t align); static void mmap_aperture_release(manageable_aperture_t *aper, void *addr, uint64_t size); @@ -204,6 +206,9 @@ typedef struct { /* whether to check userptrs on registration */ bool check_userptr; + /* whether to check reserve svm on registration */ + bool reserve_svm; + /* whether all memory is coherent (GPU cache disabled) */ bool disable_cache; } svm_t; @@ -621,6 +626,7 @@ static void reserved_aperture_release(manageable_aperture_t *app, * on entry. */ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, + void *address, uint64_t MemorySizeInBytes, uint64_t align) { @@ -641,7 +647,7 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, /* Find a big enough "hole" in the address space */ cur = NULL; next = app->vm_ranges; - start = (void *)ALIGN_UP((uint64_t)app->base, align); + start = address ? address : (void *)ALIGN_UP((uint64_t)app->base, align); while (next) { if (next->start > start && VOID_PTRS_SUB(next->start, start) >= MemorySizeInBytes) @@ -649,12 +655,17 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, cur = next; next = next->next; - start = (void *)ALIGN_UP((uint64_t)cur->end + 1, align); + if (!address) + start = (void *)ALIGN_UP((uint64_t)cur->end + 1, align); } if (!next && VOID_PTRS_SUB(app->limit, start) + 1 < MemorySizeInBytes) /* No hole found and not enough space after the last area */ return NULL; + if (cur && address && address < (void *)ALIGN_UP((uint64_t)cur->end + 1, align)) + /* Required address is not free or overlaps */ + return NULL; + if (cur && VOID_PTR_ADD(cur->end, 1) == start) { /* extend existing area */ cur->end = VOID_PTR_ADD(start, MemorySizeInBytes-1); @@ -679,11 +690,15 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, } static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, + void *address, uint64_t size, uint64_t align) { uint64_t aligned_padded_size, guard_size; void *addr, *aligned_addr, *aligned_end, *mapping_end; + if (address) + return NULL; + if (!aper->is_cpu_accessible) { pr_err("MMap Aperture must be CPU accessible\n"); return NULL; @@ -758,15 +773,16 @@ static void mmap_aperture_release(manageable_aperture_t *aper, /* Wrapper functions to call aperture-specific VA management functions */ static void *aperture_allocate_area_aligned(manageable_aperture_t *app, + void *address, uint64_t MemorySizeInBytes, uint64_t align) { - return app->ops->allocate_area_aligned(app, MemorySizeInBytes, align); + return app->ops->allocate_area_aligned(app, address, MemorySizeInBytes, align); } -static void *aperture_allocate_area(manageable_aperture_t *app, +static void *aperture_allocate_area(manageable_aperture_t *app, void *address, uint64_t MemorySizeInBytes) { - return app->ops->allocate_area_aligned(app, MemorySizeInBytes, app->align); + return app->ops->allocate_area_aligned(app, address, MemorySizeInBytes, app->align); } static void aperture_release_area(manageable_aperture_t *app, void *address, uint64_t MemorySizeInBytes) @@ -1183,7 +1199,7 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) } #define SCRATCH_ALIGN 0x10000 -void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) +void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes) { manageable_aperture_t *aperture_phy; struct kfd_ioctl_set_scratch_backing_va_args args = {0}; @@ -1205,7 +1221,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { pthread_mutex_lock(&svm.dgpu_aperture->fmm_mutex); mem = aperture_allocate_area_aligned( - svm.dgpu_aperture, + svm.dgpu_aperture, address, aligned_size, SCRATCH_ALIGN); pthread_mutex_unlock(&svm.dgpu_aperture->fmm_mutex); } else { @@ -1213,6 +1229,9 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) SCRATCH_ALIGN - PAGE_SIZE; void *padded_end, *aligned_start, *aligned_end; + if (address) + return NULL; + mem = mmap(0, aligned_padded_size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); @@ -1247,7 +1266,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes) return mem; } -static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, +static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t *mmap_offset, uint32_t flags, vm_object_t **vm_obj) { @@ -1260,7 +1279,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area(aperture, MemorySizeInBytes); + mem = aperture_allocate_area(aperture, address, MemorySizeInBytes); pthread_mutex_unlock(&aperture->fmm_mutex); /* @@ -1287,7 +1306,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, return mem; } -void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags) +void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags) { manageable_aperture_t *aperture; int32_t gpu_mem_id; @@ -1319,7 +1338,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFla if (!flags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - mem = __fmm_allocate_device(gpu_id, size, aperture, &mmap_offset, + mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); if (mem && vm_obj) { @@ -1369,7 +1388,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - mem = __fmm_allocate_device(gpu_id, MemorySizeInBytes, aperture, NULL, + mem = __fmm_allocate_device(gpu_id, NULL, MemorySizeInBytes, aperture, NULL, ioc_flags, &vm_obj); if (mem && vm_obj) { @@ -1401,13 +1420,16 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, return mem; } -static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, +static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags) { void *mem = NULL; vm_object_t *vm_obj; int mmap_prot = PROT_READ; + if (address) + return NULL; + if (flags.ui32.ExecuteAccess) mmap_prot |= PROT_EXEC; @@ -1433,8 +1455,8 @@ static void *fmm_allocate_host_cpu(uint64_t MemorySizeInBytes, return mem; } -static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, - HsaMemFlags flags) +static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, + uint64_t MemorySizeInBytes, HsaMemFlags flags) { void *mem; manageable_aperture_t *aperture; @@ -1475,7 +1497,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area(aperture, size); + mem = aperture_allocate_area(aperture, address, size); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) return NULL; @@ -1520,7 +1542,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, } } else { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; - mem = __fmm_allocate_device(gpu_id, size, aperture, + mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); if (mem && flags.ui32.HostAccess) { @@ -1555,12 +1577,12 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, uint64_t MemorySizeInBytes, return mem; } -void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, - HsaMemFlags flags) +void *fmm_allocate_host(uint32_t node_id, void *address, + uint64_t MemorySizeInBytes, HsaMemFlags flags) { if (is_dgpu) - return fmm_allocate_host_gpu(node_id, MemorySizeInBytes, flags); - return fmm_allocate_host_cpu(MemorySizeInBytes, flags); + return fmm_allocate_host_gpu(node_id, address, MemorySizeInBytes, flags); + return fmm_allocate_host_cpu(address, MemorySizeInBytes, flags); } static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) @@ -1757,7 +1779,7 @@ static HSAKMT_STATUS init_mmap_apertures(HSAuint64 base, HSAuint64 limit, /* Try to allocate one page. If it fails, we'll fall back to * managing our own reserved address range. */ - addr = aperture_allocate_area(&svm.apertures[SVM_DEFAULT], PAGE_SIZE); + addr = aperture_allocate_area(&svm.apertures[SVM_DEFAULT], NULL, PAGE_SIZE); if (addr) { aperture_release_area(&svm.apertures[SVM_DEFAULT], addr, PAGE_SIZE); @@ -1822,7 +1844,7 @@ static HSAKMT_STATUS init_svm_apertures(HSAuint64 base, HSAuint64 limit, * x86_64) or at least mmap is unlikely to run out of * addresses the GPUs can handle. */ - if (limit >= (1ULL << 47) - 1) { + if (limit >= (1ULL << 47) - 1 && !svm.reserve_svm) { HSAKMT_STATUS status = init_mmap_apertures(base, limit, align, guard_pages); @@ -1969,7 +1991,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) struct kfd_process_device_apertures *process_apertures; uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr; + char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; char *hsaDebug; unsigned int guardPages = 1; struct pci_access *pacc; @@ -1995,6 +2017,12 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) checkUserptr = getenv("HSA_CHECK_USERPTR"); svm.check_userptr = (checkUserptr && strcmp(checkUserptr, "0")); + /* If HSA_RESERVE_SVM is set to a non-0 value, + * enable packet capture and replay mode. + */ + reserveSvm = getenv("HSA_RESERVE_SVM"); + svm.reserve_svm = (reserveSvm && strcmp(reserveSvm, "0")); + /* Specify number of guard pages for SVM apertures, default is 1 */ guardPagesStr = getenv("HSA_SVM_GUARD_PAGES"); if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1) @@ -2147,6 +2175,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) */ aperture_allocate_area( &gpu_mem[gpu_mem_id].gpuvm_aperture, + NULL, gpu_mem[gpu_mem_id].gpuvm_aperture.align); } @@ -2786,7 +2815,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj fmm_check_user_memory(addr, size); /* Allocate BO, userptr address is passed in mmap_offset */ - svm_addr = __fmm_allocate_device(gpu_id, aligned_size, aperture, + svm_addr = __fmm_allocate_device(gpu_id, NULL, aligned_size, aperture, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE, &obj); @@ -2932,7 +2961,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, if (!aperture_is_valid(aperture->base, aperture->limit)) goto error_free_metadata; pthread_mutex_lock(&aperture->fmm_mutex); - mem = aperture_allocate_area_aligned(aperture, infoArgs.size, + mem = aperture_allocate_area_aligned(aperture, NULL, infoArgs.size, MAX(aperture->align, IMAGE_ALIGN)); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) @@ -3059,7 +3088,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo aperture = fmm_get_aperture(SharedMemoryStruct->ApeInfo); pthread_mutex_lock(&aperture->fmm_mutex); - reservedMem = aperture_allocate_area(aperture, + reservedMem = aperture_allocate_area(aperture, NULL, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT)); pthread_mutex_unlock(&aperture->fmm_mutex); if (!reservedMem) { diff --git a/src/fmm.h b/src/fmm.h index 1faecdd207..85ef0ef54d 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -48,10 +48,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes); void fmm_destroy_process_apertures(void); /* Memory interface */ -void *fmm_allocate_scratch(uint32_t gpu_id, uint64_t MemorySizeInBytes); -void *fmm_allocate_device(uint32_t gpu_id, uint64_t MemorySizeInBytes, HsaMemFlags flags); +void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes); +void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags); void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset); -void *fmm_allocate_host(uint32_t node_id, uint64_t MemorySizeInBytes, +void *fmm_allocate_host(uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags); void fmm_print(uint32_t node); HSAKMT_STATUS fmm_release(void *address); diff --git a/src/memory.c b/src/memory.c index 6c1996365e..b9a01c3f0f 100644 --- a/src/memory.c +++ b/src/memory.c @@ -128,8 +128,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, if (!MemoryAddress || !SizeInBytes || (SizeInBytes & (page_size-1))) return HSAKMT_STATUS_INVALID_PARAMETER; + if (MemFlags.ui32.FixedAddress) { + if (*MemoryAddress == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + } else + *MemoryAddress = NULL; + if (MemFlags.ui32.Scratch) { - *MemoryAddress = fmm_allocate_scratch(gpu_id, SizeInBytes); + *MemoryAddress = fmm_allocate_scratch(gpu_id, *MemoryAddress, SizeInBytes); if (!(*MemoryAddress)) { pr_err("[%s] failed to allocate %lu bytes from scratch\n", @@ -145,8 +151,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, /* Backwards compatibility hack: Allocate system memory if app * asks for paged memory from a GPU node. */ - *MemoryAddress = fmm_allocate_host(PreferredNode, SizeInBytes, - MemFlags); + *MemoryAddress = fmm_allocate_host(PreferredNode, *MemoryAddress, + SizeInBytes, MemFlags); if (!(*MemoryAddress)) { pr_err("[%s] failed to allocate %lu bytes from host\n", @@ -158,7 +164,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, } /* GPU allocated VRAM */ - *MemoryAddress = fmm_allocate_device(gpu_id, SizeInBytes, MemFlags); + *MemoryAddress = fmm_allocate_device(gpu_id, *MemoryAddress, SizeInBytes, MemFlags); if (!(*MemoryAddress)) { pr_err("[%s] failed to allocate %lu bytes from device\n", From 3afce42b5766cad86d2e03357ac3eccb21f7a628 Mon Sep 17 00:00:00 2001 From: Mike Li Date: Tue, 23 Oct 2018 11:57:01 -0400 Subject: [PATCH 0433/1247] Changed scripts to include running kfdtest in docker container Change-Id: I822ff4869610df6abad846542d7c290b7a5aae79 --- tests/kfdtest/scripts/run_kfdtest.sh | 44 +++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 5d692b1afa..74d617c5a3 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -32,6 +32,7 @@ GDB="" NODE="" MULTI_GPU="" FORCE_HIGH="" +RUN_IN_DOCKER="" printUsage() { echo @@ -45,6 +46,7 @@ printUsage() { "not specified test will be run on all nodes" echo " -l , --list List available nodes" echo " --high Force clocks to high for test execution" + echo " -d , --docker Run in docker container" echo " -h , --help Prints this help" echo echo "Gtest arguments will be forwarded to the app" @@ -147,6 +149,14 @@ getNodeName() { # NODE - If set tests will be run only on this NODE, else it will be # run on all available HSA Nodes runKfdTest() { + if [ "$RUN_IN_DOCKER" == "true" ]; then + if [ `sudo systemctl is-active docker` != "active" ]; then + echo "docker isn't active, install and setup docker first!!!!" + exit 0 + fi + PKG_ROOT="$(getPackageRoot)" + fi + if [ "$NODE" == "" ]; then hsaNodes=$(getHsaNodes) else @@ -164,10 +174,34 @@ runKfdTest() { gtestFilter=$(getFilter $nodeName) - echo "" - echo "++++ Starting testing node $hsaNode ($nodeName) ++++" - $GDB $KFDTEST "--node=$hsaNode" $gtestFilter $GTEST_ARGS - echo "---- Finished testing node $hsaNode ($nodeName) ----" + if [ "$RUN_IN_DOCKER" == "true" ]; then + if [ "$NODE" == "" ]; then + DEVICE_NODE="/dev/dri" + else + RENDER_NODE=$(($hsaNode + 127)) + DEVICE_NODE="/dev/dri/renderD${RENDER_NODE}" + fi + + echo "Starting testing node $hsaNode ($nodeName) in docker container" + sudo docker run -it --name kfdtest_docker --user="jenkins" --network=host \ + --device=/dev/kfd --device=${DEVICE_NODE} --group-add video --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined -v $PKG_ROOT:/home/jenkins/rocm \ + compute-artifactory.amd.com:5000/yuho/tianli-ubuntu1604-kfdtest:01 \ + /home/jenkins/rocm/utils/run_kfdtest.sh -n $hsaNode $gtestFilter $GTEST_ARGS + if [ "$?" = "0" ]; then + echo "Finished node $hsaNode ($nodeName) successfully in docker container" + else + echo "Testing failed for node $hsaNode ($nodeName) in docker container" + fi + sudo docker rm kfdtest_docker + else + echo "" + echo "++++ Starting testing node $hsaNode ($nodeName) ++++" + $GDB $KFDTEST "--node=$hsaNode" $gtestFilter $GTEST_ARGS + echo "---- Finished testing node $hsaNode ($nodeName) ----" + fi + + done } @@ -202,6 +236,8 @@ while [ "$1" != "" ]; do shift 1; NODE=$1 ;; --high) FORCE_HIGH="true" ;; + -d | --docker ) + RUN_IN_DOCKER="true" ;; -h | --help ) printUsage; exit 0 ;; *) From 5e4e19d47b9db9c263f5fbe95dd45e92ea2eb4e0 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 5 Nov 2018 20:25:45 -0500 Subject: [PATCH 0434/1247] libhsakmt: Distinguish EPERM and EACCES EPERM means "operation not permitted" and is returned when CGroup access checks fail. EACCES means "permission denied" and is returned when the device file permission bits or access control list don't allow access. EPERM can fail silently, since we assume the administrator disabled a device on purpose in the CGroup. EACCESS should produce an error message and an info message to check the device file permissions. Change-Id: Iee4c5584c5fdc4e113c3d760dede6661097b4341 Signed-off-by: Felix Kuehling --- src/fmm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 1b70c9e5f0..fdca89067f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1733,10 +1733,10 @@ int open_drm_render_device(int minor) sprintf(path, "/dev/dri/renderD%d", minor); fd = open(path, O_RDWR | O_CLOEXEC); if (fd < 0) { - if (errno != ENOENT) { - if (errno == EPERM) - pr_info("Check a) User is in \"video\" group b) cgroup permissions\n"); + if (errno != ENOENT && errno != EPERM) { pr_err("Failed to open %s: %s\n", path, strerror(errno)); + if (errno == EACCES) + pr_info("Check user is in \"video\" group\n"); } return -errno; } From 055f7c9c2c7a60023d5920f1c52b881dd2f3066c Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 6 Nov 2018 13:55:53 -0500 Subject: [PATCH 0435/1247] Use latest kfd_ioctl.h file Change-Id: Icd7da4a305581c6857e17d59fbd0c3bd5101df3b Signed-off-by: Oak Zeng --- include/linux/kfd_ioctl.h | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 4597e045f8..780d6928e1 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -35,9 +35,13 @@ struct kfd_ioctl_get_version_args { }; /* For kfd_ioctl_create_queue_args.queue_type. */ -#define KFD_IOC_QUEUE_TYPE_COMPUTE 0 -#define KFD_IOC_QUEUE_TYPE_SDMA 1 -#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 2 +#define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0 +#define KFD_IOC_QUEUE_TYPE_SDMA 0x1 +#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 0x2 +#define KFD_IOC_QUEUE_TYPE_SDMA_ENGINE(e) (0x10000 + (e)) +#define KFD_IOC_QUEUE_TYPE_SDMA_AQL_ENGINE(e) (0x20000 + (e)) +#define KFD_IOC_QUEUE_TYPE_SDMA_ENGINE_MAX (KFD_IOC_QUEUE_TYPE_SDMA_ENGINE(0xffff)) +#define KFD_IOC_QUEUE_TYPE_SDMA_AQL_ENGINE_MAX (KFD_IOC_QUEUE_TYPE_SDMA_AQL_ENGINE(0xffff)) #define KFD_MAX_QUEUE_PERCENTAGE 100 #define KFD_MAX_QUEUE_PRIORITY 15 @@ -86,7 +90,7 @@ struct kfd_ioctl_get_queue_wave_state_args { __u64 ctl_stack_address; /* to KFD */ __u32 ctl_stack_used_size; /* from KFD */ __u32 save_area_used_size; /* from KFD */ - __u32 queue_id; /* to KFD */ + __u32 queue_id; /* to KFD */ __u32 pad; }; @@ -234,6 +238,15 @@ struct kfd_ioctl_dbg_trap_args { #define KFD_SIGNAL_EVENT_LIMIT 4096 +/* For kfd_event_data.hw_exception_data.reset_type. */ +#define KFD_HW_EXCEPTION_WHOLE_GPU_RESET 0 +#define KFD_HW_EXCEPTION_PER_ENGINE_RESET 1 + +/* For kfd_event_data.hw_exception_data.reset_cause. */ +#define KFD_HW_EXCEPTION_GPU_HANG 0 +#define KFD_HW_EXCEPTION_ECC 1 + + struct kfd_ioctl_create_event_args { __u64 event_page_offset; /* from KFD */ __u32 event_trigger_data; /* from KFD - signal events only */ @@ -275,10 +288,19 @@ struct kfd_hsa_memory_exception_data { __u32 pad; }; +/* hw exception data */ +struct kfd_hsa_hw_exception_data { + __u32 reset_type; + __u32 reset_cause; + __u32 memory_lost; + __u32 gpu_id; +}; + /* Event data */ struct kfd_event_data { union { struct kfd_hsa_memory_exception_data memory_exception_data; + struct kfd_hsa_hw_exception_data hw_exception_data; }; /* From KFD */ __u64 kfd_event_data_ext; /* pointer to an extension structure for future exception types */ From 5173e7181029a809131262c3ef7fcd9e3275ea3a Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 6 Nov 2018 14:04:17 -0500 Subject: [PATCH 0436/1247] Add more SDMA queue type Those new types are used to create SDMA queue on specific engine Change-Id: I91c3bcc14fef7404cf42b256a18651432e171091 Signed-off-by: Oak Zeng --- include/hsakmttypes.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index c4d39271b9..674b965b8d 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -622,6 +622,26 @@ typedef enum _HSA_QUEUE_TYPE HSA_QUEUE_COMPUTE_AQL = 21, // HSA AQL packet compatible Compute Queue HSA_QUEUE_DMA_AQL = 22, // HSA AQL packet compatible DMA Queue + HSA_QUEUE_SDMA_ENGINE0 =0x10000, //SDMA queue created on SDMA engine 0 + HSA_QUEUE_SDMA_ENGINE1 =0x10001, //SDMA queue created on SDMA engine 1 + HSA_QUEUE_SDMA_ENGINE2 =0x10002, //SDMA queue created on SDMA engine 2 + HSA_QUEUE_SDMA_ENGINE3 =0x10003, //SDMA queue created on SDMA engine 3 + HSA_QUEUE_SDMA_ENGINE4 =0x10004, //SDMA queue created on SDMA engine 4 + HSA_QUEUE_SDMA_ENGINE5 =0x10005, //SDMA queue created on SDMA engine 5 + HSA_QUEUE_SDMA_ENGINE6 =0x10006, //SDMA queue created on SDMA engine 6 + HSA_QUEUE_SDMA_ENGINE7 =0x10007, //SDMA queue created on SDMA engine 7 + HSA_QUEUE_SDMA_ENGINE_MAX =0x1FFFF, + + + HSA_QUEUE_SDMA_AQL_ENGINE0 =0x20000, //SDMA AQL queue created on SDMA engine 0 + HSA_QUEUE_SDMA_AQL_ENGINE1 =0x20001, //SDMA AQL queue created on SDMA engine 1 + HSA_QUEUE_SDMA_AQL_ENGINE2 =0x20002, //SDMA AQL queue created on SDMA engine 2 + HSA_QUEUE_SDMA_AQL_ENGINE3 =0x20003, //SDMA AQL queue created on SDMA engine 3 + HSA_QUEUE_SDMA_AQL_ENGINE4 =0x20004, //SDMA AQL queue created on SDMA engine 4 + HSA_QUEUE_SDMA_AQL_ENGINE5 =0x20005, //SDMA AQL queue created on SDMA engine 5 + HSA_QUEUE_SDMA_AQL_ENGINE6 =0x20006, //SDMA AQL queue created on SDMA engine 6 + HSA_QUEUE_SDMA_AQL_ENGINE7 =0x20007, //SDMA AQL queue created on SDMA engine 7 + HSA_QUEUE_SDMA_AQL_ENGINE_MAX =0x2FFFF, // more types in the future HSA_QUEUE_TYPE_SIZE = 0xFFFFFFFF //aligns to 32bit enum From 58b95e0a9d81f61f9c63aafb7c7ef5b87b4e9fb5 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 6 Nov 2018 14:21:15 -0500 Subject: [PATCH 0437/1247] Create SDMA queue on specific engine Change-Id: Id651ececda55b81b45e991bd8e6616674be48d8e Signed-off-by: Oak Zeng --- src/queues.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/queues.c b/src/queues.c index 28281f5731..58f0ac0788 100644 --- a/src/queues.c +++ b/src/queues.c @@ -576,6 +576,16 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; + case HSA_QUEUE_SDMA_ENGINE0: + case HSA_QUEUE_SDMA_ENGINE1: + case HSA_QUEUE_SDMA_ENGINE2: + case HSA_QUEUE_SDMA_ENGINE3: + case HSA_QUEUE_SDMA_ENGINE4: + case HSA_QUEUE_SDMA_ENGINE5: + case HSA_QUEUE_SDMA_ENGINE6: + case HSA_QUEUE_SDMA_ENGINE7: + args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA_ENGINE(Type - HSA_QUEUE_SDMA_ENGINE0); + break; default: return HSAKMT_STATUS_INVALID_PARAMETER; } From 8d65e72045a1ee220f8147f2288bb5075db3e76a Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 8 Nov 2018 12:24:13 -0500 Subject: [PATCH 0438/1247] Move m_Type to a local variable BaseQueue class has a member function GetQueueType so m_Type is duplicated. m_Type is only used in one function. Move it to a local variable. Change-Id: Ice144cf723178dd628cb49261c23d10605f9ee7d Signed-off-by: Oak Zeng --- tests/kfdtest/src/BaseQueue.cpp | 9 ++++----- tests/kfdtest/src/BaseQueue.hpp | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index e8a65ff10a..635dd12692 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -28,8 +28,7 @@ #include "hsakmt.h" BaseQueue::BaseQueue() - :m_Type(HSA_QUEUE_TYPE_SIZE), - m_QueueBuf(NULL), + :m_QueueBuf(NULL), m_SkipWaitConsumption(true) { } @@ -39,6 +38,7 @@ BaseQueue::~BaseQueue(void) { HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint64 *pointers) { HSAKMT_STATUS status; + HSA_QUEUE_TYPE type = GetQueueType(); if (m_QueueBuf != NULL) { // Queue already exists, one queue per object @@ -47,16 +47,15 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 memset(&m_Resources, 0, sizeof(m_Resources)); - m_Type = GetQueueType(); m_QueueBuf = new HsaMemoryBuffer(size, NodeId, true/*zero*/, false/*local*/, true/*exec*/); - if (m_Type == HSA_QUEUE_COMPUTE_AQL) { + if (type == HSA_QUEUE_COMPUTE_AQL) { m_Resources.Queue_read_ptr_aql = &pointers[0]; m_Resources.Queue_write_ptr_aql = &pointers[1]; } status = hsaKmtCreateQueue(NodeId, - m_Type, + type, DEFAULT_QUEUE_PERCENTAGE, DEFAULT_PRIORITY, m_QueueBuf->As(), diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index ba0464a6e0..9d354448fa 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -95,7 +95,6 @@ class BaseQueue { unsigned int CMD_NOP; unsigned int m_pendingWptr; HSAuint64 m_pendingWptr64; - HSA_QUEUE_TYPE m_Type; HsaQueueResource m_Resources; HsaMemoryBuffer *m_QueueBuf; unsigned int m_Node; From acb80d7583fd3312235248ad8444869700ac6e4b Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 12 Nov 2018 16:06:57 -0600 Subject: [PATCH 0439/1247] Create SDMA queue on specific engine Change-Id: Iece03795510d66b03324174203faa0ac9eb4fb7d Signed-off-by: Oak Zeng --- tests/kfdtest/src/BaseQueue.cpp | 6 +++++- tests/kfdtest/src/BaseQueue.hpp | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 635dd12692..5001813f72 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -36,7 +36,7 @@ BaseQueue::~BaseQueue(void) { Destroy(); } -HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint64 *pointers) { +HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint64 *pointers, int sdmaEngineId) { HSAKMT_STATUS status; HSA_QUEUE_TYPE type = GetQueueType(); @@ -54,6 +54,10 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 m_Resources.Queue_write_ptr_aql = &pointers[1]; } + //TODO: How to get number of SDMA engine? + if (type == HSA_QUEUE_SDMA && sdmaEngineId >= 0 && sdmaEngineId < 8) + type = (HSA_QUEUE_TYPE) (HSA_QUEUE_SDMA_ENGINE0 + sdmaEngineId); + status = hsaKmtCreateQueue(NodeId, type, DEFAULT_QUEUE_PERCENTAGE, diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index 9d354448fa..23df1d1ad6 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -43,9 +43,10 @@ class BaseQueue { /** Create the queue. * @see hsaKmtCreateQueue * @param pointers is used only for creating AQL queues. Otherwise it is omitted. + * @param sdmaEngineId is used only for creating SDMA queues. -1 means let kfd kernel select a sdma engine */ virtual HSAKMT_STATUS Create(unsigned int NodeId, unsigned int size = DEFAULT_QUEUE_SIZE, - HSAuint64 *pointers = NULL); + HSAuint64 *pointers = NULL, int sdmaEngineId = -1); /** Update the queue. * @see hsaKmtUpdateQueue * @param percent New queue percentage From 29b45b8c0a880cea4525539767107b88eb23bf3a Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Fri, 16 Nov 2018 11:16:04 -0500 Subject: [PATCH 0440/1247] Thunk: make scratch memory only map to its own GPU Map scratch memory to the GPU that specified when allocate the memory Change-Id: I788f9ef0dccb63b894a75e75cac5f94a60d7ec48 Signed-off-by: shaoyunl --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index fdca89067f..28f59c4427 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2494,7 +2494,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* map to GPU */ - ret = _fmm_map_to_gpu(aperture, address, size, NULL, NULL, 0); + ret = _fmm_map_to_gpu(aperture, address, size, NULL, &gpu_id, sizeof(uint32_t)); if (ret != 0) __fmm_release(obj, aperture); From d8009b4fd386b7365ff9ef0f332777ef256c9ba0 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Mon, 19 Nov 2018 11:38:38 -0500 Subject: [PATCH 0441/1247] KFDTest: fix failure when run KFDTest on multi-GPU small bar system On small bar multi-gpu system, hsaKmtMemoryMapToGPU will fail due to latest kernel P2P sanity check. Swith to use hsaKmtMemoryMapToGPUNodes to fix the failure Change-Id: Id8b6329d1243df0e908cc9a171b5c7f9156f4a8b Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDEvictTest.cpp | 4 ++- tests/kfdtest/src/KFDIPCTest.cpp | 8 +++-- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 19 ++++++---- tests/kfdtest/src/KFDMemoryTest.cpp | 44 ++++++++++++++++-------- tests/kfdtest/src/KFDTestUtil.cpp | 8 ++++- 5 files changed, 58 insertions(+), 25 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 6fc6969f6a..0a1bf506c0 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -44,6 +44,7 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui << totalMB << ")MB VRAM in KFD" << std::endl; } + HsaMemMapFlags mapFlags = {0}; HSAKMT_STATUS ret; HSAuint32 retry = 0; @@ -56,7 +57,8 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui ret = hsaKmtAllocMemory(defaultGPUNode, vramBufSize, m_Flags, &m_pBuf); if (ret == HSAKMT_STATUS_SUCCESS) { if (is_dgpu()) { - if (hsaKmtMapMemoryToGPU(m_pBuf, vramBufSize, NULL) == HSAKMT_STATUS_ERROR) { + if (hsaKmtMapMemoryToGPUNodes(m_pBuf, vramBufSize, NULL, + mapFlags, 1, reinterpret_cast(&defaultGPUNode)) == HSAKMT_STATUS_ERROR) { EXPECT_SUCCESS(hsaKmtFreeMemory(m_pBuf, vramBufSize)); break; } diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index 1150076a6b..eba001ebb2 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -75,13 +75,15 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { HSAuint64 size = PAGE_SIZE, sharedSize; HsaMemoryBuffer tempSysBuffer(size, defaultGPUNode, false); HSAuint32 *sharedLocalBuffer = NULL; + HsaMemMapFlags mapFlags = {0}; /* Read from Pipe the shared Handle. Import shared Local Memory */ ASSERT_GE(read(pipefd[0], reinterpret_cast(&sharedHandleLM), sizeof(sharedHandleLM)), 0); ASSERT_SUCCESS(hsaKmtRegisterSharedHandle(&sharedHandleLM, reinterpret_cast(&sharedLocalBuffer), &sharedSize)); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(sharedLocalBuffer, sharedSize, NULL)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sharedLocalBuffer, sharedSize, NULL, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); /* Check for pattern in the shared Local Memory */ ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); @@ -114,9 +116,11 @@ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pip HsaMemoryBuffer tempSysBuffer(PAGE_SIZE, defaultGPUNode, false); SDMAQueue sdmaQueue; HsaSharedMemoryHandle sharedHandleLM; + HsaMemMapFlags mapFlags = {0}; /* Fill a Local Buffer with a pattern */ - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(toShareLocalBuffer.As(), toShareLocalBuffer.Size(), &AlternateVAGPU)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(toShareLocalBuffer.As(), toShareLocalBuffer.Size(), &AlternateVAGPU, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); tempSysBuffer.Fill(0xAAAAAAAA); /* Copy pattern in Local Memory before sharing it */ diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index c879080b25..0af52e9d8e 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -57,6 +57,7 @@ TEST_F(KFDLocalMemoryTest, BasicTest) { PM4Queue queue; HSAuint64 AlternateVAGPU; unsigned int BufferSize = PAGE_SIZE; + HsaMemMapFlags mapFlags = {0}; int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -76,8 +77,10 @@ TEST_F(KFDLocalMemoryTest, BasicTest) { m_pIsaGen->GetCopyDwordIsa(isaBuffer); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(srcLocalBuffer.As(), srcLocalBuffer.Size(), &AlternateVAGPU)); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dstLocalBuffer.As(), dstLocalBuffer.Size(), &AlternateVAGPU)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(srcLocalBuffer.As(), srcLocalBuffer.Size(), &AlternateVAGPU, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(dstLocalBuffer.As(), dstLocalBuffer.Size(), &AlternateVAGPU, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.SetSkipWaitConsump(0); @@ -112,6 +115,7 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { PM4Queue queue; HSAuint64 AlternateVAGPU; unsigned int BufferSize = PAGE_SIZE; + HsaMemMapFlags mapFlags = {0}; int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -134,7 +138,8 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { queue.SetSkipWaitConsump(0); if (!is_dgpu()) - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); Dispatch dispatch(isaBuffer); @@ -143,7 +148,8 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { dispatch.Sync(g_TestTimeOut); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(LocalBuffer.As())); - EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(LocalBuffer.As(), LocalBuffer.Size(), &AlternateVAGPU, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); dispatch.SetArgs(LocalBuffer.As(), SysBufferB.As()); dispatch.Submit(queue); @@ -270,6 +276,7 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { /* Allocate and test memory using the strategy explained at the top */ HSAKMT_STATUS status; HsaMemFlags memFlags = {0}; + HsaMemMapFlags mapFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 0; memFlags.ui32.NonPaged = 1; @@ -318,8 +325,8 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { + size - sizeof(unsigned)); sysBuffer.As()[0] = ++value; - status = hsaKmtMapMemoryToGPU(pages[order].pointers[p], - size, NULL); + status = hsaKmtMapMemoryToGPUNodes(pages[order].pointers[p], size, NULL, + mapFlags, 1, reinterpret_cast(&defaultGPUNode)); if (status != HSAKMT_STATUS_SUCCESS) { ASSERT_SUCCESS(hsaKmtFreeMemory(pages[order].pointers[p], size)); diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 8cc641c375..51bed05860 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -896,6 +896,16 @@ TEST_F(KFDMemoryTest, MMBench) { HSAuint64 vramSizeMB = GetVramSize(defaultGPUNode) >> 20; + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + bool is_all_large_bar = true; + + for (unsigned i = 0; i < gpuNodes.size(); i++) { + if (!m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i))) { + is_all_large_bar = false; + break; + } + } + LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; if (vramSizeMB == 0) @@ -986,22 +996,24 @@ TEST_F(KFDMemoryTest, MMBench) { IDLE_SDMA(); /* Map to all GPUs */ - start = GetSystemTickCountInMicroSec(); - for (i = 0; i < nBufs; i++) { - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(bufs[i], bufSize, &altVa)); - INTERLEAVE_SDMA(); - } - mapAllTime = GetSystemTickCountInMicroSec() - start; - IDLE_SDMA(); + if (is_all_large_bar) { + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(bufs[i], bufSize, &altVa)); + INTERLEAVE_SDMA(); + } + mapAllTime = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); - /* Unmap from all GPUs */ - start = GetSystemTickCountInMicroSec(); - for (i = 0; i < nBufs; i++) { - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); - INTERLEAVE_SDMA(); + /* Unmap from all GPUs */ + start = GetSystemTickCountInMicroSec(); + for (i = 0; i < nBufs; i++) { + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(bufs[i])); + INTERLEAVE_SDMA(); + } + unmapAllTime = GetSystemTickCountInMicroSec() - start; + IDLE_SDMA(); } - unmapAllTime = GetSystemTickCountInMicroSec() - start; - IDLE_SDMA(); /* Free */ start = GetSystemTickCountInMicroSec(); @@ -1292,6 +1304,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HsaMemMapFlags mapFlags = {0}; HsaMemFlags memFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; /* Allocate host not accessible vram */ @@ -1307,7 +1320,8 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { const HSAuint64 VRAM_OFFSET = (4 << 20) - sizeof(HSAuint64); ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, size, memFlags, &mem)); - ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(mem, size, NULL)); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(mem, size, NULL, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); /* Set the word before 4M boundary to 0xdeadbeefdeadbeef * and the word after 4M boundary to 0xcafebabecafebabe */ diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 2195d6f37e..b316909139 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -148,6 +148,9 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b m_Node(node) { m_Flags.Value = 0; + HsaMemMapFlags mapFlags = {0}; + bool map_specific_gpu = (node && !isScratch); + if (isScratch) { m_Flags.ui32.Scratch = 1; m_Flags.ui32.HostAccess = 1; @@ -170,7 +173,10 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b EXPECT_SUCCESS(hsaKmtAllocMemory(m_Node, m_Size, m_Flags, &m_pBuf)); if (is_dgpu()) { - EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(m_pBuf, m_Size, NULL)); + if (map_specific_gpu) + EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(m_pBuf, m_Size, NULL, mapFlags, 1, &m_Node)); + else + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(m_pBuf, m_Size, NULL)); m_MappedNodes = 1 << m_Node; } From af5b320c478e60b134636bf8c7bfcf932451329f Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 12 Nov 2018 16:14:14 -0600 Subject: [PATCH 0442/1247] Add test to allocate SDMA queue on specific engine Change-Id: I5b5140e4119fc01db250d63cca7389cf80ec0d16 Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDQMTest.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 53471dedd9..c6922bc014 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -105,6 +105,35 @@ TEST_F(KFDQMTest, CreateSdmaQueue) { TEST_END } +TEST_F(KFDQMTest, CreateSdmaQueueOnEngine) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + unsigned int numSdmaEngine; + + if (m_FamilyId == FAMILY_RV) + numSdmaEngine = 1; + else + numSdmaEngine = 2; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + SDMAQueue queue; + + //How to get number of sdma engine? + for (int i = 0; i < numSdmaEngine; i++) { + destBuf.Fill(0xFF); + ASSERT_SUCCESS(queue.Create(defaultGPUNode, BaseQueue::DEFAULT_QUEUE_SIZE, NULL, i)); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0x02020202)); + queue.Wait4PacketConsumption(); + ASSERT_TRUE(WaitOnValue(destBuf.As(), 0x02020202)); + ASSERT_SUCCESS(queue.Destroy()); + } + + TEST_END +} + TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { TEST_START(TESTPROFILE_RUNALL) From c15cf2e9c30848dde05fd2f11769d717758e5e6f Mon Sep 17 00:00:00 2001 From: changzhu Date: Wed, 21 Nov 2018 17:11:53 +0800 Subject: [PATCH 0443/1247] kfdtest: fix SDMACopyParams build error on redhat 7.2 in KFDTestUtilQueue.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In file included from /usr/include/c++/4.8.2/algorithm:62:0, from /home/jenkins/libhsakmt/tests/kfdtest/src/KFDTestUtilQueue.cpp:24: /usr/include/c++/4.8.2/bits/stl_algo.h: In instantiation of ‘_RandomAccessIterator std::__unguarded_partition(_RandomAccessIterator, _RandomAccessIterator, const _Tp&, _Compare) [with _RandomAccessIterator = __gnu_cxx::__normal_iterator >; _Tp = SDMACopyParams; _Compare = bool (*)(SDMACopyParams&, SDMACopyParams&)]’: /usr/include/c++/4.8.2/bits/stl_algo.h:2296:78: required from ‘_RandomAccessIterator std::__unguarded_partition_pivot(_RandomAccessIterator, _RandomAccessIterator, _Compare) [with _RandomAccessIterator = __gnu_cxx::__normal_iterator >; _Compare = bool (*)(SDMACopyParams&, SDMACopyParams&)]’ /usr/include/c++/4.8.2/bits/stl_algo.h:2337:62: required from ‘void std::__introsort_loop(_RandomAccessIterator, _RandomAccessIterator, _Size, _Compare) [with _RandomAccessIterator = __gnu_cxx::__normal_iterator >; _Size = long int; _Compare = bool (*)(SDMACopyParams&, SDMACopyParams&)]’ /usr/include/c++/4.8.2/bits/stl_algo.h:5499:44: required from ‘void std::sort(_RAIter, _RAIter, _Compare) [with _RAIter = __gnu_cxx::__normal_iterator >; _Compare = bool (*)(SDMACopyParams&, SDMACopyParams&)]’ /home/jenkins/libhsakmt/tests/kfdtest/src/KFDTestUtilQueue.cpp:351:66: required from here /usr/include/c++/4.8.2/bits/stl_algo.h:2263:35: error: invalid initialization of reference of type ‘SDMACopyParams&’ from expression of type ‘const SDMACopyParams’ while (__comp(*__first, __pivot)) ^ /usr/include/c++/4.8.2/bits/stl_algo.h:2266:34: error: invalid initialization of reference of type ‘SDMACopyParams&’ from expression of type ‘const SDMACopyParams’ while (__comp(__pivot, *__last)) ^ Change-Id: I0fce0c7e6d0a0ce93b1e6522ee8f216615765568 Signed-off-by: changzhu --- tests/kfdtest/src/KFDTestUtilQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/tests/kfdtest/src/KFDTestUtilQueue.cpp index b6604287cf..46c88cf702 100644 --- a/tests/kfdtest/src/KFDTestUtilQueue.cpp +++ b/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -323,7 +323,7 @@ class AsyncMPMQ { * SDMA queue helper functions. */ -bool sort_SDMACopyParams(SDMACopyParams &a1, SDMACopyParams &a2) { +bool sort_SDMACopyParams(const SDMACopyParams &a1, const SDMACopyParams &a2) { if (a1.node != a2.node) return a1.node < a2.node; return a1.group < a2.group; From 1fbe010354f826217c1104a6e4bf07cddd50e70d Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Tue, 20 Nov 2018 15:20:52 -0500 Subject: [PATCH 0444/1247] libhsakmt: add RAS support RAS feature enabling bit and errors return are implemented in existed topology and event mechanism. Change-Id: I9b018bba80cf4a6998e42a7bff64318c689b1d2a Signed-off-by: Eric Huang --- include/hsakmttypes.h | 47 ++++++++++++++++++++++++++++++++++++++- include/linux/kfd_ioctl.h | 29 +++++++++++++++++++----- src/events.c | 20 +++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 674b965b8d..c924d14545 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -203,7 +203,8 @@ typedef union unsigned int WaveLaunchTrapOverrideSupported: 1; // Indicates if Wave Launch Trap Override is supported on the node. unsigned int WaveLaunchModeSupported: 1; // Indicates if Wave Launch Mode is supported on the node. unsigned int PreciseMemoryOperationsSupported: 1; // Indicates if Precise Memory Operations are supported on the node. - unsigned int Reserved : 13; + unsigned int RasSupported : 1; // Indicates if GPU RAS feature is enabled + unsigned int Reserved : 12; } ui32; } HSA_CAPABILITY; @@ -916,6 +917,47 @@ typedef struct _HsaMemoryAccessFault HSA_EVENTID_MEMORYFLAGS Flags; // event flags } HsaMemoryAccessFault; +typedef enum _HSA_HW_EXCEPTION_TYPE +{ + HSA_HW_EXCEPTION_TYPE_FULL_GPU_RESET = 0, + HSA_HW_EXCEPTION_TYPE_PER_ENGINE_RESET = 1, + HSA_HW_EXCEPTION_TYPE_RAS_FATAL = 2, + HSA_HW_EXCEPTION_TYPE_RAS_NOTIFY = 3 +} HSA_HW_EXCEPTION_TYPE; + +typedef struct _HSA_HW_EXCEPTION_CAUSE +{ + unsigned int Hang : 1; // GPU hang + unsigned int Parity : 1; // RAS_ERROR_PARITY + unsigned int SingleCorrectable : 1; // ERROR_SINGLE_CORRECTABLE + unsigned int MultiUncorrectable: 1; // ERROR_MULTI_UNCORRECTABLE + unsigned int Poison : 1; // ERROR_POISON + unsigned int Reserved : 27; +} HSA_HW_EXCEPTION_CAUSE; + +typedef enum _HSA_HW_EXCEPTION_BLOCK_ID +{ + HSA_HW_EXCEPTION_BLOCK_UMC = 0, + HSA_HW_EXCEPTION_BLOCK_SDMA = 1, + HSA_HW_EXCEPTION_BLOCK_GFXHUB = 2, + HSA_HW_EXCEPTION_BLOCK_MMHUB = 3, + HSA_HW_EXCEPTION_BLOCK_ATHUB = 4, + HSA_HW_EXCEPTION_BLOCK_PCIE_BIF = 5, + HSA_HW_EXCEPTION_BLOCK_HDP = 6, + + HSA_HW_EXCEPTION_BLOCK_UNKNOWN = 0xffffffff +} HSA_HW_EXCEPTION_BLOCK_ID; + +// data associated with HSA_EVENTTYPE_HW_EXCEPTION +typedef struct _HsaHwException +{ + HSA_HW_EXCEPTION_TYPE Type; + HSA_HW_EXCEPTION_CAUSE Cause; + HSA_HW_EXCEPTION_BLOCK_ID BlockId; // Id of block on where RAS error happens + bool MemoryLost; + HSAuint32 NodeId; // Id of GPU that has hw exception +} HsaHwException; + typedef struct _HsaEventData { HSA_EVENTTYPE EventType; //event type @@ -934,6 +976,9 @@ typedef struct _HsaEventData // data associated with HSA_EVENTTYPE_MEMORY HsaMemoryAccessFault MemoryAccessFault; + // data associated with HSA_EVENTTYPE_HW_EXCEPTION + HsaHwException HwException; + } EventData; // the following data entries are internal to the KFD & thunk itself. diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 780d6928e1..d6261a9227 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -238,13 +238,28 @@ struct kfd_ioctl_dbg_trap_args { #define KFD_SIGNAL_EVENT_LIMIT 4096 -/* For kfd_event_data.hw_exception_data.reset_type. */ +/* For kfd_event_data.hw_exception_data.type. */ #define KFD_HW_EXCEPTION_WHOLE_GPU_RESET 0 #define KFD_HW_EXCEPTION_PER_ENGINE_RESET 1 +#define KFD_HW_EXCEPTION_RAS_FATAL 2 +#define KFD_HW_EXCEPTION_RAS_NOTIFY 3 -/* For kfd_event_data.hw_exception_data.reset_cause. */ -#define KFD_HW_EXCEPTION_GPU_HANG 0 -#define KFD_HW_EXCEPTION_ECC 1 +/* For kfd_event_data.hw_exception_data.cause. */ +#define KFD_HW_EXCEPTION_GPU_HANG 0x00000001 +#define KFD_HW_EXCEPTION_PARITY 0x00000002 +#define KFD_HW_EXCEPTION_SINGLE_CORRECTABLE 0x00000004 +#define KFD_HW_EXCEPTION_MULTI_UNCORRECTABLE 0x00000008 +#define KFD_HW_EXCEPTION_POISON 0x00000010 + +/*For kfd_event_data.hw_exception_data.block_id */ +#define KFD_HW_EXCEPTION_BLOCK_UMC 0 +#define HSA_HW_EXCEPTION_BLOCK_SDMA 1 +#define HSA_HW_EXCEPTION_BLOCK_GFXHUB 2 +#define HSA_HW_EXCEPTION_BLOCK_MMHUB 3 +#define HSA_HW_EXCEPTION_BLOCK_ATHUB 4 +#define HSA_HW_EXCEPTION_BLOCK_PCIE_BIF 5 +#define HSA_HW_EXCEPTION_BLOCK_HDP 6 +#define HSA_HW_EXCEPTION_BLOCK_UNKNOWN 0xffffffff struct kfd_ioctl_create_event_args { @@ -290,10 +305,12 @@ struct kfd_hsa_memory_exception_data { /* hw exception data */ struct kfd_hsa_hw_exception_data { - __u32 reset_type; - __u32 reset_cause; + __u32 type; + __u32 cause; + __u32 block_id; __u32 memory_lost; __u32 gpu_id; + __u32 pad; }; /* Event data */ diff --git a/src/events.c b/src/events.c index 43d99b8da8..a5828afa0d 100644 --- a/src/events.c +++ b/src/events.c @@ -326,6 +326,26 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], Events[i]->EventData.EventData.MemoryAccessFault.Flags = HSA_EVENTID_MEMORY_FATAL_PROCESS; analysis_memory_exception(&event_data[i].memory_exception_data); } + if (Events[i]->EventData.EventType == HSA_EVENTTYPE_HW_EXCEPTION) { + Events[i]->EventData.EventData.HwException.Type = + event_data[i].hw_exception_data.type; + Events[i]->EventData.EventData.HwException.Cause.Hang = + event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_GPU_HANG; + Events[i]->EventData.EventData.HwException.Cause.Parity = + (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_PARITY) >> 1; + Events[i]->EventData.EventData.HwException.Cause.SingleCorrectable = + (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_SINGLE_CORRECTABLE) >> 2; + Events[i]->EventData.EventData.HwException.Cause.MultiUncorrectable = + (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_MULTI_UNCORRECTABLE) >> 3; + Events[i]->EventData.EventData.HwException.Cause.Poison = + (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_POISON) >> 4; + Events[i]->EventData.EventData.HwException.BlockId = + event_data[i].hw_exception_data.block_id; + Events[i]->EventData.EventData.HwException.MemoryLost = + event_data[i].hw_exception_data.memory_lost; + Events[i]->EventData.EventData.HwException.NodeId = + event_data[i].hw_exception_data.gpu_id; + } } } out: From 381dba3932ff5d9abb1598ffe8cffc24d85a722a Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 28 Nov 2018 10:01:35 -0500 Subject: [PATCH 0445/1247] Remove SDMAConcurrentCopies from gfx906 execution This is intermittently causing VM faults and excessive evictions, which causes the rest of the tests to fail. Take it out for now until someone can investigate Change-Id: I9c43890bc9f03a4a31efbc18df0df5e40a232c58 --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 4dc6dc573a..3f8e9579b8 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -134,6 +134,7 @@ VEGA20_TESTS_BLACKLIST=\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ "KFDQMTest.GPUDoorbellWrite:"\ +"KFDQMTest.SdmaConcurrentCopies:"\ "KFDMemoryTest.MemoryRegister" RAVEN_TESTS_BLACKLIST=\ From 3a2ec0111eb78a3a48026f5e81f0647cc5fd0022 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 5 Dec 2018 08:41:16 -0500 Subject: [PATCH 0446/1247] Temporarily remove SDMA tests from gfx906 SDMA is being flaky, so remove SDMA tests from it for now Change-Id: Ia3612566813f925804ab90d6235520da7cc65926 --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 3f8e9579b8..212a8717be 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -129,12 +129,12 @@ VEGA12_TESTS_BLACKLIST=\ VEGA20_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ "KFDDBGTest.BasicAddressWatch:"\ "KFDExceptionTest.InvalidBadAddress:"\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ "KFDQMTest.GPUDoorbellWrite:"\ -"KFDQMTest.SdmaConcurrentCopies:"\ "KFDMemoryTest.MemoryRegister" RAVEN_TESTS_BLACKLIST=\ From aa7c13264a72ef8b337b40a59a8b78cf727c6a30 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 6 Dec 2018 11:14:26 -0500 Subject: [PATCH 0447/1247] Add ZeroInitializationVram test to SDMA blacklist This test uses SDMA, so add it to the SDMA list Change-Id: I2dc2b0c4328e38e593d455de2103ebe1ef0adbc2 --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 212a8717be..0627144a2a 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -42,6 +42,7 @@ BLACKLIST_ALL_ASICS=\ # disabled on non-Hawaii due to SDMA instability - SWDEV-101666 SDMA_BLACKLIST=\ "KFDIPCTest.*:"\ +"KFDLocalMemoryTest.CheckZeroInitializationVram:"\ "KFDMemoryTest.MMBench:"\ "KFDQMTest.*Sdma*:"\ "KFDQMTest.P2PTest:"\ From 54807526b94a8c7751e94ca64236b1558691b56d Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 6 Dec 2018 12:32:20 -0500 Subject: [PATCH 0448/1247] Add more SDMA-related tests to SDMA_BLACKLIST These tests all make use of an SDMAQueue in one way or another, so add them to the SDMA_BLACKLIST to be 100% certain Change-Id: Ic29e073c2f46249f3e5918145b13d276aec7bb33 --- tests/kfdtest/scripts/kfdtest.exclude | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 0627144a2a..8060068c8a 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -43,8 +43,12 @@ BLACKLIST_ALL_ASICS=\ SDMA_BLACKLIST=\ "KFDIPCTest.*:"\ "KFDLocalMemoryTest.CheckZeroInitializationVram:"\ +"KFDMemoryTest.MemoryRegister:"\ "KFDMemoryTest.MMBench:"\ +"KFDMemoryTest.SignalHandling:"\ "KFDQMTest.*Sdma*:"\ +"KFDQMTest.CreateQueueStressSingleThreaded:"\ +"KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.P2PTest:"\ "KFDPerformanceTest.P2PBandWidthTest:"\ "KFDPerformanceTest.P2POverheadTest" From 29d11d02e86c5b7aa16f0e02075a0481aa41c2fc Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Tue, 4 Dec 2018 11:15:41 -0500 Subject: [PATCH 0449/1247] Revert "libhsakmt: add RAS support" This reverts commit 1fbe010354f826217c1104a6e4bf07cddd50e70d. Change-Id: I739b17e057f2a8a0f4375741955209d2477c704a Signed-off-by: Eric Huang --- include/hsakmttypes.h | 47 +-------------------------------------- include/linux/kfd_ioctl.h | 29 +++++------------------- src/events.c | 20 ----------------- 3 files changed, 7 insertions(+), 89 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index c924d14545..674b965b8d 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -203,8 +203,7 @@ typedef union unsigned int WaveLaunchTrapOverrideSupported: 1; // Indicates if Wave Launch Trap Override is supported on the node. unsigned int WaveLaunchModeSupported: 1; // Indicates if Wave Launch Mode is supported on the node. unsigned int PreciseMemoryOperationsSupported: 1; // Indicates if Precise Memory Operations are supported on the node. - unsigned int RasSupported : 1; // Indicates if GPU RAS feature is enabled - unsigned int Reserved : 12; + unsigned int Reserved : 13; } ui32; } HSA_CAPABILITY; @@ -917,47 +916,6 @@ typedef struct _HsaMemoryAccessFault HSA_EVENTID_MEMORYFLAGS Flags; // event flags } HsaMemoryAccessFault; -typedef enum _HSA_HW_EXCEPTION_TYPE -{ - HSA_HW_EXCEPTION_TYPE_FULL_GPU_RESET = 0, - HSA_HW_EXCEPTION_TYPE_PER_ENGINE_RESET = 1, - HSA_HW_EXCEPTION_TYPE_RAS_FATAL = 2, - HSA_HW_EXCEPTION_TYPE_RAS_NOTIFY = 3 -} HSA_HW_EXCEPTION_TYPE; - -typedef struct _HSA_HW_EXCEPTION_CAUSE -{ - unsigned int Hang : 1; // GPU hang - unsigned int Parity : 1; // RAS_ERROR_PARITY - unsigned int SingleCorrectable : 1; // ERROR_SINGLE_CORRECTABLE - unsigned int MultiUncorrectable: 1; // ERROR_MULTI_UNCORRECTABLE - unsigned int Poison : 1; // ERROR_POISON - unsigned int Reserved : 27; -} HSA_HW_EXCEPTION_CAUSE; - -typedef enum _HSA_HW_EXCEPTION_BLOCK_ID -{ - HSA_HW_EXCEPTION_BLOCK_UMC = 0, - HSA_HW_EXCEPTION_BLOCK_SDMA = 1, - HSA_HW_EXCEPTION_BLOCK_GFXHUB = 2, - HSA_HW_EXCEPTION_BLOCK_MMHUB = 3, - HSA_HW_EXCEPTION_BLOCK_ATHUB = 4, - HSA_HW_EXCEPTION_BLOCK_PCIE_BIF = 5, - HSA_HW_EXCEPTION_BLOCK_HDP = 6, - - HSA_HW_EXCEPTION_BLOCK_UNKNOWN = 0xffffffff -} HSA_HW_EXCEPTION_BLOCK_ID; - -// data associated with HSA_EVENTTYPE_HW_EXCEPTION -typedef struct _HsaHwException -{ - HSA_HW_EXCEPTION_TYPE Type; - HSA_HW_EXCEPTION_CAUSE Cause; - HSA_HW_EXCEPTION_BLOCK_ID BlockId; // Id of block on where RAS error happens - bool MemoryLost; - HSAuint32 NodeId; // Id of GPU that has hw exception -} HsaHwException; - typedef struct _HsaEventData { HSA_EVENTTYPE EventType; //event type @@ -976,9 +934,6 @@ typedef struct _HsaEventData // data associated with HSA_EVENTTYPE_MEMORY HsaMemoryAccessFault MemoryAccessFault; - // data associated with HSA_EVENTTYPE_HW_EXCEPTION - HsaHwException HwException; - } EventData; // the following data entries are internal to the KFD & thunk itself. diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index d6261a9227..780d6928e1 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -238,28 +238,13 @@ struct kfd_ioctl_dbg_trap_args { #define KFD_SIGNAL_EVENT_LIMIT 4096 -/* For kfd_event_data.hw_exception_data.type. */ +/* For kfd_event_data.hw_exception_data.reset_type. */ #define KFD_HW_EXCEPTION_WHOLE_GPU_RESET 0 #define KFD_HW_EXCEPTION_PER_ENGINE_RESET 1 -#define KFD_HW_EXCEPTION_RAS_FATAL 2 -#define KFD_HW_EXCEPTION_RAS_NOTIFY 3 -/* For kfd_event_data.hw_exception_data.cause. */ -#define KFD_HW_EXCEPTION_GPU_HANG 0x00000001 -#define KFD_HW_EXCEPTION_PARITY 0x00000002 -#define KFD_HW_EXCEPTION_SINGLE_CORRECTABLE 0x00000004 -#define KFD_HW_EXCEPTION_MULTI_UNCORRECTABLE 0x00000008 -#define KFD_HW_EXCEPTION_POISON 0x00000010 - -/*For kfd_event_data.hw_exception_data.block_id */ -#define KFD_HW_EXCEPTION_BLOCK_UMC 0 -#define HSA_HW_EXCEPTION_BLOCK_SDMA 1 -#define HSA_HW_EXCEPTION_BLOCK_GFXHUB 2 -#define HSA_HW_EXCEPTION_BLOCK_MMHUB 3 -#define HSA_HW_EXCEPTION_BLOCK_ATHUB 4 -#define HSA_HW_EXCEPTION_BLOCK_PCIE_BIF 5 -#define HSA_HW_EXCEPTION_BLOCK_HDP 6 -#define HSA_HW_EXCEPTION_BLOCK_UNKNOWN 0xffffffff +/* For kfd_event_data.hw_exception_data.reset_cause. */ +#define KFD_HW_EXCEPTION_GPU_HANG 0 +#define KFD_HW_EXCEPTION_ECC 1 struct kfd_ioctl_create_event_args { @@ -305,12 +290,10 @@ struct kfd_hsa_memory_exception_data { /* hw exception data */ struct kfd_hsa_hw_exception_data { - __u32 type; - __u32 cause; - __u32 block_id; + __u32 reset_type; + __u32 reset_cause; __u32 memory_lost; __u32 gpu_id; - __u32 pad; }; /* Event data */ diff --git a/src/events.c b/src/events.c index a5828afa0d..43d99b8da8 100644 --- a/src/events.c +++ b/src/events.c @@ -326,26 +326,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], Events[i]->EventData.EventData.MemoryAccessFault.Flags = HSA_EVENTID_MEMORY_FATAL_PROCESS; analysis_memory_exception(&event_data[i].memory_exception_data); } - if (Events[i]->EventData.EventType == HSA_EVENTTYPE_HW_EXCEPTION) { - Events[i]->EventData.EventData.HwException.Type = - event_data[i].hw_exception_data.type; - Events[i]->EventData.EventData.HwException.Cause.Hang = - event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_GPU_HANG; - Events[i]->EventData.EventData.HwException.Cause.Parity = - (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_PARITY) >> 1; - Events[i]->EventData.EventData.HwException.Cause.SingleCorrectable = - (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_SINGLE_CORRECTABLE) >> 2; - Events[i]->EventData.EventData.HwException.Cause.MultiUncorrectable = - (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_MULTI_UNCORRECTABLE) >> 3; - Events[i]->EventData.EventData.HwException.Cause.Poison = - (event_data[i].hw_exception_data.cause & KFD_HW_EXCEPTION_POISON) >> 4; - Events[i]->EventData.EventData.HwException.BlockId = - event_data[i].hw_exception_data.block_id; - Events[i]->EventData.EventData.HwException.MemoryLost = - event_data[i].hw_exception_data.memory_lost; - Events[i]->EventData.EventData.HwException.NodeId = - event_data[i].hw_exception_data.gpu_id; - } } } out: From 53439669d96b402ea6a71d47f81b89cc35154c00 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 12 Dec 2018 08:36:40 -0500 Subject: [PATCH 0450/1247] libhsakmt: Add new gfx900 and gfx906 GPU IDs Change-Id: I93b2b845c3edb2da55235a56516a851145745988 --- src/topology.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/topology.c b/src/topology.c index 0e90f83640..c7f8a28214 100644 --- a/src/topology.c +++ b/src/topology.c @@ -186,7 +186,12 @@ static struct hsa_gfxip_table { { 0x6864, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6867, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6868, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6869, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x686A, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x686B, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x686D, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x686E, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, /* Vega12 */ { 0x69A0, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, @@ -201,6 +206,7 @@ static struct hsa_gfxip_table { { 0x66A1, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66A2, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66A3, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + { 0x66A4, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66A7, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66AF, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, }; From bcc348e3b9e915c1b236628434332713ff952cc5 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 12 Dec 2018 08:37:50 -0500 Subject: [PATCH 0451/1247] kfdtest: Add gfx900/gfx906 IDs to run_kfdtest.sh Change-Id: Ib6ee418a432d1de79e2306b54d702132de3d06c5 --- tests/kfdtest/scripts/run_kfdtest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 74d617c5a3..07f6c1ad94 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -118,11 +118,11 @@ deviceIdToGpuName() { platformName="pl11" ;; 6980 | 6981 | 6985 | 6986 | 6987 | 6995 | 6997 | 699f) platformName="pl12" ;; - 6860 | 6861 | 6862 | 6863 | 6864 | 6867 | 6868 | 6869 | 686a | 686b | 686c | 687f) + 6860 | 6861 | 6862 | 6863 | 6864 | 6867 | 6868 | 6869 | 686a | 686b | 686c | 687d | 687e | 687f) platformName="vg10" ;; 69a0 | 69a1 | 69a2 | 69a3 | 69af) platformName="vg12" ;; - 66a0 | 66a1 | 66a2 | 66a3 |66a7 | 66af) + 66a0 | 66a1 | 66a2 | 66a3 | 66a4 | 66a7 | 66af) platformName="vg20" ;; 15dd ) platformName="rv" ;; From 8ee93b3187540defd7fda0497e0ee323cf249866 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Tue, 4 Dec 2018 14:06:03 -0500 Subject: [PATCH 0452/1247] libhsakmt: add RAS support v2 RAS feature enabling bit and errors return are implemented in existed topology and event mechanism. v2: change library interface. Change-Id: I75807c080b5b26e8115240b05b3d7016cb05a31a Signed-off-by: Eric Huang --- include/hsakmttypes.h | 11 ++++++++--- include/linux/kfd_ioctl.h | 6 +++++- src/events.c | 3 +++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 674b965b8d..6a92796b77 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -203,7 +203,10 @@ typedef union unsigned int WaveLaunchTrapOverrideSupported: 1; // Indicates if Wave Launch Trap Override is supported on the node. unsigned int WaveLaunchModeSupported: 1; // Indicates if Wave Launch Mode is supported on the node. unsigned int PreciseMemoryOperationsSupported: 1; // Indicates if Precise Memory Operations are supported on the node. - unsigned int Reserved : 13; + unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active + unsigned int Mem_EDCSupoort: 1; // Indicates if GFX internal DRAM/HBM EDC/ECC functionality is active + unsigned int RASEventNotify: 1; // Indicates if GFX extended RASFeatures and RAS EventNotify status is available + unsigned int Reserved : 10; } ui32; } HSA_CAPABILITY; @@ -902,9 +905,11 @@ typedef struct _HsaAccessAttributeFailure unsigned int ReadOnly : 1; // Write access to a read-only page unsigned int NoExecute : 1; // Execute access to a page marked NX unsigned int GpuAccess : 1; // Host access only - unsigned int ECC : 1; // ECC failure (if supported by HW) + unsigned int ECC : 1; // RAS ECC failure (notification of DRAM ECC - non-recoverable - error, if supported by HW) unsigned int Imprecise : 1; // Can't determine the exact fault address - unsigned int Reserved : 26; // must be 0 + unsigned int ErrorType : 3; // Indicates RAS errors or other errors causing the access to GPU to fail + // 0 = no RAS error, 1 = ECC_SRAM, 2 = Link_SYNFLOOD (poison), 3 = GPU hang (not attributable to a specific cause), other values reserved + unsigned int Reserved : 23; // must be 0 } HsaAccessAttributeFailure; // data associated with HSA_EVENTID_MEMORY diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 780d6928e1..609504954d 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -285,7 +285,11 @@ struct kfd_hsa_memory_exception_data { struct kfd_memory_exception_failure failure; __u64 va; __u32 gpu_id; - __u32 pad; + __u32 ErrorType; /* 0 = no RAS error, + 1 = ECC_SRAM, + 2 = Link_SYNFLOOD (poison), + 3 = GPU hang (not attributable to a specific cause), + other values reserved */ }; /* hw exception data */ diff --git a/src/events.c b/src/events.c index 43d99b8da8..3d13341688 100644 --- a/src/events.c +++ b/src/events.c @@ -323,6 +323,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], Events[i]->EventData.EventData.MemoryAccessFault.Failure.ReadOnly = event_data[i].memory_exception_data.failure.ReadOnly; Events[i]->EventData.EventData.MemoryAccessFault.Failure.NoExecute = event_data[i].memory_exception_data.failure.NoExecute; Events[i]->EventData.EventData.MemoryAccessFault.Failure.Imprecise = event_data[i].memory_exception_data.failure.imprecise; + Events[i]->EventData.EventData.MemoryAccessFault.Failure.ErrorType = event_data[i].memory_exception_data.ErrorType; + Events[i]->EventData.EventData.MemoryAccessFault.Failure.ECC = + ((event_data[i].memory_exception_data.ErrorType == 1) || (event_data[i].memory_exception_data.ErrorType == 2)) ? 1 : 0; Events[i]->EventData.EventData.MemoryAccessFault.Flags = HSA_EVENTID_MEMORY_FATAL_PROCESS; analysis_memory_exception(&event_data[i].memory_exception_data); } From 81b8815e1a0a153454b9031deffcb82aac81a0ef Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 13 Dec 2018 17:52:46 -0500 Subject: [PATCH 0453/1247] Add -fPIC flag when building sp3 library This will support the sp3 library built on one gcc version to be compatible with another gcc version. Change-Id: If67714bd63376dc781c56ed025be335fe54b2ba5 Signed-off-by: Yong Zhao --- tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt index bb822e7beb..54a6f3f4bd 100644 --- a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt +++ b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt @@ -68,7 +68,7 @@ message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) # message(STATUS "${file}") #endforeach() -set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU" ) +set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU -fPIC" ) add_library(amdsp3 ${SRC_FILES}) From 666f90440a48927714bf06c159552035bce468ea Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 21 Dec 2018 07:29:31 -0500 Subject: [PATCH 0454/1247] Add lib requirement in CMake file Adding it to the DEBIAN/control won't work, since we use CMake to build it. Add all required packages to the CMakeLists file Change-Id: Iaf62f42e0f998d66038338fb2cf793d29c790205 --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 85de508202..f30f5b7a97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,9 +155,10 @@ set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm" ) -set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libpci3" ) +set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libnuma1, libpci3, zlib1g, libudev1" ) # RPM package specific variables +set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl-libs, pciutils-libs, libgcc" ) set ( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) From b2e026fce36f8a91b22ad274f576ef5b2209cfa6 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 20 Dec 2018 13:56:01 -0500 Subject: [PATCH 0455/1247] kfdtest: increase KFDPerformanceTest.P2PBandWidthTest timeout value KFDPerformanceTest.P2PBandWidthTest[push, push] takes about 3 seconds on 4 gfx906, the default g_TestTimeout 2 seconds is not enough to wait for sDMA queue rptr is consumed. Use kfdtest command line option --timeout=6000, the test is finished and result is reasonable twice as P2PBandWidthTest[push, none]. Change P2PBandWidthTest wait timeout to 6 seconds. Add timeout argument to function WaitOnValue, BaseQueue.Wait4PacketConsumption SDMAQueue.Wait4PacketConsumption, PM4Queue.Wait4PacketConsumption with default value is g_TestTimeOut. Change-Id: I0aa04d644339feaeea695e41647ae66568beab9e Signed-off-by: Philip Yang --- tests/kfdtest/src/BaseQueue.cpp | 4 ++-- tests/kfdtest/src/BaseQueue.hpp | 2 +- tests/kfdtest/src/KFDTestUtil.cpp | 4 +--- tests/kfdtest/src/KFDTestUtil.hpp | 2 +- tests/kfdtest/src/KFDTestUtilQueue.cpp | 5 ++++- tests/kfdtest/src/PM4Queue.cpp | 4 ++-- tests/kfdtest/src/PM4Queue.hpp | 2 +- tests/kfdtest/src/SDMAQueue.cpp | 4 ++-- tests/kfdtest/src/SDMAQueue.hpp | 2 +- 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 5001813f72..46e543ab9b 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -119,9 +119,9 @@ void BaseQueue::PlaceAndSubmitPacket(const BasePacket &packet) { SubmitPacket(); } -void BaseQueue::Wait4PacketConsumption(HsaEvent *event) { +void BaseQueue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { ASSERT_TRUE(!event) << "Not supported!" << std::endl; - ASSERT_TRUE(WaitOnValue(m_Resources.Queue_read_ptr, RptrWhenConsumed())); + ASSERT_TRUE(WaitOnValue(m_Resources.Queue_read_ptr, RptrWhenConsumed(), timeOut)); } bool BaseQueue::AllPacketsSubmitted() { diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index 23df1d1ad6..c0dd1eb840 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -64,7 +64,7 @@ class BaseQueue { /** Wait for all the packets submitted to the queue to be consumed. (i.e. wait until RPTR=WPTR). * Note that all packets being consumed is not the same as all packets being processed. */ - virtual void Wait4PacketConsumption(HsaEvent *event = NULL); + virtual void Wait4PacketConsumption(HsaEvent *event = NULL, unsigned int timeOut = g_TestTimeOut); /** @brief Place packet and submit it in one function */ virtual void PlaceAndSubmitPacket(const BasePacket &packet); diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index b316909139..af9b3b3056 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -44,9 +44,7 @@ uint64_t RoundToPowerOf2(uint64_t val) { return val; } -bool WaitOnValue(const volatile unsigned int *buf, unsigned int value) { - unsigned int timeOut = g_TestTimeOut; - +bool WaitOnValue(const volatile unsigned int *buf, unsigned int value, unsigned int timeOut) { while (timeOut > 0 && *buf != value) { Delay(1); diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index fb4937e7dc..b65dd96531 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -38,7 +38,7 @@ class BaseQueue; uint64_t RoundToPowerOf2(uint64_t val); // @brief: waits until the value is written to the buffer or until time out if received through args -bool WaitOnValue(const volatile unsigned int *buf, unsigned int value); +bool WaitOnValue(const volatile unsigned int *buf, unsigned int value, unsigned int timeOut = g_TestTimeOut); void SplitU64(const HSAuint64 value, unsigned int& rLoPart, unsigned int& rHiPart); diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/tests/kfdtest/src/KFDTestUtilQueue.cpp index 46c88cf702..8369e5a9f8 100644 --- a/tests/kfdtest/src/KFDTestUtilQueue.cpp +++ b/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -61,7 +61,10 @@ class AsyncMPSQ { /* Return only when all packets are consumed. * If there is any packet issues some IO operations, wait these IO to complete too. */ - void Wait(void) { ASSERT_NE((HSAuint64)m_queue, NULL); m_queue->Wait4PacketConsumption(m_event); } + void Wait(void) { + ASSERT_NE((HSAuint64)m_queue, NULL); + m_queue->Wait4PacketConsumption(m_event, std::max((unsigned int)6000, g_TestTimeOut)); + } /* Report the time used between packet [begin, end) in Global Counter on success. * Return 0 on failure. diff --git a/tests/kfdtest/src/PM4Queue.cpp b/tests/kfdtest/src/PM4Queue.cpp index 99707fb9ad..74c1035d01 100644 --- a/tests/kfdtest/src/PM4Queue.cpp +++ b/tests/kfdtest/src/PM4Queue.cpp @@ -71,14 +71,14 @@ void PM4Queue::SubmitPacket() { } } -void PM4Queue::Wait4PacketConsumption(HsaEvent *event) { +void PM4Queue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { if (event) { PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(0, event->EventData.HWData2, event->EventId, true)); - EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, timeOut)); } else { BaseQueue::Wait4PacketConsumption(); } diff --git a/tests/kfdtest/src/PM4Queue.hpp b/tests/kfdtest/src/PM4Queue.hpp index 64e8f4ec96..340356ee47 100644 --- a/tests/kfdtest/src/PM4Queue.hpp +++ b/tests/kfdtest/src/PM4Queue.hpp @@ -48,7 +48,7 @@ class PM4Queue : public BaseQueue { * 1) Less CPU usage (process can sleep, waiting for interrupt). * 2) Lower latency (GPU only updates RPTR in memory periodically). */ - virtual void Wait4PacketConsumption(HsaEvent *event = NULL); + virtual void Wait4PacketConsumption(HsaEvent *event = NULL, unsigned int timeOut = g_TestTimeOut); protected: virtual PACKETTYPE PacketTypeSupported() { return PACKETTYPE_PM4; } diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/tests/kfdtest/src/SDMAQueue.cpp index 047b9eae74..facbeee60d 100644 --- a/tests/kfdtest/src/SDMAQueue.cpp +++ b/tests/kfdtest/src/SDMAQueue.cpp @@ -80,13 +80,13 @@ void SDMAQueue::SubmitPacket() { } } -void SDMAQueue::Wait4PacketConsumption(HsaEvent *event) { +void SDMAQueue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { if (event) { PlacePacket(SDMAFencePacket((void*)event->EventData.HWData2, event->EventId)); PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); - EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, timeOut)); } else { BaseQueue::Wait4PacketConsumption(); } diff --git a/tests/kfdtest/src/SDMAQueue.hpp b/tests/kfdtest/src/SDMAQueue.hpp index 75a0f2a250..0a34fd56f4 100644 --- a/tests/kfdtest/src/SDMAQueue.hpp +++ b/tests/kfdtest/src/SDMAQueue.hpp @@ -41,7 +41,7 @@ class SDMAQueue : public BaseQueue { * 1) Less CPU usage (process can sleep, waiting for interrupt). * 2) Lower latency (GPU only updates RPTR in memory periodically). */ - virtual void Wait4PacketConsumption(HsaEvent *event = NULL); + virtual void Wait4PacketConsumption(HsaEvent *event = NULL, unsigned int timeOut = g_TestTimeOut); protected: // @ return Write pointer modulo queue size in dwords From 37858f23114faaa79e63831c67fa09d3b42dc87c Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Tue, 6 Nov 2018 08:06:40 -0500 Subject: [PATCH 0456/1247] Initial gfx9 debugger node suspend/resume Change-Id: I2a5dac3d02265c11f5b6985ab457e2d1caa0a033 Signed-off-by: Philip Cox --- include/hsakmt.h | 24 +++++++-------- include/hsakmttypes.h | 13 ++++++++ include/linux/kfd_ioctl.h | 30 +++++++++++++++++++ src/debug.c | 63 +++++++++++++++++++++++++++++++++++---- src/libhsakmt.ver | 2 ++ 5 files changed, 113 insertions(+), 19 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index c05e34d114..2d987d91a9 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -633,14 +633,11 @@ hsaKmtDbgAddressWatch( - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. */ HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueueSuspend( - HSAuint32 NumQueues, //IN - HSA_QUEUEID QueueId[], //IN - bool NoGracePeriod, //IN - bool MemFence //IN - ); - + HSAKMTAPI + hsaKmtNodeSuspend( + HSAuint32 Pid, + HSAuint32 NodeId, + HSAuint32 Flags); /** Resume the execution of a set of queues. If a queue is not suspended by hsaKmtDbgQueueSuspend() then it remains executing. @@ -658,12 +655,11 @@ hsaKmtQueueSuspend( - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. */ HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueueResume( - HSAuint32 NumQueues, //IN - HSA_QUEUEID QueueId[], //IN - bool MemFence //IN - ); + HSAKMTAPI + hsaKmtNodeResume( + HSAuint32 Pid, + HSAuint32 NodeId, + HSAuint32 Flags); /** Enable debug trap for NodeId. If QueueId is INVALID_QUEUEID then diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 6a92796b77..cbf3c1f579 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -75,6 +75,12 @@ typedef HSAuint64 HSA_QUEUEID; // An HSA_QUEUEID that is never a valid queue ID. #define INVALID_QUEUEID 0xFFFFFFFFFFFFFFFFULL +// A PID that is never a valid process ID. +#define INVALID_PID 0xFFFFFFFF + +// // A HSA_NODEID that is never a valid node ID. +#define INVALID_NODEID 0xFFFFFFFF + // This is included in order to force the alignments to be 4 bytes so that // it avoids extra padding added by the compiler when a 64-bit binary is generated. #pragma pack(push, hsakmttypes_h, 4) @@ -785,6 +791,13 @@ typedef enum _HSA_DBG_WAVE_LAUNCH_MODE HSA_DBG_WAVE_LAUNCH_MODE_NUM } HSA_DBG_WAVE_LAUNCH_MODE; +typedef enum HSA_DBG_NODE_CONTROL { + HSA_DBG_NODE_CONTROL_NO_GRACE_PERIOD = 0x01, + HSA_DBG_NODE_CONTROL_MEMORY_FENCE = 0x02, + HSA_DBG_NODE_CONTROL_UPDATE_CONTEXT = 0x04, +} HSA_DBG_NODE_CONTROL; + + //This structure is hardware specific and may change in the future typedef struct _HsaDbgWaveMsgAMDGen2 { diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 609504954d..1fc43aefd3 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -193,32 +193,62 @@ struct kfd_ioctl_dbg_wave_control_args { /* KFD_IOC_DBG_TRAP_ENABLE: * data1: 0=disable, 1=enable * data2: queue ID (for future use) + * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_ENABLE 0 /* KFD_IOC_DBG_TRAP_SET_TRAP_DATA: * data1: SPI_GDBG_TRAP_DATA0 * data2: SPI_GDBG_TRAP_DATA1 + * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_TRAP_DATA 1 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE: * data1: override mode: 0=OR, 1=REPLACE * data2: mask + * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 2 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE: * data1: 0=normal, 1=halt, 2=kill, 3=singlestep, 4=disable * data2: unused + * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 3 + +#define KFD__DBG_NODE_SUSPEND_NO_GRACE 0x01 +#define KFD__DBG_NODE_SUSPEND_MEMORY_FENCE 0x02 +#define KFD__DBG_NODE_SUSPEND_UPDATE_CONTEXT 0x04 +/* KFD_IOC_DBG_TRAP_NODE_SUSPEND: + * data1: pid + * data2: nodeid + * data3: flags no_grace=0x01 memory_fence=0x02 update_context=0x04 + * data4: unused + */ +#define KFD_IOC_DBG_TRAP_NODE_SUSPEND 4 + +/* KFD_IOC_DBG_TRAP_NODE_RESUME: + * data1: pid + * data2: nodeid + * data3: flags no_grace=0x01 memory_fence=0x02 update_context=0x04 + * data4: unused + */ +#define KFD_IOC_DBG_TRAP_NODE_RESUME 5 + struct kfd_ioctl_dbg_trap_args { __u32 gpu_id; /* to KFD */ __u32 op; /* to KFD */ __u32 data1; /* to KFD */ __u32 data2; /* to KFD */ + __u32 data3; /* to KFD */ + __u32 data4; /* to KFD */ }; /* Matching HSA_EVENTTYPE */ diff --git a/src/debug.c b/src/debug.c index fc32b925d8..521aaaaf14 100644 --- a/src/debug.c +++ b/src/debug.c @@ -27,6 +27,7 @@ #include "linux/kfd_ioctl.h" #include #include +#include static bool *is_device_debugged; int debug_get_reg_status(uint32_t node_id, bool *is_debugged); @@ -271,7 +272,8 @@ int debug_get_reg_status(uint32_t node_id, bool *is_debugged) static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, HSAuint32 op, HSAuint32 data1, - HSAuint32 data2) + HSAuint32 data2, + HSAuint32 data3) { uint32_t gpu_id; HSAKMT_STATUS result; @@ -296,6 +298,8 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, args.op = op; args.data1 = data1; args.data2 = data2; + args.data3 = data3; + args.data4 = 0; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); @@ -313,12 +317,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32 NodeId, if (QueueId != INVALID_QUEUEID) return HSAKMT_STATUS_NOT_SUPPORTED; - return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 1, QueueId); + return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 1, QueueId, 0); } HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId) { - return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 0, 0); + return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 0, 0, 0); } HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, @@ -328,7 +332,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, return debug_trap(NodeId, KFD_IOC_DBG_TRAP_SET_TRAP_DATA, TrapData0, - TrapData1); + TrapData1, + 0); } HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( @@ -342,7 +347,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( return debug_trap(NodeId, KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE, TrapOverride, - TrapMask); + TrapMask, + 0); } HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode( @@ -352,5 +358,52 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode( return debug_trap(NodeId, KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE, WaveLaunchMode, + 0, 0); } + +HSAKMT_STATUS HSAKMTAPI hsaKmtNodeSuspend( + HSAuint32 Pid, + HSAuint32 NodeId, + HSAuint32 Flags) +{ + pid_t current_pid = INVALID_PID; + + CHECK_KFD_OPEN(); + + if (Pid == INVALID_PID) { + current_pid = getpid(); + if (current_pid == 0) + return HSAKMT_STATUS_INVALID_HANDLE; + Pid = (HSAuint32) current_pid; + } + + return debug_trap(NodeId, + KFD_IOC_DBG_TRAP_NODE_SUSPEND, + Pid, + NodeId, + Flags); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtNodeResume( + HSAuint32 Pid, + HSAuint32 NodeId, + HSAuint32 Flags) +{ + pid_t current_pid = INVALID_PID; + + CHECK_KFD_OPEN(); + + if (Pid == INVALID_PID) { + current_pid = getpid(); + if (current_pid == 0) + return HSAKMT_STATUS_INVALID_HANDLE; + Pid = (HSAuint32) current_pid; + } + + return debug_trap(NodeId, + KFD_IOC_DBG_TRAP_NODE_RESUME, + Pid, + NodeId, + Flags); +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 91b7721764..10a66a1dca 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -61,6 +61,8 @@ hsaKmtDisableDebugTrap; hsaKmtSetDebugTrapData2; hsaKmtSetWaveLaunchTrapOverride; hsaKmtSetWaveLaunchMode; +hsaKmtNodeSuspend; +hsaKmtNodeResume; local: *; }; From 742fa5d871b805b7372b6fd210804d987604b8cf Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 21 Jan 2019 10:31:51 -0600 Subject: [PATCH 0457/1247] Revert "Add test to allocate SDMA queue on specific engine" This reverts commit af5b320c478e60b134636bf8c7bfcf932451329f. Change-Id: I262d91afc60ba2618bf4a857f162ea5236d54131 Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDQMTest.cpp | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index c6922bc014..53471dedd9 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -105,35 +105,6 @@ TEST_F(KFDQMTest, CreateSdmaQueue) { TEST_END } -TEST_F(KFDQMTest, CreateSdmaQueueOnEngine) { - TEST_START(TESTPROFILE_RUNALL) - - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - unsigned int numSdmaEngine; - - if (m_FamilyId == FAMILY_RV) - numSdmaEngine = 1; - else - numSdmaEngine = 2; - - HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); - - SDMAQueue queue; - - //How to get number of sdma engine? - for (int i = 0; i < numSdmaEngine; i++) { - destBuf.Fill(0xFF); - ASSERT_SUCCESS(queue.Create(defaultGPUNode, BaseQueue::DEFAULT_QUEUE_SIZE, NULL, i)); - queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0x02020202)); - queue.Wait4PacketConsumption(); - ASSERT_TRUE(WaitOnValue(destBuf.As(), 0x02020202)); - ASSERT_SUCCESS(queue.Destroy()); - } - - TEST_END -} - TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { TEST_START(TESTPROFILE_RUNALL) From 1923d2e335cfc75b320e68905c1839f1dc4349e7 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 21 Jan 2019 10:32:43 -0600 Subject: [PATCH 0458/1247] Revert "Create SDMA queue on specific engine" This reverts commit acb80d7583fd3312235248ad8444869700ac6e4b. Change-Id: Ia3e9db5fcba1fef80745c72c78b7c568b5c7315e Signed-off-by: Oak Zeng --- tests/kfdtest/src/BaseQueue.cpp | 6 +----- tests/kfdtest/src/BaseQueue.hpp | 3 +-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 46e543ab9b..4aa9e843d2 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -36,7 +36,7 @@ BaseQueue::~BaseQueue(void) { Destroy(); } -HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint64 *pointers, int sdmaEngineId) { +HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint64 *pointers) { HSAKMT_STATUS status; HSA_QUEUE_TYPE type = GetQueueType(); @@ -54,10 +54,6 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 m_Resources.Queue_write_ptr_aql = &pointers[1]; } - //TODO: How to get number of SDMA engine? - if (type == HSA_QUEUE_SDMA && sdmaEngineId >= 0 && sdmaEngineId < 8) - type = (HSA_QUEUE_TYPE) (HSA_QUEUE_SDMA_ENGINE0 + sdmaEngineId); - status = hsaKmtCreateQueue(NodeId, type, DEFAULT_QUEUE_PERCENTAGE, diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index c0dd1eb840..0e5b946290 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -43,10 +43,9 @@ class BaseQueue { /** Create the queue. * @see hsaKmtCreateQueue * @param pointers is used only for creating AQL queues. Otherwise it is omitted. - * @param sdmaEngineId is used only for creating SDMA queues. -1 means let kfd kernel select a sdma engine */ virtual HSAKMT_STATUS Create(unsigned int NodeId, unsigned int size = DEFAULT_QUEUE_SIZE, - HSAuint64 *pointers = NULL, int sdmaEngineId = -1); + HSAuint64 *pointers = NULL); /** Update the queue. * @see hsaKmtUpdateQueue * @param percent New queue percentage From 124f77775c1dac7b53ff333570c61ae224d654b4 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 21 Jan 2019 10:33:00 -0600 Subject: [PATCH 0459/1247] Revert "Create SDMA queue on specific engine" This reverts commit 58b95e0a9d81f61f9c63aafb7c7ef5b87b4e9fb5. Change-Id: Idc0decd86364ed3441e9037b83be8be9953f0b3e Signed-off-by: Oak Zeng --- src/queues.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/queues.c b/src/queues.c index 58f0ac0788..28281f5731 100644 --- a/src/queues.c +++ b/src/queues.c @@ -576,16 +576,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; - case HSA_QUEUE_SDMA_ENGINE0: - case HSA_QUEUE_SDMA_ENGINE1: - case HSA_QUEUE_SDMA_ENGINE2: - case HSA_QUEUE_SDMA_ENGINE3: - case HSA_QUEUE_SDMA_ENGINE4: - case HSA_QUEUE_SDMA_ENGINE5: - case HSA_QUEUE_SDMA_ENGINE6: - case HSA_QUEUE_SDMA_ENGINE7: - args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA_ENGINE(Type - HSA_QUEUE_SDMA_ENGINE0); - break; default: return HSAKMT_STATUS_INVALID_PARAMETER; } From dd6c6e7bc6900203e72e0e847ebafadc61ef5c90 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 21 Jan 2019 10:33:19 -0600 Subject: [PATCH 0460/1247] Revert "Add more SDMA queue type" This reverts commit 5173e7181029a809131262c3ef7fcd9e3275ea3a. Change-Id: I0a52a44a5d141b398d0898bea52e4dcf41dc950f Signed-off-by: Oak Zeng --- include/hsakmttypes.h | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index cbf3c1f579..210f4fc523 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -631,26 +631,6 @@ typedef enum _HSA_QUEUE_TYPE HSA_QUEUE_COMPUTE_AQL = 21, // HSA AQL packet compatible Compute Queue HSA_QUEUE_DMA_AQL = 22, // HSA AQL packet compatible DMA Queue - HSA_QUEUE_SDMA_ENGINE0 =0x10000, //SDMA queue created on SDMA engine 0 - HSA_QUEUE_SDMA_ENGINE1 =0x10001, //SDMA queue created on SDMA engine 1 - HSA_QUEUE_SDMA_ENGINE2 =0x10002, //SDMA queue created on SDMA engine 2 - HSA_QUEUE_SDMA_ENGINE3 =0x10003, //SDMA queue created on SDMA engine 3 - HSA_QUEUE_SDMA_ENGINE4 =0x10004, //SDMA queue created on SDMA engine 4 - HSA_QUEUE_SDMA_ENGINE5 =0x10005, //SDMA queue created on SDMA engine 5 - HSA_QUEUE_SDMA_ENGINE6 =0x10006, //SDMA queue created on SDMA engine 6 - HSA_QUEUE_SDMA_ENGINE7 =0x10007, //SDMA queue created on SDMA engine 7 - HSA_QUEUE_SDMA_ENGINE_MAX =0x1FFFF, - - - HSA_QUEUE_SDMA_AQL_ENGINE0 =0x20000, //SDMA AQL queue created on SDMA engine 0 - HSA_QUEUE_SDMA_AQL_ENGINE1 =0x20001, //SDMA AQL queue created on SDMA engine 1 - HSA_QUEUE_SDMA_AQL_ENGINE2 =0x20002, //SDMA AQL queue created on SDMA engine 2 - HSA_QUEUE_SDMA_AQL_ENGINE3 =0x20003, //SDMA AQL queue created on SDMA engine 3 - HSA_QUEUE_SDMA_AQL_ENGINE4 =0x20004, //SDMA AQL queue created on SDMA engine 4 - HSA_QUEUE_SDMA_AQL_ENGINE5 =0x20005, //SDMA AQL queue created on SDMA engine 5 - HSA_QUEUE_SDMA_AQL_ENGINE6 =0x20006, //SDMA AQL queue created on SDMA engine 6 - HSA_QUEUE_SDMA_AQL_ENGINE7 =0x20007, //SDMA AQL queue created on SDMA engine 7 - HSA_QUEUE_SDMA_AQL_ENGINE_MAX =0x2FFFF, // more types in the future HSA_QUEUE_TYPE_SIZE = 0xFFFFFFFF //aligns to 32bit enum From faba8950d4673c7ee64ef8f203e5839dfc57a3ca Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 22 Jan 2019 14:49:05 -0600 Subject: [PATCH 0461/1247] Delete a few SDMA queue types The design changed. Those are not needed any more Change-Id: Ibb1230d1c34d6ac5153275f9334af45c73805f37 Signed-off-by: Oak Zeng --- include/linux/kfd_ioctl.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 1fc43aefd3..6d7d14aae8 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -38,10 +38,6 @@ struct kfd_ioctl_get_version_args { #define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0 #define KFD_IOC_QUEUE_TYPE_SDMA 0x1 #define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 0x2 -#define KFD_IOC_QUEUE_TYPE_SDMA_ENGINE(e) (0x10000 + (e)) -#define KFD_IOC_QUEUE_TYPE_SDMA_AQL_ENGINE(e) (0x20000 + (e)) -#define KFD_IOC_QUEUE_TYPE_SDMA_ENGINE_MAX (KFD_IOC_QUEUE_TYPE_SDMA_ENGINE(0xffff)) -#define KFD_IOC_QUEUE_TYPE_SDMA_AQL_ENGINE_MAX (KFD_IOC_QUEUE_TYPE_SDMA_AQL_ENGINE(0xffff)) #define KFD_MAX_QUEUE_PERCENTAGE 100 #define KFD_MAX_QUEUE_PRIORITY 15 From b7649919826dd0ad3710f0076852c0ce27dc3f6e Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Fri, 11 Jan 2019 11:25:00 -0600 Subject: [PATCH 0462/1247] Set EOP buffer TC policy to non-coherent Restores regression in dispatch latency. Change-Id: I17869d3d515d8c1fa055a57afec2531903b88b16 Signed-off-by: Jay Cornwall --- src/queues.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/queues.c b/src/queues.c index 28281f5731..d3fd0602ab 100644 --- a/src/queues.c +++ b/src/queues.c @@ -388,6 +388,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, flags.ui32.ExecuteAccess = 1; flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; + flags.ui32.CoarseGrain = DeviceLocal; size = ALIGN_UP(size, align); From 41bf449e9900522f4d2f6fcbd3bfec320d05f722 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Mon, 28 Jan 2019 23:46:38 +0800 Subject: [PATCH 0463/1247] thunk: fix size overflow Some test case alloc >4gb memory. Use HSAuint64 in bytes and HSAuint32 in pages. Change-Id: I0d5e6c299903b5898cfea024178a7a26b9ba3c90 Signed-off-by: xinhui pan --- src/fmm.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 28f59c4427..0db7759b84 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3077,6 +3077,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo struct kfd_ioctl_free_memory_of_gpu_args freeArgs = {0}; const HsaSharedMemoryStruct *SharedMemoryStruct = to_const_hsa_shared_memory_struct(SharedMemoryHandle); + HSAuint64 SizeInPages = SharedMemoryStruct->SizeInPages; if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -3089,7 +3090,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo pthread_mutex_lock(&aperture->fmm_mutex); reservedMem = aperture_allocate_area(aperture, NULL, - (SharedMemoryStruct->SizeInPages << PAGE_SHIFT)); + (SizeInPages << PAGE_SHIFT)); pthread_mutex_unlock(&aperture->fmm_mutex); if (!reservedMem) { err = HSAKMT_STATUS_NO_MEMORY; @@ -3105,7 +3106,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo pthread_mutex_lock(&aperture->fmm_mutex); obj = aperture_allocate_object(aperture, reservedMem, importArgs.handle, - (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), + (SizeInPages << PAGE_SHIFT), 0); if (!obj) { err = HSAKMT_STATUS_NO_MEMORY; @@ -3124,7 +3125,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo } map_fd = importArgs.mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_mem[gpu_mem_id].drm_render_fd; - ret = mmap(reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT), + ret = mmap(reservedMem, (SizeInPages << PAGE_SHIFT), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, importArgs.mmap_offset); if (ret == MAP_FAILED) { @@ -3134,7 +3135,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo } *MemoryAddress = reservedMem; - *SizeInBytes = (SharedMemoryStruct->SizeInPages << PAGE_SHIFT); + *SizeInBytes = (SizeInPages << PAGE_SHIFT); if (gpu_id_array_size > 0) { obj->registered_device_id_array = gpu_id_array; @@ -3147,7 +3148,7 @@ err_free_obj: pthread_mutex_lock(&aperture->fmm_mutex); vm_remove_object(aperture, obj); err_free_mem: - aperture_release_area(aperture, reservedMem, (SharedMemoryStruct->SizeInPages << PAGE_SHIFT)); + aperture_release_area(aperture, reservedMem, (SizeInPages << PAGE_SHIFT)); pthread_mutex_unlock(&aperture->fmm_mutex); err_free_buffer: freeArgs.handle = importArgs.handle; From 51ee5c324a33c3dbb616ede046fedf683719a6f6 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 15 Oct 2018 18:29:08 -0400 Subject: [PATCH 0464/1247] libhsakmt: Introduce HSA_ZFB environment variable This variable is 0 by default. When set to 1, it means there is no frame buffer, so all memory allocation is routed to system memory. This mode is mainly used during emulation. Use CoarseGrain for VRAM under ZFB mode Change-Id: I29e8e98be56935e3ceb94782d70771cc45700749 Signed-off-by: Yong Zhao --- src/memory.c | 9 ++++++++- src/openclose.c | 18 +++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/memory.c b/src/memory.c index b9a01c3f0f..2bd67323d8 100644 --- a/src/memory.c +++ b/src/memory.c @@ -34,6 +34,8 @@ #include #include "fmm.h" +extern int zfb_support; + HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, HSAuint32 DefaultPolicy, HSAuint32 AlternatePolicy, @@ -147,10 +149,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, } /* GPU allocated system memory */ - if (!gpu_id || !MemFlags.ui32.NonPaged) { + if (!gpu_id || !MemFlags.ui32.NonPaged || zfb_support) { /* Backwards compatibility hack: Allocate system memory if app * asks for paged memory from a GPU node. */ + + /* If allocate VRAM under ZFB mode */ + if (zfb_support && gpu_id && MemFlags.ui32.NonPaged == 1) + MemFlags.ui32.CoarseGrain = 1; + *MemoryAddress = fmm_allocate_host(PreferredNode, *MemoryAddress, SizeInBytes, MemFlags); diff --git a/src/openclose.c b/src/openclose.c index 92848f2b91..d4eb264783 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -39,6 +39,9 @@ static const char kfd_device_name[] = "/dev/kfd"; static pid_t parent_pid = -1; int hsakmt_debug_level; +/* zfb is mainly used during emulation */ +int zfb_support; + static bool is_forked_child(void) { pid_t cur_pid = getpid(); @@ -79,14 +82,14 @@ static inline void init_page_size(void) PAGE_SHIFT = ffs(PAGE_SIZE) - 1; } -/* Normally libraries don't print messages. For debugging purpose, we'll - * print messages if an environment variable, HSAKMT_DEBUG_LEVEL, is set. - */ -static void init_debug_level(void) +static void init_vars_from_env(void) { char *envvar; int debug_level; + /* Normally libraries don't print messages. For debugging purpose, we'll + * print messages if an environment variable, HSAKMT_DEBUG_LEVEL, is set. + */ hsakmt_debug_level = HSAKMT_DEBUG_LEVEL_DEFAULT; envvar = getenv("HSAKMT_DEBUG_LEVEL"); @@ -96,6 +99,11 @@ static void init_debug_level(void) debug_level <= HSAKMT_DEBUG_LEVEL_DEBUG) hsakmt_debug_level = debug_level; } + + /* Check whether to support Zero frame buffer */ + envvar = getenv("HSA_ZFB"); + if (envvar) + zfb_support = atoi(envvar); } HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) @@ -114,7 +122,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) clear_after_fork(); if (kfd_open_count == 0) { - init_debug_level(); + init_vars_from_env(); fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); From 6857602cbc9f4f7524f94b827974b5156703e90f Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 9 Jan 2019 19:20:32 -0500 Subject: [PATCH 0465/1247] kfdtest: Include SDMA queue info in class KFDBaseComponentTest This will facilitate us to avoid using family ID to differentiate the SDMA engines and SDMA queues. Change-Id: I8d6203cc5d330e9130a1b2624997c86ba53e8ae4 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 2 ++ tests/kfdtest/src/KFDBaseComponentTest.hpp | 2 ++ tests/kfdtest/src/KFDTestUtil.cpp | 23 ++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 5 ++++- 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 44de55b93e..8d6ca73b7a 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -63,6 +63,8 @@ void KFDBaseComponentTest::SetUp() { g_TestGPUFamilyId = FamilyIdFromNode(nodeProperties); + GetSdmaInfo(nodeProperties, &m_numSdmaEngines, &m_numSdmaQueuesPerEngine); + m_FamilyId = g_TestGPUFamilyId; ROUTINE_END diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 768664e3d6..cfd291b195 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -60,6 +60,8 @@ class KFDBaseComponentTest : public testing::Test { HsaVersionInfo m_VersionInfo; HsaSystemProperties m_SystemProperties; unsigned int m_FamilyId; + unsigned int m_numSdmaEngines; + unsigned int m_numSdmaQueuesPerEngine; HsaMemFlags m_MemoryFlags; HsaNodeInfo m_NodeInfo; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index af9b3b3056..0021af2faf 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -126,6 +126,29 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { return familyId; } +void GetSdmaInfo(const HsaNodeProperties *props, + unsigned int *p_num_sdma_engines, + unsigned int *p_num_sdma_queues_per_engine) { + int num_sdma_engines = 2; + int num_sdma_queues_per_engine = 2; + + switch (props->EngineId.ui32.Major) { + case 9: + if (props->EngineId.ui32.Stepping == 2) // RAVEN + num_sdma_engines = 1; + else if (props->EngineId.ui32.Stepping == 6) // VEGA20 + num_sdma_queues_per_engine = 8; + + break; + } + + if (p_num_sdma_engines) + *p_num_sdma_engines = num_sdma_engines; + + if (p_num_sdma_queues_per_engine) + *p_num_sdma_queues_per_engine = num_sdma_queues_per_engine; +} + bool isTonga(const HsaNodeProperties *props) { /* Tonga has some workarounds in the thunk that cause certain failures */ if (props->EngineId.ui32.Major == 8 && props->EngineId.ui32.Stepping == 2) { diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index b65dd96531..48e2701be0 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -48,8 +48,11 @@ HSAKMT_STATUS CreateQueueTypeEvent(bool ManualReset, bool IsSignaled, unsigned i bool is_dgpu(); bool isTonga(const HsaNodeProperties *props); -unsigned int FamilyIdFromNode(const HsaNodeProperties * props); +unsigned int FamilyIdFromNode(const HsaNodeProperties *props); +void GetSdmaInfo(const HsaNodeProperties *props, + unsigned int *p_num_sdma_engines, + unsigned int *p_num_sdma_queues_per_engine); class HsaMemoryBuffer { public: From 7cd78301826d9b756b4ec4baa1cc754abb17573e Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 9 Jan 2019 19:29:16 -0500 Subject: [PATCH 0466/1247] kfdtest: Use SDMA queue info from class KFDBaseComponentTest Change-Id: Iacdb75006ef5f9ce6c4fbb0525d2c3a8535fdd23 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDQMTest.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 53471dedd9..d4f02b2911 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -112,23 +112,18 @@ TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { int bufSize = PAGE_SIZE; ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - unsigned int MAX_SDMA_QUEUES; - - if (m_FamilyId == FAMILY_RV) - MAX_SDMA_QUEUES = 2; - else - MAX_SDMA_QUEUES = 4; + const unsigned int numSdmaQueues = m_numSdmaEngines * m_numSdmaQueuesPerEngine; HsaMemoryBuffer destBuf(bufSize << 1 , defaultGPUNode, false); HsaMemoryBuffer srcBuf(bufSize, defaultGPUNode, false); destBuf.Fill(0xFF); - SDMAQueue queues[MAX_SDMA_QUEUES]; + std::vector queues(numSdmaQueues); - for (unsigned int qidx = 0; qidx < MAX_SDMA_QUEUES; ++qidx) + for (unsigned int qidx = 0; qidx < numSdmaQueues; ++qidx) ASSERT_SUCCESS(queues[qidx].Create(defaultGPUNode)); - for (unsigned int qidx = 0; qidx < MAX_SDMA_QUEUES; ++qidx) { + for (unsigned int qidx = 0; qidx < numSdmaQueues; ++qidx) { destBuf.Fill(0x0); srcBuf.Fill(qidx + 0xa0); queues[qidx].PlaceAndSubmitPacket( @@ -144,7 +139,7 @@ TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { destBuf.As(), srcBuf.As(), bufSize)); } - for (unsigned int qidx = 0; qidx < MAX_SDMA_QUEUES; ++qidx) + for (unsigned int qidx = 0; qidx < numSdmaQueues; ++qidx) EXPECT_SUCCESS(queues[qidx].Destroy()); TEST_END From 776077fe653b961a2f78bcbaa5bfbbd70358a8a1 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 15 Jan 2019 16:21:31 -0500 Subject: [PATCH 0467/1247] libhsakmt: Use a better name doorbell_mmap_offset The previous name doorbell_offset is used too extensively throughout the code and did not reflect the true usage. Change-Id: I50d33f5c00e82c46cdf4264a78b8f925705bed6a Signed-off-by: Yong Zhao --- src/fmm.c | 4 ++-- src/queues.c | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 0db7759b84..8c2b530649 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1369,7 +1369,7 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB } void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, - uint64_t doorbell_offset) + uint64_t doorbell_mmap_offset) { manageable_aperture_t *aperture; int32_t gpu_mem_id; @@ -1410,7 +1410,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, kfd_fd, - doorbell_offset); + doorbell_mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); return NULL; diff --git a/src/queues.c b/src/queues.c index d3fd0602ab..98869c57ac 100644 --- a/src/queues.c +++ b/src/queues.c @@ -263,12 +263,12 @@ void clear_process_doorbells(void) } static HSAKMT_STATUS map_doorbell_apu(HSAuint32 NodeId, HSAuint32 gpu_id, - HSAuint64 doorbell_offset) + HSAuint64 doorbell_mmap_offset) { void *ptr; ptr = mmap(0, doorbells[NodeId].size, PROT_READ|PROT_WRITE, - MAP_SHARED, kfd_fd, doorbell_offset); + MAP_SHARED, kfd_fd, doorbell_mmap_offset); if (ptr == MAP_FAILED) return HSAKMT_STATUS_ERROR; @@ -279,12 +279,12 @@ static HSAKMT_STATUS map_doorbell_apu(HSAuint32 NodeId, HSAuint32 gpu_id, } static HSAKMT_STATUS map_doorbell_dgpu(HSAuint32 NodeId, HSAuint32 gpu_id, - HSAuint64 doorbell_offset) + HSAuint64 doorbell_mmap_offset) { void *ptr; ptr = fmm_allocate_doorbell(gpu_id, doorbells[NodeId].size, - doorbell_offset); + doorbell_mmap_offset); if (!ptr) return HSAKMT_STATUS_ERROR; @@ -301,7 +301,7 @@ static HSAKMT_STATUS map_doorbell_dgpu(HSAuint32 NodeId, HSAuint32 gpu_id, } static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, - HSAuint64 doorbell_offset) + HSAuint64 doorbell_mmap_offset) { HSAKMT_STATUS status = HSAKMT_STATUS_SUCCESS; @@ -315,16 +315,16 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, &doorbells[NodeId]); if (doorbells[NodeId].use_gpuvm) { - status = map_doorbell_dgpu(NodeId, gpu_id, doorbell_offset); + status = map_doorbell_dgpu(NodeId, gpu_id, doorbell_mmap_offset); if (status != HSAKMT_STATUS_SUCCESS) { /* Fall back to the old method if KFD doesn't * support doorbells in GPUVM */ doorbells[NodeId].use_gpuvm = false; - status = map_doorbell_apu(NodeId, gpu_id, doorbell_offset); + status = map_doorbell_apu(NodeId, gpu_id, doorbell_mmap_offset); } } else - status = map_doorbell_apu(NodeId, gpu_id, doorbell_offset); + status = map_doorbell_apu(NodeId, gpu_id, doorbell_mmap_offset); if (status != HSAKMT_STATUS_SUCCESS) doorbells[NodeId].size = 0; From ac6f089f262b151fc29ef39b3e4c74e5197016f9 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Fri, 11 Jan 2019 08:59:39 -0500 Subject: [PATCH 0468/1247] kfdtest: Initial basic dbg suspend/resume testcase A basic sanity test that tests the codepath for the debugger suspend and resume code path. Change-Id: If4c64f7bd6a1ef45068a33965b829725a78ce492 Signed-off-by: Philip Cox --- tests/kfdtest/src/KFDDBGTest.cpp | 107 +++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 47e9da7ebb..98a0e3db0d 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -22,6 +22,7 @@ */ #include "KFDDBGTest.hpp" +#include #include "KFDQMTest.hpp" #include "PM4Queue.hpp" #include "PM4Packet.hpp" @@ -63,6 +64,32 @@ trap_present(1)\n\ end\n\ "; +static const char* iterate_isa_gfx9 = \ +"\ +shader iterate_isa\n\ +asic(GFX9)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ + flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + v_mov_b32 v5, 0\n\ +LOOP:\n\ + v_add_co_u32 v5, vcc, 1, v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ + v_cmp_lt_u32 vcc, v5, v4\n\ + s_cbranch_vccnz LOOP\n\ + flat_store_dword v[2,3], v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + s_endpgm\n\ + end\n\ +"; + + void KFDDBGTest::SetUp() { ROUTINE_START @@ -173,3 +200,83 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { } TEST_END } + +TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HSAuint32 Flags = HSA_DBG_NODE_CONTROL_NO_GRACE_PERIOD; + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer iterateBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + HsaMemoryBuffer resultBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + + unsigned int* iter = iterateBuf.As(); + unsigned int* result = resultBuf.As(); + + int suspendTimeout = 500; + int syncStatus; + + m_pIsaGen->CompileShader(iterate_isa_gfx9, "iterate_isa", isaBuffer); + + PM4Queue queue1; + + ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); + + Dispatch *dispatch1; + + dispatch1 = new Dispatch(isaBuffer); + + dispatch1->SetArgs(&iter[0], &result[0]); + dispatch1->SetDim(1, 1, 1); + + // Need a loop large enough so we don't finish before we call Suspend. + // 150000000 takes between 5 and 6 seconds, which is long enough + // to test the suspend/resume. + iter[0] = 150000000; + + ASSERT_EQ(ptrace(PTRACE_TRACEME, 0, 0, 0), 0); + ASSERT_SUCCESS(hsaKmtEnableDebugTrap(defaultGPUNode, INVALID_QUEUEID)); + + // Submit the shader, queue1 + dispatch1->Submit(queue1); + + ASSERT_SUCCESS(hsaKmtNodeSuspend(INVALID_PID, + defaultGPUNode, + Flags)); + + syncStatus = dispatch1->SyncWithStatus(suspendTimeout); + ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); + + ASSERT_NE(iter[0], result[0]); + + // The shader hasn't finished, we will wait for 20 seconds, + // and then check if it has finished. If it was suspended, + // it should not have finished. + Delay(20000); + + // Check that the shader has not finished yet. + syncStatus = dispatch1->SyncWithStatus(suspendTimeout); + ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); + + ASSERT_NE(iter[0], result[0]); + + ASSERT_SUCCESS(hsaKmtNodeResume(INVALID_PID, + defaultGPUNode, + Flags)); + + dispatch1->Sync(); + ASSERT_EQ(iter[0], result[0]); + + EXPECT_SUCCESS(queue1.Destroy()); + + ASSERT_SUCCESS(hsaKmtDisableDebugTrap(defaultGPUNode)); + + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; + } + TEST_END +} + From 90a3697e1d23e1f059086e1315831826381ef3d6 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Wed, 13 Feb 2019 10:42:52 -0500 Subject: [PATCH 0469/1247] kfdtest: Blacklist BasicDebuggerSuspendResume We need to black list this testcase temporarily because it is failing intermittently. The failure tends to only happen when the certain build machine is used to build it. This issue is being tracked by Jira ticket: ROCMOPS-389 Change-Id: Ic4682c9da389ed731cbc034dff57e6646bba0e9d Signed-off-by: Philip Cox --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 8060068c8a..9782473563 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -32,7 +32,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDPNPTest.DisableAndCreateQueue:"\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ -"KFDQMTest.SdmaEventInterrupt" +"KFDQMTest.SdmaEventInterrupt:"\ +"KFDDBGTest.BasicDebuggerSuspendResume" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From 7349276860ef5423b9470883beef28b565b4f509 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 13 Feb 2019 14:36:05 -0500 Subject: [PATCH 0470/1247] kfdtest: add RAS tests They are disabled for now. Change-Id: I9c936130cbaf8c773f4b8e94bccf4af1f45eda65 Signed-off-by: Eric Huang --- tests/kfdtest/scripts/kfdtest.exclude | 3 +- tests/kfdtest/src/KFDRASTest.cpp | 149 ++++++++++++++++++++++++++ tests/kfdtest/src/KFDRASTest.hpp | 56 ++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 tests/kfdtest/src/KFDRASTest.cpp create mode 100644 tests/kfdtest/src/KFDRASTest.hpp diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 9782473563..d496c12538 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -11,7 +11,8 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ "-KFDEventTest.MeasureInterruptConsumption:"\ "KFDLocalMemoryTest.Fragmentation:"\ "KFDQMTest.BasicCuMaskingLinear:"\ -"RDMATest.GPUDirect" +"RDMATest.GPUDirect:"\ +"KFDRASTest.*" # This is the temporary blacklist for all ASICs. This is to be used when a test is failing consistently # on every ASIC (Kaveri, Carrizo, Hawaii, Tonga, Fiji, Polaris10, Polaris11 and Vega10 . diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/tests/kfdtest/src/KFDRASTest.cpp new file mode 100644 index 0000000000..1bfdc64c9a --- /dev/null +++ b/tests/kfdtest/src/KFDRASTest.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include + +#include "linux/kfd_ioctl.h" +#include "KFDRASTest.hpp" +#include "PM4Queue.hpp" + +#define AMDGPU_DEBUGFS_NODES "/sys/kernel/debug/dri/" +#define RAS_SDMA_ERR_INJECTION "ras/sdma_err_inject" + +void KFDRASTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + char path[256]; + int renderNode; + uint32_t rasFeatures = 0; + HsaEventDescriptor eventDesc; + + m_pRasEvent = NULL; + m_setupStatus = false; + + m_defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + renderNode = KFDBaseComponentTest::FindDRMRenderNode(m_defaultGPUNode); + if (renderNode < 0) { + LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; + return; + } + + amdgpu_query_info(m_RenderNodes[renderNode].device_handle, + AMDGPU_INFO_RAS_ENABLED_FEATURES, + sizeof(uint32_t), &rasFeatures); + if (!(rasFeatures & + (AMDGPU_INFO_RAS_ENABLED_SDMA || + AMDGPU_INFO_RAS_ENABLED_UMC || + AMDGPU_INFO_RAS_ENABLED_GFX))) { + LOG() << "Skipping test: GPU doesn't support RAS features!" << std::endl; + return; + } + + snprintf(path, sizeof(path), "%s/%d/%s", AMDGPU_DEBUGFS_NODES, renderNode, RAS_SDMA_ERR_INJECTION); + + m_pFile = fopen(path, "w"); + if (!m_pFile) { + LOG() << "Skipping test: RAS error injection requires root access!" << std::endl; + return; + } + + eventDesc.EventType = HSA_EVENTTYPE_MEMORY; + eventDesc.NodeId = m_defaultGPUNode; + eventDesc.SyncVar.SyncVar.UserData = NULL; + eventDesc.SyncVar.SyncVarSize = 0; + + ASSERT_SUCCESS(hsaKmtCreateEvent(&eventDesc, true, false, &m_pRasEvent)); + + m_setupStatus = true; + + ROUTINE_END +} + +void KFDRASTest::TearDown() { + ROUTINE_START + + if (m_pRasEvent != NULL) { + EXPECT_SUCCESS(hsaKmtDestroyEvent(m_pRasEvent)); + } + + fclose(m_pFile); + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDRASTest, BasicTest) { + TEST_START(TESTPROFILE_RUNALL); + + if (!m_setupStatus) { + return; + } + + // write an uncorrectable error injection at address 1 as value 1 + ASSERT_SUCCESS(fwrite("ue 1 1", sizeof(char), 6, m_pFile)); + + EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pRasEvent, g_TestTimeOut)); + + EXPECT_EQ(1, m_pRasEvent->EventData.EventData.MemoryAccessFault.Failure.ErrorType); + + TEST_END; +} + +TEST_F(KFDRASTest, MixEventsTest) { + TEST_START(TESTPROFILE_RUNALL); + + if (!m_setupStatus) { + return; + } + + PM4Queue queue; + HsaEvent* pHsaEvent; + + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, m_defaultGPUNode, &pHsaEvent)); + ASSERT_NE(0, pHsaEvent->EventData.HWData2); + + ASSERT_SUCCESS(queue.Create(m_defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, + pHsaEvent->EventData.HWData2, pHsaEvent->EventId)); + + queue.Wait4PacketConsumption(); + + EXPECT_SUCCESS(hsaKmtWaitOnEvent(pHsaEvent, g_TestTimeOut)); + + ASSERT_SUCCESS(fwrite("ue 1 1", sizeof(char), 6, m_pFile)); + + EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pRasEvent, g_TestTimeOut)); + + EXPECT_EQ(1, m_pRasEvent->EventData.EventData.MemoryAccessFault.Failure.ErrorType); + + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(hsaKmtDestroyEvent(pHsaEvent)); + + TEST_END; +} diff --git a/tests/kfdtest/src/KFDRASTest.hpp b/tests/kfdtest/src/KFDRASTest.hpp new file mode 100644 index 0000000000..cd68e8412a --- /dev/null +++ b/tests/kfdtest/src/KFDRASTest.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDBaseComponentTest.hpp" + +#ifndef __KFD_RAS_TEST__H__ +#define __KFD_RAS_TEST__H__ + +// To be removed when amdgpu_drm.h updated with those definitions +#ifndef AMDGPU_INFO_RAS_ENABLED_FEATURES +#define AMDGPU_INFO_RAS_ENABLED_FEATURES 0x20 + +#define AMDGPU_INFO_RAS_ENABLED_UMC (1 << 0) +#define AMDGPU_INFO_RAS_ENABLED_SDMA (1 << 1) +#define AMDGPU_INFO_RAS_ENABLED_GFX (1 << 2) +#endif + +class KFDRASTest : public KFDBaseComponentTest { + public: + KFDRASTest(void) {} + ~KFDRASTest(void) {} + + // @brief Executed before every test in KFDRASTest. + virtual void SetUp(); + // @brief Executed after every test in KFDRASTest. + virtual void TearDown(); + + protected: + static const unsigned int EVENT_TIMEOUT = 5000; // 5 seconds + HsaEvent* m_pRasEvent; + HSAint32 m_defaultGPUNode; + FILE* m_pFile; + bool m_setupStatus; +}; + +#endif // __KFD_RAS_TEST__H__ From 1d478f3cf256650d54458889690b759955b25802 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 21 Feb 2019 17:42:45 -0500 Subject: [PATCH 0471/1247] kfdtest: Add a result check in CreateCpQueue test With the orginal code, CreateCpQueue will report failure if WaitOnValue return false. Add EXPECT_TRUE() so that in that case the failure is reported. Change-Id: I043d013958b452d7ccb9538dc296d99d024abf01 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDQMTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index d4f02b2911..89291c3b5a 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -73,7 +73,7 @@ TEST_F(KFDQMTest, CreateCpQueue) { queue.Wait4PacketConsumption(); - WaitOnValue(destBuf.As(), 0); + EXPECT_TRUE(WaitOnValue(destBuf.As(), 0)); EXPECT_SUCCESS(queue.Destroy()); From a0b8dd84623ca375dbda1641de06a95902b3029d Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 27 Feb 2019 07:05:42 -0500 Subject: [PATCH 0472/1247] Temporarily remove CMATest from gfx900 Per Philip Yang: For forked child process, userptr allocated on heap (through malloc) will have two vmas if child process malloc smaller size buf, free it, this is on vma cloned from parent process. Then malloc larger size buf, kernel will put some pages on previous freed space from vma cloned, create new vma for the rest of pages. This is what IPCTest does. Change-Id: I054771e20880f975d7cc774225f19aad5363843f --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index d496c12538..a4498ef1af 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -127,6 +127,7 @@ LEXA_TESTS_BLACKLIST=\ # KFDDBG.BasicAddressWatch is not supported on GFXv9 and later. FIXME: it should disable itself VEGA10_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ +"KFDIPCTest.CrossMemoryAttachTest:"\ "KFDDBGTest.BasicAddressWatch" VEGA12_TESTS_BLACKLIST=\ From d00ec779cebd0e422f3000fd5a63c248fd3559c6 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 28 Feb 2019 07:29:47 -0500 Subject: [PATCH 0473/1247] kfdtest.exclude: Temporarily blacklist IPC on gfx900 Due to the recent HMM changes, the KFDIPCTest can intermittently fail, combined with CrossMemoryAttach consistently failing. Remove it for now while Philip Yang investigates Change-Id: Icf272100bb7882eff4202ad6f4ced63b569f4e7d --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index a4498ef1af..493ef684c6 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -125,9 +125,10 @@ LEXA_TESTS_BLACKLIST=\ "$SDMA_BLACKLIST" # KFDDBG.BasicAddressWatch is not supported on GFXv9 and later. FIXME: it should disable itself +# KFDIPCTest is temporarily blacklisted due to recent HMM changes. Philip Yang is investigating VEGA10_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDIPCTest.CrossMemoryAttachTest:"\ +"KFDIPCTest.*:"\ "KFDDBGTest.BasicAddressWatch" VEGA12_TESTS_BLACKLIST=\ From f8028a40fde893ae7cb96b603f2ac6dbc32a94ae Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 1 Mar 2019 16:42:30 -0500 Subject: [PATCH 0474/1247] libhsakmt: Support x2APIC in topology Current processor/cache topology code implements xAPIC architecture, which is 8 bits addressability. This is not enough for a system having more than 255 processors. x2APIC is the extension of xAPIC architecture to support 32 bit addressability of processors. This patch detects the x2APIC enablement and uses the extension leaf to get apicid when detected. Change-Id: I0826585d02f696a46cd5efb9a6630c60af01e2d8 Signed-off-by: Amber Lin --- src/topology.c | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/topology.c b/src/topology.c index c7f8a28214..31f9797d09 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1163,6 +1163,7 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list uint32_t cpuid_op_cache; uint32_t eax, ebx, ecx = 0, edx; /* cpuid registers */ cpu_cacheinfo_t *cpu_ci_list, *this_cpu; + bool x2apic = false; if (!temp_cpu_ci_list) { ret = HSAKMT_STATUS_ERROR; @@ -1217,10 +1218,28 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list goto exit; } - eax = 0x1; + /* Detect the availability of the extended topology leaf */ + eax = 0x0; cpuid(&eax, &ebx, &ecx, &edx); - this_cpu->apicid = (ebx >> 24) & 0xff; - this_cpu->max_num_apicid = (ebx >> 16) & 0x0FF; + if (eax >= 11) { + eax = 0xb; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx) + x2apic = true; + } + + if (x2apic) { + eax = 0xb; + cpuid(&eax, &ebx, &ecx, &edx); + this_cpu->apicid = edx; + cpuid_count(4, 0, &eax, &ebx, &ecx, &edx); + this_cpu->max_num_apicid = (eax >> 26) + 1; + } else { + eax = 0x1; + cpuid(&eax, &ebx, &ecx, &edx); + this_cpu->max_num_apicid = (ebx >> 16) & 0x0FF; + } this_cpu->num_caches = cpuid_find_num_cache_leaves(cpuid_op_cache); this_cpu->num_duplicated_caches = 0; this_cpu->cache_info = calloc( From e4109de26d9706478b6345ab2f5b5dcef044c67c Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 11 Feb 2019 15:41:16 -0600 Subject: [PATCH 0475/1247] Use latest kfd_ioctl.h A new SDMA queue type for XGMI was added Change-Id: Iad065c1a7c053a58e0d86becfb374215e316a611 Signed-off-by: Oak Zeng --- include/linux/kfd_ioctl.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 6d7d14aae8..485be6456a 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -38,6 +38,7 @@ struct kfd_ioctl_get_version_args { #define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0 #define KFD_IOC_QUEUE_TYPE_SDMA 0x1 #define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 0x2 +#define KFD_IOC_QUEUE_TYPE_SDMA_XGMI 0x3 #define KFD_MAX_QUEUE_PERCENTAGE 100 #define KFD_MAX_QUEUE_PRIORITY 15 From 414a3508d6ee7fa301ee1ca9ef0587c7e0c4d146 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Wed, 13 Feb 2019 10:22:32 -0600 Subject: [PATCH 0476/1247] Thunk interface to get SDMA engine info Add SDMA engine info fields to node properties and modify get node properties API to read SDMA engine info from sysfs Change-Id: Iea877b5bc008cc9df9405daf564a359535f1bc9f Signed-off-by: Oak Zeng --- include/hsakmttypes.h | 4 +++- src/topology.c | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 210f4fc523..f74641d181 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -299,7 +299,9 @@ typedef struct _HsaNodeProperties HSA_DEBUG_PROPERTIES DebugProperties; // Debug properties of this node. HSAuint64 HiveID; // XGMI Hive the GPU node belongs to in the system. It is an opaque and static // number hash created by the PSP - HSAuint8 Reserved[44]; + HSAuint32 NumSdmaEngines; // number of PCIe optimized SDMA engines + HSAuint32 NumSdmaXgmiEngines;// number of XGMI optimized SDMA engines + HSAuint8 Reserved[36]; } HsaNodeProperties; diff --git a/src/topology.c b/src/topology.c index 31f9797d09..4e3290c7ea 100644 --- a/src/topology.c +++ b/src/topology.c @@ -990,6 +990,10 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->uCodeEngineVersions.Value = (uint32_t)prop_val & 0x3ff; else if (strcmp(prop_name, "hive_id") == 0) props->HiveID = prop_val; + else if (strcmp(prop_name, "num_sdma_engines") == 0) + props->NumSdmaEngines = prop_val; + else if (strcmp(prop_name, "num_sdma_xgmi_engines") == 0) + props->NumSdmaXgmiEngines = prop_val; } hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); From 1046a1fd7240f421633eff303b7e38da1ef6f0a7 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 11 Feb 2019 15:52:00 -0600 Subject: [PATCH 0477/1247] Introduce XGMI SDMA queue type Change-Id: I8c6ff04f92c2bbea0bab94ddb8cc4cceb5d74d02 Signed-off-by: Oak Zeng --- include/hsakmttypes.h | 4 +++- src/queues.c | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index f74641d181..06d650c5e8 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -617,9 +617,10 @@ typedef enum _HSA_QUEUE_PRIORITY typedef enum _HSA_QUEUE_TYPE { HSA_QUEUE_COMPUTE = 1, // AMD PM4 compatible Compute Queue - HSA_QUEUE_SDMA = 2, // SDMA Queue, used for data transport and format conversion (e.g. (de-)tiling, etc). + HSA_QUEUE_SDMA = 2, // PCIe optimized SDMA Queue, used for data transport and format conversion (e.g. (de-)tiling, etc). HSA_QUEUE_MULTIMEDIA_DECODE = 3, // reserved, for HSA multimedia decode queue HSA_QUEUE_MULTIMEDIA_ENCODE = 4, // reserved, for HSA multimedia encode queue + HSA_QUEUE_SDMA_XGMI = 5, // XGMI optimized SDMA Queue // the following values indicate a queue type permitted to reference OS graphics // resources through the interoperation API. See [5] "HSA Graphics Interoperation @@ -632,6 +633,7 @@ typedef enum _HSA_QUEUE_TYPE HSA_QUEUE_COMPUTE_AQL = 21, // HSA AQL packet compatible Compute Queue HSA_QUEUE_DMA_AQL = 22, // HSA AQL packet compatible DMA Queue + HSA_QUEUE_DMA_AQL_XGMI = 23, // HSA AQL packet compatible XGMI optimized DMA Queue // more types in the future diff --git a/src/queues.c b/src/queues.c index 98869c57ac..aaee01cb83 100644 --- a/src/queues.c +++ b/src/queues.c @@ -465,7 +465,8 @@ static int handle_concrete_asic(struct queue *q, const struct device_info *dev_info = q->dev_info; bool ret; - if (!dev_info || args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) + if (!dev_info || args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA || + args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI) return HSAKMT_STATUS_SUCCESS; if (dev_info->eop_buffer_size > 0) { @@ -574,6 +575,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, case HSA_QUEUE_SDMA: args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA; break; + case HSA_QUEUE_SDMA_XGMI: + args.queue_type = KFD_IOC_QUEUE_TYPE_SDMA_XGMI; + break; case HSA_QUEUE_COMPUTE_AQL: args.queue_type = KFD_IOC_QUEUE_TYPE_COMPUTE_AQL; break; From cfa47ac1f9302588180e9713a5c34e4e9b74444f Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 11 Mar 2019 13:02:56 -0400 Subject: [PATCH 0478/1247] libhsakmt: Fix missing apicid in topology While adding x2APIC support, apicid for non-x2apic was missing out by mistake. Change-Id: I25eed362c035c0e9fb9ea948899c49f70311f269 Signed-off-by: Amber Lin --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index 4e3290c7ea..ec034bcf52 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1242,6 +1242,7 @@ static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list } else { eax = 0x1; cpuid(&eax, &ebx, &ecx, &edx); + this_cpu->apicid = (ebx >> 24) & 0xff; this_cpu->max_num_apicid = (ebx >> 16) & 0x0FF; } this_cpu->num_caches = cpuid_find_num_cache_leaves(cpuid_op_cache); From 0bd9f35563aa895e8c4f731aabe6022843b6cd87 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 18 Mar 2019 09:58:00 -0400 Subject: [PATCH 0479/1247] Revert "kfdtest.exclude: Temporarily blacklist IPC on gfx900" This reverts commit d00ec779cebd0e422f3000fd5a63c248fd3559c6. Fixes for HMM change corner cases are merged in from drm-next. Tests are passed on gfx900 with the latest amd-kfd-staging. Change-Id: I6c00d1eacf6b3f1ce715e085ae622b4e9ff1b7ff Signed-off-by: Philip Yang --- tests/kfdtest/scripts/kfdtest.exclude | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 493ef684c6..d496c12538 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -125,10 +125,8 @@ LEXA_TESTS_BLACKLIST=\ "$SDMA_BLACKLIST" # KFDDBG.BasicAddressWatch is not supported on GFXv9 and later. FIXME: it should disable itself -# KFDIPCTest is temporarily blacklisted due to recent HMM changes. Philip Yang is investigating VEGA10_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDIPCTest.*:"\ "KFDDBGTest.BasicAddressWatch" VEGA12_TESTS_BLACKLIST=\ From c7439a003943e98869d3560c82ddf033fe651714 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 21 Mar 2019 06:11:49 -0400 Subject: [PATCH 0480/1247] libhsakmt: Add another gfx902 GPU ID Change-Id: I967f16bf548171df73d2e721f16c1aac52e99852 Signed-off-by: Kent Russell --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index ec034bcf52..b5997016f0 100644 --- a/src/topology.c +++ b/src/topology.c @@ -201,6 +201,7 @@ static struct hsa_gfxip_table { { 0x69Af, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, /* Raven */ { 0x15DD, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, + { 0x15D8, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, /* Vega20 */ { 0x66A0, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66A1, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, From 1304a92e17a66f71ae993e528f0077f085863dab Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 21 Mar 2019 06:05:08 -0400 Subject: [PATCH 0481/1247] libhsakmt: Add Vega M support While this may not be supported in the runtime, the kernel/firmware support it Change-Id: I7fe4536a6b3055f39e25f453060e899938645d91 Signed-off-by: Kent Russell --- src/libhsakmt.h | 1 + src/pmc_table.c | 1 + src/queues.c | 7 +++++++ src/topology.c | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 0bc971fc0a..a34889c967 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -97,6 +97,7 @@ enum asic_family_type { CHIP_POLARIS10, CHIP_POLARIS11, CHIP_POLARIS12, + CHIP_VEGAM, CHIP_VEGA10, CHIP_VEGA12, CHIP_VEGA20, diff --git a/src/pmc_table.c b/src/pmc_table.c index 046fec91d4..0459d67240 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1743,6 +1743,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, case CHIP_POLARIS10: case CHIP_POLARIS11: case CHIP_POLARIS12: + case CHIP_VEGAM: *block = polaris_blocks[block_id]; break; case CHIP_VEGA10: diff --git a/src/queues.c b/src/queues.c index aaee01cb83..a1af2f39e7 100644 --- a/src/queues.c +++ b/src/queues.c @@ -98,6 +98,12 @@ const struct device_info polaris12_device_info = { .doorbell_size = DOORBELL_SIZE_GFX8, }; +const struct device_info vegam_device_info = { + .asic_family = CHIP_VEGAM, + .eop_buffer_size = TONGA_PAGE_SIZE, + .doorbell_size = DOORBELL_SIZE_GFX8, +}; + const struct device_info vega10_device_info = { .asic_family = CHIP_VEGA10, .eop_buffer_size = 4096, @@ -132,6 +138,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_POLARIS10] = &polaris10_device_info, [CHIP_POLARIS11] = &polaris11_device_info, [CHIP_POLARIS12] = &polaris12_device_info, + [CHIP_VEGAM] = &vegam_device_info, [CHIP_VEGA10] = &vega10_device_info, [CHIP_VEGA12] = &vega12_device_info, [CHIP_VEGA20] = &vega20_device_info, diff --git a/src/topology.c b/src/topology.c index b5997016f0..a730d1b2d7 100644 --- a/src/topology.c +++ b/src/topology.c @@ -178,6 +178,10 @@ static struct hsa_gfxip_table { { 0x6995, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, { 0x6997, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, { 0x699F, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + /* VegaM */ + { 0x694C, 8, 0, 3, 1, "VegaM", CHIP_VEGAM }, + { 0x694E, 8, 0, 3, 1, "VegaM", CHIP_VEGAM }, + { 0x694F, 8, 0, 3, 1, "VegaM", CHIP_VEGAM }, /* Vega10 */ { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, { 0x6861, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, From c7f1277013cb04745ae2a5ff1caa5dfb4556985b Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Tue, 26 Mar 2019 02:22:29 -0500 Subject: [PATCH 0482/1247] Add hsaKmtRegisterMemoryWithFlags. API follows hsaKmtRegisterMemory but allows passing HsaMemFlags. Change-Id: I66a230a87c8b085f27c769bdf2cb4d0d96a5d6dd --- include/hsakmt.h | 14 +++++++++++++- src/fmm.c | 11 +++++++---- src/fmm.h | 3 ++- src/libhsakmt.ver | 1 + src/memory.c | 29 +++++++++++++++++++++++++++-- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 2d987d91a9..031ab1dc57 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -400,6 +400,18 @@ hsaKmtRegisterMemoryToNodes( ); +/** + Registers with KFD a memory buffer with memory attributes +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRegisterMemoryWithFlags( + void *MemoryAddress, // IN (cache-aligned) + HSAuint64 MemorySizeInBytes, // IN (cache-aligned) + HsaMemFlags MemFlags // IN + ); + /** Registers with KFD a graphics buffer and returns graphics metadata */ @@ -515,7 +527,7 @@ hsaKmtMapMemoryToGPUNodes( void* MemoryAddress, //IN (page-aligned) HSAuint64 MemorySizeInBytes, //IN (page-aligned) HSAuint64* AlternateVAGPU, //OUT (page-aligned) - HsaMemMapFlags MemMapFlags, //IN + HsaMemMapFlags MemMapFlags, //IN HSAuint64 NumberOfNodes, //IN HSAuint32* NodeArray //IN ); diff --git a/src/fmm.c b/src/fmm.c index 8c2b530649..079ee55664 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2794,7 +2794,8 @@ bool fmm_get_handle(void *address, uint64_t *handle) return found; } -static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_object_t **obj_ret) +static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, + vm_object_t **obj_ret, bool coarse_grain) { manageable_aperture_t *aperture = svm.dgpu_aperture; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); @@ -2818,7 +2819,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj svm_addr = __fmm_allocate_device(gpu_id, NULL, aligned_size, aperture, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | - KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE, &obj); + KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | + (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; @@ -2841,7 +2843,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, vm_obj HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, - uint32_t gpu_id_array_size) + uint32_t gpu_id_array_size, + bool coarse_grain) { manageable_aperture_t *aperture = NULL; vm_object_t *object = NULL; @@ -2857,7 +2860,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, return HSAKMT_STATUS_SUCCESS; /* Register a new user ptr */ - ret = fmm_register_user_memory(address, size_in_bytes, &object); + ret = fmm_register_user_memory(address, size_in_bytes, &object, coarse_grain); if (ret != HSAKMT_STATUS_SUCCESS) return ret; if (gpu_id_array_size == 0) diff --git a/src/fmm.h b/src/fmm.h index 85ef0ef54d..9709b6e4ce 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -69,7 +69,8 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, - uint32_t gpu_id_array_size); + uint32_t gpu_id_array_size, + bool coarse_grain); HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo *GraphicsResourceInfo, uint32_t *gpu_id_array, diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 10a66a1dca..12bb825f7f 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -26,6 +26,7 @@ hsaKmtAllocMemory; hsaKmtFreeMemory; hsaKmtRegisterMemory; hsaKmtRegisterMemoryToNodes; +hsaKmtRegisterMemoryWithFlags; hsaKmtRegisterGraphicsHandleToNodes; hsaKmtShareMemory; hsaKmtRegisterSharedHandle; diff --git a/src/memory.c b/src/memory.c index 2bd67323d8..63b88e4d5d 100644 --- a/src/memory.c +++ b/src/memory.c @@ -210,7 +210,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress, return HSAKMT_STATUS_SUCCESS; return fmm_register_memory(MemoryAddress, MemorySizeInBytes, - NULL, 0); + NULL, 0, true); } HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, @@ -235,7 +235,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, if (ret == HSAKMT_STATUS_SUCCESS) { ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, gpu_id_array, - NumberOfNodes*sizeof(uint32_t)); + NumberOfNodes*sizeof(uint32_t), + true); if (ret != HSAKMT_STATUS_SUCCESS) free(gpu_id_array); } @@ -243,6 +244,30 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, return ret; } +HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryWithFlags(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + HsaMemFlags MemFlags) +{ + CHECK_KFD_OPEN(); + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + pr_debug("[%s] address %p\n", + __func__, MemoryAddress); + + // Registered memory should be ordinary paged host memory. + if ((MemFlags.ui32.HostAccess != 1) || (MemFlags.ui32.NonPaged == 1)) + return HSAKMT_STATUS_NOT_SUPPORTED; + + if (!is_dgpu) + /* TODO: support mixed APU and dGPU configurations */ + return HSAKMT_STATUS_NOT_SUPPORTED; + + ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, + NULL, 0, MemFlags.ui32.CoarseGrain); + + return ret; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes(HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo *GraphicsResourceInfo, HSAuint64 NumberOfNodes, From d6cde5bf087dccb27081e6583b52b960c32fc97f Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Thu, 28 Mar 2019 14:41:23 -0400 Subject: [PATCH 0483/1247] libhsakmt: update kfd_ioctl.h regarding RAS interface It is aligned with RAS changes in KFD. Change-Id: I52816da01a4001158a40a1207d1fbe6ec3271343 Signed-off-by: Eric Huang --- include/linux/kfd_ioctl.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 485be6456a..62c7ee41e2 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -273,6 +273,11 @@ struct kfd_ioctl_dbg_trap_args { #define KFD_HW_EXCEPTION_GPU_HANG 0 #define KFD_HW_EXCEPTION_ECC 1 +/* For kfd_hsa_memory_exception_data.ErrorType */ +#define KFD_MEM_ERR_NO_RAS 0 +#define KFD_MEM_ERR_SRAM_ECC 1 +#define KFD_MEM_ERR_POISON_CONSUMED 2 +#define KFD_MEM_ERR_GPU_HANG 3 struct kfd_ioctl_create_event_args { __u64 event_page_offset; /* from KFD */ @@ -313,10 +318,11 @@ struct kfd_hsa_memory_exception_data { __u64 va; __u32 gpu_id; __u32 ErrorType; /* 0 = no RAS error, - 1 = ECC_SRAM, - 2 = Link_SYNFLOOD (poison), - 3 = GPU hang (not attributable to a specific cause), - other values reserved */ + * 1 = ECC_SRAM, + * 2 = Link_SYNFLOOD (poison), + * 3 = GPU hang (not attributable to a specific cause), + * other values reserved + */ }; /* hw exception data */ From e5b215570b462b8271db012196ce866f5e685742 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Thu, 28 Mar 2019 14:51:06 -0400 Subject: [PATCH 0484/1247] kfdtest: fix and change in RAS test 1. RAS error injection debugfs interface has been changed which is using ras_ctrl instead of *_err_inject. 2. Remove ASSERT_SUCCESS for fwrite, because fwrite returns the size of written item but not the error number. 3. Using throw exception instead of return to avoid a segment fault. Change-Id: I6c4d9c2f7e66719faec99abd1552105a08c238a4 Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDRASTest.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/tests/kfdtest/src/KFDRASTest.cpp index 1bfdc64c9a..2fb96094f8 100644 --- a/tests/kfdtest/src/KFDRASTest.cpp +++ b/tests/kfdtest/src/KFDRASTest.cpp @@ -29,7 +29,7 @@ #include "PM4Queue.hpp" #define AMDGPU_DEBUGFS_NODES "/sys/kernel/debug/dri/" -#define RAS_SDMA_ERR_INJECTION "ras/sdma_err_inject" +#define RAS_CONTROL "ras/ras_ctrl" void KFDRASTest::SetUp() { ROUTINE_START @@ -49,7 +49,7 @@ void KFDRASTest::SetUp() { renderNode = KFDBaseComponentTest::FindDRMRenderNode(m_defaultGPUNode); if (renderNode < 0) { LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; - return; + throw; } amdgpu_query_info(m_RenderNodes[renderNode].device_handle, @@ -60,15 +60,15 @@ void KFDRASTest::SetUp() { AMDGPU_INFO_RAS_ENABLED_UMC || AMDGPU_INFO_RAS_ENABLED_GFX))) { LOG() << "Skipping test: GPU doesn't support RAS features!" << std::endl; - return; + throw; } - snprintf(path, sizeof(path), "%s/%d/%s", AMDGPU_DEBUGFS_NODES, renderNode, RAS_SDMA_ERR_INJECTION); + snprintf(path, sizeof(path), "%s/%d/%s", AMDGPU_DEBUGFS_NODES, renderNode, RAS_CONTROL); m_pFile = fopen(path, "w"); if (!m_pFile) { LOG() << "Skipping test: RAS error injection requires root access!" << std::endl; - return; + throw; } eventDesc.EventType = HSA_EVENTTYPE_MEMORY; @@ -105,7 +105,7 @@ TEST_F(KFDRASTest, BasicTest) { } // write an uncorrectable error injection at address 1 as value 1 - ASSERT_SUCCESS(fwrite("ue 1 1", sizeof(char), 6, m_pFile)); + fwrite("inject umc ue 1 1", sizeof(char), 17, m_pFile); EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pRasEvent, g_TestTimeOut)); @@ -136,7 +136,7 @@ TEST_F(KFDRASTest, MixEventsTest) { EXPECT_SUCCESS(hsaKmtWaitOnEvent(pHsaEvent, g_TestTimeOut)); - ASSERT_SUCCESS(fwrite("ue 1 1", sizeof(char), 6, m_pFile)); + fwrite("inject umc ue 1 1", sizeof(char), 17, m_pFile); EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pRasEvent, g_TestTimeOut)); From 006c2c248d84707b53b13c52f641e75ccb4c844e Mon Sep 17 00:00:00 2001 From: Tom Stellard Date: Thu, 28 Feb 2019 21:32:45 -0800 Subject: [PATCH 0485/1247] Separate build version from library version This patch separates the build version (i.e. ROCm version) from the library version used to set the SONAME of the shared objects. This prevents the SONAME from getting bumped each time there is a new ROCm release without any change to the libhsakmt ABI. 1.0.6 was choosen as the library version since this was the last library version used prior to switching to the ROCm version numbers. Change-Id: I7c29ae84d8a362a831e804569d8147ca65155cad --- CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f30f5b7a97..ede67dcc10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,11 @@ get_version ( "1.0.0" ) set ( BUILD_VERSION_MAJOR ${VERSION_MAJOR} ) set ( BUILD_VERSION_MINOR ${VERSION_MINOR} ) set ( BUILD_VERSION_PATCH ${VERSION_PATCH} ) -set ( LIB_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) + +set ( LIB_VERSION_MAJOR 1) +set ( LIB_VERSION_MINOR 0) +set ( LIB_VERSION_PATCH 6) +set ( LIB_VERSION_STRING "${LIB_VERSION_MAJOR}.${LIB_VERSION_MINOR}.${LIB_VERSION_PATCH}" ) if ( DEFINED VERSION_BUILD AND NOT ${VERSION_BUILD} STREQUAL "" ) message ( "VERSION BUILD DEFINED ${VERSION_BUILD}" ) @@ -73,7 +77,7 @@ endif () set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) ## Linker Flags -set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.$(PROJECT_VERSION_MAJOR) -Wl,-z,nodelete -Wl,-no-undefined" ) +set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) ## Source files set ( HSAKMT_SRC "src/debug.c" @@ -100,7 +104,7 @@ add_library ( ${HSAKMT_TARGET} SHARED ${HSAKMT_SRC} ) ## Set the VERSION and SOVERSION values set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" ) -set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${BUILD_VERSION_MAJOR}" ) +set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" ) find_package(PkgConfig) pkg_check_modules(PC_LIBPCI REQUIRED libpci) From b8a1331763c2af7f2403e540aa81433ff421feba Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Tue, 25 Sep 2018 19:16:43 +0200 Subject: [PATCH 0486/1247] cmake: Create cmake config file Another cmake project like hsa-runtime could just use: find_package(hsakmt REQUIRED 1.9.0) Change-Id: Ia1c9a80ef287facdd607382d69649b0718d687b4 --- CMakeLists.txt | 4 +++- hsakmt-config.cmake.in | 9 +++++++++ hsakmt-dev.txt | 28 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 hsakmt-config.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index ede67dcc10..59a29cea8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ set ( HSAKMT_PACKAGE "hsakmt-roct" ) set ( HSAKMT_COMPONENT "lib${HSAKMT}" ) set ( HSAKMT_TARGET "${HSAKMT}" ) -project ( ${HSAKMT_TARGET} ) +project ( ${HSAKMT_TARGET} VERSION 1.9.0) ## Set default module path if not already set if ( NOT DEFINED CMAKE_MODULE_PATH ) @@ -133,6 +133,8 @@ add_custom_target ( build-dev -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" -DCPACK_RPM_PACKAGE_REQUIRES="hsakmt-roct" -DCPACK_PACKAGING_INSTALL_PREFIX="${CPACK_PACKAGING_INSTALL_PREFIX}" + -DVERSION="${BUILD_VERSION_STRING}" + -DHSAKMT_BIN_NAME="${HSAKMT}" WORKING_DIRECTORY ${DEV_BUILD_DIR} ) ## Custom targets for the devel package diff --git a/hsakmt-config.cmake.in b/hsakmt-config.cmake.in new file mode 100644 index 0000000000..91f73898b5 --- /dev/null +++ b/hsakmt-config.cmake.in @@ -0,0 +1,9 @@ +@PACKAGE_INIT@ + +set_and_check(HSAKMT_INCLUDE_DIR @PACKAGE_INCLUDE_INSTALL_DIR@) +set_and_check(HSAKMT_LIB_DIR @PACKAGE_LIB_INSTALL_DIR@) + +set(HSAKMT_LIBRARY @PACKAGE_LIB_INSTALL_DIR@/@HSAKMT_LIBRARY_NAME@) +set(HSAKMT_LIBRARIES @PACKAGE_LIB_INSTALL_DIR@/@HSAKMT_LIBRARY_NAME@) + +mark_as_advanced(HSAKMT_LIBRARY HSAKMT_INCLUDE_DIR) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index 79924fde39..eb0d3e2acb 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -67,4 +67,32 @@ set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-T # RPM package specific variables set ( CPACK_RPM_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} = ${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) +# Create cmake configuration files +include(CMakePackageConfigHelpers) + +set(HSAKMT_LIBRARY_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${PROJECT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) + +set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME} CACHE INTERNAL "") +set(LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE INTERNAL "") + +configure_package_config_file(${SOURCE_DIR}/${HSAKMT_BIN_NAME}-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_BIN_NAME}-config.cmake + PATH_VARS + INCLUDE_INSTALL_DIR LIB_INSTALL_DIR + INSTALL_DESTINATION + ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_BIN_NAME}) + +write_basic_package_version_file(${HSAKMT_BIN_NAME}-config-version.cmake + VERSION ${VERSION} + COMPATIBILITY + AnyNewerVersion) + +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_BIN_NAME}-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_BIN_NAME}-config-version.cmake + DESTINATION + ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_BIN_NAME} + COMPONENT + devel) + include ( CPack ) From 8ccfa4c75c366cb8271f97fb6e43cb4029839602 Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Tue, 25 Sep 2018 18:47:57 +0200 Subject: [PATCH 0487/1247] cmake: Do not strip targets in the release build Distributions want to generate debuginfo packages, do not strip them! If you want to do it during installation use 'make install/strip'! Change-Id: I3983af24ce4f4ddb189ede0ed0820dfee83b6280 --- CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59a29cea8f..33b0f26779 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,11 +113,6 @@ target_link_libraries ( ${HSAKMT_TARGET} pthread rt numa ${PC_LIBPCI_LIBRARIES} ) -## If the library is a release, strip the target library -if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) - add_custom_command ( TARGET ${HSAKMT_TARGET} POST_BUILD COMMAND ${CMAKE_STRIP} ${HSAKMT_COMPONENT}.so ) -endif () - ## Define default variable and variables for the optional build target hsakmt-dev set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) From 0045974858b606ad1442275b48f80975ea66f902 Mon Sep 17 00:00:00 2001 From: Andreas Schneider Date: Fri, 7 Sep 2018 10:31:59 +0200 Subject: [PATCH 0488/1247] cmake: Do not mess with CMAKE flags Specifically, don't mess with CMAKE_SHARED_LINKER_FLAGS or CMAKE_C_FLAGS Change-Id: I73e287df5b80d440079c6b3abe8c401d492d11dd --- CMakeLists.txt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33b0f26779..e3141d3ccb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,21 +63,20 @@ set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUIL #set ( CMAKE_VERBOSE_MAKEFILE on ) ## Compiler flags -set ( CMAKE_C_FLAGS "-fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden" ) +set (HSAKMT_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden) if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") - set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror" ) + set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -Werror ) endif () - if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) - set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2" ) + set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -O2 ) else () - set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g" ) + set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -g ) endif () set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) ## Linker Flags -set ( CMAKE_SHARED_LINKER_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) +set (HSAKMT_LINK_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) ## Source files set ( HSAKMT_SRC "src/debug.c" @@ -101,6 +100,8 @@ include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/src ) ## Add the library add_library ( ${HSAKMT_TARGET} SHARED ${HSAKMT_SRC} ) +target_compile_options(${HSAKMT_TARGET} PRIVATE ${HSAKMT_C_FLAGS}) +set_property(TARGET ${HSAKMT_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_LINK_FLAGS}) ## Set the VERSION and SOVERSION values set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" ) From fdadae67456901ae4fe64cbd4b7a988c0a29bf72 Mon Sep 17 00:00:00 2001 From: Alex Voicu Date: Fri, 12 Apr 2019 12:23:21 +0300 Subject: [PATCH 0489/1247] Fix Clang build Change-Id: I0b51699c0a1368cf5813bd9d3cd4479139d23d6a --- CMakeLists.txt | 7 +++++-- src/fmm.c | 1 + src/libhsakmt.h | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e3141d3ccb..b8b565beab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,10 @@ set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUIL #set ( CMAKE_VERBOSE_MAKEFILE on ) ## Compiler flags -set (HSAKMT_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wlogical-op -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden) +set (HSAKMT_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden) +if ( CMAKE_COMPILER_IS_GNUCC ) + set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -Wlogical-op) +endif () if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -Werror ) endif () @@ -133,7 +136,7 @@ add_custom_target ( build-dev -DHSAKMT_BIN_NAME="${HSAKMT}" WORKING_DIRECTORY ${DEV_BUILD_DIR} ) -## Custom targets for the devel package +## Custom targets for the devel package add_custom_target ( install-dev DEPENDS build-dev COMMAND make install WORKING_DIRECTORY ${DEV_BUILD_DIR} ) add_custom_target ( package-dev DEPENDS build-dev COMMAND rm -rf *.deb *.rpm *.tar.gz diff --git a/src/fmm.c b/src/fmm.c index 079ee55664..f335ef4293 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -277,6 +277,7 @@ static inline HsaSharedMemoryStruct *to_hsa_shared_memory_struct( return (HsaSharedMemoryStruct *)SharedMemoryHandle; } +__attribute__((unused)) static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( HsaSharedMemoryStruct *SharedMemoryStruct) { diff --git a/src/libhsakmt.h b/src/libhsakmt.h index a34889c967..1a2ba979ff 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -67,7 +67,7 @@ extern int PAGE_SHIFT; #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) #define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1)) #define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) -#define BITMASK(n) (((n) < sizeof(1ULL) * CHAR_BIT ? (1ULL << (n)) : 0) - 1ULL) +#define BITMASK(n) ((n) ? (UINT64_MAX >> (sizeof(UINT64_MAX) * CHAR_BIT - n)) : n) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) /* HSA Thunk logging usage */ From ee9831779cf88f7d6b913a485afb856a88dd29c4 Mon Sep 17 00:00:00 2001 From: Alex Voicu Date: Fri, 12 Apr 2019 15:40:09 +0300 Subject: [PATCH 0490/1247] Macros are devious, wrap argument to prevent surprises. Change-Id: Ib99e7d2ec1e7a2802f4ae7946ba1fa92c9940a85 --- src/libhsakmt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 1a2ba979ff..d9eab68b51 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -67,7 +67,7 @@ extern int PAGE_SHIFT; #define ALIGN_UP(x,align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) #define ALIGN_UP_32(x,align) (((uint32_t)(x) + (align) - 1) & ~(uint32_t)((align)-1)) #define PAGE_ALIGN_UP(x) ALIGN_UP(x,PAGE_SIZE) -#define BITMASK(n) ((n) ? (UINT64_MAX >> (sizeof(UINT64_MAX) * CHAR_BIT - n)) : n) +#define BITMASK(n) ((n) ? (UINT64_MAX >> (sizeof(UINT64_MAX) * CHAR_BIT - (n))) : 0) #define ARRAY_LEN(array) (sizeof(array) / sizeof(array[0])) /* HSA Thunk logging usage */ From e109ce541c9fcfd636f775df171322953852af82 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 23 Apr 2019 07:11:12 -0400 Subject: [PATCH 0491/1247] Fix test if Render Node can't be found If a Render Node can't be found, we should finish off all child processes immediately, then return. Trying to do the check before forking the children results in the test failing as well, regardless of the status of finding the render node, which is likely why the forking occurred first in the test's initial creation. This way we ensure that things finish cleanly before moving to the next test Change-Id: I2e1b62fed25c30ff1f179612127c23960da4ee5e --- tests/kfdtest/src/KFDEvictTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 0a1bf506c0..cd530c7464 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -341,6 +341,7 @@ TEST_F(KFDEvictTest, BasicTest) { int rn = FindDRMRenderNode(defaultGPUNode); if (rn < 0) { LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; + WaitChildProcesses(); return; } @@ -551,6 +552,7 @@ TEST_F(KFDEvictTest, QueueTest) { int rn = FindDRMRenderNode(defaultGPUNode); if (rn < 0) { LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; + WaitChildProcesses(); return; } From 5b18614eafa1bc943c702cd190f2bc820748e0af Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 18 Apr 2019 13:25:32 -0400 Subject: [PATCH 0492/1247] kfdtest: Add CreateDestroyCpQueue and CreateDestroySdmaQueue test Those two tests cover the basic queue creation and destruction without submitting packets to CP and SDMA user queues. During bringup, they bring values in term of untangling the issues arising in queue creation and packet execution, which are two very different kinds. Because of those two tests, we also rename some existing tests as follows: CreateCpQueue -> SubmitPacketCpQueue CreateSdmaQueue -> SubmitPacketSdmaQueue CreateMultipleCpQueues -> MultipleCpQueues CreateMultipleSdmaQueues -> MultipleSdmaQueues Lastly, move MultipleCpQueues test closer to the CP queue section rather than leaving it behind the SDMA queue section. Change-Id: I110fb3f3fb21878339045dd1d1c8c9d61b8988b7 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDQMTest.cpp | 98 +++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 89291c3b5a..5f9dada9f0 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -55,7 +55,22 @@ void KFDQMTest::TearDown() { ROUTINE_END } -TEST_F(KFDQMTest, CreateCpQueue) { +TEST_F(KFDQMTest, CreateDestroyCpQueue) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, SubmitPacketCpQueue) { TEST_START(TESTPROFILE_RUNALL) int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); @@ -80,7 +95,53 @@ TEST_F(KFDQMTest, CreateCpQueue) { TEST_END } -TEST_F(KFDQMTest, CreateSdmaQueue) { +TEST_F(KFDQMTest, MultipleCpQueues) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + static const unsigned int MAX_CP_QUEUES = 16; + + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); + + destBuf.Fill(0xFF); + + PM4Queue queues[MAX_CP_QUEUES]; + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + ASSERT_SUCCESS(queues[qidx].Create(defaultGPUNode)) << " QueueId=" << qidx; + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) { + queues[qidx].PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As()+qidx*2, qidx, qidx)); + + queues[qidx].Wait4PacketConsumption(); + + WaitOnValue(destBuf.As()+qidx*2, qidx); + } + + for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + EXPECT_SUCCESS(queues[qidx].Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, CreateDestroySdmaQueue) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + SDMAQueue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDQMTest, SubmitPacketSdmaQueue) { TEST_START(TESTPROFILE_RUNALL) int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); @@ -105,7 +166,7 @@ TEST_F(KFDQMTest, CreateSdmaQueue) { TEST_END } -TEST_F(KFDQMTest, CreateMultipleSdmaQueues) { +TEST_F(KFDQMTest, MultipleSdmaQueues) { TEST_START(TESTPROFILE_RUNALL) int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); @@ -208,37 +269,6 @@ TEST_F(KFDQMTest, SdmaConcurrentCopies) { TEST_END } -TEST_F(KFDQMTest, CreateMultipleCpQueues) { - TEST_START(TESTPROFILE_RUNALL) - - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - static const unsigned int MAX_CP_QUEUES = 16; - - HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); - - destBuf.Fill(0xFF); - - PM4Queue queues[MAX_CP_QUEUES]; - - for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) - ASSERT_SUCCESS(queues[qidx].Create(defaultGPUNode)) << " QueueId=" << qidx; - - for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) { - queues[qidx].PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As()+qidx*2, qidx, qidx)); - - queues[qidx].Wait4PacketConsumption(); - - WaitOnValue(destBuf.As()+qidx*2, qidx); - } - - for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) - EXPECT_SUCCESS(queues[qidx].Destroy()); - - TEST_END -} - TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) { TEST_START(TESTPROFILE_RUNALL) From ae111689f0105ca2bdedc965cec60a7e23250e88 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 23 Apr 2019 15:08:30 -0500 Subject: [PATCH 0493/1247] Map remapped mmio page to process space HDP conherence registers are remapped at driver level to an empty page in mmio space (the remapped mmio page). This change allocate and map the remapped mmio page to process space. Change-Id: I89c405c41870a79c5b58eea0d8e564aa35f55182 Signed-off-by: Oak Zeng --- include/linux/kfd_ioctl.h | 9 +++++ src/fmm.c | 76 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 62c7ee41e2..2eb86860da 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -400,6 +400,7 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) +#define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP (1 << 4) /* Allocation flags: attributes/access options */ #define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31) #define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30) @@ -437,6 +438,14 @@ struct kfd_ioctl_free_memory_of_gpu_args { __u64 handle; /* to KFD */ }; +/* Register offset inside the remapped mmio page + */ +enum kfd_mmio_remap { + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0, + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4, +}; + + /* Map memory to one or more GPUs * * @handle: memory handle returned by alloc diff --git a/src/fmm.c b/src/fmm.c index f335ef4293..d6d039f6fe 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -175,6 +175,7 @@ typedef struct { uint64_t local_mem_size; aperture_t lds_aperture; aperture_t scratch_aperture; + aperture_t mmio_aperture; manageable_aperture_t scratch_physical; /* For dGPU, scratch physical is allocated from * dgpu_aperture. When requested by RT, each * GPU will get a differnt range @@ -1983,6 +1984,67 @@ static void fmm_init_rbtree(void) } } +static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) +{ + void *mem; + manageable_aperture_t *aperture = svm.dgpu_alt_aperture; + uint32_t ioc_flags; + vm_object_t *vm_obj = NULL; + HsaMemFlags flags; + void *ret; + uint64_t mmap_offset; + + /* Allocate physical memory and vm object*/ + ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP | + KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | + KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + mem = __fmm_allocate_device(gpu_id, NULL, PAGE_SIZE, aperture, + &mmap_offset, ioc_flags, &vm_obj); + + if (!mem || !vm_obj) + return NULL; + + flags.Value = 0; + flags.ui32.NonPaged = 1; + flags.ui32.HostAccess = 1; + flags.ui32.Reserved = 0; + pthread_mutex_lock(&aperture->fmm_mutex); + vm_obj->flags = flags.Value; + vm_obj->node_id = node_id; + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* Map for CPU access*/ + ret = mmap(mem, PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, mmap_fd, + mmap_offset); + if (ret == MAP_FAILED) { + __fmm_release(vm_obj, aperture); + return NULL; + } + + /* Map for GPU access*/ + if (fmm_map_to_gpu(mem, PAGE_SIZE, NULL)) { + __fmm_release(vm_obj, aperture); + return NULL; + } + + return mem; +} + +static void release_mmio(void) +{ + uint32_t gpu_mem_id; + + for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) { + if (!gpu_mem[gpu_mem_id].mmio_aperture.base) + continue; + fmm_unmap_from_gpu(gpu_mem[gpu_mem_id].mmio_aperture.base); + munmap(gpu_mem[gpu_mem_id].mmio_aperture.base, PAGE_SIZE); + fmm_release(gpu_mem[gpu_mem_id].mmio_aperture.base); + } +} + HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i; @@ -2229,6 +2291,19 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) fmm_init_rbtree(); + for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) { + if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) + continue; + gpu_mem[gpu_mem_id].mmio_aperture.base = map_mmio( + gpu_mem[gpu_mem_id].node_id, + gpu_mem[gpu_mem_id].gpu_id, + gpu_mem[gpu_mem_id].drm_render_fd); + if (gpu_mem[gpu_mem_id].mmio_aperture.base) + gpu_mem[gpu_mem_id].mmio_aperture.limit = (void *) + ((char *)gpu_mem[gpu_mem_id].mmio_aperture.base + + PAGE_SIZE - 1); + } + free(process_apertures); return ret; @@ -2247,6 +2322,7 @@ sysfs_parse_failed: void fmm_destroy_process_apertures(void) { + release_mmio(); if (gpu_mem) { free(gpu_mem); gpu_mem = NULL; From e4a6a01389882a0aac8ab0608673b1c890559c7e Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 30 Apr 2019 09:33:29 -0500 Subject: [PATCH 0494/1247] Fix return value of fmm_get_aperture_base_and_limit Only return success when the aperture is valid Change-Id: I63b97fd0450e1ff277cf45abc7a1be9f7a0c0d50 --- src/fmm.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index d6d039f6fe..cef9212e4d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2333,7 +2333,7 @@ void fmm_destroy_process_apertures(void) HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, HSAuint64 *aperture_base, HSAuint64 *aperture_limit) { - HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; + HSAKMT_STATUS err = HSAKMT_STATUS_ERROR; int32_t slot = gpu_mem_find_by_gpu_id(gpu_id); if (slot < 0) @@ -2345,6 +2345,7 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA gpu_mem[slot].gpuvm_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.base); *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].gpuvm_aperture.limit); + err = HSAKMT_STATUS_SUCCESS; } break; @@ -2353,6 +2354,7 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA gpu_mem[slot].scratch_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.base); *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].scratch_aperture.limit); + err = HSAKMT_STATUS_SUCCESS; } break; @@ -2361,6 +2363,7 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA gpu_mem[slot].lds_aperture.limit)) { *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.base); *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].lds_aperture.limit); + err = HSAKMT_STATUS_SUCCESS; } break; @@ -2372,11 +2375,12 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA svm.dgpu_aperture->limit)) { *aperture_base = PORT_VPTR_TO_UINT64(svm.dgpu_alt_aperture->base); *aperture_limit = PORT_VPTR_TO_UINT64(svm.dgpu_aperture->limit); + err = HSAKMT_STATUS_SUCCESS; } break; default: - err = HSAKMT_STATUS_ERROR; + break; } return err; From 804aa90a221bb0317de68fe0d82dbee3b6f91af7 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Fri, 26 Apr 2019 17:42:16 -0500 Subject: [PATCH 0495/1247] Add MMIO_REMAP heap type Add a MMIO_REMAP heap type and expose mmio virtual address through HsaKmtGetNodeMemoryProperties Change-Id: I1e585e6dfbec8fa7c85f1dda7b89b763a8e2c439 Signed-off-by: Oak Zeng --- include/hsakmttypes.h | 1 + src/fmm.c | 9 +++++++++ src/fmm.h | 1 + src/topology.c | 20 +++++++++++++++++++- 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 06d650c5e8..f70f46e4e9 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -316,6 +316,7 @@ typedef enum _HSA_HEAPTYPE HSA_HEAPTYPE_GPU_LDS = 4, // GPU internal memory (LDS) HSA_HEAPTYPE_GPU_SCRATCH = 5, // GPU special memory (scratch) HSA_HEAPTYPE_DEVICE_SVM = 6, // sys-memory mapped by device page tables + HSA_HEAPTYPE_MMIO_REMAP = 7, // remapped mmio, such as hdp flush registers HSA_HEAPTYPE_NUMHEAPTYPES, HSA_HEAPTYPE_SIZE = 0xFFFFFFFF diff --git a/src/fmm.c b/src/fmm.c index cef9212e4d..fc2b50d394 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2379,6 +2379,15 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA } break; + case FMM_MMIO: + if (aperture_is_valid(gpu_mem[slot].mmio_aperture.base, + gpu_mem[slot].mmio_aperture.limit)) { + *aperture_base = PORT_VPTR_TO_UINT64(gpu_mem[slot].mmio_aperture.base); + *aperture_limit = PORT_VPTR_TO_UINT64(gpu_mem[slot].mmio_aperture.limit); + err = HSAKMT_STATUS_SUCCESS; + } + break; + default: break; } diff --git a/src/fmm.h b/src/fmm.h index 9709b6e4ce..e06acfde24 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -35,6 +35,7 @@ typedef enum { FMM_LDS, FMM_SCRATCH, FMM_SVM, + FMM_MMIO, FMM_LAST_APERTURE_TYPE } aperture_type_e; diff --git a/src/topology.c b/src/topology.c index a730d1b2d7..606918d2bc 100644 --- a/src/topology.c +++ b/src/topology.c @@ -39,7 +39,12 @@ #include "libhsakmt.h" #include "fmm.h" -/* Number of memory banks added by thunk on top of topology */ +/* Number of memory banks added by thunk on top of topology + * This only includes static heaps like LDS, scratch and SVM, + * not for MMIO_REMAP heap. MMIO_REMAP memory bank is reported + * dynamically based on whether mmio aperture was mapped + * successfully on this node. + */ #define NUM_OF_IGPU_HEAPS 3 #define NUM_OF_DGPU_HEAPS 3 /* SYSFS related */ @@ -2057,10 +2062,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, *NodeProperties = g_props[NodeId].node; /* For CPU only node don't add any additional GPU memory banks. */ if (gpu_id) { + uint64_t base, limit; if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) NodeProperties->NumMemoryBanks += NUM_OF_DGPU_HEAPS; else NodeProperties->NumMemoryBanks += NUM_OF_IGPU_HEAPS; + if (fmm_get_aperture_base_and_limit(FMM_MMIO, gpu_id, &base, + &limit) == HSAKMT_STATUS_SUCCESS) + NodeProperties->NumMemoryBanks += 1; } err = HSAKMT_STATUS_SUCCESS; @@ -2151,6 +2160,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, i++; } + /* Add mmio aperture */ + if (i < NumBanks && + fmm_get_aperture_base_and_limit(FMM_MMIO, gpu_id, + &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { + MemoryProperties[i].HeapType = HSA_HEAPTYPE_MMIO_REMAP; + MemoryProperties[i].SizeInBytes = (aperture_limit - MemoryProperties[i].VirtualBaseAddress) + 1; + i++; + } + out: pthread_mutex_unlock(&hsakmt_mutex); return err; From c8d823eb108663d086855f75b6633f41cc92d741 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 30 Apr 2019 23:34:18 -0400 Subject: [PATCH 0496/1247] kfdtest: Make eviction tests more robust - Run more graphics command submissions with shorter delay between them - Synchronize after every graphics command submission - Include the big VRAM BO in the BOList of the command submission to trigger more evictions - In QueueTest, run AMDGPU command submissions concurrently with compute shader on the user mode queue - Submit AMDGPU commands to GFX queue instead of compute queue to avoid deadlocks between user-mode and kernel-mode queues on the same pipe - Allocate slightly less memory from KFD to avoid allocation errors due to fragmentation or memory leaks in previous tests - Running only two processes maximizes the number of KFD evictions (probably because of lower chances of evicting non-KFD BOs) Change-Id: If05d53f5fcf690b6488998a3f933f120ddaa71ee Signed-off-by: Felix Kuehling --- tests/kfdtest/src/KFDEvictTest.cpp | 49 +++++++++++++++--------------- tests/kfdtest/src/KFDEvictTest.hpp | 2 +- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index cd530c7464..3a3ab03f27 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -30,7 +30,7 @@ #include "SDMAQueue.hpp" #include "Dispatch.hpp" -#define N_PROCESSES (8) /* Number of processes running in parallel, must be at least 2 */ +#define N_PROCESSES (2) /* Number of processes running in parallel, must be at least 2 */ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) @@ -176,7 +176,7 @@ static inline int amdgpu_get_bo_list(amdgpu_device_handle dev, amdgpu_bo_handle return amdgpu_bo_list_create(dev, bo2 ? 2 : 1, resources, NULL, list); } -void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { +void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle) { amdgpu_context_handle contextHandle; amdgpu_bo_handle ibResultHandle; void *ibResultCpu; @@ -197,7 +197,7 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { &ibResultHandle, &ibResultCpu, &ibResultMcAddress, &vaHandle)); - ASSERT_EQ(0, amdgpu_get_bo_list(m_RenderNodes[rn].device_handle, ibResultHandle, NULL, + ASSERT_EQ(0, amdgpu_get_bo_list(m_RenderNodes[rn].device_handle, ibResultHandle, handle, &boList)); /* Fill Nop cammands in IB */ @@ -210,7 +210,7 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { ibInfo.size = 16; memset(&ibsRequest, 0, sizeof(struct amdgpu_cs_request)); - ibsRequest.ip_type = AMDGPU_HW_IP_COMPUTE; + ibsRequest.ip_type = AMDGPU_HW_IP_GFX; ibsRequest.ring = 0; ibsRequest.number_of_ibs = 1; ibsRequest.ibs = &ibInfo; @@ -218,21 +218,23 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn) { ibsRequest.fence_info.handle = NULL; memset(&fenceStatus, 0, sizeof(struct amdgpu_cs_fence)); - for (int i = 0; i < ALLOCATE_RETRY_TIMES; i++) { + for (int i = 0; i < 100; i++) { ASSERT_EQ(0, amdgpu_cs_submit(contextHandle, 0, &ibsRequest, 1)); - sleep(1); + Delay(50); + + fenceStatus.context = contextHandle; + fenceStatus.ip_type = AMDGPU_HW_IP_GFX; + fenceStatus.ip_instance = 0; + fenceStatus.ring = 0; + fenceStatus.fence = ibsRequest.seq_no; + + EXPECT_EQ(0, amdgpu_cs_query_fence_status(&fenceStatus, + g_TestTimeOut*1000000, + 0, &expired)); + if (!expired) + WARN() << "CS did not signal completion" << std::endl; } - fenceStatus.context = contextHandle; - fenceStatus.ip_type = AMDGPU_HW_IP_COMPUTE; - fenceStatus.ip_instance = 0; - fenceStatus.ring = 0; - fenceStatus.fence = ibsRequest.seq_no; - - EXPECT_EQ(0, amdgpu_cs_query_fence_status(&fenceStatus, - g_TestTimeOut, - 0, &expired)); - EXPECT_EQ(0, amdgpu_bo_list_destroy(boList)); EXPECT_EQ(0, amdgpu_bo_unmap_and_free(ibResultHandle, vaHandle, @@ -331,7 +333,8 @@ TEST_F(KFDEvictTest, BasicTest) { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; } - HSAint32 count = vramSize / vramBufSize / N_PROCESSES; + // Use 7/8 of VRAM between all processes + HSAuint32 count = vramSize * 7 / (8* vramBufSize * N_PROCESSES); LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; @@ -353,7 +356,7 @@ TEST_F(KFDEvictTest, BasicTest) { amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); - AmdgpuCommandSubmissionComputeNop(rn); + AmdgpuCommandSubmissionComputeNop(rn, handle); FreeAmdgpuBo(handle); LOG() << m_psName << "free buffer" << std::endl; @@ -533,7 +536,8 @@ TEST_F(KFDEvictTest, QueueTest) { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; } - HSAuint32 count = vramSize / vramBufSize / N_PROCESSES; + // Use 7/8 of VRAM between all processes + HSAuint32 count = vramSize * 7 / (8 * vramBufSize * N_PROCESSES); LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; @@ -568,8 +572,6 @@ TEST_F(KFDEvictTest, QueueTest) { amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); - AmdgpuCommandSubmissionComputeNop(rn); - unsigned int wavefront_num = pBuffers.size(); LOG() << m_psName << "wavefront number " << wavefront_num << std::endl; @@ -590,8 +592,7 @@ TEST_F(KFDEvictTest, QueueTest) { /* Submit the packet and start shader */ dispatch0.Submit(pm4Queue); - /* Doing evict/restore queue test for 5 seconds while queue is running */ - sleep(5); + AmdgpuCommandSubmissionComputeNop(rn, handle); /* Uncomment this line for debugging */ // LOG() << m_psName << "notify shader to quit" << std::endl; @@ -600,7 +601,7 @@ TEST_F(KFDEvictTest, QueueTest) { addrBuffer.Fill(0x5678); /* Wait for shader to finish or timeout if shader has vm page fault */ - dispatch0.SyncWithStatus(120000); + EXPECT_EQ(0, dispatch0.SyncWithStatus(120000)); EXPECT_SUCCESS(pm4Queue.Destroy()); diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index 0ab4630763..e6d8953068 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -56,7 +56,7 @@ class KFDEvictTest : public KFDLocalMemoryTest { void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); void AllocAmdgpuBo(int rn, HSAuint64 vramBufSize, amdgpu_bo_handle &handle); void FreeAmdgpuBo(amdgpu_bo_handle handle); - void AmdgpuCommandSubmissionComputeNop(int rn); + void AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle); void ForkChildProcesses(int nprocesses); void WaitChildProcesses(); From b26580788b8f923b0f259d5e420b5f6369c426e9 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 30 Apr 2019 15:32:01 -0500 Subject: [PATCH 0497/1247] Host HDP flush test Change-Id: I396ac021d15da972f4841d6d8f90d4b175e64ecd Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDMemoryTest.cpp | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 51bed05860..1328047b73 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -36,6 +36,7 @@ #include "PM4Packet.hpp" #include "SDMAQueue.hpp" #include "SDMAPacket.hpp" +#include "linux/kfd_ioctl.h" const char* gfx8_ScratchCopyDword = "\ @@ -104,6 +105,32 @@ type(CS)\n\ end\n\ "; +/* Input: A buffer of at least 3 dwords. + * DW0: used as a signal b/t host and device. Host + * write 0xcafe to signal device. + * DW1: Input buffer for host/device to read/write. + * DW2: Output buffer for device to write. + * This shader continously poll the signal buffer, + * Once signal buffer is signaled, it copies input buffer + * to output buffer + */ +const char* gfx9_CopyOnSignal = +"\ +shader CopyOnSignal\n\ +asic(GFX9)\n\ +type(CS)\n\ +/* Assume input buffer in s0, s1 */\n\ + s_movk_i32 s18, 0xcafe\n\ + POLLSIGNAL:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 POLLSIGNAL\n\ + s_load_dword s17, s[0:1], 0x4 glc\n\ + s_store_dword s17, s[0:1], 0x8 glc\n\ + s_endpgm\n\ + end\n\ +"; + void KFDMemoryTest::SetUp() { ROUTINE_START @@ -1689,3 +1716,90 @@ TEST_F(KFDMemoryTest, MMBandWidth) { TEST_END } + +/* For the purpose of testing HDP flush from CPU. + * Use CPU to write to coherent vram and check + * from shader. + * Asic before gfx9 doesn't support user space + * HDP flush so only run on vega10 and after. + * This should only run on large bar system. + */ +TEST_F(KFDMemoryTest, HostHdpFlush) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HsaMemFlags memoryFlags = m_MemoryFlags; + /* buffer[0]: signal; buffer[1]: Input to shader; buffer[2]: Output to + * shader + */ + unsigned int *buffer = NULL; + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); + HSAuint32 *mmioBase = NULL; + unsigned int *nullPtr = NULL; + + if (!pNodeProperties) { + LOG() << "Failed to get gpu node properties." << std::endl; + return; + } + + if (m_FamilyId < FAMILY_AI) { + LOG() << "Skipping test: Test requires gfx9 and later asics." << std::endl; + return; + } + HSAuint64 vramSizeMB = GetVramSize(defaultGPUNode) >> 20; + + if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode) || !vramSizeMB) { + LOG() << "Skipping test: Test requires a large bar GPU." << std::endl; + return; + } + + HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(defaultGPUNode, pNodeProperties->NumMemoryBanks, + memoryProperties)); + for (unsigned int bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) { + if (memoryProperties[bank].HeapType == HSA_HEAPTYPE_MMIO_REMAP) { + mmioBase = (unsigned int *)memoryProperties[bank].VirtualBaseAddress; + break; + } + } + ASSERT_NE(mmioBase, nullPtr) << "mmio base is NULL"; + + memoryFlags.ui32.NonPaged = 1; + memoryFlags.ui32.CoarseGrain = 0; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, memoryFlags, + reinterpret_cast(&buffer))); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(buffer, PAGE_SIZE, NULL)); + + /* Signal is dead from the beginning*/ + buffer[0] = 0xdead; + buffer[1] = 0xfeeb; + buffer[2] = 0xfeed; + /* Submit a shader to poll the signal*/ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9_CopyOnSignal,"CopyOnSignal", isaBuffer); + Dispatch dispatch0(isaBuffer); + dispatch0.SetArgs(buffer, NULL); + dispatch0.Submit(queue); + + buffer[1] = 0xbeef; + /* Flush HDP */ + mmioBase[KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL/4] = 0x1; + /* Give cafe to wake up */ + buffer[0] = 0xcafe; + + /* Check test result*/ + dispatch0.Sync(); + EXPECT_EQ(0xbeef, buffer[2]); + + // Clean up + EXPECT_SUCCESS(queue.Destroy()); + delete [] memoryProperties; + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(buffer)); + EXPECT_SUCCESS(hsaKmtFreeMemory(buffer, PAGE_SIZE)); + + TEST_END +} From d21e9d5bbd13ffc9fd89a54ef40ac1f4a0518c37 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Wed, 10 Apr 2019 14:45:18 -0400 Subject: [PATCH 0498/1247] libhsakmt: Update HsaQueueInfo for GetQueueInfo hsaKmtGetQueueInfo needs to return the control stack size, and the wave state size for the debugger. These changes are needed to support returning the new values. Change-Id: Ib4c60e0ea34446c06aef4a86996250989f348a69 Signed-off-by: Philip Cox --- include/hsakmttypes.h | 30 +++++++++++++++++++++++++++++- src/queues.c | 8 ++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index f70f46e4e9..8b0d232597 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -641,6 +641,34 @@ typedef enum _HSA_QUEUE_TYPE HSA_QUEUE_TYPE_SIZE = 0xFFFFFFFF //aligns to 32bit enum } HSA_QUEUE_TYPE; +/** + The user context save area starts at offset 0 with the + HsaUserContextSaveAreaHeader header followed by the space for a + user space copy of the control stack and the user space wave save + state. The area must be dword aligned. The context save area is + valid for the duration that the associated queue exists. When a + context save occurs, the HsaUserContextSaveAreaHeader header will + be updated with information about the context save. The context save + area is not modified by any other operation, including a context + resume. + */ + +typedef struct +{ + HSAuint32 ControlStackOffset; // Byte offset from start of user context + // save area to the last saved top (lowest + // address) of control stack data. Must be + // 4 byte aligned. + HSAuint32 ControlStackSize; // Byte size of the last saved control stack + // data. Must be 4 byte aligned. + HSAuint32 WaveStateOffset; // Byte offset from start of user context save + // area to the last saved base (lowest address) + // of wave state data. Must be 4 byte aligned. + HSAuint32 WaveStateSize; // Byte size of the last saved wave state data. + // Must be 4 byte aligned. +} HsaUserContextSaveAreaHeader; + + typedef struct { HSAuint32 QueueDetailError; // HW specific queue error state @@ -656,7 +684,7 @@ typedef struct HSAuint64 SaveAreaSizeInBytes; // Must be 4-Byte aligned HSAuint32* ControlStackTop; // ptr to the TOS HSAuint64 ControlStackUsedInBytes; // Must be 4-Byte aligned - HSAuint64 Reserved1; // runtime/system CU assignment + HsaUserContextSaveAreaHeader *SaveAreaHeader; HSAuint64 Reserved2; // runtime/system CU assignment } HsaQueueInfo; diff --git a/src/queues.c b/src/queues.c index a1af2f39e7..4472e6f63a 100644 --- a/src/queues.c +++ b/src/queues.c @@ -373,9 +373,12 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) ctl_stack_size = cu_num * WAVES_PER_CU_VI * 8 + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU_VI; - q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size); - q->ctx_save_restore_size = q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size); + q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size + + sizeof(HsaUserContextSaveAreaHeader)); + + q->ctx_save_restore_size = q->ctl_stack_size + + PAGE_ALIGN_UP(wg_data_size); return true; } return false; @@ -767,6 +770,7 @@ hsaKmtGetQueueInfo( QueueInfo->CUMaskInfo = q->cu_mask; QueueInfo->QueueDetailError = 0; QueueInfo->QueueTypeExtended = 0; + QueueInfo->SaveAreaHeader = q->ctx_save_restore; return HSAKMT_STATUS_SUCCESS; } From c2c1385e290d811beede3cb094bfa6ea6e606dcf Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Fri, 29 Mar 2019 14:42:53 -0400 Subject: [PATCH 0499/1247] libhsakmt: Update wave suspend/resume API This is updating to the new suspend and resume API for the KFD and the thunk. We now support passing in a list of queues to suspend, and not just all of the queues for the process. The kfdtest testcase was also updated so it still compiles. Change-Id: I71d1b178476bd9df0c311bdedaa6a891528cebcf Signed-off-by: Philip Cox --- include/hsakmt.h | 86 ++++++++++---- include/hsakmttypes.h | 7 +- include/linux/kfd_ioctl.h | 41 +++---- src/debug.c | 197 +++++++++++++++++++++++-------- src/libhsakmt.ver | 4 +- tests/kfdtest/src/KFDDBGTest.cpp | 17 ++- 6 files changed, 244 insertions(+), 108 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 031ab1dc57..6d4b1f3a26 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -624,54 +624,88 @@ hsaKmtDbgAddressWatch( /** Suspend the execution of a set of queues. A queue that is suspended - allows the context save state to be inspected and modified. If a + allows the wave context save state to be inspected and modified. If a queue is already suspended it remains suspended. A suspended queue can be resumed by hsaKmtDbgQueueResume(). - If NoGracePeriod is false then the default grace period used for - waiting for waves to complete before context switching is used. If - NoGracePeriod is true then no grace period us used and waves are - context saved as soon as possible. - - If MemFence is true all queues being suspended will perform a - sequentially consistent system scope release that synchronizes with - a sequentially consistent system scope acquire performed by this + For each node that has a queue suspended, a sequentially consistent + system scope release will be performed that synchronizes with a + sequentially consistent system scope acquire performed by this call. This ensures any memory updates performed by the suspended queues are visible to the thread calling this operation. + Pid is the process that owns the queues that are to be supended or + resumed. If the value is -1 then the Pid of the process calling + hsaKmtQueueSuspend or hsaKmtQueueResume is used. + + NumQueues is the number of queues that are being requested to + suspend or resume. + + Queues is a pointer to an array with NumQueues entries of + HSA_QUEUEID. The queues in the list must be for queues that exist + for Pid, and can be a mixture of queues for different nodes. + + GracePeriod to wait after initialiating context save before forcing + waves to context save. A value of 0 indicates no grace period. + It is ignored by hsaKmtQueueResume. + + Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. Returns: - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. + - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid for Pid. */ + HSAKMT_STATUS - HSAKMTAPI - hsaKmtNodeSuspend( - HSAuint32 Pid, - HSAuint32 NodeId, - HSAuint32 Flags); +HSAKMTAPI +hsaKmtQueueSuspend( + HSAuint32 Pid, // IN + HSAuint32 NumQueues, // IN + HSA_QUEUEID *Queues, // IN + HSAuint32 GracePeriod, // IN + HSAuint32 Flags); // IN + /** Resume the execution of a set of queues. If a queue is not - suspended by hsaKmtDbgQueueSuspend() then it remains executing. + suspended by hsaKmtDbgQueueSuspend() then it remains executing. Any + changes to the wave state data will be used when the waves are + restored. Changes to the control stack data will have no effect. - If MemFence is true this call will perform a sequentially - consistent system scope release that synchronizes with a + For each node that has a queue resumed, a sequentially consistent + system scope release will be performed that synchronizes with a sequentially consistent system scope acquire performed by all queues being resumed. This ensures any memory updates performed by the thread calling this operation are visible to the resumed queues. - Returns: - - HSAKMT_STATUS_SUCCESS if successful. + For each node that has a queue resumed, the instruction cache will + be invalidated. This ensures any instruction code updates performed + by the thread calling this operation are visible to the resumed + queues. + Pid is the process that owns the queues that are to be supended or + resumed. If the value is -1 then the Pid of the process calling + hsaKmtQueueSuspend or hsaKmtQueueResume is used. + + NumQueues is the number of queues that are being requested to + suspend or resume. + + Queues is a pointer to an array with NumQueues entries of + HSA_QUEUEID. The queues in the list must be for queues that exist + for Pid, and can be a mixture of queues for different nodes. + + Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. + Returns: + - HSAKMT_STATUS_SUCCESS if successful - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. */ + HSAKMT_STATUS - HSAKMTAPI - hsaKmtNodeResume( - HSAuint32 Pid, - HSAuint32 NodeId, - HSAuint32 Flags); +HSAKMTAPI +hsaKmtQueueResume( + HSAuint32 Pid, // IN + HSAuint32 NumQueues, // IN + HSA_QUEUEID *Queues, // IN + HSAuint32 Flags); // IN /** Enable debug trap for NodeId. If QueueId is INVALID_QUEUEID then diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 8b0d232597..84292a70f0 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -804,10 +804,11 @@ typedef enum _HSA_DBG_WAVE_LAUNCH_MODE HSA_DBG_WAVE_LAUNCH_MODE_NUM } HSA_DBG_WAVE_LAUNCH_MODE; +/** + * There are no flags currently defined. + */ typedef enum HSA_DBG_NODE_CONTROL { - HSA_DBG_NODE_CONTROL_NO_GRACE_PERIOD = 0x01, - HSA_DBG_NODE_CONTROL_MEMORY_FENCE = 0x02, - HSA_DBG_NODE_CONTROL_UPDATE_CONTEXT = 0x04, + HSA_DBG_NODE_CONTROL_FLAG_MAX = 0x01 } HSA_DBG_NODE_CONTROL; diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 2eb86860da..177365d17b 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -188,64 +188,61 @@ struct kfd_ioctl_dbg_wave_control_args { }; /* KFD_IOC_DBG_TRAP_ENABLE: + * ptr: unused * data1: 0=disable, 1=enable * data2: queue ID (for future use) * data3: unused - * data4: unused */ #define KFD_IOC_DBG_TRAP_ENABLE 0 /* KFD_IOC_DBG_TRAP_SET_TRAP_DATA: + * ptr: unused * data1: SPI_GDBG_TRAP_DATA0 * data2: SPI_GDBG_TRAP_DATA1 * data3: unused - * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_TRAP_DATA 1 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE: + * ptr: unused * data1: override mode: 0=OR, 1=REPLACE * data2: mask * data3: unused - * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 2 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE: + * ptr: unused * data1: 0=normal, 1=halt, 2=kill, 3=singlestep, 4=disable * data2: unused * data3: unused - * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 3 - -#define KFD__DBG_NODE_SUSPEND_NO_GRACE 0x01 -#define KFD__DBG_NODE_SUSPEND_MEMORY_FENCE 0x02 -#define KFD__DBG_NODE_SUSPEND_UPDATE_CONTEXT 0x04 /* KFD_IOC_DBG_TRAP_NODE_SUSPEND: - * data1: pid - * data2: nodeid - * data3: flags no_grace=0x01 memory_fence=0x02 update_context=0x04 - * data4: unused + * ptr: pointer to an array of Queues IDs + * data1: flags + * data2: number of queues + * data3: grace period */ #define KFD_IOC_DBG_TRAP_NODE_SUSPEND 4 /* KFD_IOC_DBG_TRAP_NODE_RESUME: - * data1: pid - * data2: nodeid - * data3: flags no_grace=0x01 memory_fence=0x02 update_context=0x04 - * data4: unused + * ptr: pointer to an array of Queues IDs + * data1: flags + * data2: number of queues + * data3: unused */ #define KFD_IOC_DBG_TRAP_NODE_RESUME 5 struct kfd_ioctl_dbg_trap_args { - __u32 gpu_id; /* to KFD */ - __u32 op; /* to KFD */ - __u32 data1; /* to KFD */ - __u32 data2; /* to KFD */ - __u32 data3; /* to KFD */ - __u32 data4; /* to KFD */ + __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ + __u32 pid; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 op; /* to KFD */ + __u32 data1; /* to KFD */ + __u32 data2; /* to KFD */ + __u32 data3; /* to KFD */ }; /* Matching HSA_EVENTTYPE */ diff --git a/src/debug.c b/src/debug.c index 521aaaaf14..64e68891c6 100644 --- a/src/debug.c +++ b/src/debug.c @@ -273,7 +273,10 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, HSAuint32 op, HSAuint32 data1, HSAuint32 data2, - HSAuint32 data3) + HSAuint32 data3, + HSAuint32 pid, + HSAuint64 pointer + ) { uint32_t gpu_id; HSAKMT_STATUS result; @@ -282,16 +285,29 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, CHECK_KFD_OPEN(); - if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) - return HSAKMT_STATUS_INVALID_HANDLE; + if (op == KFD_IOC_DBG_TRAP_NODE_RESUME || + op == KFD_IOC_DBG_TRAP_NODE_RESUME) { + if (NodeId != INVALID_NODEID) + return HSAKMT_STATUS_INVALID_HANDLE; - result = hsaKmtGetNodeProperties(NodeId, &NodeProperties); + // gpu_id is ignored for suspend/resume queues. + gpu_id = INVALID_NODEID; + } else { + if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) + return HSAKMT_STATUS_INVALID_HANDLE; - if (result != HSAKMT_STATUS_SUCCESS) - return result; + result = hsaKmtGetNodeProperties(NodeId, &NodeProperties); - if (!NodeProperties.Capability.ui32.DebugTrapSupported) - return HSAKMT_STATUS_NOT_SUPPORTED; + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + if (!NodeProperties.Capability.ui32.DebugTrapSupported) + return HSAKMT_STATUS_NOT_SUPPORTED; + } + + if (pid == INVALID_PID) { + pid = (HSAuint32) getpid(); + } memset(&args, 0x00, sizeof(args)); args.gpu_id = gpu_id; @@ -299,7 +315,8 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, args.data1 = data1; args.data2 = data2; args.data3 = data3; - args.data4 = 0; + args.pid = pid; + args.ptr = pointer; long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); @@ -317,12 +334,24 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32 NodeId, if (QueueId != INVALID_QUEUEID) return HSAKMT_STATUS_NOT_SUPPORTED; - return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 1, QueueId, 0); + return debug_trap(NodeId, + KFD_IOC_DBG_TRAP_ENABLE, + 1, + QueueId, + 0, + INVALID_PID, + 0); } HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId) { - return debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 0, 0, 0); + return debug_trap(NodeId, + KFD_IOC_DBG_TRAP_ENABLE, + 0, + 0, + 0, + INVALID_PID, + 0); } HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, @@ -333,6 +362,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, KFD_IOC_DBG_TRAP_SET_TRAP_DATA, TrapData0, TrapData1, + 0, + INVALID_PID, 0); } @@ -348,6 +379,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE, TrapOverride, TrapMask, + 0, + INVALID_PID, 0); } @@ -359,51 +392,115 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode( KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE, WaveLaunchMode, 0, + 0, + INVALID_PID, 0); } -HSAKMT_STATUS HSAKMTAPI hsaKmtNodeSuspend( - HSAuint32 Pid, - HSAuint32 NodeId, - HSAuint32 Flags) -{ - pid_t current_pid = INVALID_PID; +/** + * Suspend the execution of a set of queues. A queue that is suspended + * allows the wave context save state to be inspected and modified. If a + * queue is already suspended it remains suspended. A suspended queue + * can be resumed by hsaKmtDbgQueueResume(). + * + * For each node that has a queue suspended, a sequentially consistent + * system scope release will be performed that synchronizes with a + * sequentially consistent system scope acquire performed by this + * call. This ensures any memory updates performed by the suspended + * queues are visible to the thread calling this operation. + * + * Pid is the process that owns the queues that are to be supended or + * resumed. If the value is -1 then the Pid of the process calling + * hsaKmtQueueSuspend or hsaKmtQueueResume is used. + * + * NumQueues is the number of queues that are being requested to + * suspend or resume. + * + * Queues is a pointer to an array with NumQueues entries of + * HSA_QUEUEID. The queues in the list must be for queues the exist + * for Pid, and can be a mixture of queues for different nodes. + * + * GracePeriod is the number of milliseconds to wait after + * initialiating context save before forcing waves to context save. A + * value of 0 indicates no grace period. It is ignored by + * hsaKmtQueueResume. + * + * Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. + * Returns: + * - HSAKMT_STATUS_SUCCESS if successful. + * - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid for Pid. + */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueueSuspend( + HSAuint32 Pid, // IN + HSAuint32 NumQueues, // IN + HSA_QUEUEID *Queues, // IN + HSAuint32 GracePeriod, // IN + HSAuint32 Flags) // IN +{ CHECK_KFD_OPEN(); - if (Pid == INVALID_PID) { - current_pid = getpid(); - if (current_pid == 0) - return HSAKMT_STATUS_INVALID_HANDLE; - Pid = (HSAuint32) current_pid; - } - - return debug_trap(NodeId, - KFD_IOC_DBG_TRAP_NODE_SUSPEND, - Pid, - NodeId, - Flags); -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtNodeResume( - HSAuint32 Pid, - HSAuint32 NodeId, - HSAuint32 Flags) -{ - pid_t current_pid = INVALID_PID; - - CHECK_KFD_OPEN(); - - if (Pid == INVALID_PID) { - current_pid = getpid(); - if (current_pid == 0) - return HSAKMT_STATUS_INVALID_HANDLE; - Pid = (HSAuint32) current_pid; - } - - return debug_trap(NodeId, + return debug_trap(INVALID_NODEID, KFD_IOC_DBG_TRAP_NODE_RESUME, + Flags, + NumQueues, + GracePeriod, Pid, - NodeId, - Flags); + (HSAuint64)Queues); +} +/** + * Resume the execution of a set of queues. If a queue is not + * suspended by hsaKmtDbgQueueSuspend() then it remains executing. Any + * changes to the wave state data will be used when the waves are + * restored. Changes to the control stack data will have no effect. + * + * For each node that has a queue resumed, a sequentially consistent + * system scope release will be performed that synchronizes with a + * sequentially consistent system scope acquire performed by all + * queues being resumed. This ensures any memory updates performed by + * the thread calling this operation are visible to the resumed + * queues. + * + * For each node that has a queue resumed, the instruction cache will + * be invalidated. This ensures any instruction code updates performed + * by the thread calling this operation are visible to the resumed + * queues. + * + * Pid is the process that owns the queues that are to be supended or + * resumed. If the value is -1 then the Pid of the process calling + * hsaKmtQueueSuspend or hsaKmtQueueResume is used. + * + * NumQueues is the number of queues that are being requested to + * suspend or resume. + * + * Queues is a pointer to an array with NumQueues entries of + * HSA_QUEUEID. The queues in the list must be for queues the exist + * for Pid, and can be a mixture of queues for different nodes. + * + * Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. + * Returns: + * - HSAKMT_STATUS_SUCCESS if successful + * - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. + */ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueueResume( + HSAuint32 Pid, // IN + HSAuint32 NumQueues, // IN + HSA_QUEUEID *Queues, // IN + HSAuint32 Flags) // IN +{ + + CHECK_KFD_OPEN(); + + return debug_trap(INVALID_NODEID, + KFD_IOC_DBG_TRAP_NODE_RESUME, + Flags, + NumQueues, + 0, + Pid, + (HSAuint64) Queues); } diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 12bb825f7f..8e9b745042 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -62,8 +62,8 @@ hsaKmtDisableDebugTrap; hsaKmtSetDebugTrapData2; hsaKmtSetWaveLaunchTrapOverride; hsaKmtSetWaveLaunchMode; -hsaKmtNodeSuspend; -hsaKmtNodeResume; +hsaKmtQueueSuspend; +hsaKmtQueueResume; local: *; }; diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 98a0e3db0d..e27868856b 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -208,7 +208,7 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HSAuint32 Flags = HSA_DBG_NODE_CONTROL_NO_GRACE_PERIOD; + HSAuint32 Flags = 0; HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer iterateBuf(PAGE_SIZE, defaultGPUNode, true, false, false); HsaMemoryBuffer resultBuf(PAGE_SIZE, defaultGPUNode, true, false, false); @@ -222,6 +222,7 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { m_pIsaGen->CompileShader(iterate_isa_gfx9, "iterate_isa", isaBuffer); PM4Queue queue1; + HSA_QUEUEID queue_ids[2]; ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); @@ -242,9 +243,13 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { // Submit the shader, queue1 dispatch1->Submit(queue1); + queue_ids[0] = 0; - ASSERT_SUCCESS(hsaKmtNodeSuspend(INVALID_PID, - defaultGPUNode, + ASSERT_SUCCESS(hsaKmtQueueSuspend( + INVALID_PID, + 1, // one queue + queue_ids, + 10, // grace period Flags)); syncStatus = dispatch1->SyncWithStatus(suspendTimeout); @@ -263,8 +268,10 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { ASSERT_NE(iter[0], result[0]); - ASSERT_SUCCESS(hsaKmtNodeResume(INVALID_PID, - defaultGPUNode, + ASSERT_SUCCESS(hsaKmtQueueResume( + INVALID_PID, + 1, // Num queues + queue_ids, Flags)); dispatch1->Sync(); From b0d23aee167b1667df54b0f36e1b68cb0dc72a8b Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Tue, 7 May 2019 06:56:00 -0400 Subject: [PATCH 0500/1247] fix suspend/resume logic in debug_trap code There was a mistake and RESUME was used when it should have been suspend in two places in the suspend resume code. This fixes that error. Change-Id: I69be733d7ae7c14ce5ee8af57a307976e4212d62 --- src/debug.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 64e68891c6..244c13dcaa 100644 --- a/src/debug.c +++ b/src/debug.c @@ -285,7 +285,7 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, CHECK_KFD_OPEN(); - if (op == KFD_IOC_DBG_TRAP_NODE_RESUME || + if (op == KFD_IOC_DBG_TRAP_NODE_SUSPEND || op == KFD_IOC_DBG_TRAP_NODE_RESUME) { if (NodeId != INVALID_NODEID) return HSAKMT_STATUS_INVALID_HANDLE; @@ -443,7 +443,7 @@ hsaKmtQueueSuspend( CHECK_KFD_OPEN(); return debug_trap(INVALID_NODEID, - KFD_IOC_DBG_TRAP_NODE_RESUME, + KFD_IOC_DBG_TRAP_NODE_SUSPEND, Flags, NumQueues, GracePeriod, From 8f10c9375d1606b4eb136864094813f2f9fc8177 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 7 May 2019 16:06:51 -0400 Subject: [PATCH 0501/1247] libhsakmt: Disable -Werror by default This can cause build failures on unknown of future compiler versions. Only enable it if explicitly enabled by an environment variable. This allows us to continue building with -Werror in internal builds with known compiler versions. Change-Id: Ic1cd9d223218cc4e4cddba49df93bb357c1cbd40 Signed-off-by: Felix Kuehling --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8b565beab..e020132ac1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,7 @@ set (HSAKMT_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-securi if ( CMAKE_COMPILER_IS_GNUCC ) set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -Wlogical-op) endif () -if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") +if ( ${HSAKMT_WERROR} ) set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -Werror ) endif () if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) From 78e4ef17c2bbab6e6cb0e2bb2e86f69dd05eed23 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 7 May 2019 23:22:12 -0500 Subject: [PATCH 0502/1247] Temporarily disable HostHdpFlush test Change-Id: I070cb3523a33b4efbfa7041fa2623059e1ff37bb Signed-off-by: Oak Zeng --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index d496c12538..dee66b6555 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -34,7 +34,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ "KFDQMTest.SdmaEventInterrupt:"\ -"KFDDBGTest.BasicDebuggerSuspendResume" +"KFDDBGTest.BasicDebuggerSuspendResume:"\ +"KFDMemoryTest.HostHdpFlush" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From 54e042eee1cef16b39e37eb2a99a6d67fc9f51fe Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 13 May 2019 09:03:06 -0400 Subject: [PATCH 0503/1247] Add missing gfx803 ID Change-Id: I9eca81f0f149ea924c3b81bd80680d7fd1ad7a6c --- src/topology.c | 1 + tests/kfdtest/scripts/run_kfdtest.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 606918d2bc..3ce5a83f70 100644 --- a/src/topology.c +++ b/src/topology.c @@ -164,6 +164,7 @@ static struct hsa_gfxip_table { { 0x67CF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67D0, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, { 0x67DF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x6FDF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, /* Polaris11 */ { 0x67E0, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, { 0x67E1, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 07f6c1ad94..2648e18b4d 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -112,7 +112,7 @@ deviceIdToGpuName() { platformName="tg" ;; 7300 | 730f) platformName="fj" ;; - 67c0 | 67c1 | 67c2 | 67c4 | 67c7 | 67c8 | 67c9 | 67ca | 67cc | 67cf | 67d0 | 67df ) + 67c0 | 67c1 | 67c2 | 67c4 | 67c7 | 67c8 | 67c9 | 67ca | 67cc | 67cf | 67d0 | 67df | 6fdf ) platformName="pl10" ;; 67e0 | 67e1 | 67e3 | 67e7 | 67e8 | 67e9 | 67eb | 67ef | 67ff ) platformName="pl11" ;; From 608bc7c3a0c926b6ad79edf1c152235dc070ff63 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Tue, 14 May 2019 09:37:50 -0400 Subject: [PATCH 0504/1247] Fix type mismatch passed to queue suspend/resume The queue IDs passed over to the kernel via kfd_ioctl_dbg_trap_args->ptr should be a list of uint32_t's. Need to convert from the passed in 64 bit HSA_QUEUEID to 32 bit uint32_t's. Change-Id: I8718566d9f9ffc90ce0b2ecc129b10c49d73186a Signed-off-by: Philip Cox --- src/debug.c | 26 ++++++++++++++++++++++---- src/libhsakmt.h | 1 + src/queues.c | 17 +++++++++++++++++ tests/kfdtest/src/KFDDBGTest.cpp | 4 +++- 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/debug.c b/src/debug.c index 244c13dcaa..3457da61de 100644 --- a/src/debug.c +++ b/src/debug.c @@ -440,15 +440,25 @@ hsaKmtQueueSuspend( HSAuint32 GracePeriod, // IN HSAuint32 Flags) // IN { + HSAKMT_STATUS result; + uint32_t *queue_ids_ptr; + CHECK_KFD_OPEN(); - return debug_trap(INVALID_NODEID, + queue_ids_ptr = convert_queue_ids(NumQueues, Queues); + if (!queue_ids_ptr) + return HSAKMT_STATUS_NO_MEMORY; + + result = debug_trap(INVALID_NODEID, KFD_IOC_DBG_TRAP_NODE_SUSPEND, Flags, NumQueues, GracePeriod, Pid, - (HSAuint64)Queues); + (HSAuint64)queue_ids_ptr); + + free(queue_ids_ptr); + return result; } /** * Resume the execution of a set of queues. If a queue is not @@ -493,14 +503,22 @@ hsaKmtQueueResume( HSA_QUEUEID *Queues, // IN HSAuint32 Flags) // IN { + HSAKMT_STATUS result; + uint32_t *queue_ids_ptr; CHECK_KFD_OPEN(); - return debug_trap(INVALID_NODEID, + queue_ids_ptr = convert_queue_ids(NumQueues, Queues); + if (!queue_ids_ptr) + return HSAKMT_STATUS_NO_MEMORY; + + result = debug_trap(INVALID_NODEID, KFD_IOC_DBG_TRAP_NODE_RESUME, Flags, NumQueues, 0, Pid, - (HSAuint64) Queues); + (HSAuint64)queue_ids_ptr); + free(queue_ids_ptr); + return result; } diff --git a/src/libhsakmt.h b/src/libhsakmt.h index d9eab68b51..91e6c21e72 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -136,6 +136,7 @@ HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes); void destroy_device_debugging_memory(void); HSAKMT_STATUS init_counter_props(unsigned int NumNodes); void destroy_counter_props(void); +uint32_t *convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues); extern int kmtIoctl(int fd, unsigned long request, void *arg); diff --git a/src/queues.c b/src/queues.c index 4472e6f63a..69b96ba6e8 100644 --- a/src/queues.c +++ b/src/queues.c @@ -800,3 +800,20 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetTrapHandler(HSAuint32 Node, return (err == -1) ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } +uint32_t *convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues) +{ + uint32_t *queue_ids_ptr; + unsigned int i; + + queue_ids_ptr = malloc(NumQueues * sizeof(uint32_t)); + if (!queue_ids_ptr) + return NULL; + + for (i = 0; i < NumQueues; i++) { + struct queue *q = PORT_UINT64_TO_VPTR(Queues[i]); + + queue_ids_ptr[i] = q->queue_id; + } + return queue_ids_ptr; +} + diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index e27868856b..a20db2b15b 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -222,6 +222,7 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { m_pIsaGen->CompileShader(iterate_isa_gfx9, "iterate_isa", isaBuffer); PM4Queue queue1; + HsaQueueResource *qResources; HSA_QUEUEID queue_ids[2]; ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); @@ -243,7 +244,8 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { // Submit the shader, queue1 dispatch1->Submit(queue1); - queue_ids[0] = 0; + qResources = queue1.GetResource(); + queue_ids[0] = qResources->QueueId; ASSERT_SUCCESS(hsaKmtQueueSuspend( INVALID_PID, From 64b90261d94ce112b89afe558bf1092dc135e4d4 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 17 May 2019 18:25:10 -0400 Subject: [PATCH 0505/1247] libhsakmt: Enable invisible debug VRAM mappings by default Remove the HSA_DEBUG environment variable that controlled the creation of these mappings. This should allow the debugger to attach to a running process and access VRAM buffers through ptrace without having to do anything special. On processes that create many small VRAM mappings, this may cause regressions due to the per-process mmap limit. However, the sub-allocator in ROCr should consolidate most small allocations into 2MB blocks nowadays, for good TLB efficiency. So this is unlikely to cause problems. Change-Id: I929da1be0f6cb51ec00a02f3f241d16083e4d95f Signed-off-by: Felix Kuehling --- src/fmm.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index fc2b50d394..64677804b7 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -221,7 +221,6 @@ static gpu_mem_t *gpu_mem; static unsigned int gpu_mem_count; static gpu_mem_t *g_first_gpu_mem; -static bool hsa_debug; static void *dgpu_shared_aperture_base; static void *dgpu_shared_aperture_limit; @@ -1351,7 +1350,7 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB pthread_mutex_unlock(&aperture->fmm_mutex); } - if (mem && (flags.ui32.HostAccess || hsa_debug)) { + if (mem) { int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_mem[gpu_mem_id].drm_render_fd; int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : @@ -2055,15 +2054,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; - char *hsaDebug; unsigned int guardPages = 1; struct pci_access *pacc; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; - hsaDebug = getenv("HSA_DEBUG"); - hsa_debug = hsaDebug && strcmp(hsaDebug, "0"); - /* If HSA_DISABLE_CACHE is set to a non-0 value, disable caching */ disableCache = getenv("HSA_DISABLE_CACHE"); svm.disable_cache = (disableCache && strcmp(disableCache, "0")); From d278b2579e1a5b0744bbee3816e74f9f13689e2f Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 29 May 2019 11:13:41 -0400 Subject: [PATCH 0506/1247] kfdtest: fix debugfs path bug in RAS test The path was wrong based on assumption that GPU dri render node starts from 0, because if there is a VGA device on board, node 0 will be VGA and node 1 will be GPU. So the fix will look at the name of GPU minor node and find the correct primary node on which RAS debugfs entry exists. Change-Id: Icc5e63ce48698d5d29105c0417e3bec8afa0a7c8 Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDRASTest.cpp | 40 +++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/tests/kfdtest/src/KFDRASTest.cpp index 2fb96094f8..697726129b 100644 --- a/tests/kfdtest/src/KFDRASTest.cpp +++ b/tests/kfdtest/src/KFDRASTest.cpp @@ -30,14 +30,16 @@ #define AMDGPU_DEBUGFS_NODES "/sys/kernel/debug/dri/" #define RAS_CONTROL "ras/ras_ctrl" +#define DRM_RENDER_NUMBER 64 void KFDRASTest::SetUp() { ROUTINE_START KFDBaseComponentTest::SetUp(); - char path[256]; - int renderNode; + char path[256], name[128], tmp[128]; + int renderNode, minor, i; + FILE *pDriMinor, *pDriPrimary; uint32_t rasFeatures = 0; HsaEventDescriptor eventDesc; @@ -63,8 +65,40 @@ void KFDRASTest::SetUp() { throw; } - snprintf(path, sizeof(path), "%s/%d/%s", AMDGPU_DEBUGFS_NODES, renderNode, RAS_CONTROL); + minor = renderNode + 128; + snprintf(path, sizeof(path), "%s%d/%s", AMDGPU_DEBUGFS_NODES, minor, "name"); + pDriMinor = fopen(path, "r"); + if (!pDriMinor) { + LOG() << "Skipping test: DRM render debugfs node requires root access!" << std::endl; + throw; + } + + memset(name, 0, sizeof(name)); + fread(name, sizeof(name), 1, pDriMinor); + + fclose(pDriMinor); + + for (i = 0; i < DRM_RENDER_NUMBER; i++) { + snprintf(path, sizeof(path), "%s%d/%s", AMDGPU_DEBUGFS_NODES, i, "name"); + pDriPrimary = fopen(path, "r"); + if (!pDriPrimary) + continue; + memset(tmp, 0, sizeof(tmp)); + fread(tmp, sizeof(tmp), 1, pDriPrimary); + if (!strcmp(name, tmp)) { + fclose(pDriPrimary); + break; + } + fclose(pDriPrimary); + } + + if (i == DRM_RENDER_NUMBER) { + LOG() << "Skipping test: Could not find the debugfs node!" << std::endl; + throw; + } + + snprintf(path, sizeof(path), "%s%d/%s", AMDGPU_DEBUGFS_NODES, i, RAS_CONTROL); m_pFile = fopen(path, "w"); if (!m_pFile) { LOG() << "Skipping test: RAS error injection requires root access!" << std::endl; From 47d1c1759216c11b14fefe3806893b6f21bb4cee Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 29 May 2019 11:41:30 -0400 Subject: [PATCH 0507/1247] kfdtest: fix error injection failure in RAS test 1. umc error injection only accepts parameter "0 0". 2. flush output to file in order to make writing happen immediately. Change-Id: I8d3bde287caee6b90b6eec56c760f5a228be7595 Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDRASTest.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/tests/kfdtest/src/KFDRASTest.cpp index 697726129b..09d61a03fa 100644 --- a/tests/kfdtest/src/KFDRASTest.cpp +++ b/tests/kfdtest/src/KFDRASTest.cpp @@ -138,8 +138,9 @@ TEST_F(KFDRASTest, BasicTest) { return; } - // write an uncorrectable error injection at address 1 as value 1 - fwrite("inject umc ue 1 1", sizeof(char), 17, m_pFile); + // write an uncorrectable error injection at address 0 as value 0 + fwrite("inject umc ue 0 0", sizeof(char), 17, m_pFile); + fflush(m_pFile); EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pRasEvent, g_TestTimeOut)); @@ -170,7 +171,8 @@ TEST_F(KFDRASTest, MixEventsTest) { EXPECT_SUCCESS(hsaKmtWaitOnEvent(pHsaEvent, g_TestTimeOut)); - fwrite("inject umc ue 1 1", sizeof(char), 17, m_pFile); + fwrite("inject umc ue 0 0", sizeof(char), 17, m_pFile); + fflush(m_pFile); EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pRasEvent, g_TestTimeOut)); From f5a094bc96046f0013f8ecadf52ef7ef6078dc33 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 28 May 2019 15:05:41 -0400 Subject: [PATCH 0508/1247] libhsakmt: Update kfd_ioctl.h Change-Id: Ibf165023b98787fdf295f50324e19aa062f2421d Signed-off-by: Felix Kuehling --- include/linux/kfd_ioctl.h | 62 +++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 177365d17b..fe355090ee 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -236,13 +236,13 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_IOC_DBG_TRAP_NODE_RESUME 5 struct kfd_ioctl_dbg_trap_args { - __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ - __u32 pid; /* to KFD */ - __u32 gpu_id; /* to KFD */ - __u32 op; /* to KFD */ - __u32 data1; /* to KFD */ - __u32 data2; /* to KFD */ - __u32 data3; /* to KFD */ + __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ + __u32 pid; /* to KFD */ + __u32 gpu_id; /* to KFD */ + __u32 op; /* to KFD */ + __u32 data1; /* to KFD */ + __u32 data2; /* to KFD */ + __u32 data3; /* to KFD */ }; /* Matching HSA_EVENTTYPE */ @@ -435,14 +435,6 @@ struct kfd_ioctl_free_memory_of_gpu_args { __u64 handle; /* to KFD */ }; -/* Register offset inside the remapped mmio page - */ -enum kfd_mmio_remap { - KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0, - KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4, -}; - - /* Map memory to one or more GPUs * * @handle: memory handle returned by alloc @@ -476,11 +468,27 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { __u32 n_success; /* to/from KFD */ }; +/* Allocate GWS for specific queue + * + * @gpu_id: device identifier + * @queue_id: queue's id that GWS is allocated for + * @num_gws: how many GWS to allocate + * @first_gws: index of the first GWS allocated. + * only support contiguous GWS allocation + */ +struct kfd_ioctl_alloc_queue_gws_args { + __u32 gpu_id; /* to KFD */ + __u32 queue_id; /* to KFD */ + __u32 num_gws; /* to KFD */ + __u32 first_gws; /* from KFD */ +}; + struct kfd_ioctl_get_dmabuf_info_args { __u64 size; /* from KFD */ __u64 metadata_ptr; /* to KFD */ __u32 metadata_size; /* to KFD (space allocated by user) - * from KFD (actual metadata size) */ + * from KFD (actual metadata size) + */ __u32 gpu_id; /* from KFD */ __u32 flags; /* from KFD (KFD_IOC_ALLOC_MEM_FLAGS) */ __u32 dmabuf_fd; /* to KFD */ @@ -493,6 +501,13 @@ struct kfd_ioctl_import_dmabuf_args { __u32 dmabuf_fd; /* to KFD */ }; +/* Register offset inside the remapped mmio page + */ +enum kfd_mmio_remap { + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0, + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4, +}; + struct kfd_ioctl_ipc_export_handle_args { __u64 handle; /* to KFD */ __u32 share_handle[4]; /* from KFD */ @@ -634,18 +649,21 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_IMPORT_DMABUF \ AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args) -#define AMDKFD_IOC_IPC_IMPORT_HANDLE \ - AMDKFD_IOWR(0x1E, struct kfd_ioctl_ipc_import_handle_args) +#define AMDKFD_IOC_ALLOC_QUEUE_GWS \ + AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) + +#define AMDKFD_IOC_IPC_IMPORT_HANDLE \ + AMDKFD_IOWR(0x1F, struct kfd_ioctl_ipc_import_handle_args) #define AMDKFD_IOC_IPC_EXPORT_HANDLE \ - AMDKFD_IOWR(0x1F, struct kfd_ioctl_ipc_export_handle_args) - -#define AMDKFD_IOC_CROSS_MEMORY_COPY \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_cross_memory_copy_args) + AMDKFD_IOWR(0x20, struct kfd_ioctl_ipc_export_handle_args) #define AMDKFD_IOC_DBG_TRAP \ AMDKFD_IOW(0x21, struct kfd_ioctl_dbg_trap_args) +#define AMDKFD_IOC_CROSS_MEMORY_COPY \ + AMDKFD_IOWR(0x22, struct kfd_ioctl_cross_memory_copy_args) + #define AMDKFD_COMMAND_START 0x01 #define AMDKFD_COMMAND_END 0x22 From 6f5379d315d7c0517ff9aacde314546980c9798a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 29 May 2019 00:28:51 -0400 Subject: [PATCH 0509/1247] kfdtest: Pass timeout parameter to BaseQueue::Wait4PacketConsumption Change-Id: I0e88db5ca8e6712e9efc419a10eb4c49cedb6f62 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/PM4Queue.cpp | 2 +- tests/kfdtest/src/SDMAQueue.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/PM4Queue.cpp b/tests/kfdtest/src/PM4Queue.cpp index 74c1035d01..548e35986c 100644 --- a/tests/kfdtest/src/PM4Queue.cpp +++ b/tests/kfdtest/src/PM4Queue.cpp @@ -80,6 +80,6 @@ void PM4Queue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, timeOut)); } else { - BaseQueue::Wait4PacketConsumption(); + BaseQueue::Wait4PacketConsumption(NULL, timeOut); } } diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/tests/kfdtest/src/SDMAQueue.cpp index facbeee60d..a6b41ff31a 100644 --- a/tests/kfdtest/src/SDMAQueue.cpp +++ b/tests/kfdtest/src/SDMAQueue.cpp @@ -88,6 +88,6 @@ void SDMAQueue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, timeOut)); } else { - BaseQueue::Wait4PacketConsumption(); + BaseQueue::Wait4PacketConsumption(NULL, timeOut); } } From 6984f3e3b4c6c333781f9ea5e71e2ca11dba6862 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 17 May 2019 18:21:18 -0400 Subject: [PATCH 0510/1247] kfdtest: Add KFDEvictionTest.BurstyTest Change-Id: I748603b0b204ffc3ea33399ecbc022233a7447d3 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/KFDEvictTest.cpp | 73 +++++++++++++++++++++++++++++- tests/kfdtest/src/KFDEvictTest.hpp | 4 +- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 3a3ab03f27..2de8b12358 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -176,7 +176,8 @@ static inline int amdgpu_get_bo_list(amdgpu_device_handle dev, amdgpu_bo_handle return amdgpu_bo_list_create(dev, bo2 ? 2 : 1, resources, NULL, list); } -void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle) { +void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle, + PM4Queue *computeQueue = NULL) { amdgpu_context_handle contextHandle; amdgpu_bo_handle ibResultHandle; void *ibResultCpu; @@ -233,6 +234,15 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle ha 0, &expired)); if (!expired) WARN() << "CS did not signal completion" << std::endl; + + /* If a compute queue is given, submit a short compute job + * every 16 loops (about once a second). If the process was + * evicted, restore can take quite long. + */ + if (computeQueue && (i & 0xf) == 0) { + computeQueue->PlaceAndSubmitPacket(PM4NopPacket()); + computeQueue->Wait4PacketConsumption(NULL, 10000); + } } EXPECT_EQ(0, amdgpu_bo_list_destroy(boList)); @@ -622,3 +632,64 @@ TEST_F(KFDEvictTest, QueueTest) { TEST_END } +/* Evict a queue running in bursts, so that the process has a chance + * to be idle when restored but the queue needs to resume to perform + * more work later. This test is designed to stress the idle process + * eviction optimization in KFD that leaves idle processes evicted + * until the next time the doorbell page is accessed. + */ +TEST_F(KFDEvictTest, BurstyTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + + if (!vramSize) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } else { + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; + } + + // Use 7/8 of VRAM between all processes + HSAuint32 count = vramSize * 7 / (8* vramBufSize * N_PROCESSES); + + LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; + + /* Fork the child processes */ + ForkChildProcesses(N_PROCESSES); + + int rn = FindDRMRenderNode(defaultGPUNode); + if (rn < 0) { + LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; + WaitChildProcesses(); + return; + } + + PM4Queue pm4Queue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + + std::vector pBuffers; + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + + /* Allocate gfx vram size of at most one third system memory */ + HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; + amdgpu_bo_handle handle; + AllocAmdgpuBo(rn, size, handle); + + AmdgpuCommandSubmissionComputeNop(rn, handle, &pm4Queue); + + FreeAmdgpuBo(handle); + LOG() << m_psName << "free buffer" << std::endl; + FreeBuffers(pBuffers, vramBufSize); + + EXPECT_SUCCESS(pm4Queue.Destroy()); + + WaitChildProcesses(); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index e6d8953068..a602980fbe 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -29,6 +29,7 @@ #include "KFDLocalMemoryTest.hpp" #include "KFDBaseComponentTest.hpp" #include "IsaGenerator.hpp" +#include "PM4Queue.hpp" // @class KFDEvictTest // Test eviction and restore procedure using two processes @@ -56,7 +57,8 @@ class KFDEvictTest : public KFDLocalMemoryTest { void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); void AllocAmdgpuBo(int rn, HSAuint64 vramBufSize, amdgpu_bo_handle &handle); void FreeAmdgpuBo(amdgpu_bo_handle handle); - void AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle); + void AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle, + PM4Queue *computeQueue); void ForkChildProcesses(int nprocesses); void WaitChildProcesses(); From f474cf21cdca29f3c18d7de72d9688b7e50a5e7b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 29 May 2019 00:15:13 -0400 Subject: [PATCH 0511/1247] kfdtest: Reduce libdrm VRAM usage in eviction tests This reduces thrashing due to graphics submissions only and significantly speeds up the BasicTest when keeping idle compute processes evicted. In the BasicTest compute is always idle, so only one compute eviction and no restore is triggered. Then graphics submissions complete quickly without thrashing each other. Change-Id: Iae6da98903b20424a5097f235e1d09cf13e4b41b Signed-off-by: Felix Kuehling --- tests/kfdtest/src/KFDEvictTest.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 2de8b12358..5ad8dea59a 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -334,7 +334,7 @@ TEST_F(KFDEvictTest, BasicTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; - HSAuint64 vramSize = GetVramSize(defaultGPUNode); + HSAuint64 vramSize = GetVramSize(defaultGPUNode) * 7 / 8; if (!vramSize) { LOG() << "Skipping test: No VRAM found." << std::endl; @@ -344,7 +344,7 @@ TEST_F(KFDEvictTest, BasicTest) { } // Use 7/8 of VRAM between all processes - HSAuint32 count = vramSize * 7 / (8* vramBufSize * N_PROCESSES); + HSAuint32 count = vramSize / (vramBufSize * N_PROCESSES); LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; @@ -537,7 +537,7 @@ TEST_F(KFDEvictTest, QueueTest) { } HSAuint32 i; - HSAuint64 vramSize = GetVramSize(defaultGPUNode); + HSAuint64 vramSize = GetVramSize(defaultGPUNode) * 7 / 8; if (!vramSize) { LOG() << "Skipping test: No VRAM found." << std::endl; @@ -547,7 +547,7 @@ TEST_F(KFDEvictTest, QueueTest) { } // Use 7/8 of VRAM between all processes - HSAuint32 count = vramSize * 7 / (8 * vramBufSize * N_PROCESSES); + HSAuint32 count = vramSize / (vramBufSize * N_PROCESSES); LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; @@ -646,7 +646,7 @@ TEST_F(KFDEvictTest, BurstyTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; - HSAuint64 vramSize = GetVramSize(defaultGPUNode); + HSAuint64 vramSize = GetVramSize(defaultGPUNode) * 7 / 8; if (!vramSize) { LOG() << "Skipping test: No VRAM found." << std::endl; @@ -656,7 +656,7 @@ TEST_F(KFDEvictTest, BurstyTest) { } // Use 7/8 of VRAM between all processes - HSAuint32 count = vramSize * 7 / (8* vramBufSize * N_PROCESSES); + HSAuint32 count = vramSize / (vramBufSize * N_PROCESSES); LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; From 396a85e97b665f0d345444937149232f90988183 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 29 May 2019 00:23:30 -0400 Subject: [PATCH 0512/1247] kfdtest: Allocate PM4 queue and dispatch earlier KFDEvictTest.QueueTest Allocating these before the big memory allocations minimizes the chances of spurious out of memory errors. Change-Id: I94aff9ec7ea34d4dc98ae08ac4cf9dc335b3df7f Signed-off-by: Felix Kuehling --- tests/kfdtest/src/KFDEvictTest.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 5ad8dea59a..cdc6c792b0 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -574,6 +574,13 @@ TEST_F(KFDEvictTest, QueueTest) { HsaMemoryBuffer addrBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer resultBuffer(PAGE_SIZE, defaultGPUNode); + m_pIsaGen->CompileShader(CreateShader().c_str(), "ReadMemory", isaBuffer); + + PM4Queue pm4Queue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + + Dispatch dispatch0(isaBuffer); + std::vector pBuffers; AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); @@ -591,12 +598,6 @@ TEST_F(KFDEvictTest, QueueTest) { for (i = 0; i < wavefront_num; i++) *(localBufAddr + i) = pBuffers[i]; - m_pIsaGen->CompileShader(CreateShader().c_str(), "ReadMemory", isaBuffer); - - PM4Queue pm4Queue; - ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); - - Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(localBufAddr, result); dispatch0.SetDim(wavefront_num, 1, 1); /* Submit the packet and start shader */ From 45d717d860de903befbaf8b0fd777ff850f28f72 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Fri, 10 May 2019 12:08:07 -0500 Subject: [PATCH 0513/1247] Add node property to report number of GWS Change-Id: I81263ca7ebfa3c0f9f1be78acfa0920e47d551b1 Signed-off-by: Oak Zeng --- include/hsakmttypes.h | 3 ++- src/topology.c | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 84292a70f0..cebf5ac699 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -301,7 +301,8 @@ typedef struct _HsaNodeProperties // number hash created by the PSP HSAuint32 NumSdmaEngines; // number of PCIe optimized SDMA engines HSAuint32 NumSdmaXgmiEngines;// number of XGMI optimized SDMA engines - HSAuint8 Reserved[36]; + HSAuint32 NumGws; // number of GWS barriers + HSAuint8 Reserved[32]; } HsaNodeProperties; diff --git a/src/topology.c b/src/topology.c index 3ce5a83f70..6fde6e588b 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1005,6 +1005,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->NumSdmaEngines = prop_val; else if (strcmp(prop_name, "num_sdma_xgmi_engines") == 0) props->NumSdmaXgmiEngines = prop_val; + else if (strcmp(prop_name, "num_gws") == 0) + props->NumGws = prop_val; } hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); From 65d554f5e4495d52f1f68debff68af681db22d6e Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Fri, 10 May 2019 12:38:12 -0500 Subject: [PATCH 0514/1247] Thunk API to allocate queue GWS Change-Id: I6c5b109e2567cb71aed9245923cfcbeee6295ab2 Signed-off-by: Oak Zeng --- include/hsakmt.h | 12 ++++++++++++ src/libhsakmt.ver | 1 + src/queues.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index 6d4b1f3a26..6d666a931c 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -571,6 +571,18 @@ hsaKmtUnmapGraphicHandle( HSAuint64 SizeInBytes //IN ); +/** + Allocate GWS resource for a queue + */ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtAllocQueueGWS( + HSAuint32 NodeId, //IN + HSA_QUEUEID QueueId, //IN + HSAuint32 nGWS, //IN + HSAuint32 *firstGWS //OUT + ); /** Notifies the kernel driver that a process wants to use GPU debugging facilities diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 8e9b745042..54d79b440d 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -64,6 +64,7 @@ hsaKmtSetWaveLaunchTrapOverride; hsaKmtSetWaveLaunchMode; hsaKmtQueueSuspend; hsaKmtQueueResume; +hsaKmtAllocQueueGWS; local: *; }; diff --git a/src/queues.c b/src/queues.c index 69b96ba6e8..f967805c22 100644 --- a/src/queues.c +++ b/src/queues.c @@ -34,6 +34,7 @@ #include #include #include +#include /* 1024 doorbells, 4 or 8 bytes each doorbell depending on ASIC generation */ #define DOORBELL_SIZE_GFX7 4 @@ -817,3 +818,44 @@ uint32_t *convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues) return queue_ids_ptr; } +HSAKMT_STATUS +HSAKMTAPI +hsaKmtAllocQueueGWS( + HSAuint32 NodeId, + HSA_QUEUEID QueueId, + HSAuint32 nGWS, + HSAuint32 *firstGWS) +{ + struct kfd_ioctl_alloc_queue_gws_args args = {0}; + struct queue *q = PORT_UINT64_TO_VPTR(QueueId); + HSAKMT_STATUS result; + uint32_t gpu_id; + + CHECK_KFD_OPEN(); + + result = validate_nodeid(NodeId, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); + return result; + } + + args.gpu_id = gpu_id; + args.queue_id = (HSAuint32)q->queue_id; + args.num_gws = nGWS; + + int err = kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_QUEUE_GWS, &args); + + if (!err && firstGWS) + *firstGWS = args.first_gws; + + if (!err) + return HSAKMT_STATUS_SUCCESS; + else if (err == -EINVAL) + return HSAKMT_STATUS_INVALID_PARAMETER; + else if (err == -EBUSY) + return HSAKMT_STATUS_OUT_OF_RESOURCES; + else if (err == -ENODEV) + return HSAKMT_STATUS_NOT_SUPPORTED; + else + return HSAKMT_STATUS_ERROR; +} From 888e1a7ae71c84ab117688d5b62ec29e81afab20 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 23 May 2019 17:09:57 -0500 Subject: [PATCH 0515/1247] Use kfd fd to mmap mmio Change-Id: Iadd2e1ea46d0951aaa5a6cefbc7d42d1b2c1f653 Signed-off-by: Oak Zeng --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 64677804b7..aaad772101 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2292,7 +2292,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_id].mmio_aperture.base = map_mmio( gpu_mem[gpu_mem_id].node_id, gpu_mem[gpu_mem_id].gpu_id, - gpu_mem[gpu_mem_id].drm_render_fd); + kfd_fd); if (gpu_mem[gpu_mem_id].mmio_aperture.base) gpu_mem[gpu_mem_id].mmio_aperture.limit = (void *) ((char *)gpu_mem[gpu_mem_id].mmio_aperture.base + From fe4db3387564652728d199ead5f7a17dfbcda27e Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Mon, 10 Jun 2019 11:42:26 -0400 Subject: [PATCH 0516/1247] Remove passing blacklisted kfd tests This relates to the following commits: 1. commit aa7c13264a72ef8b337b40a59a8b78cf727c6a30 2. commit 54807526b94a8c7751e94ca64236b1558691b56d 3. commit 6df62c78b829a1469fe6dcbbb7e40861c200e5a9 Change-Id: I3d0d3214baba403b4709b358132b6756a15f42d7 Signed-off-by: Ori Messinger --- tests/kfdtest/scripts/kfdtest.exclude | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index dee66b6555..5a5bc5fcee 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -41,17 +41,14 @@ BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ "$TEMPORARY_BLACKLIST_ALL_ASICS" -# SDMA-based tests (KFDIPCTest.BasicTest, KFDQM.*Sdma*, KFDMemoryTest.MMBench) are all +# SDMA-based tests (KFDIPCTest.BasicTest, KFDQMTest.SdmaEventInterrupt) are both # disabled on non-Hawaii due to SDMA instability - SWDEV-101666 +# KFDLocalMemoryTest.CheckZeroInitializationVram Passes, but it +# takes nearly 4 min (229385 ms) to run so it remains blacklisted SDMA_BLACKLIST=\ "KFDIPCTest.*:"\ "KFDLocalMemoryTest.CheckZeroInitializationVram:"\ -"KFDMemoryTest.MemoryRegister:"\ -"KFDMemoryTest.MMBench:"\ -"KFDMemoryTest.SignalHandling:"\ -"KFDQMTest.*Sdma*:"\ -"KFDQMTest.CreateQueueStressSingleThreaded:"\ -"KFDQMTest.GPUDoorbellWrite:"\ +"KFDQMTest.SdmaEventInterrupt:"\ "KFDQMTest.P2PTest:"\ "KFDPerformanceTest.P2PBandWidthTest:"\ "KFDPerformanceTest.P2POverheadTest" @@ -141,9 +138,7 @@ VEGA20_TESTS_BLACKLIST=\ "KFDDBGTest.BasicAddressWatch:"\ "KFDExceptionTest.InvalidBadAddress:"\ "KFDExceptionTest.InvalidWriteAddress:"\ -"KFDQMTest.CreateQueueStressSingleThreaded:"\ -"KFDQMTest.GPUDoorbellWrite:"\ -"KFDMemoryTest.MemoryRegister" +"KFDQMTest.GPUDoorbellWrite:" RAVEN_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ From 3f2d2e67c9631a4d4420a0415007cc7745333e44 Mon Sep 17 00:00:00 2001 From: Cole Nelson Date: Thu, 13 Jun 2019 10:47:23 -0700 Subject: [PATCH 0517/1247] kfdtest: Blacklist multiple tests on gfx900/20 PSDB and other jenkins jobs are currently failing on several kfd tests. This is blocking user throughput for screening patches by PSDB. Blacklist multiple tests and submit JIRA's. KFDIPCTest.BasicTest (ROCMOPS-459) .CMABasicTest (ROCMOPS-460) .CrossMemoryAttachTest (ROCMOPS-461) KFDMemoryTest.BigBufferStressTest (ROCMOPS-462) KFDQMTest.MultipleSdmaQueues (ROCMOPS-463) (ROCMOPS-416) KFDEvictTest.BurstyTest (ROCMOPS-464) Change-Id: I2c7cdeabc26654f39823201ce86d4113b3a98a0e Signed-off-by: Cole Nelson --- tests/kfdtest/scripts/kfdtest.exclude | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 5a5bc5fcee..0042c568f6 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -123,22 +123,34 @@ LEXA_TESTS_BLACKLIST=\ "$SDMA_BLACKLIST" # KFDDBG.BasicAddressWatch is not supported on GFXv9 and later. FIXME: it should disable itself +# KFDIPCTest.BasicTest (ROCMOPS-459) .CMABasicTest (ROCMOPS-460) .CrossMemoryAttachTest (ROCMOPS-461) +# KFDMemoryTest.BigBufferStressTest (ROCMOPS-462) +# KFDEvictTest.BurstyTest (ROCMOPS-464) +# KFDQMTest.MultipleSdmaQueues (ROCMOPS-463) VEGA10_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDDBGTest.BasicAddressWatch" +"KFDDBGTest.BasicAddressWatch:"\ +"KFDIPCTest.BasicTest:"\ +"KFDEvictTest.BurstyTest:"\ +"KFDIPCTest.CMABasicTest:"\ +"KFDIPCTest.CrossMemoryAttachTest:"\ +"KFDMemoryTest.BigBufferStressTest:"\ +"KFDQMTest.MultipleSdmaQueues" VEGA12_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDDBGTest.BasicAddressWatch" +# KFDEvictTest.BurstyTest (ROCMOPS-464) VEGA20_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDDBGTest.BasicAddressWatch:"\ +"KFDEvictTest.BurstyTest:"\ "KFDExceptionTest.InvalidBadAddress:"\ "KFDExceptionTest.InvalidWriteAddress:"\ -"KFDQMTest.GPUDoorbellWrite:" +"KFDQMTest.GPUDoorbellWrite" RAVEN_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ From 36776e99176cf71356b6ea86f44781dab96834af Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 13 Jun 2019 15:22:52 -0400 Subject: [PATCH 0518/1247] kfdtest: avoid BigBufStressTest run on NUMA node 0 Because dma32 zone is on node 0, use all system memory on node 0 will cause TTM eviction to free dma32 zone for other devices which only work with 32bit physical address. The TTM eviction and restore may take too long and cause queue timeout. Running on other NUMA nodes, the NUMA default memory policy is MPOL_PREFERRED, means TTM will get pages from local node first, and then get remaining pages from other nodes. Check /proc/buddyinfo can confirm this. Reset NUMA bind to all after the test. Change-Id: I39b373c07a2d5aa396f5c7602bffabab0481930f Signed-off-by: Philip Yang --- tests/kfdtest/CMakeLists.txt | 2 +- tests/kfdtest/src/KFDMemoryTest.cpp | 29 +++++++++++++++++++++++++++++ tests/kfdtest/src/KFDMemoryTest.hpp | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index b79f880a74..523d70356f 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -104,7 +104,7 @@ link_directories(${SP3_DIR}) add_executable(kfdtest ${SRC_FILES}) -target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LIBRARIES} ${DRM_AMDGPU_LIBRARIES} pthread m stdc++ rt amdsp3) +target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LIBRARIES} ${DRM_AMDGPU_LIBRARIES} pthread m stdc++ rt amdsp3 numa) configure_file ( scripts/kfdtest.exclude kfdtest.exclude COPYONLY ) configure_file ( scripts/run_kfdtest.sh run_kfdtest.sh COPYONLY ) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 1328047b73..bc076f73f1 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "Dispatch.hpp" #include "PM4Queue.hpp" @@ -809,6 +810,24 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, << vramSizeMB * 15 / 16 << "MB" << std::endl; } +void KFDMemoryTest::NumaNodeBind(const char *nodeStr) { + if (numa_available() != -1) { + int num_node = numa_num_task_nodes(); + + if (num_node > 1) { + struct bitmask *nodemask; + + LOG() << "NUMA total nodes " << num_node << ", bind to " << nodeStr << std::endl; + + nodemask = numa_parse_nodestring(nodeStr); + if (nodemask) { + numa_bind(nodemask); + numa_free_nodemask(nodemask); + } + } + } +} + /* BigBufferStressTest allocs, maps/unmaps, and frees the biggest possible system * buffers. Its size is found using binary search in the range (0, RAM SIZE) with * a granularity of 128M. Repeat the similar logic on local buffers (VRAM). @@ -839,6 +858,13 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + /* Don't run on node 0 on multiple NUMA node machine because dma32 zone is on node 0, + * Use all memory including dma32 zone on node 0 will cause TTM eviction to free dma32 + * zone for other devices which supports 32bit physical address. The eviction and + * restore may retry if busy and cause queue timeout and test failure. + */ + NumaNodeBind("!0"); + BigBufferSystemMemory(defaultGPUNode, granularityMB, NULL); BigBufferVRAM(defaultGPUNode, granularityMB, NULL); @@ -893,6 +919,9 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { } EXPECT_SUCCESS(queue.Destroy()); + /* Reset to run on all task nodes */ + NumaNodeBind("all"); + TEST_END } diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp index 947968999e..8f1f78b9ac 100644 --- a/tests/kfdtest/src/KFDMemoryTest.hpp +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -44,6 +44,7 @@ class KFDMemoryTest : public KFDBaseComponentTest { void BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); void BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); + void NumaNodeBind(const char *nodeStr); }; #endif // __KFD_MEMORY_TEST__H__ From 4066dcd542ac4ec7f49359fb1e61c61388e172d5 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 13 Jun 2019 15:46:05 -0400 Subject: [PATCH 0519/1247] kfdtest: increase BigBufStressTest timeout and avoid VM fault If TTM eviction and restore happens, it may takes very long time if retry, the longest time is 5 minutes during my test. There is chance packet is submited to queue while eviction, we have to increase the Wait4PacketConsumption timeout. The queue will continue to execute after eviction and restore. If we upmap the memory from GPU while queue is evicted, this will cause VM fault. Change to unmap memory after queue is destroyed. Change-Id: I1b44e2274ea7b83398b2e3293578dad6947cb5af Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDMemoryTest.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index bc076f73f1..99f53563cd 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -879,10 +879,11 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { HSAuint64 block_size_mb = 128; HSAuint64 block_size = block_size_mb * 1024 * 1024; PM4Queue queue; - ASSERT_SUCCESS(queue.Create(defaultGPUNode)); /* Test 4 times to see if there is any memory leak.*/ for (int repeat = 1; repeat < 5; repeat++) { + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + for (i = 0; i < ARRAY_ENTRIES; i++) { ret = hsaKmtAllocMemory(0 /* system */, block_size, m_MemoryFlags, reinterpret_cast(&pDb_array[i])); @@ -904,20 +905,23 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { allocationCount = i; EXPECT_GE(i, allocationCount) << "There might be memory leak!" << std::endl; - while (i--) { + for (int j = 0; j < i; j++) { /* To see if GPU can access the memory correctly*/ - unsigned int *begin = pDb_array[i]; + unsigned int *begin = pDb_array[j]; *begin = 0; queue.PlaceAndSubmitPacket( PM4WriteDataPacket(begin, 0xdeadbeaf)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(NULL, 300000); EXPECT_TRUE(WaitOnValue(begin, 0xdeadbeaf)); + } - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[i])); - EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); + EXPECT_SUCCESS(queue.Destroy()); + + for (int j = 0; j < i; j++) { + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[j])); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[j], block_size)); } } - EXPECT_SUCCESS(queue.Destroy()); /* Reset to run on all task nodes */ NumaNodeBind("all"); From 5d163cd82181ab2b8b1c8adb4f465eec52a0c91a Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 27 May 2019 14:57:57 -0500 Subject: [PATCH 0520/1247] Fix HostHdpFlush shader 1. Use s_mov_b32 to move 0xcafe to s18. s_movk_i32 is a sign extention move instruction. Oxcafe will be extended to 0xffffcafe which is not desired 2. Add wait to s_load_dword instruction to make sure memory read finish before the next store instruction. Change-Id: I665d1d471019edfaba5693e07cdc567d4103573f Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDMemoryTest.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 99f53563cd..ad1b24a6f4 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -121,13 +121,15 @@ shader CopyOnSignal\n\ asic(GFX9)\n\ type(CS)\n\ /* Assume input buffer in s0, s1 */\n\ - s_movk_i32 s18, 0xcafe\n\ - POLLSIGNAL:\n\ + s_mov_b32 s18, 0xcafe\n\ +POLLSIGNAL:\n\ s_load_dword s16, s[0:1], 0x0 glc\n\ s_cmp_eq_i32 s16, s18\n\ s_cbranch_scc0 POLLSIGNAL\n\ s_load_dword s17, s[0:1], 0x4 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ s_store_dword s17, s[0:1], 0x8 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ s_endpgm\n\ end\n\ "; @@ -1794,7 +1796,7 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { for (unsigned int bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) { if (memoryProperties[bank].HeapType == HSA_HEAPTYPE_MMIO_REMAP) { mmioBase = (unsigned int *)memoryProperties[bank].VirtualBaseAddress; - break; + break; } } ASSERT_NE(mmioBase, nullPtr) << "mmio base is NULL"; @@ -1821,11 +1823,11 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { buffer[1] = 0xbeef; /* Flush HDP */ mmioBase[KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL/4] = 0x1; - /* Give cafe to wake up */ buffer[0] = 0xcafe; /* Check test result*/ dispatch0.Sync(); + mmioBase[KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL/4] = 0x1; EXPECT_EQ(0xbeef, buffer[2]); // Clean up From be9ac578efe0a290c04254de5e7dcb3450f46e8c Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 27 May 2019 15:13:08 -0500 Subject: [PATCH 0521/1247] Re-enable HostHdpFlush test The bus error bug was fixed from kfd driver and Thunk Change-Id: Id02617fdc26f1c49307f90a0a939e05f22d739e7 Signed-off-by: Oak Zeng --- tests/kfdtest/scripts/kfdtest.exclude | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 0042c568f6..8f83b03210 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -34,8 +34,7 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ "KFDQMTest.SdmaEventInterrupt:"\ -"KFDDBGTest.BasicDebuggerSuspendResume:"\ -"KFDMemoryTest.HostHdpFlush" +"KFDDBGTest.BasicDebuggerSuspendResume" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From 62ee7b411216fc9fe1cc458bb9757abf76b36f9e Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 21 Jun 2019 15:14:45 -0400 Subject: [PATCH 0522/1247] Restore SDMA blacklist The SDMA blacklist should contain all tests that use SDMA. It will be applied to all ASICs that are know to have SDMA stability issues. Change-Id: I53e723382c12f99bddf9c535000e27737a7ea1f6 Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/kfdtest.exclude | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 8f83b03210..75d315859b 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -40,14 +40,17 @@ BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ "$TEMPORARY_BLACKLIST_ALL_ASICS" -# SDMA-based tests (KFDIPCTest.BasicTest, KFDQMTest.SdmaEventInterrupt) are both +# SDMA-based tests (KFDIPCTest.BasicTest, KFDQM.*Sdma*, KFDMemoryTest.MMBench) are all # disabled on non-Hawaii due to SDMA instability - SWDEV-101666 -# KFDLocalMemoryTest.CheckZeroInitializationVram Passes, but it -# takes nearly 4 min (229385 ms) to run so it remains blacklisted SDMA_BLACKLIST=\ "KFDIPCTest.*:"\ "KFDLocalMemoryTest.CheckZeroInitializationVram:"\ -"KFDQMTest.SdmaEventInterrupt:"\ +"KFDMemoryTest.MemoryRegister:"\ +"KFDMemoryTest.MMBench:"\ +"KFDMemoryTest.SignalHandling:"\ +"KFDQMTest.*Sdma*:"\ +"KFDQMTest.CreateQueueStressSingleThreaded:"\ +"KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.P2PTest:"\ "KFDPerformanceTest.P2PBandWidthTest:"\ "KFDPerformanceTest.P2POverheadTest" From be6ff2cdff5f7c9951aa4b6ea94ba43681bbd369 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 27 Jun 2019 10:19:46 -0400 Subject: [PATCH 0523/1247] Remove failing tests due to gfx1010 kernel merge BasicAddressWatch causes issues where KFDEvictTest and KFDQMTest.OverSubscribeCpQueues fails, and results in a GPU hang/reset. PM4EventInterrupt just hangs indefinitely. Remove them for now to allow the kernel merges to resume, and figure out what happened in the nv10 merge to cause it Change-Id: I418f9561ecb3e71bc52ac48ea363fcbde82a8e2b --- tests/kfdtest/scripts/kfdtest.exclude | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 75d315859b..8e7787e172 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -25,6 +25,7 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # KFDPNPTest.DisableAndCreateQueue (KFD-329) # KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) # KFDQMTest.mGPUShareBO (KFD-334) +# KFDQMTest.PM4EventInterrupt/KFDDBGTest.BasicAddressWatch (KFD-431) TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDGraphicsInterop.RegisterForeignDeviceMem:"\ "KFDPMTest.SuspendWithActiveProcess:"\ @@ -33,6 +34,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDPNPTest.DisableAndCreateQueue:"\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ +"KFDDBGTest.BasicAddressWatch*:"\ +"KFDQMTest.PM4EventInterrupt:"\ "KFDQMTest.SdmaEventInterrupt:"\ "KFDDBGTest.BasicDebuggerSuspendResume" From b507911ccdfd0cd85c9b8c39b597673359fd4505 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 7 Jun 2019 17:03:34 -0400 Subject: [PATCH 0524/1247] kfdtest: Use SDMA engine information directly from the node Change-Id: Icd391c8e821fb0ff5a1094f21b880a97e6d417a3 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 3 ++- tests/kfdtest/src/KFDBaseComponentTest.hpp | 1 + tests/kfdtest/src/KFDTestUtil.cpp | 16 +++++++--------- tests/kfdtest/src/KFDTestUtil.hpp | 1 + 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 8d6ca73b7a..c56e0ea239 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -63,7 +63,8 @@ void KFDBaseComponentTest::SetUp() { g_TestGPUFamilyId = FamilyIdFromNode(nodeProperties); - GetSdmaInfo(nodeProperties, &m_numSdmaEngines, &m_numSdmaQueuesPerEngine); + GetSdmaInfo(nodeProperties, &m_numSdmaEngines, &m_numSdmaXgmiEngines, + &m_numSdmaQueuesPerEngine); m_FamilyId = g_TestGPUFamilyId; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index cfd291b195..867718e06d 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -61,6 +61,7 @@ class KFDBaseComponentTest : public testing::Test { HsaSystemProperties m_SystemProperties; unsigned int m_FamilyId; unsigned int m_numSdmaEngines; + unsigned int m_numSdmaXgmiEngines; unsigned int m_numSdmaQueuesPerEngine; HsaMemFlags m_MemoryFlags; HsaNodeInfo m_NodeInfo; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 0021af2faf..da3da02d42 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -128,22 +128,20 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { void GetSdmaInfo(const HsaNodeProperties *props, unsigned int *p_num_sdma_engines, + unsigned int *p_num_sdma_xgmi_engines, unsigned int *p_num_sdma_queues_per_engine) { - int num_sdma_engines = 2; int num_sdma_queues_per_engine = 2; - switch (props->EngineId.ui32.Major) { - case 9: - if (props->EngineId.ui32.Stepping == 2) // RAVEN - num_sdma_engines = 1; - else if (props->EngineId.ui32.Stepping == 6) // VEGA20 + if (props->EngineId.ui32.Major == 9) { + if (props->EngineId.ui32.Stepping == 6) // VEGA20 num_sdma_queues_per_engine = 8; - - break; } if (p_num_sdma_engines) - *p_num_sdma_engines = num_sdma_engines; + *p_num_sdma_engines = props->NumSdmaEngines; + + if (p_num_sdma_xgmi_engines) + *p_num_sdma_xgmi_engines = props->NumSdmaXgmiEngines; if (p_num_sdma_queues_per_engine) *p_num_sdma_queues_per_engine = num_sdma_queues_per_engine; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 48e2701be0..1e40e8b4e4 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -52,6 +52,7 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props); void GetSdmaInfo(const HsaNodeProperties *props, unsigned int *p_num_sdma_engines, + unsigned int *p_num_sdma_xgmi_engines, unsigned int *p_num_sdma_queues_per_engine); class HsaMemoryBuffer { From c27704ded95ff6e105fd2f4435e89f89ce8f8a1c Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 30 Oct 2018 16:37:56 -0400 Subject: [PATCH 0525/1247] kfdtest: Add core test category This will faciliate ASIC bringup, including under simulation environment. Change-Id: Ie027a77a2498cba739fea51f404d9843ce8dbeae Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 11 +++++++++++ tests/kfdtest/scripts/run_kfdtest.sh | 1 + 2 files changed, 12 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 8e7787e172..42d763e849 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -162,3 +162,14 @@ RAVEN_TESTS_BLACKLIST=\ "KFDDBGTest.BasicAddressWatch:"\ "KFDQMTest.QueueLatency:"\ "KFDQMTest.SdmaEventInterrupt" + +# Core tests, used in scenarios like bringup +CORE_TESTS=\ +"KFDQMTest.CreateDestroyCpQueue:"\ +"KFDQMTest.SubmitPacketCpQueue:"\ +"KFDQMTest.MultipleCpQueues:"\ +"KFDQMTest.CreateDestroySdmaQueue:"\ +"KFDQMTest.SubmitPacketSdmaQueue:"\ +"KFDQMTest.MultipleSdmaQueues:"\ +"KFDCWSRTest.BasicTest:"\ +"KFDEventTest.SignalEvent" diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 2648e18b4d..5ac5afa44f 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -75,6 +75,7 @@ getFilter() { vg12 ) FILTER="--gtest_filter=$VEGA12_TESTS_BLACKLIST" ;; vg20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; rv ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; + core ) FILTER="--gtest_filter=$CORE_TESTS" ;; all ) FILTER="" ;; *) die "Unsupported platform $PLATFORM or node $NODE. Exiting" ;; esac From 664c6617ad744b60b3b6b82634043488ddee60c3 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 4 Jul 2019 13:48:21 -0400 Subject: [PATCH 0526/1247] Added SP3 assembler support for gfx10 Change-Id: I31c1df0f6d5243089e2ec3db381a19362be18d6c Signed-off-by: Oak Zeng Signed-off-by: shaoyunl --- tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt index 54a6f3f4bd..db59e3716e 100644 --- a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt +++ b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt @@ -58,8 +58,10 @@ aux_source_directory(${SCLIB_SRC}/sp3/ci SRC_FILES) aux_source_directory(${SCLIB_SRC}/sp3/gfx8 SRC_FILES) aux_source_directory(${SCLIB_SRC}/sp3/gfx81 SRC_FILES) aux_source_directory(${SCLIB_SRC}/sp3/gfx9 SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/gfx10 SRC_FILES) aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx81 SRC_FILES) aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx9 SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx10 SRC_FILES) message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) @@ -68,7 +70,7 @@ message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) # message(STATUS "${file}") #endforeach() -set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU -fPIC" ) +set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU -fPIC -DGFX10_BUILD" ) add_library(amdsp3 ${SRC_FILES}) From 3b014adccc401a6ae42fb52480c2408edd33e3b6 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 30 May 2019 16:09:06 -0500 Subject: [PATCH 0527/1247] Device HDP flush test Change-Id: I1c19e44caeee4a6e59200dceb718896fcff9bf82 Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDMemoryTest.cpp | 150 +++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index ad1b24a6f4..51579f3c79 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -107,10 +107,10 @@ type(CS)\n\ "; /* Input: A buffer of at least 3 dwords. - * DW0: used as a signal b/t host and device. Host - * write 0xcafe to signal device. - * DW1: Input buffer for host/device to read/write. + * DW0: used as a signal. 0xcafe means it is signaled + * DW1: Input buffer for device to read. * DW2: Output buffer for device to write. + * Once receive signal, device will copy DW1 to DW2 * This shader continously poll the signal buffer, * Once signal buffer is signaled, it copies input buffer * to output buffer @@ -134,6 +134,27 @@ POLLSIGNAL:\n\ end\n\ "; +/* Input0: A buffer of at least 2 dwords. + * DW0: used as a signal. Write 0xcafe to signal + * DW1: Write to this buffer for other device to read. + * Input1: mmio base address + */ +const char* gfx9_WriteAndSignal = +"\ +shader WriteAndSignal\n\ +asic(GFX9)\n\ +type(CS)\n\ +/* Assume input buffer in s0, s1 */\n\ + s_mov_b32 s18, 0xbeef\n\ + s_store_dword s18, s[0:1], 0x4 glc\n\ + s_mov_b32 s18, 0x1\n\ + s_store_dword s18, s[2:3], 0 glc\n\ + s_mov_b32 s18, 0xcafe\n\ + s_store_dword s18, s[0:1], 0x0 glc\n\ + s_endpgm\n\ + end\n\ +"; + void KFDMemoryTest::SetUp() { ROUTINE_START @@ -1838,3 +1859,126 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { TEST_END } + +/* Test HDP flush from device. + * Use shader on device 1 to write vram of device 0 + * and flush HDP of device 0. Read vram from device 0 + * and write back to vram to check the result from CPU. + * Asic before gfx9 doesn't support device HDP flush + * so only run on vega10 and after. + * This should only run on system with at least one + * large bar node (which is used as device 0). + */ +TEST_F(KFDMemoryTest, DeviceHdpFlush) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HsaMemFlags memoryFlags = m_MemoryFlags; + /* buffer is physically on device 0. + * buffer[0]: Use as signaling b/t devices; + * buffer[1]: Device 1 write to buffer[1] and device 0 read it + * buffer[2]: Device 0 copy buffer[1] to buffer[2] for CPU to check + */ + unsigned int *buffer = NULL; + const HsaNodeProperties *pNodeProperties; + HSAuint32 *mmioBase = NULL; + unsigned int *nullPtr = NULL; + std::vector nodes; + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: At least two GPUs are required." << std::endl; + return; + } + + /* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */ + if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { + nodes.push_back(g_TestNodeId); + nodes.push_back(g_TestDstNodeId); + if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[0])) { + LOG() << "Skipping test: first GPU specified is not a large bar GPU." << std::endl; + return; + } + if (nodes[0] == nodes[1]) { + LOG() << "Skipping test: Different GPUs must be specified (2 GPUs required)." << std::endl; + return; + } + } else { + HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); + if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPU)) { + LOG() << "Skipping test: Default GPUs must be large bar." << std::endl; + return; + } + nodes.push_back(defaultGPU); + for (unsigned i = 0; i < gpuNodes.size(); i++) + if (gpuNodes.at(i) != defaultGPU) + nodes.push_back(gpuNodes.at(i)); + if (nodes.size() < 2) { + LOG() << "Skipping test: At least 2 GPUs required." << std::endl; + return; + } + } + + pNodeProperties = m_NodeInfo.GetNodeProperties(nodes[0]); + if (!pNodeProperties) { + LOG() << "Failed to get gpu node properties." << std::endl; + return; + } + + if (m_FamilyId < FAMILY_AI) { + LOG() << "Skipping test: Test requires gfx9 and later asics." << std::endl; + return; + } + + HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; + EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(nodes[0], pNodeProperties->NumMemoryBanks, + memoryProperties)); + for (unsigned int bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) { + if (memoryProperties[bank].HeapType == HSA_HEAPTYPE_MMIO_REMAP) { + mmioBase = (unsigned int *)memoryProperties[bank].VirtualBaseAddress; + break; + } + } + ASSERT_NE(mmioBase, nullPtr) << "mmio base is NULL"; + + memoryFlags.ui32.NonPaged = 1; + memoryFlags.ui32.CoarseGrain = 0; + ASSERT_SUCCESS(hsaKmtAllocMemory(nodes[0], PAGE_SIZE, memoryFlags, + reinterpret_cast(&buffer))); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(buffer, PAGE_SIZE, NULL)); + + /* Signal is dead from the beginning*/ + buffer[0] = 0xdead; + buffer[1] = 0xfeeb; + buffer[2] = 0xfeeb; + /* Submit shaders*/ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(nodes[0])); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, nodes[0], true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9_CopyOnSignal, "CopyOnSignal", isaBuffer); + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(buffer, NULL); + dispatch.Submit(queue); + + PM4Queue queue0; + ASSERT_SUCCESS(queue0.Create(nodes[1])); + HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9_WriteAndSignal, "WriteAndSignal", isaBuffer0); + Dispatch dispatch0(isaBuffer0); + dispatch0.SetArgs(buffer, mmioBase); + dispatch0.Submit(queue0); + + /* Check test result*/ + dispatch0.Sync(); + dispatch.Sync(); + EXPECT_EQ(0xbeef, buffer[2]); + + // Clean up + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue0.Destroy()); + delete [] memoryProperties; + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(buffer)); + EXPECT_SUCCESS(hsaKmtFreeMemory(buffer, PAGE_SIZE)); + + TEST_END +} From 6cad92de6fcfdccefe150f019f5f5efbc099a541 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 4 Jul 2019 16:23:37 -0400 Subject: [PATCH 0528/1247] Added family ID for gfx1010 Change-Id: I1b9a2b5270e70d12f066906f4e6cfea2cbfc2110 Signed-off-by: Oak Zeng Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDTestFlags.hpp | 1 + tests/kfdtest/src/KFDTestUtil.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index 04522a5b46..ecd309b9c7 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -59,6 +59,7 @@ enum KfdFamilyId { FAMILY_CZ, // Carrizo, Nolan, Amur FAMILY_AI, // Arctic Islands FAMILY_RV, // Raven + FAMILY_NV, // Navi10 }; #endif // __KFD_TEST_FLAGS__H__ diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index da3da02d42..68bf8268fb 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -116,6 +116,9 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { if (props->EngineId.ui32.Stepping == 2) familyId = FAMILY_RV; break; + case 10: + familyId = FAMILY_NV; + break; } if (props->NumCPUCores && props->NumFComputeCores) From c6ed539b44d4f2f7334a781bc7a229decbdcc797 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 4 Jul 2019 18:49:34 -0400 Subject: [PATCH 0529/1247] Add gfx IsaGenerator Change-Id: I93ccb889b4bb7f0f5921a90cebbc0550d1eb3f7d Signed-off-by: Oak Zeng Signed-off-by: shaoyunl --- tests/kfdtest/src/IsaGenerator.cpp | 4 + tests/kfdtest/src/IsaGenerator_Gfx10.cpp | 142 +++++++++++++++++++++++ tests/kfdtest/src/IsaGenerator_Gfx10.hpp | 49 ++++++++ 3 files changed, 195 insertions(+) create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx10.cpp create mode 100644 tests/kfdtest/src/IsaGenerator_Gfx10.hpp diff --git a/tests/kfdtest/src/IsaGenerator.cpp b/tests/kfdtest/src/IsaGenerator.cpp index 83f34b948e..10cd994028 100644 --- a/tests/kfdtest/src/IsaGenerator.cpp +++ b/tests/kfdtest/src/IsaGenerator.cpp @@ -29,6 +29,7 @@ #include "IsaGenerator_Gfx72.hpp" #include "IsaGenerator_Gfx8.hpp" #include "IsaGenerator_Gfx9.hpp" +#include "IsaGenerator_Gfx10.hpp" #include "GoogleTestExtension.hpp" @@ -91,6 +92,9 @@ IsaGenerator* IsaGenerator::Create(unsigned int familyId) { case FAMILY_AI: case FAMILY_RV: return new IsaGenerator_Gfx9; + case FAMILY_NV: + return new IsaGenerator_Gfx10; + default: LOG() << "Error: Invalid ISA" << std::endl; return NULL; diff --git a/tests/kfdtest/src/IsaGenerator_Gfx10.cpp b/tests/kfdtest/src/IsaGenerator_Gfx10.cpp new file mode 100644 index 0000000000..d8d33086e5 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx10.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "IsaGenerator_Gfx10.hpp" + +#include +#include + +/* The binaries are generated from following ISA */ +const std::string IsaGenerator_Gfx10::ASIC_NAME = "GFX10"; +#if 0 +static const char * atomic_add = \ +"\ +shader atomic_add \n\ +asic(GFX10) \n\ +wave_size(32) \n\ +type(CS) \n\ + v_mov_b32 v0, s0 \n\ + v_mov_b32 v1, s1 \n\ + v_mov_b32 v2, 1 \n\ + flat_atomic_add v3, v[0:1], v2 slc glc \n\ + s_waitcnt 0 \n\ + s_endpgm \n\ +end \n\ +"; + +static const char * copy_dword = \ +"\ +shader copy_dword \n\ +asic(GFX10) \n\ +wave_size(32) \n\ +type(CS) \n\ + v_mov_b32 v0, s0 \n\ + v_mov_b32 v1, s1 \n\ + v_mov_b32 v2, s2 \n\ + v_mov_b32 v3, s3 \n\ + flat_load_dword v4, v[0:1] slc glc \n\ + s_waitcnt 0 \n\ + flat_store_dword v[2:3], v4 slc glc \n\ + s_endpgm \n\ +end \n\ +"; + +static const char * loop= \ +"\ +shader loop \n\ +asic(GFX10) \n\ +type(CS) \n\ +wave_size(32) \n\ +loop: \n\ + s_branch loop \n\ + s_endpgm \n\ +end \n\ +"; + +static const char * noop= \ +"\ +shader noop \n\ +asic(GFX10) \n\ +type(CS) \n\ +wave_size(32) \n\ + s_endpgm \n\ +end \n\ +"; +#endif + +const uint32_t IsaGenerator_Gfx10::NOOP_ISA[] = { +0xb0804004, 0xbf810000, +0xbf9f0000, 0xbf9f0000, +0xbf9f0000, 0xbf9f0000, +0xbf9f0000 +}; + +const uint32_t IsaGenerator_Gfx10::COPY_DWORD_ISA[] = { +0xb0804004, 0x7e000200, +0x7e020201, 0x7e040202, +0x7e060203, 0xdc330000, +0x47d0000, 0xbf8c0000, +0xdc730000, 0x7d0402, +0xbf810000, 0xbf9f0000, +0xbf9f0000, 0xbf9f0000, +0xbf9f0000, 0xbf9f0000 +}; + +const uint32_t IsaGenerator_Gfx10::INFINITE_LOOP_ISA[] = { +0xbf82ffff, 0xb0804004, +0xbf810000, 0xbf9f0000, +0xbf9f0000, 0xbf9f0000, +0xbf9f0000, 0xbf9f0000 +}; + +const uint32_t IsaGenerator_Gfx10::ATOMIC_ADD_ISA[] = { +0xb0804004, 0x7e000200, +0x7e020201, 0x7e040281, +0xdccb0000, 0x37d0200, +0xbf8c0000, 0xbf810000, +0xbf9f0000, 0xbf9f0000, +0xbf9f0000, 0xbf9f0000, +0xbf9f0000 +}; + + +void IsaGenerator_Gfx10::GetNoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx10::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { + std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx10::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Gfx10::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { + std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); +} + +const std::string& IsaGenerator_Gfx10::GetAsicName() { + return ASIC_NAME; +} + diff --git a/tests/kfdtest/src/IsaGenerator_Gfx10.hpp b/tests/kfdtest/src/IsaGenerator_Gfx10.hpp new file mode 100644 index 0000000000..e4a57cda56 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Gfx10.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _ISAGENERATOR_GFX10_H_ +#define _ISAGENERATOR_GFX10_H_ + +#include +#include "IsaGenerator.hpp" + +class IsaGenerator_Gfx10 : public IsaGenerator { + public: + virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); + virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); + + protected: + virtual const std::string& GetAsicName(); + + private: + static const std::string ASIC_NAME; + + static const uint32_t NOOP_ISA[]; + static const uint32_t COPY_DWORD_ISA[]; + static const uint32_t INFINITE_LOOP_ISA[]; + static const uint32_t ATOMIC_ADD_ISA[]; +}; + +#endif // _ISAGENERATOR_GFX9_H_ From 67f366243d8f47adcc1c7fd038a59ec533c3e4ca Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 9 Jul 2019 14:54:49 -0400 Subject: [PATCH 0530/1247] fix mbind on NUMA system mbind walks through pages to setup vma memory policy. So we need do mmap to create vma mappings first, then call mbind. mbind will do nothing if vma does not exist. And add numa available check before executing mbind, and return NULL to hsaKmtAllocMemory if mbind failed. Change-Id: I28ab661885d807ca51ef90e87230669dc80f10ec Signed-off-by: Philip Yang --- src/fmm.c | 76 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index aaad772101..4aced94956 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include "rbtree.h" #ifndef MPOL_F_STATIC_NODES @@ -1456,6 +1457,39 @@ static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, return mem; } +static int bind_mem_to_numa(uint32_t node_id, void *mem, + uint64_t MemorySizeInBytes, HsaMemFlags flags) +{ + int mode = MPOL_F_STATIC_NODES; + struct bitmask *node_mask; + int num_node; + + if (numa_available() == -1) + return 0; + + num_node = numa_num_task_nodes(); + + if (num_node > 1) { + node_mask = numa_bitmask_alloc(num_node); + if (!node_mask) + return -ENOMEM; + + numa_bitmask_setbit(node_mask, node_id); + mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; + if (mbind(mem, MemorySizeInBytes, mode, node_mask->maskp, + num_node + 1, 0)) { + pr_warn("Failed to set NUMA policy for %p\n", mem); + + numa_bitmask_free(node_mask); + return -EFAULT; + } + + numa_bitmask_free(node_mask); + } + + return 0; +} + static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags) { @@ -1492,10 +1526,6 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, * memory is allocated from KFD */ if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { - const unsigned int bits_per_long = sizeof(unsigned long) * 8; - unsigned long node_mask[node_id / bits_per_long + 1]; - int mode = MPOL_F_STATIC_NODES; - /* Allocate address space */ pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, address, size); @@ -1503,24 +1533,15 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, if (!mem) return NULL; - /* Bind to NUMA node */ - memset(node_mask, 0, sizeof(node_mask)); - node_mask[node_id / bits_per_long] = 1UL << (node_id % bits_per_long); - mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; - if (mbind(mem, MemorySizeInBytes, mode, node_mask, node_id+1, 0)) - pr_warn("Failed to set NUMA policy for %lu pages at %p\n", - MemorySizeInBytes >> 12, mem); - /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) - == MAP_FAILED) { - /* Release address space */ - pthread_mutex_lock(&aperture->fmm_mutex); - aperture_release_area(aperture, mem, size); - pthread_mutex_unlock(&aperture->fmm_mutex); - return NULL; - } + == MAP_FAILED) + goto out_release_area; + + /* Bind to NUMA node */ + if (bind_mem_to_numa(node_id, mem, MemorySizeInBytes, flags)) + goto out_release_area; /* Mappings in the DGPU aperture don't need to be copied on * fork. This avoids MMU notifiers and evictions due to user @@ -1534,13 +1555,8 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, vm_obj = fmm_allocate_memory_object(gpu_id, mem, size, aperture, &mmap_offset, ioc_flags); - if (!vm_obj) { - /* Release address space */ - pthread_mutex_lock(&aperture->fmm_mutex); - aperture_release_area(aperture, mem, size); - pthread_mutex_unlock(&aperture->fmm_mutex); - return NULL; - } + if (!vm_obj) + goto out_release_area; } else { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; mem = __fmm_allocate_device(gpu_id, address, size, aperture, @@ -1576,6 +1592,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, } return mem; + +out_release_area: + /* Release address space */ + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, size); + pthread_mutex_unlock(&aperture->fmm_mutex); + + return NULL; } void *fmm_allocate_host(uint32_t node_id, void *address, From 6704b051d265980e087bdc75606822ac66cd0cf7 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 11 Jun 2019 20:12:34 -0400 Subject: [PATCH 0531/1247] kfdtest: Factor out multi-process test into a base class Create KFDMultiProcessTest base class for tests forking multiple child processes. Derive KFDEvictTest from that class. Change-Id: Ie5f3362c45be2b807bf7a83839ab3820352a67f9 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/KFDEvictTest.cpp | 73 +++++++-------------- tests/kfdtest/src/KFDEvictTest.hpp | 30 +++------ tests/kfdtest/src/KFDMultiProcessTest.cpp | 77 +++++++++++++++++++++++ tests/kfdtest/src/KFDMultiProcessTest.hpp | 62 ++++++++++++++++++ 4 files changed, 169 insertions(+), 73 deletions(-) create mode 100644 tests/kfdtest/src/KFDMultiProcessTest.cpp create mode 100644 tests/kfdtest/src/KFDMultiProcessTest.hpp diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index cdc6c792b0..f4a4f8cde6 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -34,6 +34,28 @@ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +void KFDEvictTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDEvictTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, std::vector &pBuffers) { HSAuint64 totalMB; @@ -253,57 +275,6 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle ha EXPECT_EQ(0, amdgpu_cs_ctx_free(contextHandle)); } -void KFDEvictTest::ForkChildProcesses(int nprocesses) { - int i; - - for (i = 0; i < nprocesses - 1; ++i) { - pid_t pid = fork(); - ASSERT_GE(pid, 0); - - if (pid == 0) { - /* Child process */ - /* Cleanup file descriptors copied from parent process - * then call SetUp->hsaKmtOpenKFD to create new process - */ - m_psName = "Test process " + std::to_string(i) + " "; - TearDown(); - SetUp(); - m_ChildPids.clear(); - m_IsParent = false; - return; - } - - /* Parent process */ - m_ChildPids.push_back(pid); - } - - m_psName = "Test process " + std::to_string(i) + " "; -} - -void KFDEvictTest::WaitChildProcesses() { - if (m_IsParent) { - /* Only run by parent process */ - int childStatus; - int childExitOkNum = 0; - int size = m_ChildPids.size(); - - for (HSAuint32 i = 0; i < size; i++) { - pid_t pid = m_ChildPids.front(); - - waitpid(pid, &childStatus, 0); - if (WIFEXITED(childStatus) == 1 && WEXITSTATUS(childStatus) == 0) - childExitOkNum++; - - m_ChildPids.erase(m_ChildPids.begin()); - } - - EXPECT_EQ(childExitOkNum, size); - } - - /* Child process or parent process finished successfully */ - m_ChildStatus = HSAKMT_STATUS_SUCCESS; -} - /* Evict and restore procedure basic test * * Use N_PROCESSES processes to allocate vram buf size larger than total vram size diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index a602980fbe..b5e7e1c3bf 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -26,31 +26,22 @@ #include #include -#include "KFDLocalMemoryTest.hpp" -#include "KFDBaseComponentTest.hpp" +#include "KFDMultiProcessTest.hpp" #include "IsaGenerator.hpp" #include "PM4Queue.hpp" // @class KFDEvictTest // Test eviction and restore procedure using two processes -class KFDEvictTest : public KFDLocalMemoryTest { +class KFDEvictTest : public KFDMultiProcessTest { public: - KFDEvictTest(void): m_ChildStatus(HSAKMT_STATUS_ERROR), m_IsParent(true) {} + KFDEvictTest(void): m_pIsaGen(NULL) {} - ~KFDEvictTest(void) { - if (!m_IsParent) { - /* Child process has to exit - * otherwise gtest will continue other tests - */ - exit(m_ChildStatus); - } - - try { - WaitChildProcesses(); - } catch (...) {} - } + ~KFDEvictTest(void) {} protected: + virtual void SetUp(); + virtual void TearDown(); + std::string CreateShader(); void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, std::vector &pBuffers); @@ -59,16 +50,11 @@ class KFDEvictTest : public KFDLocalMemoryTest { void FreeAmdgpuBo(amdgpu_bo_handle handle); void AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle, PM4Queue *computeQueue); - void ForkChildProcesses(int nprocesses); - void WaitChildProcesses(); protected: // Members - std::string m_psName; - std::vector m_ChildPids; + IsaGenerator* m_pIsaGen; HsaMemFlags m_Flags; void* m_pBuf; - HSAKMT_STATUS m_ChildStatus; - bool m_IsParent; }; #endif // __KFD_EVICT_TEST__H__ diff --git a/tests/kfdtest/src/KFDMultiProcessTest.cpp b/tests/kfdtest/src/KFDMultiProcessTest.cpp new file mode 100644 index 0000000000..2934438f28 --- /dev/null +++ b/tests/kfdtest/src/KFDMultiProcessTest.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDMultiProcessTest.hpp" + +void KFDMultiProcessTest::ForkChildProcesses(int nprocesses) { + int i; + + for (i = 0; i < nprocesses - 1; ++i) { + pid_t pid = fork(); + ASSERT_GE(pid, 0); + + if (pid == 0) { + /* Child process */ + /* Cleanup file descriptors copied from parent process + * then call SetUp->hsaKmtOpenKFD to create new process + */ + m_psName = "Test process " + std::to_string(i) + " "; + TearDown(); + SetUp(); + m_ChildPids.clear(); + m_IsParent = false; + m_ProcessIndex = i; + return; + } + + /* Parent process */ + m_ChildPids.push_back(pid); + } + + m_psName = "Test process " + std::to_string(i) + " "; + m_ProcessIndex = i; +} + +void KFDMultiProcessTest::WaitChildProcesses() { + if (m_IsParent) { + /* Only run by parent process */ + int childStatus; + int childExitOkNum = 0; + int size = m_ChildPids.size(); + + for (HSAuint32 i = 0; i < size; i++) { + pid_t pid = m_ChildPids.front(); + + waitpid(pid, &childStatus, 0); + if (WIFEXITED(childStatus) == 1 && WEXITSTATUS(childStatus) == 0) + childExitOkNum++; + + m_ChildPids.erase(m_ChildPids.begin()); + } + + EXPECT_EQ(childExitOkNum, size); + } + + /* Child process or parent process finished successfully */ + m_ChildStatus = HSAKMT_STATUS_SUCCESS; +} diff --git a/tests/kfdtest/src/KFDMultiProcessTest.hpp b/tests/kfdtest/src/KFDMultiProcessTest.hpp new file mode 100644 index 0000000000..3907571545 --- /dev/null +++ b/tests/kfdtest/src/KFDMultiProcessTest.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_MULTI_PROCESS_TEST__H__ +#define __KFD_MULTI_PROCESS_TEST__H__ + +#include +#include +#include "KFDBaseComponentTest.hpp" + +// @class KFDMultiProcessTest +// Base class for tests forking multiple child processes +class KFDMultiProcessTest : public KFDBaseComponentTest { + public: + KFDMultiProcessTest(void): m_ChildStatus(HSAKMT_STATUS_ERROR), m_IsParent(true) {} + + ~KFDMultiProcessTest(void) { + if (!m_IsParent) { + /* Child process has to exit + * otherwise gtest will continue other tests + */ + exit(m_ChildStatus); + } + + try { + WaitChildProcesses(); + } catch (...) {} + } + + protected: + void ForkChildProcesses(int nprocesses); + void WaitChildProcesses(); + + protected: // Members + std::string m_psName; + int m_ProcessIndex; + std::vector m_ChildPids; + HSAKMT_STATUS m_ChildStatus; + bool m_IsParent; +}; + +#endif // __KFD_MULTI_PROCESS_TEST__H__ From 5475e618e5d29978381d379d6648a1d8910e1ee8 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 12 Jun 2019 21:44:46 -0400 Subject: [PATCH 0532/1247] kfdtest: Add multi-process oversubscription test This test is designed to reproduce soft-hangs cause by HWS running with oversubscription. Change-Id: I49861522b3ff5ba50df5ddc968545c35ccb25353 Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/kfdtest.exclude | 4 +- tests/kfdtest/src/KFDHWSTest.cpp | 133 ++++++++++++++++++++++++++ tests/kfdtest/src/KFDHWSTest.hpp | 50 ++++++++++ 3 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 tests/kfdtest/src/KFDHWSTest.cpp create mode 100644 tests/kfdtest/src/KFDHWSTest.hpp diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 42d763e849..68f8afccf0 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -26,6 +26,7 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) # KFDQMTest.mGPUShareBO (KFD-334) # KFDQMTest.PM4EventInterrupt/KFDDBGTest.BasicAddressWatch (KFD-431) +# KFDHWSTest.* (SWDEV-193035) TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDGraphicsInterop.RegisterForeignDeviceMem:"\ "KFDPMTest.SuspendWithActiveProcess:"\ @@ -37,7 +38,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDDBGTest.BasicAddressWatch*:"\ "KFDQMTest.PM4EventInterrupt:"\ "KFDQMTest.SdmaEventInterrupt:"\ -"KFDDBGTest.BasicDebuggerSuspendResume" +"KFDDBGTest.BasicDebuggerSuspendResume:"\ +"KFDHWSTest.*" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ diff --git a/tests/kfdtest/src/KFDHWSTest.cpp b/tests/kfdtest/src/KFDHWSTest.cpp new file mode 100644 index 0000000000..66c0b5d8ff --- /dev/null +++ b/tests/kfdtest/src/KFDHWSTest.cpp @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDHWSTest.hpp" + +void KFDHWSTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDHWSTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +void KFDHWSTest::RunTest(unsigned nProcesses, unsigned nQueues, unsigned nLoops) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + unsigned q, l; + bool timeout = false; + + /* Fork the child processes */ + ForkChildProcesses(nProcesses); + + // Create queues + PM4Queue *queues = new PM4Queue[nQueues]; + for (q = 0; q < nQueues; q++) + ASSERT_SUCCESS(queues[q].Create(defaultGPUNode)); + + // Create dispatch pointers. Each loop iteration creates fresh dispatches + Dispatch **dispatch = new Dispatch*[nQueues]; + for (q = 0; q < nQueues; q++) + dispatch[q] = NULL; + + // Logging: Each process prints its index after each loop iteration, all in one line. + std::ostream &log = LOG() << std::dec << "Process " << m_ProcessIndex << " starting." << std::endl; + + // Run work on all queues + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->GetNoopIsa(isaBuffer); + for (l = 0; l < nLoops; l++) { + for (q = 0; q < nQueues; q++) { + if (dispatch[q]) + delete dispatch[q]; + dispatch[q] = new Dispatch(isaBuffer); + dispatch[q]->SetArgs(NULL, NULL); + dispatch[q]->SetDim(1, 1, 1); + dispatch[q]->Submit(queues[q]); + } + for (q = 0; q < nQueues; q++) { + timeout = dispatch[q]->SyncWithStatus(g_TestTimeOut); + if (timeout) + goto timeout; + } + log << m_ProcessIndex; + } + +timeout: + log << std::endl; + if (timeout) { + WARN() << "Process " << m_ProcessIndex << " timeout." << std::endl; + } else { + LOG() << "Process " << m_ProcessIndex << " done. Waiting ..." << std::endl; + + // Wait here before destroying queues. If another process' queues + // are soft-hanging, destroying queues can resolve the soft-hang + // by changing the run list. Make sure the other process's + // dispatches have a chance to time out first. + Delay(g_TestTimeOut+1000); + } + + // Destroy queues and dispatches. Destroying the queues first + // ensures that the memory allocated by the Dispatch is no longer + // accessed by the GPU. + LOG() << "Process " << m_ProcessIndex << " cleaning up." << std::endl; + for (q = 0; q < nQueues; q++) { + EXPECT_SUCCESS(queues[q].Destroy()); + if (dispatch[q]) + delete dispatch[q]; + } + delete[] queues; + delete[] dispatch; + + // This is after all the cleanup to avoid leaving any garbage + // behind, but before WaitChildProcesses to ensure a child process + // with a timeout exits with an error that can be detected by the + // parent. + ASSERT_FALSE(timeout); + + WaitChildProcesses(); +} + +TEST_F(KFDHWSTest, MultiProcessOversubscribed) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + RunTest(3, 13, 40); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDHWSTest.hpp b/tests/kfdtest/src/KFDHWSTest.hpp new file mode 100644 index 0000000000..e3ea5155c2 --- /dev/null +++ b/tests/kfdtest/src/KFDHWSTest.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_HWS_TEST__H__ +#define __KFD_HWS_TEST__H__ + +#include + +#include "PM4Queue.hpp" +#include "IsaGenerator.hpp" +#include "KFDMultiProcessTest.hpp" +#include "Dispatch.hpp" + +class KFDHWSTest : public KFDMultiProcessTest { + public: + KFDHWSTest():m_pIsaGen(NULL) {} + + ~KFDHWSTest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); + + void RunTest(unsigned nProcesses, unsigned nQueues, unsigned nLoops); + + protected: // Members + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_QCM_TEST__H__ From 67c4fe230b97c592f932ab303b44adad5dbab27e Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 10 Jul 2019 00:07:17 -0400 Subject: [PATCH 0533/1247] kfdtest: Disable CheckZeroInitializationVram test KFD will soon stop initializing VRAM allocations. Change-Id: I901c736886bb3bd3b1b54a21d383ccd7907928fd Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/kfdtest.exclude | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 68f8afccf0..6b21c640e0 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -7,12 +7,16 @@ # # CU Masking Linear/Split are not working correctly due to how the HW distributes work over CUs. # They are available for testing but are not currently expected to pass on CI/VI/AI. +# +# The CheckZeroInitializationVram test is no longer expected to pass as KFD no longer +# clears memory at allocation time. PERMANENT_BLACKLIST_ALL_ASICS=\ "-KFDEventTest.MeasureInterruptConsumption:"\ "KFDLocalMemoryTest.Fragmentation:"\ "KFDQMTest.BasicCuMaskingLinear:"\ "RDMATest.GPUDirect:"\ -"KFDRASTest.*" +"KFDRASTest.*:"\ +"KFDLocalMemoryTest.CheckZeroInitializationVram" # This is the temporary blacklist for all ASICs. This is to be used when a test is failing consistently # on every ASIC (Kaveri, Carrizo, Hawaii, Tonga, Fiji, Polaris10, Polaris11 and Vega10 . From 02ccb9eb571abe3faf9bc5a0a933a7e826d64985 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Fri, 5 Jul 2019 16:12:53 -0400 Subject: [PATCH 0534/1247] Thunk: Add gfx1010 initial support Add gfx1010 basic support on Thunk Change-Id: Ie4c0922158c7f5e2951f8694f4b204f371f1aa23 Signed-off-by: shaoyunl --- src/libhsakmt.h | 3 ++- src/queues.c | 8 +++++++- src/topology.c | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 91e6c21e72..da419f3029 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -101,7 +101,8 @@ enum asic_family_type { CHIP_VEGA10, CHIP_VEGA12, CHIP_VEGA20, - CHIP_RAVEN + CHIP_RAVEN, + CHIP_NAVI10 }; #define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) diff --git a/src/queues.c b/src/queues.c index f967805c22..d8fbc7f786 100644 --- a/src/queues.c +++ b/src/queues.c @@ -129,6 +129,11 @@ const struct device_info vega20_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info navi10_device_info = { + .asic_family = CHIP_NAVI10, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, @@ -143,7 +148,8 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_VEGA10] = &vega10_device_info, [CHIP_VEGA12] = &vega12_device_info, [CHIP_VEGA20] = &vega20_device_info, - [CHIP_RAVEN] = &raven_device_info + [CHIP_RAVEN] = &raven_device_info, + [CHIP_NAVI10] = &navi10_device_info, }; struct queue { diff --git a/src/topology.c b/src/topology.c index 6fde6e588b..fb4ad66f66 100644 --- a/src/topology.c +++ b/src/topology.c @@ -220,6 +220,12 @@ static struct hsa_gfxip_table { { 0x66A4, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66A7, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66AF, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + /* Navi10 */ + { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, + { 0x7312, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, + { 0x7318, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, + { 0x731A, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, + { 0x731F, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, }; enum cache_type { From 4e9ff4393da224828d4597b719a376e8ec5c44d9 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 11 Jul 2019 16:41:00 -0400 Subject: [PATCH 0535/1247] kfdtest: MMBench: Test a more useful range of buffer sizes Currently the test only covers relatively small buffers sizes. It's useful to test buffer sizes up to 1GB to see the impact of features that target the efficiency of large buffer allocations and mappings. Change-Id: I2e8d5afd482894dbe2166f32d38091199b9c15e6 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/KFDMemoryTest.cpp | 64 ++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 51579f3c79..7fe844c14d 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -956,17 +956,30 @@ TEST_F(KFDMemoryTest, MMBench) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); - const unsigned nBufs = 1000; /* measure us, report ns */ unsigned testIndex, sizeIndex, memType, nMemTypes; const char *memTypeStrings[2] = {"SysMem", "VRAM"}; - const unsigned nSizes = 4; - const unsigned bufSizes[nSizes] = {PAGE_SIZE, PAGE_SIZE*4, PAGE_SIZE*16, PAGE_SIZE*64}; + const struct { + unsigned size; + unsigned num; + } bufParams[] = { + /* Buffer sizes in x16 increments. Limit memory usage to about + * 1GB. For small sizes we use 1000 buffers, which means we + * conveniently measure microseconds and report nanoseconds. + */ + {PAGE_SIZE , 1000}, /* 4KB */ + {PAGE_SIZE << 4, 1000}, /* 64KB */ + {PAGE_SIZE << 9, 500}, /* 2MB */ + {PAGE_SIZE << 13, 32}, /* 32MB */ + {PAGE_SIZE << 18, 1}, /* 1GB */ + }; + const unsigned nSizes = sizeof(bufParams) / sizeof(bufParams[0]); const unsigned nTests = nSizes << 2; -#define TEST_BUFSIZE(index) (bufSizes[(index) & (nSizes-1)]) +#define TEST_BUFSIZE(index) (bufParams[(index) % nSizes].size) +#define TEST_NBUFS(index) (bufParams[(index) % nSizes].num) #define TEST_MEMTYPE(index) ((index / nSizes) & 0x1) #define TEST_SDMA(index) (((index / nSizes) >> 1) & 0x1) - void *bufs[nBufs]; + void *bufs[1000]; HSAuint64 start, end; unsigned i; HSAKMT_STATUS ret; @@ -1022,16 +1035,17 @@ TEST_F(KFDMemoryTest, MMBench) { } \ } while (0) - LOG() << "Test (avg. ns)\t alloc mapOne umapOne mapAll umapAll free" << std::endl; + LOG() << "Test (avg. ns)\t alloc mapOne umapOne mapAll umapAll free" << std::endl; for (testIndex = 0; testIndex < nTests; testIndex++) { unsigned bufSize = TEST_BUFSIZE(testIndex); + unsigned nBufs = TEST_NBUFS(testIndex); unsigned memType = TEST_MEMTYPE(testIndex); bool interleaveSDMA = TEST_SDMA(testIndex); HSAuint64 allocTime, map1Time, unmap1Time, mapAllTime, unmapAllTime, freeTime; HSAuint32 allocNode; - if ((testIndex & (nSizes-1)) == 0) - LOG() << "--------------------------------------------------------------------" << std::endl; + if ((testIndex % nSizes) == 0) + LOG() << "--------------------------------------------------------------------------" << std::endl; if (memType >= nMemTypes) continue; // skip unsupported mem types @@ -1107,16 +1121,36 @@ TEST_F(KFDMemoryTest, MMBench) { freeTime = GetSystemTickCountInMicroSec() - start; IDLE_SDMA(); + allocTime = allocTime * 1000 / nBufs; + map1Time = map1Time * 1000 / nBufs; + unmap1Time = unmap1Time * 1000 / nBufs; + mapAllTime = mapAllTime * 1000 / nBufs; + unmapAllTime = unmapAllTime * 1000 / nBufs; + freeTime = freeTime * 1000 / nBufs; + + unsigned bufSizeLog; + char bufSizeUnit; + if (bufSize < (1 << 20)) { + bufSizeLog = bufSize >> 10; + bufSizeUnit = 'K'; + } else if (bufSize < (1 << 30)) { + bufSizeLog = bufSize >> 20; + bufSizeUnit = 'M'; + } else { + bufSizeLog = bufSize >> 30; + bufSizeUnit = 'G'; + } + LOG() << std::dec << std::setiosflags(std::ios::right) - << std::setw(3) << (bufSize >> 10) << "K-" + << std::setw(3) << bufSizeLog << bufSizeUnit << "-" << memTypeStrings[memType] << "-" << (interleaveSDMA ? "SDMA\t" : "noSDMA\t") - << std::setw(8) << allocTime - << std::setw(8) << map1Time - << std::setw(8) << unmap1Time - << std::setw(8) << mapAllTime - << std::setw(8) << unmapAllTime - << std::setw(8) << freeTime << std::endl; + << std::setw(9) << allocTime + << std::setw(9) << map1Time + << std::setw(9) << unmap1Time + << std::setw(9) << mapAllTime + << std::setw(9) << unmapAllTime + << std::setw(9) << freeTime << std::endl; #define MMBENCH_KEY_PREFIX memTypeStrings[memType] << "-" \ << (interleaveSDMA ? "SDMA" : "noSDMA") << "-" \ From ff82d3a354d81ef41c5bae8bac3f79e66b40e02d Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Mon, 8 Jul 2019 15:53:31 -0400 Subject: [PATCH 0536/1247] kfdtest: Add Gfx10 pm4 packet format Add release_mem and acquire_mem pm4 packet format for nv Change-Id: I172407c3418005922c17937e1e43f57d153ea732 Signed-off-by: shaoyunl --- tests/kfdtest/include/pm4_pkt_struct_nv.h | 173 ++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 tests/kfdtest/include/pm4_pkt_struct_nv.h diff --git a/tests/kfdtest/include/pm4_pkt_struct_nv.h b/tests/kfdtest/include/pm4_pkt_struct_nv.h new file mode 100644 index 0000000000..df21bc670f --- /dev/null +++ b/tests/kfdtest/include/pm4_pkt_struct_nv.h @@ -0,0 +1,173 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __PM4__PKT__STRUCT__NV__HPP__ +#define __PM4__PKT__STRUCT__NV__HPP__ + +#include "pm4_pkt_struct_ai.h" + +typedef struct _PM4_ACQUIRE_MEM_NV +{ + union + { + PM4_TYPE_3_HEADER header; ///header + unsigned int ordinal1; + }; + + unsigned int reserved; + + unsigned int coher_size; + + + union + { + struct + { + unsigned int coher_size_hi:8; + unsigned int reserved1:24; + } bitfields3; + unsigned int ordinal4; + }; + + unsigned int coher_base_lo; + + + union + { + struct + { + unsigned int coher_base_hi:24; + unsigned int reserved2:8; + } bitfields4; + unsigned int ordinal6; + }; + + union + { + struct + { + unsigned int poll_interval:16; + unsigned int reserved3:16; + } bitfields5; + unsigned int ordinal7; + }; + + union + { + struct + { + unsigned int gcr_cntl:18; + unsigned int reserved4:14; + } bitfields6; + unsigned int ordinal8; + }; + + +} PM4ACQUIRE_MEM_NV, *PPM4ACQUIRE_MEM_NV; + +typedef struct PM4_MEC_RELEASE_MEM_NV { + union { + PM4_TYPE_3_HEADER header; + unsigned int ordinal1; + }; + + union { + struct { + unsigned int event_type:6; + unsigned int reserved1:2; + AI_MEC_RELEASE_MEM_event_index_enum event_index:4; + unsigned int gcr_cntl:12; + unsigned int reserved4:1; + AI_MEC_RELEASE_MEM_cache_policy_enum cache_policy:2; + unsigned int reserved5:1; + AI_MEC_RELEASE_MEM_pq_exe_status_enum pq_exe_status:1; + unsigned int reserved6:3; + } bitfields2; + unsigned int ordinal2; + }; + + union { + struct { + unsigned int reserved7:16; + AI_MEC_RELEASE_MEM_dst_sel_enum dst_sel:2; + unsigned int reserved8:6; + AI_MEC_RELEASE_MEM_int_sel_enum int_sel:3; + unsigned int reserved9:2; + AI_MEC_RELEASE_MEM_data_sel_enum data_sel:3; + } bitfields3; + unsigned int ordinal3; + }; + + union { + struct { + unsigned int reserved10:2; + unsigned int address_lo_32b:30; + } bitfields4a; + struct { + unsigned int reserved11:3; + unsigned int address_lo_64b:29; + } bitfields4b; + unsigned int reserved12; + + unsigned int ordinal4; + }; + + union { + unsigned int address_hi; + + unsigned int reserved13; + + unsigned int ordinal5; + }; + + union { + unsigned int data_lo; + + unsigned int cmp_data_lo; + + struct { + unsigned int dw_offset:16; + unsigned int num_dwords:16; + } bitfields6c; + unsigned int reserved14; + + unsigned int ordinal6; + }; + + union { + unsigned int data_hi; + + unsigned int cmp_data_hi; + + unsigned int reserved15; + + unsigned int reserved16; + + unsigned int ordinal7; + }; + + unsigned int int_ctxid; +} PM4MEC_RELEASE_MEM_NV, *PPM4MEC_RELEASE_MEM_NV; + + +#endif // __PM4__PKT__STRUCT__NV__HPP__ From e9882daf11a9d2e6c7a30e708e61d8b06d289015 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Wed, 17 Jul 2019 14:25:02 -0400 Subject: [PATCH 0537/1247] KFDTest : Add gfx1xxx release_mem and acquire_mem packet support use family ID as parameter when construct the packets Change-Id: I6c1706954ab7b8cbb8bef2aab16edf21f5e1abf0 Signed-off-by: shaoyunl --- tests/kfdtest/src/Dispatch.cpp | 6 +- tests/kfdtest/src/KFDEventTest.cpp | 8 +- tests/kfdtest/src/KFDMemoryTest.cpp | 4 +- tests/kfdtest/src/KFDQMTest.cpp | 18 +- tests/kfdtest/src/KFDRASTest.cpp | 2 +- tests/kfdtest/src/KFDTestUtilQueue.cpp | 4 +- tests/kfdtest/src/PM4Packet.cpp | 231 ++++++++++++++++--------- tests/kfdtest/src/PM4Packet.hpp | 34 ++-- tests/kfdtest/src/PM4Queue.cpp | 2 +- 9 files changed, 196 insertions(+), 113 deletions(-) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index 6fc9ffe56c..802841060c 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -80,7 +80,7 @@ void Dispatch::Submit(BaseQueue& queue) { EventData.EventData.SyncVar.SyncVar.UserData, m_pEop->EventId)); } - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, m_pEop->EventData.HWData2, m_pEop->EventId)); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, m_pEop->EventData.HWData2, m_pEop->EventId)); if (!queue.GetSkipWaitConsump()) queue.Wait4PacketConsumption(); @@ -200,7 +200,7 @@ void Dispatch::BuildIb() { // ORDERED_APPEND_MODE=0, USE_THREAD_DIMENSIONS=1, ORDER_MODE=0, DISPATCH_CACHE_CNTL=0, // SCALAR_L1_INV_VOL=0, VECTOR_L1_INV_VOL=0, DATA_ATC=?, RESTORE=0} - m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket()); + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId)); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_START_X, COMPUTE_DISPATCH_DIMS_VALUES, ARRAY_SIZE(COMPUTE_DISPATCH_DIMS_VALUES))); @@ -225,5 +225,5 @@ void Dispatch::BuildIb() { m_IndirectBuf.AddPacket(PM4PartialFlushPacket()); - m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket()); + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId)); } diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index 16e53613fc..9aebb53b46 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -103,7 +103,7 @@ TEST_F(KFDEventTest, SignalEvent) { /* From gfx9 onward, m_pHsaEvent->EventId will also be passed to int_ctxid in * the Release Mem packet, which is used as context id in ISR. */ - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); queue.Wait4PacketConsumption(); @@ -148,7 +148,7 @@ class QueueAndSignalBenchmark { PM4Queue queue; HsaEvent** pHsaEvent = reinterpret_cast(calloc(eventCount, sizeof(HsaEvent*))); - size_t packetSize = PM4ReleaseMemoryPacket(false, 0, 0).SizeInBytes(); + size_t packetSize = PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, 0, 0).SizeInBytes(); int qSize = fmax(PAGE_SIZE, pow2_round_up(packetSize*eventCount + 1)); time = 0; @@ -162,7 +162,7 @@ class QueueAndSignalBenchmark { if (r != HSAKMT_STATUS_SUCCESS) goto exit; - queue.PlacePacket(PM4ReleaseMemoryPacket(false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); + queue.PlacePacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); } startTime = gettime(); @@ -278,7 +278,7 @@ TEST_F(KFDEventTest, SignalMultipleEventsWaitForAll) { unsigned int pktSizeDwords = 0; for (i = 0; i < EVENT_NUMBER; i++) { - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, pHsaEvent[i]->EventData.HWData2, + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); queue.Wait4PacketConsumption(); diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 7fe844c14d..69b0e5e477 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -564,7 +564,7 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { mem[2] = 0x0; queue.PlaceAndSubmitPacket(PM4WriteDataPacket(reinterpret_cast(gpuva2), 0xdeadbeef)); - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(true, 0, 0)); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, true, 0, 0)); queue.Wait4PacketConsumption(); EXPECT_EQ(true, WaitOnValue((unsigned int *)(&mem[2]), 0xdeadbeef)); EXPECT_SUCCESS(queue.Destroy()); @@ -1450,7 +1450,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { data0[0], data0[1])); queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)mem1, data1[0], data1[1])); - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(true, 0, 0)); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, true, 0, 0)); queue.Wait4PacketConsumption(); /* Allow any process to trace this one. If kernel is built without diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 5f9dada9f0..07b83c6dea 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1092,7 +1092,7 @@ TEST_F(KFDQMTest, QueueLatency) { PM4Queue queue; const int queueSize = PAGE_SIZE * 2; - const int packetSize = PM4ReleaseMemoryPacket(0, 0, 0, 0, 0).SizeInBytes(); + const int packetSize = PM4ReleaseMemoryPacket(m_FamilyId, 0, 0, 0, 0, 0).SizeInBytes(); /* We always leave one NOP(dword) empty after packet which is required by ring itself. * We also place NOPs when queue wraparound to avoid crossing buffer end. See PlacePacket(). * So the worst case is that we need two packetSize space to place one packet. @@ -1141,7 +1141,7 @@ TEST_F(KFDQMTest, QueueLatency) { /* Submit packets serially*/ i = 0; do { - queue.PlacePacket(PM4ReleaseMemoryPacket(true, + queue.PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)&qts[i], 0, true, @@ -1168,7 +1168,7 @@ TEST_F(KFDQMTest, QueueLatency) { /* Workload of queue packet itself */ i = 0; do { - queue.PlacePacket(PM4ReleaseMemoryPacket(true, + queue.PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)&qts[i], 0, true, @@ -1549,7 +1549,7 @@ TEST_F(KFDQMTest, PM4EventInterrupt) { const HSAuint64 bufSize = PAGE_SIZE; const int packetCount = bufSize / sizeof(unsigned int); const int totalPacketSize = packetCount * PM4WriteDataPacket(0, 0).SizeInBytes() + - PM4ReleaseMemoryPacket(0, 0, 0).SizeInBytes(); + PM4ReleaseMemoryPacket(m_FamilyId, 0, 0, 0).SizeInBytes(); const int queueSize = RoundToPowerOf2(totalPacketSize); /* 4 PM4 queues will be running at same time.*/ @@ -1575,7 +1575,7 @@ TEST_F(KFDQMTest, PM4EventInterrupt) { queue[i].PlacePacket(PM4WriteDataPacket(buf[i] + index, 0xdeadbeaf)); /* releaseMemory packet makes sure all previous written data is visible.*/ - queue[i].PlacePacket(PM4ReleaseMemoryPacket(0, + queue[i].PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, 0, reinterpret_cast(event[i]->EventData.HWData2), event[i]->EventId, true)); @@ -1748,10 +1748,10 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { * the first queue. */ otherQueue.PlacePacket( - PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_write_ptr, + PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_write_ptr, pendingWptr, false)); otherQueue.PlacePacket( - PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_DoorBell, + PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_DoorBell, pendingWptr, false)); #endif @@ -1775,10 +1775,10 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { * the PM4 packet on the first queue. */ otherQueue.PlacePacket( - PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_write_ptr, + PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_write_ptr, pendingWptr64, true)); otherQueue.PlacePacket( - PM4ReleaseMemoryPacket(true, (HSAuint64)qRes->Queue_DoorBell, + PM4ReleaseMemoryPacket(m_FamilyId, true, (HSAuint64)qRes->Queue_DoorBell, pendingWptr64, true)); #endif diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/tests/kfdtest/src/KFDRASTest.cpp index 09d61a03fa..428db0e16d 100644 --- a/tests/kfdtest/src/KFDRASTest.cpp +++ b/tests/kfdtest/src/KFDRASTest.cpp @@ -164,7 +164,7 @@ TEST_F(KFDRASTest, MixEventsTest) { ASSERT_SUCCESS(queue.Create(m_defaultGPUNode)); - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(false, + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, pHsaEvent->EventData.HWData2, pHsaEvent->EventId)); queue.Wait4PacketConsumption(); diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/tests/kfdtest/src/KFDTestUtilQueue.cpp index 8369e5a9f8..991485fbd4 100644 --- a/tests/kfdtest/src/KFDTestUtilQueue.cpp +++ b/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -103,7 +103,7 @@ class AsyncMPSQ { if (m_queueType == HSA_QUEUE_SDMA) return SDMATimePacket(0).SizeInBytes(); else if (m_queueType == HSA_QUEUE_COMPUTE) - return PM4ReleaseMemoryPacket(0, 0, 0, 0, 0).SizeInBytes(); + return PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0, 0, 0, 0, 0).SizeInBytes(); return 0; } @@ -126,7 +126,7 @@ class AsyncMPSQ { PlacePacket(SDMATimePacket(addr)); else if (m_queueType == HSA_QUEUE_COMPUTE) PlacePacket( - PM4ReleaseMemoryPacket(true, (HSAuint64)addr, 0, true, true)); + PM4ReleaseMemoryPacket(g_TestGPUFamilyId, true, (HSAuint64)addr, 0, true, true)); else WARN() << "Unsupported queue type!" << std::endl; } diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index 71e919d1b7..503e1b799c 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -73,41 +73,46 @@ void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) { memcpy(m_pPacketData->data, data, m_ndw * sizeof(uint32_t)); } -PM4ReleaseMemoryPacket::~PM4ReleaseMemoryPacket(void) { - if (m_pPacketData) - free(m_pPacketData); +PM4ReleaseMemoryPacket::PM4ReleaseMemoryPacket(unsigned int familyId, bool isPolling, + uint64_t address, uint64_t data, bool is64bit, bool isTimeStamp):m_pPacketData(NULL) { + + if (familyId < FAMILY_AI) + InitPacketCI(isPolling, address, data, is64bit, isTimeStamp); + else if (familyId < FAMILY_NV) + InitPacketAI(isPolling, address, data, is64bit, isTimeStamp); + else + InitPacketNV(isPolling, address, data, is64bit, isTimeStamp); } -void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, - uint64_t data, bool is64bit, bool isTimeStamp) { - if (g_TestGPUFamilyId < FAMILY_AI) { - PM4_RELEASE_MEM_CI *pkt; +void PM4ReleaseMemoryPacket::InitPacketCI(bool isPolling, uint64_t address, + uint64_t data, bool is64bit, bool isTimeStamp) { + PM4_RELEASE_MEM_CI *pkt; - m_packetSize = sizeof(PM4_RELEASE_MEM_CI); - pkt = reinterpret_cast(calloc(1, m_packetSize)); - m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); + m_packetSize = sizeof(PM4_RELEASE_MEM_CI); + pkt = reinterpret_cast(calloc(1, m_packetSize)); + m_pPacketData = pkt; + EXPECT_NOTNULL(m_pPacketData); - InitPM4Header(pkt->header, IT_RELEASE_MEM); + InitPM4Header(pkt->header, IT_RELEASE_MEM); - pkt->bitfields2.event_type = 0x14; - pkt->bitfields2.event_index = event_index_mec_release_mem_EVENT_WRITE_EOP_5; + pkt->bitfields2.event_type = 0x14; + pkt->bitfields2.event_index = event_index_mec_release_mem_EVENT_WRITE_EOP_5; // Possible values: // 0101(5): EVENT_WRITE_EOP event types // 0110(6): Reserved for EVENT_WRITE_EOS packet. // 0111(7): Reserved (previously) for EVENT_WRITE packet. - pkt->bitfields2.l2_wb = 1; - pkt->bitfields2.l2_inv = 1; - pkt->bitfields2.cache_policy = cache_policy_mec_release_mem_BYPASS_2; - pkt->bitfields2.atc = is_dgpu() ? + pkt->bitfields2.l2_wb = 1; + pkt->bitfields2.l2_inv = 1; + pkt->bitfields2.cache_policy = cache_policy_mec_release_mem_BYPASS_2; + pkt->bitfields2.atc = is_dgpu() ? atc_mec_release_mem_ci_NOT_USE_ATC_0 : atc_mec_release_mem_ci_USE_ATC_1; // ATC setting for fences and timestamps to the MC or TCL2. - pkt->bitfields3.dst_sel = dst_sel_mec_release_mem_MEMORY_CONTROLLER_0; + pkt->bitfields3.dst_sel = dst_sel_mec_release_mem_MEMORY_CONTROLLER_0; // Possible values: // 0 - memory_controller. // 1 - tc_l2. - if (address) { - pkt->bitfields3.int_sel = (isPolling ? + if (address) { + pkt->bitfields3.int_sel = (isPolling ? int_sel_mec_release_mem_SEND_DATA_AFTER_WRITE_CONFIRM_3 : int_sel_mec_release_mem_SEND_INTERRUPT_AFTER_WRITE_CONFIRM_2); // Possible values: @@ -116,10 +121,10 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, // 2 - Send Interrupt when Write Confirm (WC) is received from the MC. // 3 - Wait for WC, but dont send interrupt (applicable to 7.3+) [g73_1] // 4 - Reserved for INTERRUPT packet - if (isTimeStamp && is64bit) - pkt->bitfields3.data_sel = data_sel_mec_release_mem_SEND_GPU_CLOCK_COUNTER_3; - else - pkt->bitfields3.data_sel = is64bit ? + if (isTimeStamp && is64bit) + pkt->bitfields3.data_sel = data_sel_mec_release_mem_SEND_GPU_CLOCK_COUNTER_3; + else + pkt->bitfields3.data_sel = is64bit ? data_sel_mec_release_mem_SEND_64_BIT_DATA_2 : data_sel_mec_release_mem_SEND_32_BIT_LOW_1; // Possible values: @@ -131,62 +136,108 @@ void PM4ReleaseMemoryPacket::InitPacket(bool isPolling, uint64_t address, // 5 - Store GDS Data to memory. // 6 - Reserved for use by the CP for Signal Semaphore. // 7 - Reserved for use by the CP for Wait Semaphore. - } else { - pkt->bitfields3.int_sel = (isPolling ? + } else { + pkt->bitfields3.int_sel = (isPolling ? int_sel_mec_release_mem_NONE_0 : int_sel_mec_release_mem_SEND_INTERRUPT_ONLY_1); - pkt->bitfields3.data_sel = data_sel_mec_release_mem_NONE_0; - } + pkt->bitfields3.data_sel = data_sel_mec_release_mem_NONE_0; + } - pkt->bitfields4a.address_lo_dword_aligned = static_cast((address&0xffffffff) >> 2); - pkt->addr_hi = static_cast(address>>32); + pkt->bitfields4a.address_lo_dword_aligned = static_cast((address&0xffffffff) >> 2); + pkt->addr_hi = static_cast(address>>32); - pkt->data_lo = static_cast(data); - pkt->data_hi = static_cast(data >> 32); - } else { - PM4MEC_RELEASE_MEM_AI *pkt; + pkt->data_lo = static_cast(data); + pkt->data_hi = static_cast(data >> 32); +} +void PM4ReleaseMemoryPacket::InitPacketAI(bool isPolling, uint64_t address, + uint64_t data, bool is64bit, bool isTimeStamp) { + PM4MEC_RELEASE_MEM_AI *pkt; - m_packetSize = sizeof(PM4MEC_RELEASE_MEM_AI); - pkt = reinterpret_cast(calloc(1, m_packetSize)); - m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); + m_packetSize = sizeof(PM4MEC_RELEASE_MEM_AI); + pkt = reinterpret_cast(calloc(1, m_packetSize)); + m_pPacketData = pkt; + EXPECT_NOTNULL(m_pPacketData); - InitPM4Header(pkt->header, IT_RELEASE_MEM); + InitPM4Header(pkt->header, IT_RELEASE_MEM); - pkt->bitfields2.event_type = 0x14; - pkt->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe; - pkt->bitfields2.tc_wb_action_ena = 1; - pkt->bitfields2.tc_action_ena = 1; - pkt->bitfields2.cache_policy = cache_policy__mec_release_mem__lru; + pkt->bitfields2.event_type = 0x14; + pkt->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe; + pkt->bitfields2.tc_wb_action_ena = 1; + pkt->bitfields2.tc_action_ena = 1; + pkt->bitfields2.cache_policy = cache_policy__mec_release_mem__lru; - pkt->bitfields3.dst_sel = dst_sel__mec_release_mem__memory_controller; + pkt->bitfields3.dst_sel = dst_sel__mec_release_mem__memory_controller; - if (address) { - pkt->bitfields3.int_sel = (isPolling ? + if (address) { + pkt->bitfields3.int_sel = (isPolling ? int_sel__mec_release_mem__send_data_after_write_confirm: int_sel__mec_release_mem__send_interrupt_after_write_confirm); - if (isTimeStamp && is64bit) - pkt->bitfields3.data_sel = data_sel__mec_release_mem__send_gpu_clock_counter; - else - pkt->bitfields3.data_sel = is64bit ? + if (isTimeStamp && is64bit) + pkt->bitfields3.data_sel = data_sel__mec_release_mem__send_gpu_clock_counter; + else + pkt->bitfields3.data_sel = is64bit ? data_sel__mec_release_mem__send_64_bit_data : data_sel__mec_release_mem__send_32_bit_low; - } else { - pkt->bitfields3.int_sel = (isPolling ? + } else { + pkt->bitfields3.int_sel = (isPolling ? int_sel__mec_release_mem__none: int_sel__mec_release_mem__send_interrupt_only); - pkt->bitfields3.data_sel = data_sel__mec_release_mem__none; - } - - pkt->bitfields4a.address_lo_32b = static_cast((address&0xffffffff) >> 2); - pkt->address_hi = static_cast(address>>32); - - pkt->data_lo = static_cast(data); - pkt->data_hi = static_cast(data >> 32); - - pkt->int_ctxid = static_cast(data); + pkt->bitfields3.data_sel = data_sel__mec_release_mem__none; } + + pkt->bitfields4a.address_lo_32b = static_cast((address&0xffffffff) >> 2); + pkt->address_hi = static_cast(address>>32); + + pkt->data_lo = static_cast(data); + pkt->data_hi = static_cast(data >> 32); + + pkt->int_ctxid = static_cast(data); +} + +void PM4ReleaseMemoryPacket::InitPacketNV(bool isPolling, uint64_t address, + uint64_t data, bool is64bit, bool isTimeStamp) { + PM4MEC_RELEASE_MEM_NV *pkt; + + m_packetSize = sizeof(PM4_MEC_RELEASE_MEM_NV); + pkt = reinterpret_cast(calloc(1, m_packetSize)); + m_pPacketData = pkt; + EXPECT_NOTNULL(m_pPacketData); + + InitPM4Header(pkt->header, IT_RELEASE_MEM); + + pkt->bitfields2.event_type = 0x14; + pkt->bitfields2.event_index = event_index__mec_release_mem__end_of_pipe; + pkt->bitfields2.gcr_cntl = (1<<10) | (1<<9) | (1<<8) | (1<<3) | (1<<2); + pkt->bitfields2.cache_policy = cache_policy__mec_release_mem__lru; + + pkt->bitfields3.dst_sel = dst_sel__mec_release_mem__memory_controller; + + if (address) { + pkt->bitfields3.int_sel = (isPolling ? + int_sel__mec_release_mem__send_data_after_write_confirm: + int_sel__mec_release_mem__send_interrupt_after_write_confirm); + + if (isTimeStamp && is64bit) + pkt->bitfields3.data_sel = data_sel__mec_release_mem__send_gpu_clock_counter; + else + pkt->bitfields3.data_sel = is64bit ? + data_sel__mec_release_mem__send_64_bit_data : + data_sel__mec_release_mem__send_32_bit_low; + } else { + pkt->bitfields3.int_sel = (isPolling ? + int_sel__mec_release_mem__none: + int_sel__mec_release_mem__send_interrupt_only); + pkt->bitfields3.data_sel = data_sel__mec_release_mem__none; + } + + pkt->bitfields4a.address_lo_32b = static_cast((address&0xffffffff) >> 2); + pkt->address_hi = static_cast(address>>32); + + pkt->data_lo = static_cast(data); + pkt->data_hi = static_cast(data >> 32); + + pkt->int_ctxid = static_cast(data); } PM4IndirectBufPacket::PM4IndirectBufPacket(IndirectBuffer *pIb) { @@ -211,22 +262,48 @@ void PM4IndirectBufPacket::InitPacket(IndirectBuffer *pIb) { m_packetData.bitfields4.vmid = 0; // in iommutest: vmid = queueParams.VMID; m_packetData.bitfields4.cache_policy = cache_policy_indirect_buffer_BYPASS_2; } +PM4AcquireMemoryPacket::PM4AcquireMemoryPacket(unsigned int familyId):m_pPacketData(NULL) +{ -PM4AcquireMemoryPacket::PM4AcquireMemoryPacket(void) { - memset(&m_packetData, 0, SizeInBytes()); - InitPM4Header(m_packetData.header, IT_ACQUIRE_MEM); - - m_packetData.bitfields2.coher_cntl = 0x28c00000; // copied from the way the HSART does this. - m_packetData.bitfields2.engine = engine_acquire_mem_PFP_0; - m_packetData.coher_size = 0xFFFFFFFF; - m_packetData.bitfields3.coher_size_hi = 0; - m_packetData.coher_base_lo = 0; - m_packetData.bitfields4.coher_base_hi = 0; - m_packetData.bitfields5.poll_interval = 4; // copied from the way the HSART does this. + if (familyId < FAMILY_NV) + InitPacketAI(); + else + InitPacketNV(); } -unsigned int PM4AcquireMemoryPacket::SizeInBytes() const { - return sizeof(PM4ACQUIRE_MEM); +void PM4AcquireMemoryPacket::InitPacketAI(void) { + + PM4ACQUIRE_MEM *pkt; + m_packetSize = sizeof(PM4ACQUIRE_MEM); + pkt = reinterpret_cast(calloc(1, m_packetSize)); + m_pPacketData = pkt; + EXPECT_NOTNULL(m_pPacketData); + InitPM4Header(pkt->header, IT_ACQUIRE_MEM); + pkt->bitfields2.coher_cntl = 0x28c00000; // copied from the way the HSART does this. + pkt->bitfields2.engine = engine_acquire_mem_PFP_0; + pkt->coher_size = 0xFFFFFFFF; + pkt->bitfields3.coher_size_hi = 0; + pkt->coher_base_lo = 0; + pkt->bitfields4.coher_base_hi = 0; + pkt->bitfields5.poll_interval = 4; // copied from the way the HSART does this. +} +void PM4AcquireMemoryPacket::InitPacketNV(void) { + PM4ACQUIRE_MEM_NV *pkt; + m_packetSize = sizeof(PM4ACQUIRE_MEM_NV); + pkt = reinterpret_cast(calloc(1, m_packetSize)); + m_pPacketData = pkt; + EXPECT_NOTNULL(m_pPacketData); + InitPM4Header(pkt->header, IT_ACQUIRE_MEM); + pkt->coher_size = 0xFFFFFFFF; + pkt->bitfields3.coher_size_hi = 0; + pkt->coher_base_lo = 0; + pkt->bitfields4.coher_base_hi = 0; + pkt->bitfields5.poll_interval = 4; //copied from the way the HSART does this. + /* Invalidate gL2, gL1 with range base + * Invalidate GLV, GLK (L0$) + * Invalidate all Icache (GLI) + */ + pkt->bitfields6.gcr_cntl = (1<<14|1<<9|1<<8|1<<7|1); } PM4SetShaderRegPacket::PM4SetShaderRegPacket(void) diff --git a/tests/kfdtest/src/PM4Packet.hpp b/tests/kfdtest/src/PM4Packet.hpp index 8f2f5753b5..8624db0dd4 100644 --- a/tests/kfdtest/src/PM4Packet.hpp +++ b/tests/kfdtest/src/PM4Packet.hpp @@ -29,6 +29,7 @@ #include "pm4_pkt_struct_common.h" #include "pm4_pkt_struct_ci.h" #include "pm4_pkt_struct_ai.h" +#include "pm4_pkt_struct_nv.h" #include "IndirectBuffer.hpp" // @class PM4Packet: Marks a group of all PM4 packets @@ -87,21 +88,24 @@ class PM4ReleaseMemoryPacket : public PM4Packet { // Empty constructor, before using the packet call the init func PM4ReleaseMemoryPacket(void): m_pPacketData(NULL) {} // This contructor will also init the packet, no need for additional calls - PM4ReleaseMemoryPacket(bool isPolling, uint64_t address, uint64_t data, - bool is64bit = false, bool isTimeStamp = false): m_pPacketData(NULL) { - InitPacket(isPolling, address, data, is64bit, isTimeStamp); - } + PM4ReleaseMemoryPacket(unsigned int familyId, bool isPolling, uint64_t address, uint64_t data, + bool is64bit = false, bool isTimeStamp = false); - virtual ~PM4ReleaseMemoryPacket(void); + virtual ~PM4ReleaseMemoryPacket(void) {if (m_pPacketData)free(m_pPacketData);} // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return m_packetSize; } // @returns Pointer to the packet virtual const void *GetPacket() const { return m_pPacketData; } // @brief Initialise the packet - void InitPacket(bool isPolling, uint64_t address, uint64_t data, - bool is64bit = false, bool isTimeStamp = false); private: + void InitPacketCI(bool isPolling, uint64_t address, uint64_t data, + bool is64bit = false, bool isTimeStamp = false); + void InitPacketAI(bool isPolling, uint64_t address, uint64_t data, + bool is64bit = false, bool isTimeStamp = false); + void InitPacketNV(bool isPolling, uint64_t address, uint64_t data, + bool is64bit = false, bool isTimeStamp = false); + void *m_pPacketData; unsigned int m_packetSize; }; @@ -130,17 +134,19 @@ class PM4IndirectBufPacket : public PM4Packet { // @class PM4AcquireMemoryPacket class PM4AcquireMemoryPacket : public PM4Packet { public: - PM4AcquireMemoryPacket(void); - virtual ~PM4AcquireMemoryPacket(void) {} + PM4AcquireMemoryPacket(unsigned int familyId); + virtual ~PM4AcquireMemoryPacket(void) {if (m_pPacketData)free(m_pPacketData);} // @returns the packet size in bytes - virtual unsigned int SizeInBytes() const; - // @returns a pointer to the packet - virtual const void *GetPacket() const { return &m_packetData; } + virtual unsigned int SizeInBytes() const { return m_packetSize; } + // @returns Pointer to the packet + virtual const void *GetPacket() const { return m_pPacketData; } private: - // PM4ACQUIRE_MEM struct contains all the packet's data - PM4ACQUIRE_MEM m_packetData; + void InitPacketAI(void); + void InitPacketNV(void); + void *m_pPacketData; + unsigned int m_packetSize; }; // @class PM4SetShaderRegPacket Packet that writes to consecutive registers starting at baseOffset. diff --git a/tests/kfdtest/src/PM4Queue.cpp b/tests/kfdtest/src/PM4Queue.cpp index 548e35986c..6613ec8909 100644 --- a/tests/kfdtest/src/PM4Queue.cpp +++ b/tests/kfdtest/src/PM4Queue.cpp @@ -73,7 +73,7 @@ void PM4Queue::SubmitPacket() { void PM4Queue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { if (event) { - PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(0, + PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0, event->EventData.HWData2, event->EventId, true)); From b4e834ab616451690a75519e9bb3e3a271b09453 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Tue, 16 Jul 2019 16:46:39 -0400 Subject: [PATCH 0538/1247] KFDTest: remove the usage global g_TestGPUFamilyId Adjust the KFDTest for multi-gou support Change-Id: Ib3491e3f645d35fdba6ab702d65fcc86f48d3958 Signed-off-by: shaoyunl --- tests/kfdtest/src/BasePacket.cpp | 5 +++++ tests/kfdtest/src/BasePacket.hpp | 3 ++- tests/kfdtest/src/BaseQueue.cpp | 2 ++ tests/kfdtest/src/BaseQueue.hpp | 2 ++ tests/kfdtest/src/Dispatch.cpp | 15 ++++++++++----- tests/kfdtest/src/Dispatch.hpp | 1 + tests/kfdtest/src/KFDBaseComponentTest.cpp | 10 ++++++++-- tests/kfdtest/src/KFDBaseComponentTest.hpp | 3 +++ tests/kfdtest/src/KFDEventTest.cpp | 5 +++-- tests/kfdtest/src/KFDTestFlags.hpp | 1 - tests/kfdtest/src/KFDTestMain.cpp | 1 + tests/kfdtest/src/KFDTestUtilQueue.cpp | 4 ++-- tests/kfdtest/src/PM4Packet.cpp | 5 ++++- tests/kfdtest/src/PM4Queue.cpp | 4 ++-- tests/kfdtest/src/SDMAPacket.cpp | 2 +- tests/kfdtest/src/SDMAQueue.cpp | 2 +- 16 files changed, 47 insertions(+), 18 deletions(-) diff --git a/tests/kfdtest/src/BasePacket.cpp b/tests/kfdtest/src/BasePacket.cpp index fd714ca756..daae6691ae 100644 --- a/tests/kfdtest/src/BasePacket.cpp +++ b/tests/kfdtest/src/BasePacket.cpp @@ -23,6 +23,11 @@ #include "BasePacket.hpp" #include "KFDTestUtil.hpp" +#include "KFDBaseComponentTest.hpp" + +BasePacket::BasePacket(void) { + m_FamilyId = g_baseTest->GetFamilyIdFromDefaultNode(); +} void BasePacket::Dump() const { unsigned int size = SizeInDWords(); diff --git a/tests/kfdtest/src/BasePacket.hpp b/tests/kfdtest/src/BasePacket.hpp index 1802f901df..efbebc88f1 100644 --- a/tests/kfdtest/src/BasePacket.hpp +++ b/tests/kfdtest/src/BasePacket.hpp @@ -37,7 +37,7 @@ enum PACKETTYPE { // @class BasePacket class BasePacket { public: - BasePacket(void) {} + BasePacket(void); virtual ~BasePacket(void) {} // @returns Packet type @@ -52,6 +52,7 @@ class BasePacket { void Dump() const; protected: + unsigned int m_FamilyId; }; #endif // __KFD_BASE_PACKET__H__ diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 4aa9e843d2..835ab4c71d 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -26,6 +26,7 @@ #include "PM4Queue.hpp" #include "AqlQueue.hpp" #include "hsakmt.h" +#include "KFDBaseComponentTest.hpp" BaseQueue::BaseQueue() :m_QueueBuf(NULL), @@ -81,6 +82,7 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 m_pendingWptr = 0; m_pendingWptr64 = 0; m_Node = NodeId; + m_FamilyId = g_baseTest->GetFamilyIdFromNodeId(NodeId); return status; } diff --git a/tests/kfdtest/src/BaseQueue.hpp b/tests/kfdtest/src/BaseQueue.hpp index 0e5b946290..570af95c8c 100644 --- a/tests/kfdtest/src/BaseQueue.hpp +++ b/tests/kfdtest/src/BaseQueue.hpp @@ -87,6 +87,7 @@ class BaseQueue { HSAuint64 GetPendingWptr64() { return m_pendingWptr64; } virtual _HSA_QUEUE_TYPE GetQueueType() = 0; unsigned int GetNodeId() { return m_Node; } + unsigned int GetFamilyId() { return m_FamilyId; } protected: static const unsigned int CMD_NOP_TYPE_2 = 0x80000000; @@ -98,6 +99,7 @@ class BaseQueue { HsaQueueResource m_Resources; HsaMemoryBuffer *m_QueueBuf; unsigned int m_Node; + unsigned int m_FamilyId; // @return Write pointer modulo queue size in dwords virtual unsigned int Wptr() = 0; diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index 802841060c..559594ee44 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -28,6 +28,8 @@ #include "asic_reg/gfx_7_2_d.h" #include "asic_reg/gfx_7_2_sh_mask.h" +#include "KFDBaseComponentTest.hpp" + Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset) :m_IsaBuf(isaBuf), m_IndirectBuf(PACKETTYPE_PM4, PAGE_SIZE / sizeof(unsigned int), isaBuf.Node()), m_DimX(1), m_DimY(1), m_DimZ(1), m_pArg1(NULL), m_pArg2(NULL), m_pEop(NULL), m_ScratchEn(false), @@ -39,6 +41,8 @@ Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset) eventDesc.SyncVar.SyncVarSize = 0; hsaKmtCreateEvent(&eventDesc, !eventAutoReset, false, &m_pEop); + + m_FamilyId = g_baseTest->GetFamilyIdFromNodeId(isaBuf.Node()); } Dispatch::~Dispatch() { @@ -69,6 +73,7 @@ void Dispatch::SetSpiPriority(unsigned int priority) { void Dispatch::Submit(BaseQueue& queue) { ASSERT_NE(m_pEop, (void*)0); + EXPECT_EQ(m_FamilyId, queue.GetFamilyId()); BuildIb(); @@ -80,7 +85,7 @@ void Dispatch::Submit(BaseQueue& queue) { EventData.EventData.SyncVar.SyncVar.UserData, m_pEop->EventId)); } - queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, m_pEop->EventData.HWData2, m_pEop->EventId)); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, m_pEop->EventData.HWData2, m_pEop->EventId)); if (!queue.GetSkipWaitConsump()) queue.Wait4PacketConsumption(); @@ -200,14 +205,14 @@ void Dispatch::BuildIb() { // ORDERED_APPEND_MODE=0, USE_THREAD_DIMENSIONS=1, ORDER_MODE=0, DISPATCH_CACHE_CNTL=0, // SCALAR_L1_INV_VOL=0, VECTOR_L1_INV_VOL=0, DATA_ATC=?, RESTORE=0} - m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId)); + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId)); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_START_X, COMPUTE_DISPATCH_DIMS_VALUES, ARRAY_SIZE(COMPUTE_DISPATCH_DIMS_VALUES))); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_LO, - (g_TestGPUFamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8, - (g_TestGPUFamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8))); + (m_FamilyId >= FAMILY_AI) ? COMPUTE_PGM_VALUES_GFX9 : COMPUTE_PGM_VALUES_GFX8, + (m_FamilyId >= FAMILY_AI) ? ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX9) : ARRAY_SIZE(COMPUTE_PGM_VALUES_GFX8))); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC, ARRAY_SIZE(COMPUTE_PGM_RSRC))); @@ -225,5 +230,5 @@ void Dispatch::BuildIb() { m_IndirectBuf.AddPacket(PM4PartialFlushPacket()); - m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(g_TestGPUFamilyId)); + m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId)); } diff --git a/tests/kfdtest/src/Dispatch.hpp b/tests/kfdtest/src/Dispatch.hpp index b89306c02d..ce8ffba21b 100644 --- a/tests/kfdtest/src/Dispatch.hpp +++ b/tests/kfdtest/src/Dispatch.hpp @@ -70,6 +70,7 @@ class Dispatch { HSAuint64 m_scratch_base; unsigned int m_SpiPriority; + unsigned int m_FamilyId; }; #endif // __KFD_DISPATCH__H__ diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index c56e0ea239..27f3db1cda 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -61,12 +61,12 @@ void KFDBaseComponentTest::SetUp() { const HsaNodeProperties *nodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); ASSERT_NOTNULL(nodeProperties) << "failed to get HSA default GPU Node properties"; - g_TestGPUFamilyId = FamilyIdFromNode(nodeProperties); + m_FamilyId = FamilyIdFromNode(nodeProperties); GetSdmaInfo(nodeProperties, &m_numSdmaEngines, &m_numSdmaXgmiEngines, &m_numSdmaQueuesPerEngine); - m_FamilyId = g_TestGPUFamilyId; + g_baseTest = this; ROUTINE_END } @@ -84,6 +84,7 @@ void KFDBaseComponentTest::TearDown() { EXPECT_SUCCESS(hsaKmtReleaseSystemProperties()); EXPECT_SUCCESS(hsaKmtCloseKFD()); + g_baseTest = NULL; ROUTINE_END } @@ -128,6 +129,11 @@ HSAuint64 KFDBaseComponentTest::GetVramSize(int defaultGPUNode) { return 0; } +unsigned int KFDBaseComponentTest::GetFamilyIdFromNodeId(unsigned int nodeId) +{ + return FamilyIdFromNode(m_NodeInfo.GetNodeProperties(nodeId)); +} + int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) { HsaNodeProperties *nodeProperties; _HSAKMT_STATUS status; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 867718e06d..fce5c0d884 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -55,6 +55,8 @@ class KFDBaseComponentTest : public testing::Test { // @brief Finds DRM Render node corresponding to gpuNode // @return DRM Render Node if successful or -1 on failure int FindDRMRenderNode(int gpuNode); + unsigned int GetFamilyIdFromNodeId(unsigned int nodeId); + unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; } protected: HsaVersionInfo m_VersionInfo; @@ -76,4 +78,5 @@ class KFDBaseComponentTest : public testing::Test { virtual void TearDown(); }; +extern KFDBaseComponentTest* g_baseTest; #endif // __KFD_BASE_COMPONENT_TEST__H__ diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index 9aebb53b46..ca16e871ff 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -147,8 +147,9 @@ class QueueAndSignalBenchmark { uint64_t startTime; PM4Queue queue; + unsigned int familyId = g_baseTest->GetFamilyIdFromNodeId(node); HsaEvent** pHsaEvent = reinterpret_cast(calloc(eventCount, sizeof(HsaEvent*))); - size_t packetSize = PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, 0, 0).SizeInBytes(); + size_t packetSize = PM4ReleaseMemoryPacket(familyId, false, 0, 0).SizeInBytes(); int qSize = fmax(PAGE_SIZE, pow2_round_up(packetSize*eventCount + 1)); time = 0; @@ -162,7 +163,7 @@ class QueueAndSignalBenchmark { if (r != HSAKMT_STATUS_SUCCESS) goto exit; - queue.PlacePacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); + queue.PlacePacket(PM4ReleaseMemoryPacket(familyId, false, pHsaEvent[i]->EventData.HWData2, pHsaEvent[i]->EventId)); } startTime = gettime(); diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index ecd309b9c7..1514695890 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -30,7 +30,6 @@ extern unsigned int g_TestTimeOut; extern int g_TestNodeId; extern int g_TestDstNodeId; extern bool g_IsChildProcess; -extern unsigned int g_TestGPUFamilyId; // Each test should call TEST_START with the test custom profile and HW scheduling enum TESTPROFILE{ diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index 7715bd06a1..a8837f95c4 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -54,6 +54,7 @@ int g_TestNodeId; int g_TestDstNodeId; bool g_IsChildProcess; unsigned int g_TestGPUFamilyId; +class KFDBaseComponentTest *g_baseTest; GTEST_API_ int main(int argc, char **argv) { // Default values for run parameters diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/tests/kfdtest/src/KFDTestUtilQueue.cpp index 991485fbd4..e1fe07d4df 100644 --- a/tests/kfdtest/src/KFDTestUtilQueue.cpp +++ b/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -103,7 +103,7 @@ class AsyncMPSQ { if (m_queueType == HSA_QUEUE_SDMA) return SDMATimePacket(0).SizeInBytes(); else if (m_queueType == HSA_QUEUE_COMPUTE) - return PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0, 0, 0, 0, 0).SizeInBytes(); + return PM4ReleaseMemoryPacket(m_queue->GetFamilyId(), 0, 0, 0, 0, 0).SizeInBytes(); return 0; } @@ -126,7 +126,7 @@ class AsyncMPSQ { PlacePacket(SDMATimePacket(addr)); else if (m_queueType == HSA_QUEUE_COMPUTE) PlacePacket( - PM4ReleaseMemoryPacket(g_TestGPUFamilyId, true, (HSAuint64)addr, 0, true, true)); + PM4ReleaseMemoryPacket(m_queue->GetFamilyId(), true, (HSAuint64)addr, 0, true, true)); else WARN() << "Unsupported queue type!" << std::endl; } diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index 503e1b799c..195256fbaf 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -26,6 +26,7 @@ #include #include "PM4Packet.hpp" #include "hsakmttypes.h" +#include "KFDBaseComponentTest.hpp" #include "asic_reg/gfx_7_2_enum.h" @@ -75,7 +76,7 @@ void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) { PM4ReleaseMemoryPacket::PM4ReleaseMemoryPacket(unsigned int familyId, bool isPolling, uint64_t address, uint64_t data, bool is64bit, bool isTimeStamp):m_pPacketData(NULL) { - + m_FamilyId = familyId; if (familyId < FAMILY_AI) InitPacketCI(isPolling, address, data, is64bit, isTimeStamp); else if (familyId < FAMILY_NV) @@ -264,6 +265,7 @@ void PM4IndirectBufPacket::InitPacket(IndirectBuffer *pIb) { } PM4AcquireMemoryPacket::PM4AcquireMemoryPacket(unsigned int familyId):m_pPacketData(NULL) { + m_FamilyId = familyId; if (familyId < FAMILY_NV) InitPacketAI(); @@ -278,6 +280,7 @@ void PM4AcquireMemoryPacket::InitPacketAI(void) { pkt = reinterpret_cast(calloc(1, m_packetSize)); m_pPacketData = pkt; EXPECT_NOTNULL(m_pPacketData); + InitPM4Header(pkt->header, IT_ACQUIRE_MEM); pkt->bitfields2.coher_cntl = 0x28c00000; // copied from the way the HSART does this. pkt->bitfields2.engine = engine_acquire_mem_PFP_0; diff --git a/tests/kfdtest/src/PM4Queue.cpp b/tests/kfdtest/src/PM4Queue.cpp index 6613ec8909..9bc6402fac 100644 --- a/tests/kfdtest/src/PM4Queue.cpp +++ b/tests/kfdtest/src/PM4Queue.cpp @@ -56,7 +56,7 @@ unsigned int PM4Queue::RptrWhenConsumed() { void PM4Queue::SubmitPacket() { // m_pending Wptr is in dwords - if (g_TestGPUFamilyId < FAMILY_AI) { + if (m_FamilyId < FAMILY_AI) { // Pre-Vega10 uses 32-bit wptr and doorbell MemoryBarrier(); *m_Resources.Queue_write_ptr = m_pendingWptr; @@ -73,7 +73,7 @@ void PM4Queue::SubmitPacket() { void PM4Queue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { if (event) { - PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(g_TestGPUFamilyId, 0, + PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, 0, event->EventData.HWData2, event->EventId, true)); diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index 81ed3ba73d..f78745037b 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -30,7 +30,7 @@ /* Byte/dword count in many SDMA packets is 1-based in AI, meaning a * count of 1 is encoded as 0. */ -#define SDMA_COUNT(c) (g_TestGPUFamilyId < FAMILY_AI ? (c) : (c)-1) +#define SDMA_COUNT(c) (m_FamilyId < FAMILY_AI ? (c) : (c)-1) SDMAWriteDataPacket::SDMAWriteDataPacket(void): packetData(NULL) { diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/tests/kfdtest/src/SDMAQueue.cpp index a6b41ff31a..9b77f78310 100644 --- a/tests/kfdtest/src/SDMAQueue.cpp +++ b/tests/kfdtest/src/SDMAQueue.cpp @@ -63,7 +63,7 @@ unsigned int SDMAQueue::RptrWhenConsumed() { void SDMAQueue::SubmitPacket() { // m_pending Wptr is in dwords - if (g_TestGPUFamilyId < FAMILY_AI) { + if (m_FamilyId < FAMILY_AI) { // Pre-Vega10 uses 32-bit wptr and doorbell unsigned int wPtrInBytes = m_pendingWptr * sizeof(unsigned int); MemoryBarrier(); From 4baeef356f6e8e44f75c1a8d73fd66bd55a2a4f7 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 5 Jul 2019 04:51:37 -0400 Subject: [PATCH 0539/1247] kfdtest: Sumbit to SDMA ring when using libdrm command submission Because not all ASICs (like gfx908) have GFX rings, we should use SDMA rings instead of GFX rings. Change-Id: Ibcc9f9e555302ba4ce25ac76c2ca73b8c3962a58 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDEvictTest.cpp | 16 +++++++++------- tests/kfdtest/src/KFDEvictTest.hpp | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index f4a4f8cde6..2777d91c1b 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -34,6 +34,8 @@ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +#define SDMA_NOP 0x0 + void KFDEvictTest::SetUp() { ROUTINE_START @@ -198,7 +200,7 @@ static inline int amdgpu_get_bo_list(amdgpu_device_handle dev, amdgpu_bo_handle return amdgpu_bo_list_create(dev, bo2 ? 2 : 1, resources, NULL, list); } -void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle, +void KFDEvictTest::AmdgpuCommandSubmissionSdmaNop(int rn, amdgpu_bo_handle handle, PM4Queue *computeQueue = NULL) { amdgpu_context_handle contextHandle; amdgpu_bo_handle ibResultHandle; @@ -226,14 +228,14 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle ha /* Fill Nop cammands in IB */ ptr = reinterpret_cast(ibResultCpu); for (int i = 0; i < 16; i++) - ptr[i] = 0xffff1000; + ptr[i] = SDMA_NOP; memset(&ibInfo, 0, sizeof(struct amdgpu_cs_ib_info)); ibInfo.ib_mc_address = ibResultMcAddress; ibInfo.size = 16; memset(&ibsRequest, 0, sizeof(struct amdgpu_cs_request)); - ibsRequest.ip_type = AMDGPU_HW_IP_GFX; + ibsRequest.ip_type = AMDGPU_HW_IP_DMA; ibsRequest.ring = 0; ibsRequest.number_of_ibs = 1; ibsRequest.ibs = &ibInfo; @@ -246,7 +248,7 @@ void KFDEvictTest::AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle ha Delay(50); fenceStatus.context = contextHandle; - fenceStatus.ip_type = AMDGPU_HW_IP_GFX; + fenceStatus.ip_type = AMDGPU_HW_IP_DMA; fenceStatus.ip_instance = 0; fenceStatus.ring = 0; fenceStatus.fence = ibsRequest.seq_no; @@ -337,7 +339,7 @@ TEST_F(KFDEvictTest, BasicTest) { amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); - AmdgpuCommandSubmissionComputeNop(rn, handle); + AmdgpuCommandSubmissionSdmaNop(rn, handle); FreeAmdgpuBo(handle); LOG() << m_psName << "free buffer" << std::endl; @@ -574,7 +576,7 @@ TEST_F(KFDEvictTest, QueueTest) { /* Submit the packet and start shader */ dispatch0.Submit(pm4Queue); - AmdgpuCommandSubmissionComputeNop(rn, handle); + AmdgpuCommandSubmissionSdmaNop(rn, handle); /* Uncomment this line for debugging */ // LOG() << m_psName << "notify shader to quit" << std::endl; @@ -653,7 +655,7 @@ TEST_F(KFDEvictTest, BurstyTest) { amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); - AmdgpuCommandSubmissionComputeNop(rn, handle, &pm4Queue); + AmdgpuCommandSubmissionSdmaNop(rn, handle, &pm4Queue); FreeAmdgpuBo(handle); LOG() << m_psName << "free buffer" << std::endl; diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index b5e7e1c3bf..2b838a5388 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -48,7 +48,7 @@ class KFDEvictTest : public KFDMultiProcessTest { void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); void AllocAmdgpuBo(int rn, HSAuint64 vramBufSize, amdgpu_bo_handle &handle); void FreeAmdgpuBo(amdgpu_bo_handle handle); - void AmdgpuCommandSubmissionComputeNop(int rn, amdgpu_bo_handle handle, + void AmdgpuCommandSubmissionSdmaNop(int rn, amdgpu_bo_handle handle, PM4Queue *computeQueue); protected: // Members From 395750264d7e39f49c453508e345a6707682dfb3 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 18 Jul 2019 18:52:56 -0400 Subject: [PATCH 0540/1247] KFDTest : Add family ID when building SDMA packet Some SDMA packet format might be different among asic versions Change-Id: Ic7eda7554c23e3972e168480874ca67a92677346 Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDIPCTest.cpp | 8 +++--- tests/kfdtest/src/KFDMemoryTest.cpp | 10 +++---- tests/kfdtest/src/KFDQMTest.cpp | 37 +++++++++++++------------- tests/kfdtest/src/KFDTestUtil.cpp | 10 ++++--- tests/kfdtest/src/KFDTestUtilQueue.cpp | 3 ++- tests/kfdtest/src/SDMAPacket.cpp | 30 ++++++++++----------- tests/kfdtest/src/SDMAPacket.hpp | 15 +++++------ tests/kfdtest/src/SDMAQueue.cpp | 2 +- 8 files changed, 57 insertions(+), 58 deletions(-) diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index eba001ebb2..ea5a23a94e 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -88,13 +88,13 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { /* Check for pattern in the shared Local Memory */ ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); size = size < sharedSize ? size : sharedSize; - sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(tempSysBuffer.As(), + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), tempSysBuffer.As(), sharedLocalBuffer, size)); sdmaQueue.Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xAAAAAAAA)); /* Fill in the Local Memory with different pattern */ - sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sharedLocalBuffer, 0xBBBBBBBB)); + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaQueue.GetFamilyId(), sharedLocalBuffer, 0xBBBBBBBB)); sdmaQueue.Wait4PacketConsumption(); /* Clean up */ @@ -125,7 +125,7 @@ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pip /* Copy pattern in Local Memory before sharing it */ ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); - sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(toShareLocalBuffer.As(), + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), toShareLocalBuffer.As(), tempSysBuffer.As(), size)); sdmaQueue.Wait4PacketConsumption(); @@ -141,7 +141,7 @@ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pip EXPECT_EQ(WEXITSTATUS(status), 0); /* Check for the new pattern filled in by child process */ - sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(tempSysBuffer.As(), + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), tempSysBuffer.As(), toShareLocalBuffer.As(), size)); sdmaQueue.Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xBBBBBBBB)); diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 69b0e5e477..801ba64afa 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -453,7 +453,7 @@ TEST_F(KFDMemoryTest, MemoryRegister) { dispatch0.Submit(pm4Queue); dispatch0.Sync(g_TestTimeOut); - sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaBuffer.As(), 0x12345678)); + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaQueue.GetFamilyId(), sdmaBuffer.As(), 0x12345678)); sdmaQueue.Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(&stackData[sdmaOffset], 0x12345678)); @@ -496,7 +496,7 @@ TEST_F(KFDMemoryTest, MemoryRegister) { dispatch1.Submit(pm4Queue); dispatch1.Sync(g_TestTimeOut); - sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaBuffer.As(), 0xD0BED0BE)); + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaQueue.GetFamilyId(), sdmaBuffer.As(), 0xD0BED0BE)); sdmaQueue.Wait4PacketConsumption(); EXPECT_SUCCESS(pm4Queue.Destroy()); @@ -1021,10 +1021,10 @@ TEST_F(KFDMemoryTest, MMBench) { #define INTERLEAVE_SDMA() do { \ if (interleaveSDMA) { \ sdmaQueue[0].PlaceAndSubmitPacket( \ - SDMAWriteDataPacket(sdmaBuffer.As(), \ + SDMAWriteDataPacket(sdmaQueue[0].GetFamilyId(), sdmaBuffer.As(), \ 0x12345678)); \ sdmaQueue[1].PlaceAndSubmitPacket( \ - SDMAWriteDataPacket(sdmaBuffer.As()+16, \ + SDMAWriteDataPacket(sdmaQueue[1].GetFamilyId(), sdmaBuffer.As()+16, \ 0x12345678)); \ } \ } while (0) @@ -1601,7 +1601,7 @@ TEST_F(KFDMemoryTest, SignalHandling) { pDb[0] = 0x02020202; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(pDb, 0x01010101) ); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), pDb, 0x01010101) ); queue.Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(pDb, 0x01010101)); EXPECT_SUCCESS(queue.Destroy()); diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 07b83c6dea..942f426ff7 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -155,7 +155,7 @@ TEST_F(KFDQMTest, SubmitPacketSdmaQueue) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0x02020202)); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As(), 0x02020202)); queue.Wait4PacketConsumption(); @@ -188,9 +188,9 @@ TEST_F(KFDQMTest, MultipleSdmaQueues) { destBuf.Fill(0x0); srcBuf.Fill(qidx + 0xa0); queues[qidx].PlaceAndSubmitPacket( - SDMACopyDataPacket(destBuf.As(), srcBuf.As(), bufSize)); + SDMACopyDataPacket(queues[qidx].GetFamilyId(), destBuf.As(), srcBuf.As(), bufSize)); queues[qidx].PlaceAndSubmitPacket( - SDMAWriteDataPacket(destBuf.As() + bufSize/4, 0x02020202)); + SDMAWriteDataPacket(queues[qidx].GetFamilyId(), destBuf.As() + bufSize/4, 0x02020202)); queues[qidx].Wait4PacketConsumption(); @@ -241,7 +241,7 @@ TEST_F(KFDQMTest, SdmaConcurrentCopies) { for (unsigned j = 0; j < NPACKETS; j++) queue.PlacePacket( - SDMACopyDataPacket(dstBuf.As()+COPY_SIZE*j, + SDMACopyDataPacket(queue.GetFamilyId(), dstBuf.As()+COPY_SIZE*j, srcBuf.As()+COPY_SIZE*j, COPY_SIZE)); queue.SubmitPacket(); @@ -260,7 +260,7 @@ TEST_F(KFDQMTest, SdmaConcurrentCopies) { } log << "Done." << std::endl; - queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(srcBuf.As(), 0x02020202)); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), srcBuf.As(), 0x02020202)); queue.Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(srcBuf.As(), 0x02020202)); @@ -326,7 +326,7 @@ TEST_F(KFDQMTest, DisableSdmaQueueByUpdateWithNullAddress) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0)); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As(), 0)); WaitOnValue(destBuf.As(), 0); @@ -334,7 +334,7 @@ TEST_F(KFDQMTest, DisableSdmaQueueByUpdateWithNullAddress) { EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, true)); - queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), 0)); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As(), 0)); // Don't sync since we don't expect rptr to change when the queue is disabled. Delay(2000); @@ -1268,9 +1268,9 @@ TEST_F(KFDQMTest, SdmaQueueWraparound) { destBuf.Fill(0x0); srcBuf.Fill(pktIdx); queue.PlaceAndSubmitPacket( - SDMACopyDataPacket(destBuf.As(), srcBuf.As(), bufSize)); + SDMACopyDataPacket(queue.GetFamilyId(), destBuf.As(), srcBuf.As(), bufSize)); queue.PlaceAndSubmitPacket( - SDMAWriteDataPacket(destBuf.As() + bufSize/4, 0x02020202)); + SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As() + bufSize/4, 0x02020202)); queue.Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); @@ -1280,7 +1280,7 @@ TEST_F(KFDQMTest, SdmaQueueWraparound) { } for (unsigned int pktIdx = 0; pktIdx <= queue.Size()/sizeof(SDMA_PKT_WRITE_UNTILED); ++pktIdx) { - queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(destBuf.As(), pktIdx)); + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), destBuf.As(), pktIdx)); queue.Wait4PacketConsumption(); WaitOnValue(destBuf.As(), pktIdx); } @@ -1406,7 +1406,7 @@ sdma_copy(HSAuint32 node, void *src, void *const dst[], int n, HSAuint64 size) { HsaEvent *event; ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, node, &event)); ASSERT_SUCCESS(sdmaQueue.Create(node)); - sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(dst, src, n, size)); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), dst, src, n, size)); sdmaQueue.Wait4PacketConsumption(event); EXPECT_SUCCESS(sdmaQueue.Destroy()); hsaKmtDestroyEvent(event); @@ -1418,7 +1418,7 @@ sdma_fill(HSAint32 node, void *dst, unsigned int data, HSAuint64 size) { HsaEvent *event; ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, node, &event)); ASSERT_SUCCESS(sdmaQueue.Create(node)); - sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(dst, data, size)); + sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(sdmaQueue.GetFamilyId(), dst, data, size)); sdmaQueue.Wait4PacketConsumption(event); EXPECT_SUCCESS(sdmaQueue.Destroy()); hsaKmtDestroyEvent(event); @@ -1647,10 +1647,11 @@ TEST_F(KFDQMTest, SdmaEventInterrupt) { /* Let sDMA have some workload first.*/ queue[i].PlacePacket(SDMATimePacket(&ts[1])); queue[i].PlacePacket( - SDMACopyDataPacket(dst[i], src, bufSize)); + SDMACopyDataPacket(queue[i].GetFamilyId(), dst[i], src, bufSize)); queue[i].PlacePacket(SDMATimePacket(&ts[2])); queue[i].PlacePacket( - SDMAFencePacket(reinterpret_cast(event[i]->EventData.HWData2), event[i]->EventId)); + SDMAFencePacket(queue[i].GetFamilyId(), + reinterpret_cast(event[i]->EventData.HWData2), event[i]->EventId)); queue[i].PlacePacket(SDMATimePacket(&ts[3])); queue[i].PlacePacket(SDMATrapPacket(event[i]->EventId)); queue[i].PlacePacket(SDMATimePacket(&ts[4])); @@ -1738,9 +1739,9 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { * engine. This should submit the PM4 packet on the first * queue. */ - otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_write_ptr, + otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_write_ptr, pendingWptr)); - otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_DoorBell, + otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_DoorBell, pendingWptr)); #else /* Write the wptr and doorbell update using WRITE_DATA packets @@ -1764,9 +1765,9 @@ TEST_F(KFDQMTest, GPUDoorbellWrite) { * engine. This should submit the PM4 packet on the first * queue. */ - otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_write_ptr, + otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_write_ptr, 2, &pendingWptr64)); - otherQueue.PlacePacket(SDMAWriteDataPacket(qRes->Queue_DoorBell, + otherQueue.PlacePacket(SDMAWriteDataPacket(otherQueue.GetFamilyId(), qRes->Queue_DoorBell, 2, &pendingWptr64)); #else /* Write the 64-bit wptr and doorbell update using RELEASE_MEM diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 68bf8268fb..144a16b926 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -272,8 +272,10 @@ void HsaMemoryBuffer::Fill(HSAuint32 value, BaseQueue& baseQueue, HSAuint64 offs size = size ? size : m_Size; ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; - baseQueue.PlacePacket(SDMAFillDataPacket((reinterpret_cast(this->As() + offset)), value, size)); - baseQueue.PlacePacket(SDMAFencePacket(reinterpret_cast(event->EventData.HWData2), event->EventId)); + baseQueue.PlacePacket(SDMAFillDataPacket(baseQueue.GetFamilyId(), + (reinterpret_cast(this->As() + offset)), value, size)); + baseQueue.PlacePacket(SDMAFencePacket(baseQueue.GetFamilyId(), + reinterpret_cast(event->EventData.HWData2), event->EventId)); baseQueue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); EXPECT_SUCCESS(hsaKmtWaitOnEvent(event, g_TestTimeOut)); @@ -326,10 +328,10 @@ bool HsaMemoryBuffer::IsPattern(HSAuint64 location, HSAuint32 pattern, BaseQueue return false; *tmp = ~pattern; - baseQueue.PlacePacket(SDMACopyDataPacket((void *)tmp, + baseQueue.PlacePacket(SDMACopyDataPacket(baseQueue.GetFamilyId(), (void *)tmp, reinterpret_cast(this->As() + location), sizeof(HSAuint32))); - baseQueue.PlacePacket(SDMAFencePacket(reinterpret_cast(event->EventData.HWData2), + baseQueue.PlacePacket(SDMAFencePacket(baseQueue.GetFamilyId(), reinterpret_cast(event->EventData.HWData2), event->EventId)); baseQueue.PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/tests/kfdtest/src/KFDTestUtilQueue.cpp index e1fe07d4df..92283f71fb 100644 --- a/tests/kfdtest/src/KFDTestUtilQueue.cpp +++ b/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -31,6 +31,7 @@ #include "PM4Packet.hpp" #include "KFDTestUtil.hpp" #include "KFDTestUtilQueue.hpp" +#include "KFDBaseComponentTest.hpp" #define MB_PER_SEC(size, time) ((((size) * 1ULL) >> 20) * 1000ULL * 1000ULL * 1000ULL / (time)) @@ -355,7 +356,7 @@ void sdma_multicopy(std::vector &array, int mashup, TSPattern ts for (i = 0; i < array.size(); i++) { sharedPacket packet(new - SDMACopyDataPacket(array[i].dst, array[i].src, array[i].size)); + SDMACopyDataPacket(g_baseTest->GetFamilyIdFromNodeId(array[i].node), array[i].dst, array[i].src, array[i].size)); packetList.push_back(packet); /* We put the real queue_id in local handle[] to reduce some assignment.*/ diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index f78745037b..2cd13f0823 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -32,18 +32,16 @@ */ #define SDMA_COUNT(c) (m_FamilyId < FAMILY_AI ? (c) : (c)-1) -SDMAWriteDataPacket::SDMAWriteDataPacket(void): +SDMAWriteDataPacket::SDMAWriteDataPacket(unsigned int familyId, void* destAddr, unsigned int data): packetData(NULL) { + m_FamilyId = familyId; + InitPacket(destAddr, 1, &data); } -SDMAWriteDataPacket::SDMAWriteDataPacket(void* destAddr, unsigned int data): - packetData(NULL) { - InitPacket(destAddr, data); -} - -SDMAWriteDataPacket::SDMAWriteDataPacket(void* destAddr, unsigned int ndw, +SDMAWriteDataPacket::SDMAWriteDataPacket(unsigned int familyId, void* destAddr, unsigned int ndw, void *data): packetData(NULL) { + m_FamilyId = familyId; InitPacket(destAddr, ndw, data); } @@ -52,10 +50,6 @@ SDMAWriteDataPacket::~SDMAWriteDataPacket(void) { free(packetData); } -void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int data) { - InitPacket(destAddr, 1, &data); -} - void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int ndw, void *data) { packetSize = sizeof(SDMA_PKT_WRITE_UNTILED) + @@ -79,7 +73,8 @@ SDMACopyDataPacket::~SDMACopyDataPacket(void) { free(packetData); } -SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, unsigned int surfsize) { +SDMACopyDataPacket::SDMACopyDataPacket(unsigned int familyId, + void *const dsts[], void *src, int n, unsigned int surfsize) { int32_t size = 0, i; void **dst = reinterpret_cast(malloc(sizeof(void*) * n)); const int singlePacketSize = sizeof(SDMA_PKT_COPY_LINEAR) + @@ -88,6 +83,7 @@ SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, uns if (n > 2) WARN() << "SDMACopyDataPacket does not support more than 2 dst addresses!" << std::endl; + m_FamilyId = familyId; memcpy(dst, dsts, sizeof(void*) * n); packetSize = ((surfsize + TWO_MEG - 1) >> BITS) * singlePacketSize; @@ -125,18 +121,19 @@ SDMACopyDataPacket::SDMACopyDataPacket(void *const dsts[], void *src, int n, uns free(dst); } -SDMACopyDataPacket::SDMACopyDataPacket(void* dst, void *src, unsigned int surfsize) { - new (this)SDMACopyDataPacket(&dst, src, 1, surfsize); +SDMACopyDataPacket::SDMACopyDataPacket(unsigned int familyId, void* dst, void *src, unsigned int surfsize) { + new (this)SDMACopyDataPacket(familyId, &dst, src, 1, surfsize); } SDMAFillDataPacket::~SDMAFillDataPacket() { free(m_PacketData); } -SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned int size) { +SDMAFillDataPacket::SDMAFillDataPacket(unsigned int familyId, void *dst, unsigned int data, unsigned int size) { unsigned int copy_size; SDMA_PKT_CONSTANT_FILL *pSDMA; + m_FamilyId = familyId; /* SDMA support maximum 0x3fffe0 byte in one copy. Use 2M copy_size */ m_PacketSize = ((size + TWO_MEG - 1) >> BITS) * sizeof(SDMA_PKT_CONSTANT_FILL); pSDMA = reinterpret_cast(calloc(1, m_PacketSize)); @@ -174,7 +171,8 @@ SDMAFillDataPacket::SDMAFillDataPacket(void *dst, unsigned int data, unsigned in SDMAFencePacket::SDMAFencePacket(void) { } -SDMAFencePacket::SDMAFencePacket(void* destAddr, unsigned int data) { +SDMAFencePacket::SDMAFencePacket(unsigned int familyId, void* destAddr, unsigned int data) { + m_FamilyId = familyId; InitPacket(destAddr, data); } diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index b1afb374ce..cb27c944ac 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -38,18 +38,15 @@ class SDMAPacket : public BasePacket { class SDMAWriteDataPacket : public SDMAPacket { public: - // Empty constructor, before using the packet call the init func - SDMAWriteDataPacket(void); // This contructor will also init the packet, no need for additional calls - SDMAWriteDataPacket(void* destAddr, unsigned int data); - SDMAWriteDataPacket(void* destAddr, unsigned int ndw, void *data); + SDMAWriteDataPacket(unsigned int familyId, void* destAddr, unsigned int data); + SDMAWriteDataPacket(unsigned int familyId, void* destAddr, unsigned int ndw, void *data); virtual ~SDMAWriteDataPacket(void); // @returns Pointer to the packet virtual const void *GetPacket() const { return packetData; } // @breif Initialise the packet - void InitPacket(void* destAddr, unsigned int data); void InitPacket(void* destAddr, unsigned int ndw, void *data); // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return packetSize; } @@ -63,8 +60,8 @@ class SDMAWriteDataPacket : public SDMAPacket { class SDMACopyDataPacket : public SDMAPacket { public: // This contructor will also init the packet, no need for additional calls - SDMACopyDataPacket(void *dest, void *src, unsigned int size); - SDMACopyDataPacket(void *const dst[], void *src, int n, unsigned int surfsize); + SDMACopyDataPacket(unsigned int familyId, void *dest, void *src, unsigned int size); + SDMACopyDataPacket(unsigned int familyId, void *const dst[], void *src, int n, unsigned int surfsize); virtual ~SDMACopyDataPacket(void); @@ -84,7 +81,7 @@ class SDMACopyDataPacket : public SDMAPacket { class SDMAFillDataPacket : public SDMAPacket { public: // This contructor will also init the packet, no need for additional calls - SDMAFillDataPacket(void *dest, unsigned int data, unsigned int size); + SDMAFillDataPacket(unsigned int familyId, void *dest, unsigned int data, unsigned int size); virtual ~SDMAFillDataPacket(void); @@ -106,7 +103,7 @@ class SDMAFencePacket : public SDMAPacket { // Empty constructor, before using the packet call the init func SDMAFencePacket(void); // This contructor will also init the packet, no need for additional calls - SDMAFencePacket(void* destAddr, unsigned int data); + SDMAFencePacket(unsigned int familyId, void* destAddr, unsigned int data); virtual ~SDMAFencePacket(void); diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/tests/kfdtest/src/SDMAQueue.cpp index 9b77f78310..ece6cc5237 100644 --- a/tests/kfdtest/src/SDMAQueue.cpp +++ b/tests/kfdtest/src/SDMAQueue.cpp @@ -82,7 +82,7 @@ void SDMAQueue::SubmitPacket() { void SDMAQueue::Wait4PacketConsumption(HsaEvent *event, unsigned int timeOut) { if (event) { - PlacePacket(SDMAFencePacket((void*)event->EventData.HWData2, event->EventId)); + PlacePacket(SDMAFencePacket(m_FamilyId, (void*)event->EventData.HWData2, event->EventId)); PlaceAndSubmitPacket(SDMATrapPacket(event->EventId)); From 5b44be190784ea6eaa99676a77213f3d0f7ed325 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 18 Jul 2019 19:06:43 -0400 Subject: [PATCH 0541/1247] KFDTest: Added gfx1010 SDMA fence packet support Change-Id: I33d824353d77317363b73ddc52cd182f86b8bc66 Signed-off-by: shaoyunl Signed-off-by: Oak Zeng --- tests/kfdtest/src/SDMAPacket.cpp | 24 ++++++++++++++++++++++-- tests/kfdtest/src/SDMAPacket.hpp | 4 +++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index 2cd13f0823..df5eb6b18f 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -173,13 +173,16 @@ SDMAFencePacket::SDMAFencePacket(void) { SDMAFencePacket::SDMAFencePacket(unsigned int familyId, void* destAddr, unsigned int data) { m_FamilyId = familyId; - InitPacket(destAddr, data); + if (m_FamilyId < FAMILY_NV) + InitPacketCI(destAddr, data); + else + InitPacketNV(destAddr, data); } SDMAFencePacket::~SDMAFencePacket(void) { } -void SDMAFencePacket::InitPacket(void* destAddr, unsigned int data) { +void SDMAFencePacket::InitPacketCI(void* destAddr, unsigned int data) { memset(&packetData, 0, SizeInBytes()); packetData.HEADER_UNION.op = SDMA_OP_FENCE; @@ -191,6 +194,23 @@ void SDMAFencePacket::InitPacket(void* destAddr, unsigned int data) { packetData.DATA_UNION.data = data; } +void SDMAFencePacket::InitPacketNV(void * destAddr,unsigned int data) { + memset(&packetData, 0, SizeInBytes()); + + /* GPA=0 becaue we use virtual address + * Snoop = 1 because we want the write be CPU coherent + * System = 1 because the memory is system memory + * mtype = uncached, for the purpose of CPU coherent, L2 policy doesn't matter in this case + */ + packetData.HEADER_UNION.DW_0_DATA = (0 << 23) | (1 << 22) | (1 << 20) | (3 << 15) | SDMA_OP_FENCE; + + SplitU64(reinterpret_cast(destAddr), + packetData.ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ + packetData.ADDR_HI_UNION.DW_2_DATA); /*dst_addr_63_32*/ + + packetData.DATA_UNION.data = data; +} + SDMATrapPacket::SDMATrapPacket(unsigned int eventID) { InitPacket(eventID); diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index cb27c944ac..03f837806b 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -110,7 +110,9 @@ class SDMAFencePacket : public SDMAPacket { // @returns Pointer to the packet virtual const void *GetPacket() const { return &packetData; } // @brief Initialise the packet - void InitPacket(void* destAddr, unsigned int data); + void InitPacketCI(void* destAddr, unsigned int data); + void InitPacketNV(void* destAddr, unsigned int data); + // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return sizeof(SDMA_PKT_FENCE ); } From 1ca1825b84b1761eb9f28208ccd846f191e4a786 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 2 Jul 2019 11:54:28 -0400 Subject: [PATCH 0542/1247] Remove KFDDBGTest.BasicAddressWatch This functionality doesn't work on GFX9+, and was disabled for gfx802. Remove the test altogether for now, especially since some kernel changes broke it on gfx803, and the functionality is deprecated now anyways. Leave the code for reference, but "#if 0" it to prevent it from compiling or being in the kfdtest binary Change-Id: I848b4f23201f18612cbdc122a5b46e4010c4af2a --- tests/kfdtest/scripts/kfdtest.exclude | 10 +--------- tests/kfdtest/src/KFDDBGTest.cpp | 3 +++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 6b21c640e0..9bbb1f926f 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -29,7 +29,6 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # KFDPNPTest.DisableAndCreateQueue (KFD-329) # KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) # KFDQMTest.mGPUShareBO (KFD-334) -# KFDQMTest.PM4EventInterrupt/KFDDBGTest.BasicAddressWatch (KFD-431) # KFDHWSTest.* (SWDEV-193035) TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDGraphicsInterop.RegisterForeignDeviceMem:"\ @@ -39,7 +38,6 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDPNPTest.DisableAndCreateQueue:"\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ -"KFDDBGTest.BasicAddressWatch*:"\ "KFDQMTest.PM4EventInterrupt:"\ "KFDQMTest.SdmaEventInterrupt:"\ "KFDDBGTest.BasicDebuggerSuspendResume:"\ @@ -114,7 +112,6 @@ TONGA_TESTS_BLACKLIST=\ "KFDMemoryTest.MemoryRegister*:"\ "KFDMemoryTest.QueryPointerInfo:"\ "KFDPerfCountersTest.*:"\ -"KFDDBGTest.BasicAddressWatch:"\ "KFDQMTest.*" FIJI_TESTS_BLACKLIST=\ @@ -133,14 +130,12 @@ LEXA_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST" -# KFDDBG.BasicAddressWatch is not supported on GFXv9 and later. FIXME: it should disable itself # KFDIPCTest.BasicTest (ROCMOPS-459) .CMABasicTest (ROCMOPS-460) .CrossMemoryAttachTest (ROCMOPS-461) # KFDMemoryTest.BigBufferStressTest (ROCMOPS-462) # KFDEvictTest.BurstyTest (ROCMOPS-464) # KFDQMTest.MultipleSdmaQueues (ROCMOPS-463) VEGA10_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDDBGTest.BasicAddressWatch:"\ "KFDIPCTest.BasicTest:"\ "KFDEvictTest.BurstyTest:"\ "KFDIPCTest.CMABasicTest:"\ @@ -150,14 +145,12 @@ VEGA10_TESTS_BLACKLIST=\ VEGA12_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"$SDMA_BLACKLIST:"\ -"KFDDBGTest.BasicAddressWatch" +"$SDMA_BLACKLIST"\ # KFDEvictTest.BurstyTest (ROCMOPS-464) VEGA20_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ -"KFDDBGTest.BasicAddressWatch:"\ "KFDEvictTest.BurstyTest:"\ "KFDExceptionTest.InvalidBadAddress:"\ "KFDExceptionTest.InvalidWriteAddress:"\ @@ -165,7 +158,6 @@ VEGA20_TESTS_BLACKLIST=\ RAVEN_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDDBGTest.BasicAddressWatch:"\ "KFDQMTest.QueueLatency:"\ "KFDQMTest.SdmaEventInterrupt" diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index a20db2b15b..e8ab3df780 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -114,6 +114,8 @@ void KFDDBGTest::TearDown() { ROUTINE_END } +#if 0 +// Functionality is deprecated now, keeping code for reference TEST_F(KFDDBGTest, BasicAddressWatch) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId >= FAMILY_VI) { @@ -200,6 +202,7 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { } TEST_END } +#endif TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { TEST_START(TESTPROFILE_RUNALL) From 79a3995816f1b155c15e65c28abd5620189d9b27 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 2 Jul 2019 11:05:17 -0400 Subject: [PATCH 0543/1247] Restore PM4EventInterrupt to non-gfx803 testing The test is viable still on VG10/20. Phil is investigating why it takes so long on gfx803 Change-Id: I61669b29dc0e8407858a5c73cfa69c5ea923846f --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 9bbb1f926f..435e46790d 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -38,7 +38,6 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDPNPTest.DisableAndCreateQueue:"\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ -"KFDQMTest.PM4EventInterrupt:"\ "KFDQMTest.SdmaEventInterrupt:"\ "KFDDBGTest.BasicDebuggerSuspendResume:"\ "KFDHWSTest.*" @@ -114,8 +113,10 @@ TONGA_TESTS_BLACKLIST=\ "KFDPerfCountersTest.*:"\ "KFDQMTest.*" +# Since Navi10 was merged, the PM4Event test takes 6min to run FIJI_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.PM4EventInterrupt:"\ "$SDMA_BLACKLIST" ELLESMERE_TESTS_BLACKLIST=\ From 85a9821519b7680ed6ded417af705562b8a85c46 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 18 Jul 2019 19:34:55 -0400 Subject: [PATCH 0544/1247] KFDTest: Shader modification for gfx10 Modified shaders in KFD memory test to support gfx10. There is no gprs register for flat_scratch on gfx10. Use s_setreg_b32 instruction to set flat scratch base address register Change-Id: I505156a046056b61ce2d873343feb50ce635274a Signed-off-by: shaoyunl Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDMemoryTest.cpp | 116 +++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 11 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 801ba64afa..58c14c94c0 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -84,6 +84,31 @@ type(CS)\n\ \n\ end\n\ "; +const char* gfx10_ScratchCopyDword = +"\ +shader ScratchCopyDword\n\ +asic(GFX10)\n\ +type(CS)\n\ +wave_size(32)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ +/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ + s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO), s4\n\ + s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s5\n\ +/*copy a dword between the passed addresses*/\n\ + flat_load_dword v4, v[0:1] slc\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + flat_store_dword v[2:3], v4 slc\n\ + \n\ + s_endpgm\n\ + \n\ +end\n\ +"; + + /* Continuously poll src buffer and check buffer value * After src buffer is filled with specific value (0x5678, @@ -106,6 +131,22 @@ type(CS)\n\ end\n\ "; +const char* gfx10_PollMemory = +"\ +shader ReadMemory\n\ +asic(GFX10)\n\ +type(CS)\n\ +/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ + s_movk_i32 s18, 0x5678\n\ + LOOP:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 LOOP\n\ + s_store_dword s18, s[2:3], 0x0 glc\n\ + s_endpgm\n\ + end\n\ +"; + /* Input: A buffer of at least 3 dwords. * DW0: used as a signal. 0xcafe means it is signaled * DW1: Input buffer for device to read. @@ -134,6 +175,27 @@ POLLSIGNAL:\n\ end\n\ "; +const char* gfx10_CopyOnSignal = +"\ +shader CopyOnSignal\n\ +asic(GFX10)\n\ +type(CS)\n\ +/* Assume input buffer in s0, s1 */\n\ + s_mov_b32 s18, 0xcafe\n\ +POLLSIGNAL:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 POLLSIGNAL\n\ + s_load_dword s17, s[0:1], 0x4 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_store_dword s17, s[0:1], 0x8 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_endpgm\n\ + end\n\ +"; + + + /* Input0: A buffer of at least 2 dwords. * DW0: used as a signal. Write 0xcafe to signal * DW1: Write to this buffer for other device to read. @@ -155,6 +217,22 @@ type(CS)\n\ end\n\ "; +const char* gfx10_WriteAndSignal = +"\ +shader WriteAndSignal\n\ +asic(GFX10)\n\ +type(CS)\n\ +/* Assume input buffer in s0, s1 */\n\ + s_mov_b32 s18, 0xbeef\n\ + s_store_dword s18, s[0:1], 0x4 glc\n\ + s_mov_b32 s18, 0x1\n\ + s_store_dword s18, s[2:3], 0 glc\n\ + s_mov_b32 s18, 0xcafe\n\ + s_store_dword s18, s[0:1], 0x0 glc\n\ + s_endpgm\n\ + end\n\ +"; + void KFDMemoryTest::SetUp() { ROUTINE_START @@ -259,8 +337,8 @@ TEST_F(KFDMemoryTest, MMapLarge) { */ TEST_F(KFDMemoryTest, MapUnmapToNodes) { TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId != FAMILY_AI) { - LOG() << "Skipping test: GFX9-based shader not supported on other ASICs." << std::endl; + if (m_FamilyId < FAMILY_AI) { + LOG() << "Skipping test: Test requires gfx9 and later asics." << std::endl; return; } @@ -286,7 +364,7 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + m_pIsaGen->CompileShader((m_FamilyId < FAMILY_NV) ? gfx9_PollMemory : gfx10_PollMemory, "ReadMemory", isaBuffer); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); @@ -623,9 +701,14 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { // Initialize the srcBuffer to some fixed value srcMemBuffer.Fill(0x01010101); - // Initialize a buffer with a dword copy ISA - m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? gfx9_ScratchCopyDword : gfx8_ScratchCopyDword, - "ScratchCopyDword", isaBuffer); + const char *pScratchCopyDword; + if (m_FamilyId < FAMILY_AI) + pScratchCopyDword = gfx8_ScratchCopyDword; + else if (m_FamilyId < FAMILY_NV) + pScratchCopyDword = gfx9_ScratchCopyDword; + else + pScratchCopyDword = gfx10_ScratchCopyDword; + m_pIsaGen->CompileShader(pScratchCopyDword, "ScratchCopyDword", isaBuffer); const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); @@ -1520,8 +1603,16 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); // dstBuffer is cpu accessible gtt memory HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? gfx9_ScratchCopyDword : gfx8_ScratchCopyDword, - "ScratchCopyDword", isaBuffer); + + const char *pScratchCopyDword; + if (m_FamilyId < FAMILY_AI) + pScratchCopyDword = gfx8_ScratchCopyDword; + else if (m_FamilyId < FAMILY_NV) + pScratchCopyDword = gfx9_ScratchCopyDword; + else + pScratchCopyDword = gfx10_ScratchCopyDword; + + m_pIsaGen->CompileShader(pScratchCopyDword, "ScratchCopyDword", isaBuffer); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(mem0, dstBuffer.As()); dispatch0.Submit(queue); @@ -1870,7 +1961,8 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_CopyOnSignal,"CopyOnSignal", isaBuffer); + m_pIsaGen->CompileShader((m_FamilyId < FAMILY_NV) ? gfx9_CopyOnSignal : gfx10_CopyOnSignal, + "CopyOnSignal", isaBuffer); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(buffer, NULL); dispatch0.Submit(queue); @@ -1989,7 +2081,8 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(nodes[0])); HsaMemoryBuffer isaBuffer(PAGE_SIZE, nodes[0], true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_CopyOnSignal, "CopyOnSignal", isaBuffer); + m_pIsaGen->CompileShader((queue.GetFamilyId() < FAMILY_NV) ? gfx9_CopyOnSignal : gfx10_CopyOnSignal, + "CopyOnSignal", isaBuffer); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, NULL); dispatch.Submit(queue); @@ -1997,7 +2090,8 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue0; ASSERT_SUCCESS(queue0.Create(nodes[1])); HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_WriteAndSignal, "WriteAndSignal", isaBuffer0); + m_pIsaGen->CompileShader((queue0.GetFamilyId() < FAMILY_NV) ? gfx9_WriteAndSignal : gfx10_WriteAndSignal, + "WriteAndSignal", isaBuffer0); Dispatch dispatch0(isaBuffer0); dispatch0.SetArgs(buffer, mmioBase); dispatch0.Submit(queue0); From 33bd5c520b113e528a050d10e7abeb680eac8f51 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Fri, 19 Jul 2019 10:53:52 -0400 Subject: [PATCH 0545/1247] KFDTest: Test 8 SDMA queues per engine for gfx1010 Change-Id: I709112ab10b1f5684e6411f44682bdef14745b50 Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDTestUtil.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 144a16b926..3f5c687a0b 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -138,6 +138,8 @@ void GetSdmaInfo(const HsaNodeProperties *props, if (props->EngineId.ui32.Major == 9) { if (props->EngineId.ui32.Stepping == 6) // VEGA20 num_sdma_queues_per_engine = 8; + } else if (props->EngineId.ui32.Major == 10) { //NAVIi + num_sdma_queues_per_engine = 8; } if (p_num_sdma_engines) From 157f5b945f5b43197129bd46d160374f43c88b4d Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Fri, 19 Jul 2019 11:00:02 -0400 Subject: [PATCH 0546/1247] KFDTest: Add CWSR test for gfx1010 Change-Id: I708b2fd5f06e0c7591ebe04c40576845cf2a891a Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDCWSRTest.cpp | 38 +++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 497065b6a6..d06ebc1d26 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -77,6 +77,33 @@ LOOP:\n\ end\n\ "; +static const char* iterate_isa_gfx10 = \ +"\ +shader iterate_isa\n\ +asic(GFX10)\n\ +wave_size(32)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ + flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + v_mov_b32 v5, 0\n\ +LOOP:\n\ + v_add_co_u32 v5, vcc, 1, v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ + v_cmp_lt_u32 vcc, v5, v4\n\ + s_cbranch_vccnz LOOP\n\ + flat_store_dword v[2,3], v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + s_endpgm\n\ +end\n\ +"; + + /* v[0:1] = target iteration value v[2:3] = iterate result @@ -129,9 +156,16 @@ TEST_F(KFDCWSRTest, BasicTest) { unsigned int* iter = iterateBuf.As(); unsigned int* result = resultBuf.As(); + const char *pIterateIsa; - m_pIsaGen->CompileShader((m_FamilyId >= FAMILY_AI) ? iterate_isa_gfx9 : iterate_isa_gfx8 , - "iterate_isa", isaBuffer); + if (m_FamilyId < FAMILY_AI) + pIterateIsa = iterate_isa_gfx8; + else if (m_FamilyId < FAMILY_NV) + pIterateIsa = iterate_isa_gfx9; + else + pIterateIsa = iterate_isa_gfx10; + + m_pIsaGen->CompileShader(pIterateIsa, "iterate_isa", isaBuffer); PM4Queue queue1, queue2; From c0663be7e8f97bfe50d205d59db128c631eb89df Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Tue, 23 Jul 2019 15:28:48 -0400 Subject: [PATCH 0547/1247] KFDTest: Enable KFDEvictTest.QueueTest for gfx1010 v_add_u32 was removed from gfx10, use carry-out explicit instruction v_add_co_u32 instead on both gfx9 and gfx10 Change-Id: I1fcd5956844457a676757ad13bdce7f5304bb34b Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDEvictTest.cpp | 68 +++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 2777d91c1b..01efd86573 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -376,13 +376,13 @@ static const char* gfx9_ReadMemory = v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ v_mov_b32 v5, s3\n\ - v_add_u32 v5, vcc_lo, v5\n\ + v_add_co_u32 v5, vcc, v5, vcc_lo\n\ \n\ // compute input buffer offset used to store corresponding local buffer address\n\ v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ v_mov_b32 v3, s1\n\ - v_add_u32 v3, vcc_lo, v3\n\ + v_add_co_u32 v3, vcc, v3, vcc_lo\n\ \n\ // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ flat_load_dwordx2 v[6:7], v[2:3] slc\n\ @@ -404,9 +404,9 @@ L_REPEAT:\n\ v_mov_b32 v13, v7\n\ L_LOOP_READ:\n\ flat_load_dwordx2 v[14:15], v[12:13] slc\n\ - v_add_u32 v9, v9, v10 \n\ + v_add_co_u32 v9, vcc, v9, v10 \n\ v_add_co_u32 v12, vcc, v12, v10\n\ - v_add_u32 v13, vcc_lo, v13\n\ + v_add_co_u32 v13, vcc, v13, vcc_lo\n\ v_cmp_lt_u32 vcc, v9, v11\n\ s_cbranch_vccnz L_LOOP_READ\n\ s_branch L_REPEAT\n\ @@ -469,11 +469,67 @@ L_QUIT:\n\ end\n\ "; + +static const char* gfx10_ReadMemory = +"\ + shader ReadMemory\n\ + asic(GFX10)\n\ + wave_size(32)\n\ + type(CS)\n\ + \n\ + // compute address of corresponding output buffer\n\ + v_mov_b32 v0, s4 // use workgroup id as index\n\ + v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ + v_mov_b32 v5, s3\n\ + v_add_co_u32 v5, vcc, v5, vcc_lo\n\ + \n\ + // compute input buffer offset used to store corresponding local buffer address\n\ + v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ + v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ + v_mov_b32 v3, s1\n\ + v_add_co_u32 v3, vcc, v3, vcc_lo\n\ + \n\ + // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ + flat_load_dwordx2 v[6:7], v[2:3] slc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + \n\ + v_mov_b32 v8, 0x5678\n\ + s_movk_i32 s8, 0x5678\n\ +L_REPEAT:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + s_cmp_eq_i32 s16, s8\n\ + s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ + // loop read 64M local buffer starting at v[6:7]\n\ + // every 4k page only read once\n\ + v_mov_b32 v9, 0\n\ + v_mov_b32 v10, 0x1000 // 4k page\n\ + v_mov_b32 v11, 0x4000000 // 64M size\n\ + v_mov_b32 v12, v6\n\ + v_mov_b32 v13, v7\n\ +L_LOOP_READ:\n\ + flat_load_dwordx2 v[14:15], v[12:13] slc\n\ + v_add_co_u32 v9, vcc, v9, v10 \n\ + v_add_co_u32 v12, vcc, v12, v10\n\ + v_add_co_u32 v13, vcc, v13, vcc_lo\n\ + v_cmp_lt_u32 vcc, v9, v11\n\ + s_cbranch_vccnz L_LOOP_READ\n\ + s_branch L_REPEAT\n\ +L_QUIT:\n\ + flat_store_dword v[4:5], v8\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ + s_endpgm\n\ + end\n\ +"; + std::string KFDEvictTest::CreateShader() { - if (m_FamilyId >= FAMILY_AI) + if (m_FamilyId < FAMILY_AI) + return gfx8_ReadMemory; + else if (m_FamilyId < FAMILY_NV) return gfx9_ReadMemory; else - return gfx8_ReadMemory; + return gfx10_ReadMemory; } /* Evict and restore queue test From b6f6d9da1cf753e6f553fc1a0e2ec3b45d3e427c Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Thu, 25 Jul 2019 14:18:03 -0400 Subject: [PATCH 0548/1247] Enable KFDGraphicsInterop.RegisterForeignDeviceMem KFDGraphicsInterop.RegisterForeignDeviceMem looks like it is running now. Re-enable it for kfdtest for all platforms. Change-Id: I6f6ee9cd11da793c5d525d8676bfc6d5bd8007bb Signed-off-by: Philip Cox --- tests/kfdtest/scripts/kfdtest.exclude | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 435e46790d..031bf9e7c3 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -24,14 +24,12 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # failures can be found # NOTE: If you update this alphabetical listing, add the corresponding JIRA ticket for reference # -# KFDGraphicsInterop.RegisterForeignDeviceMem (KFD-366) # KFDPMTest.SuspendWith[ActiveProcess|IdleQueue|IdleQueueAfterWork] (KFD-326|327|328) # KFDPNPTest.DisableAndCreateQueue (KFD-329) # KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) # KFDQMTest.mGPUShareBO (KFD-334) # KFDHWSTest.* (SWDEV-193035) TEMPORARY_BLACKLIST_ALL_ASICS=\ -"KFDGraphicsInterop.RegisterForeignDeviceMem:"\ "KFDPMTest.SuspendWithActiveProcess:"\ "KFDPMTest.SuspendWithIdleQueue:"\ "KFDPMTest.SuspendWithIdleQueueAfterWork:"\ From 78e754ca5ba05812ee581758b871ddd94253bd1d Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 25 Jul 2019 12:02:46 -0400 Subject: [PATCH 0549/1247] KFDTest: Make shader compatiable for gfx9 and gfx10 Remove the CHIP name from the shader ISA and add wave_size(32) to make the same shader can be used for both GFX9 and GFX10 Change-Id: I16ea72f87980c3d9c11298e20c06a0a073fe9a28 Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDCWSRTest.cpp | 31 +------------ tests/kfdtest/src/KFDEvictTest.cpp | 64 +++----------------------- tests/kfdtest/src/KFDMemoryTest.cpp | 70 ++++------------------------- tests/kfdtest/src/KFDQMTest.cpp | 2 + 4 files changed, 18 insertions(+), 149 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index d06ebc1d26..0fbb425f72 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -52,35 +52,10 @@ LOOP:\n\ end\n\ "; +//This shader can be used by gfx9 and gfx10 static const char* iterate_isa_gfx9 = \ "\ shader iterate_isa\n\ -asic(GFX9)\n\ -type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ - flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - v_mov_b32 v5, 0\n\ -LOOP:\n\ - v_add_co_u32 v5, vcc, 1, v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ - v_cmp_lt_u32 vcc, v5, v4\n\ - s_cbranch_vccnz LOOP\n\ - flat_store_dword v[2,3], v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - s_endpgm\n\ -end\n\ -"; - -static const char* iterate_isa_gfx10 = \ -"\ -shader iterate_isa\n\ -asic(GFX10)\n\ wave_size(32)\n\ type(CS)\n\ /*copy the parameters from scalar registers to vector registers*/\n\ @@ -160,10 +135,8 @@ TEST_F(KFDCWSRTest, BasicTest) { if (m_FamilyId < FAMILY_AI) pIterateIsa = iterate_isa_gfx8; - else if (m_FamilyId < FAMILY_NV) - pIterateIsa = iterate_isa_gfx9; else - pIterateIsa = iterate_isa_gfx10; + pIterateIsa = iterate_isa_gfx9; m_pIsaGen->CompileShader(pIterateIsa, "iterate_isa", isaBuffer); diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 01efd86573..b74aedba35 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -364,11 +364,15 @@ TEST_F(KFDEvictTest, BasicTest) { * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 * v[6:7] - local buf address used for read test + * + * This shader can be used by gfx9 and gfx10 + * */ + static const char* gfx9_ReadMemory = "\ shader ReadMemory\n\ - asic(GFX9)\n\ + wave_size(32)\n\ type(CS)\n\ \n\ // compute address of corresponding output buffer\n\ @@ -469,67 +473,11 @@ L_QUIT:\n\ end\n\ "; - -static const char* gfx10_ReadMemory = -"\ - shader ReadMemory\n\ - asic(GFX10)\n\ - wave_size(32)\n\ - type(CS)\n\ - \n\ - // compute address of corresponding output buffer\n\ - v_mov_b32 v0, s4 // use workgroup id as index\n\ - v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ - v_mov_b32 v5, s3\n\ - v_add_co_u32 v5, vcc, v5, vcc_lo\n\ - \n\ - // compute input buffer offset used to store corresponding local buffer address\n\ - v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ - v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ - v_mov_b32 v3, s1\n\ - v_add_co_u32 v3, vcc, v3, vcc_lo\n\ - \n\ - // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ - flat_load_dwordx2 v[6:7], v[2:3] slc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - \n\ - v_mov_b32 v8, 0x5678\n\ - s_movk_i32 s8, 0x5678\n\ -L_REPEAT:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - s_cmp_eq_i32 s16, s8\n\ - s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ - // loop read 64M local buffer starting at v[6:7]\n\ - // every 4k page only read once\n\ - v_mov_b32 v9, 0\n\ - v_mov_b32 v10, 0x1000 // 4k page\n\ - v_mov_b32 v11, 0x4000000 // 64M size\n\ - v_mov_b32 v12, v6\n\ - v_mov_b32 v13, v7\n\ -L_LOOP_READ:\n\ - flat_load_dwordx2 v[14:15], v[12:13] slc\n\ - v_add_co_u32 v9, vcc, v9, v10 \n\ - v_add_co_u32 v12, vcc, v12, v10\n\ - v_add_co_u32 v13, vcc, v13, vcc_lo\n\ - v_cmp_lt_u32 vcc, v9, v11\n\ - s_cbranch_vccnz L_LOOP_READ\n\ - s_branch L_REPEAT\n\ -L_QUIT:\n\ - flat_store_dword v[4:5], v8\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ - s_endpgm\n\ - end\n\ -"; - std::string KFDEvictTest::CreateShader() { if (m_FamilyId < FAMILY_AI) return gfx8_ReadMemory; - else if (m_FamilyId < FAMILY_NV) - return gfx9_ReadMemory; else - return gfx10_ReadMemory; + return gfx9_ReadMemory; } /* Evict and restore queue test diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 58c14c94c0..aa3b75f9b3 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -118,23 +118,7 @@ end\n\ const char* gfx9_PollMemory = "\ shader ReadMemory\n\ -asic(GFX9)\n\ -type(CS)\n\ -/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ - s_movk_i32 s18, 0x5678\n\ - LOOP:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 LOOP\n\ - s_store_dword s18, s[2:3], 0x0 glc\n\ - s_endpgm\n\ - end\n\ -"; - -const char* gfx10_PollMemory = -"\ -shader ReadMemory\n\ -asic(GFX10)\n\ +wave_size(32)\n\ type(CS)\n\ /* Assume src address in s0, s1 and dst address in s2, s3*/\n\ s_movk_i32 s18, 0x5678\n\ @@ -159,7 +143,7 @@ type(CS)\n\ const char* gfx9_CopyOnSignal = "\ shader CopyOnSignal\n\ -asic(GFX9)\n\ +wave_size(32)\n\ type(CS)\n\ /* Assume input buffer in s0, s1 */\n\ s_mov_b32 s18, 0xcafe\n\ @@ -175,27 +159,6 @@ POLLSIGNAL:\n\ end\n\ "; -const char* gfx10_CopyOnSignal = -"\ -shader CopyOnSignal\n\ -asic(GFX10)\n\ -type(CS)\n\ -/* Assume input buffer in s0, s1 */\n\ - s_mov_b32 s18, 0xcafe\n\ -POLLSIGNAL:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 POLLSIGNAL\n\ - s_load_dword s17, s[0:1], 0x4 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_store_dword s17, s[0:1], 0x8 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; - - - /* Input0: A buffer of at least 2 dwords. * DW0: used as a signal. Write 0xcafe to signal * DW1: Write to this buffer for other device to read. @@ -204,7 +167,7 @@ POLLSIGNAL:\n\ const char* gfx9_WriteAndSignal = "\ shader WriteAndSignal\n\ -asic(GFX9)\n\ +wave_size(32)\n\ type(CS)\n\ /* Assume input buffer in s0, s1 */\n\ s_mov_b32 s18, 0xbeef\n\ @@ -217,21 +180,7 @@ type(CS)\n\ end\n\ "; -const char* gfx10_WriteAndSignal = -"\ -shader WriteAndSignal\n\ -asic(GFX10)\n\ -type(CS)\n\ -/* Assume input buffer in s0, s1 */\n\ - s_mov_b32 s18, 0xbeef\n\ - s_store_dword s18, s[0:1], 0x4 glc\n\ - s_mov_b32 s18, 0x1\n\ - s_store_dword s18, s[2:3], 0 glc\n\ - s_mov_b32 s18, 0xcafe\n\ - s_store_dword s18, s[0:1], 0x0 glc\n\ - s_endpgm\n\ - end\n\ -"; +//These gfx9_PullMemory, gfx9_CopyOnSignal, gfx9_WriteAndSignal shaders can be used by both gfx9 and gfx10 void KFDMemoryTest::SetUp() { ROUTINE_START @@ -364,7 +313,7 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - m_pIsaGen->CompileShader((m_FamilyId < FAMILY_NV) ? gfx9_PollMemory : gfx10_PollMemory, "ReadMemory", isaBuffer); + m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); @@ -1961,8 +1910,7 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader((m_FamilyId < FAMILY_NV) ? gfx9_CopyOnSignal : gfx10_CopyOnSignal, - "CopyOnSignal", isaBuffer); + m_pIsaGen->CompileShader(gfx9_CopyOnSignal, "CopyOnSignal", isaBuffer); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(buffer, NULL); dispatch0.Submit(queue); @@ -2081,8 +2029,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(nodes[0])); HsaMemoryBuffer isaBuffer(PAGE_SIZE, nodes[0], true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader((queue.GetFamilyId() < FAMILY_NV) ? gfx9_CopyOnSignal : gfx10_CopyOnSignal, - "CopyOnSignal", isaBuffer); + m_pIsaGen->CompileShader(gfx9_CopyOnSignal, "CopyOnSignal", isaBuffer); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, NULL); dispatch.Submit(queue); @@ -2090,8 +2037,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue0; ASSERT_SUCCESS(queue0.Create(nodes[1])); HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader((queue0.GetFamilyId() < FAMILY_NV) ? gfx9_WriteAndSignal : gfx10_WriteAndSignal, - "WriteAndSignal", isaBuffer0); + m_pIsaGen->CompileShader(gfx9_WriteAndSignal, "WriteAndSignal", isaBuffer0); Dispatch dispatch0(isaBuffer0); dispatch0.SetArgs(buffer, mmioBase); dispatch0.Submit(queue0); diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 942f426ff7..48e4fbb098 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -489,10 +489,12 @@ TEST_F(KFDQMTest, OverSubscribeCpQueues) { /* A simple isa loop program with dense mathematic operations * s1 controls the number iterations of the loop + * This shader can be used by GFX8, GFX9 and GFX10 */ static const char *loop_isa = \ "\ shader loop_isa\n\ +wave_size(32)\n\ type(CS)\n\ s_movk_i32 s0, 0x0008\n\ s_movk_i32 s1, 0x00ff\n\ From 20cd954fe83639aefaa127da4389549abecf3074 Mon Sep 17 00:00:00 2001 From: Cole Nelson Date: Mon, 29 Jul 2019 13:10:18 -0700 Subject: [PATCH 0550/1247] KFDTest: restore KFDMemoryTest.BigBufferStressTest on gfx900 Addressed by: 4066dcd kfdtest: increase BigBufStressTest timeout and avoid VM fault 36776e9 kfdtest: avoid BigBufStressTest run on NUMA node 0 Change-Id: If21c6e42b4cf6aada1f74e77f0d8d1a2fdebcdb8 Signed-off-by: Cole Nelson --- tests/kfdtest/scripts/kfdtest.exclude | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 031bf9e7c3..333b82f838 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -130,7 +130,6 @@ LEXA_TESTS_BLACKLIST=\ "$SDMA_BLACKLIST" # KFDIPCTest.BasicTest (ROCMOPS-459) .CMABasicTest (ROCMOPS-460) .CrossMemoryAttachTest (ROCMOPS-461) -# KFDMemoryTest.BigBufferStressTest (ROCMOPS-462) # KFDEvictTest.BurstyTest (ROCMOPS-464) # KFDQMTest.MultipleSdmaQueues (ROCMOPS-463) VEGA10_TESTS_BLACKLIST=\ @@ -139,7 +138,6 @@ VEGA10_TESTS_BLACKLIST=\ "KFDEvictTest.BurstyTest:"\ "KFDIPCTest.CMABasicTest:"\ "KFDIPCTest.CrossMemoryAttachTest:"\ -"KFDMemoryTest.BigBufferStressTest:"\ "KFDQMTest.MultipleSdmaQueues" VEGA12_TESTS_BLACKLIST=\ From 4d7b0990e404e7ef06b579bfce99cee43c0747fb Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 12 Jul 2019 17:48:27 -0400 Subject: [PATCH 0551/1247] libhsakmt: Sanity check node_id for NUMA binding Ignore requests to bind to invalid NUMA nodes. This affects only legacy applications (such as KFDTest) that allocate system memory as paged memory with a GPU node ID. Change-Id: I81e514af6d0c1ab2ed5229adeeca1fa0ab2a0685 Signed-off-by: Felix Kuehling --- src/fmm.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index 4aced94956..660ff234b3 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1469,6 +1469,12 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, num_node = numa_num_task_nodes(); + /* Ignore binding requests to invalid nodes IDs */ + if (node_id >= (unsigned)num_node) { + pr_warn("node_id %d >= num_node %d\n", node_id, num_node); + return 0; + } + if (num_node > 1) { node_mask = numa_bitmask_alloc(num_node); if (!node_mask) From 53e4eca48581800d137857f1fb5db38217ccca65 Mon Sep 17 00:00:00 2001 From: Konstantin Zhuravlyov Date: Wed, 31 Jul 2019 14:18:27 -0400 Subject: [PATCH 0552/1247] Allow ccache enabled builds if -DROCM_CCACHE_BUILD=ON Change-Id: Ia80965feb6e4ff22d466f1ff3142fb2ef303f99c --- CMakeLists.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e020132ac1..f75e55d142 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,17 @@ set ( HSAKMT_TARGET "${HSAKMT}" ) project ( ${HSAKMT_TARGET} VERSION 1.9.0) +# Optionally, build HSAKMT with ccache. +set(ROCM_CCACHE_BUILD OFF CACHE BOOL "Set to ON for a ccache enabled build") +if (ROCM_CCACHE_BUILD) + find_program(CCACHE_PROGRAM ccache) + if (CCACHE_PROGRAM) + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_PROGRAM}) + else() + message(WARNING "Unable to find ccache. Falling back to real compiler") + endif() # if (CCACHE_PROGRAM) +endif() # if (ROCM_CCACHE_BUILD) + ## Set default module path if not already set if ( NOT DEFINED CMAKE_MODULE_PATH ) set ( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/" ) From 23db2c658d8bfb22d892bc8b86fb2f5691b6099d Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 4 Oct 2018 18:28:18 -0400 Subject: [PATCH 0553/1247] libhsakmt: Add gfx908 support Change-Id: Icced5ca4c68eb6cc3978e0d8e836d0ccfc8c980d Signed-off-by: Yong Zhao --- src/libhsakmt.h | 3 ++- src/pmc_table.c | 1 + src/queues.c | 7 +++++++ src/topology.c | 4 ++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index da419f3029..98ebadd6e8 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -102,7 +102,8 @@ enum asic_family_type { CHIP_VEGA12, CHIP_VEGA20, CHIP_RAVEN, - CHIP_NAVI10 + CHIP_ARCTURUS, + CHIP_NAVI10, }; #define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) diff --git a/src/pmc_table.c b/src/pmc_table.c index 0459d67240..dd4c69b618 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1750,6 +1750,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, case CHIP_VEGA12: case CHIP_VEGA20: case CHIP_RAVEN: + case CHIP_ARCTURUS: *block = vega_blocks[block_id]; break; default: diff --git a/src/queues.c b/src/queues.c index d8fbc7f786..be3df8f733 100644 --- a/src/queues.c +++ b/src/queues.c @@ -129,6 +129,12 @@ const struct device_info vega20_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info arcturus_device_info = { + .asic_family = CHIP_ARCTURUS, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + const struct device_info navi10_device_info = { .asic_family = CHIP_NAVI10, .eop_buffer_size = 4096, @@ -149,6 +155,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_VEGA12] = &vega12_device_info, [CHIP_VEGA20] = &vega20_device_info, [CHIP_RAVEN] = &raven_device_info, + [CHIP_ARCTURUS] = &arcturus_device_info, [CHIP_NAVI10] = &navi10_device_info, }; diff --git a/src/topology.c b/src/topology.c index fb4ad66f66..da95c01dfa 100644 --- a/src/topology.c +++ b/src/topology.c @@ -220,6 +220,10 @@ static struct hsa_gfxip_table { { 0x66A4, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66A7, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66AF, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + /* Arcturus */ + { 0x7388, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, + { 0x738C, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, + { 0x738E, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, /* Navi10 */ { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x7312, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, From 5ae585430214b1acc7aa2aeede773a2438f8d4d2 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 31 Jul 2019 14:32:45 -0400 Subject: [PATCH 0554/1247] kfdtest: Improve FlatScratchAccess by not hardcoding the value We should use the SE number reflected by NumShaderBanks of the node rather than hardcoding it. Change-Id: I945fb001f81ce506249cf485a7ce25aee8219bc7 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDMemoryTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index aa3b75f9b3..acaa224007 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -674,7 +674,7 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { for (unsigned int bank = 0; bank < pNodeProperties->NumMemoryBanks; bank++) { if (memoryProperties[bank].HeapType == HSA_HEAPTYPE_GPU_SCRATCH) { - int numWaves = 4; // WAVES must be >= # SE + int numWaves = pNodeProperties->NumShaderBanks; // WAVES must be >= # SE int waveSize = 1; // Amount of space used by each wave in units of 256 dwords PM4Queue queue; From d954db828d37e219e9b297c53dc87e9a591859d5 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 25 Jul 2019 11:14:36 -0400 Subject: [PATCH 0555/1247] kfdtest: Add gfx908 test cases Change-Id: I427bb8381ea0690d2b0ae496acd4902fba0d18b4 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 7 +++++++ tests/kfdtest/scripts/run_kfdtest.sh | 3 +++ 2 files changed, 10 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 333b82f838..9008339a3e 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -158,6 +158,13 @@ RAVEN_TESTS_BLACKLIST=\ "KFDQMTest.QueueLatency:"\ "KFDQMTest.SdmaEventInterrupt" +# KFDPNPTest.DisableWithActiveProcess (KFD-432) +# KFDQMTest.BasicCuMaskingEven (KFD-433) +ARCT_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDPNPTest.DisableWithActiveProcess" + # Core tests, used in scenarios like bringup CORE_TESTS=\ "KFDQMTest.CreateDestroyCpQueue:"\ diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 5ac5afa44f..f81e3e2687 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -75,6 +75,7 @@ getFilter() { vg12 ) FILTER="--gtest_filter=$VEGA12_TESTS_BLACKLIST" ;; vg20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; rv ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; + arct ) FILTER="--gtest_filter=$ARCT_TESTS_BLACKLIST" ;; core ) FILTER="--gtest_filter=$CORE_TESTS" ;; all ) FILTER="" ;; *) die "Unsupported platform $PLATFORM or node $NODE. Exiting" ;; @@ -127,6 +128,8 @@ deviceIdToGpuName() { platformName="vg20" ;; 15dd ) platformName="rv" ;; + 7380 | 46 | 47 | 48 | 738c | 7388 | 738e) + platformName="arct" ;; * ) return ;; esac From 65ab296840ba1a0e899a1b4fdb1751a5018d4924 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Mon, 29 Jul 2019 15:02:32 -0400 Subject: [PATCH 0556/1247] KFDTest: Set CS_W32_EN bit for wave32 workloads for GFX10 On GFX10, the wave size is determined by the COMPUTE_DISPATCH_INITIATOR value passed to DISPATCH_DIRECT.CS_W32_EN, default 0 value was giving 64 lane waves Change-Id: Ie8c407a24bd2825757ec481be62247b35047e5ca Signed-off-by: shaoyunl --- tests/kfdtest/src/Dispatch.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index 559594ee44..dc75c4c671 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -200,10 +200,12 @@ void Dispatch::BuildIb() { 0, // COMPUTE_USER_DATA_15 - - unused }; - const unsigned int DISPATCH_INIT_VALUE = 0x00000021 | (is_dgpu() ? 0 : 0x1000); + const unsigned int DISPATCH_INIT_VALUE = 0x00000021 | (is_dgpu() ? 0 : 0x1000) | + ((m_FamilyId >= FAMILY_NV) ? 0x8000 : 0); // {COMPUTE_SHADER_EN=1, PARTIAL_TG_EN=0, FORCE_START_AT_000=0, ORDERED_APPEND_ENBL=0, // ORDERED_APPEND_MODE=0, USE_THREAD_DIMENSIONS=1, ORDER_MODE=0, DISPATCH_CACHE_CNTL=0, // SCALAR_L1_INV_VOL=0, VECTOR_L1_INV_VOL=0, DATA_ATC=?, RESTORE=0} + // Set CS_W32_EN for wave32 workloads for gfx10 since all the shaders used in KFDTest is 32 bit . m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId)); From e65685df1953640e263624144bac9cf54b917987 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Tue, 30 Jul 2019 15:34:41 -0400 Subject: [PATCH 0557/1247] KFDTest: simply P2PBandwidthTest case for specified source/destination Only run the P2P test over the specified source and destination nodes if user already specify them Change-Id: Ia3c0195cead7f46e3e28507f3255d8c59a287ab8 Signed-off-by: shaoyunl --- tests/kfdtest/src/KFDPerformanceTest.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index 4d49f392cc..e4f5af2b51 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -191,6 +191,21 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { std::stringstream msg; char str[64]; + if (isSpecified) { + HSAuint32 n1 = g_TestNodeId; + HSAuint32 n2 = g_TestDstNodeId; + HSAuint64 speed, speed2; + + LOG() << "Copy from node to node by [push, pull]" << std::endl; + snprintf(str, sizeof(str), "[%d -> %d] ", n1, n2); + testNodeToNodes(n1, &n2, 1, OUT, IN, size, &speed, &speed2, &msg); + + LOG() << std::dec << str << (float)speed / 1024 << " - " << + (float)speed2 / 1024 << " GB/s" << std::endl; + goto exit; + + } + for (; s < twoNodesIdx; s++) { LOG() << test_suits_string[s] << std::endl; msg << test_suits_string[s] << std::endl; @@ -270,6 +285,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { } } +exit: /* New line.*/ LOG() << std::endl << msg.str() << std::endl; From f40f166e20883195d51801e172f45507fc3c7b21 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 25 Jul 2019 15:59:14 -0500 Subject: [PATCH 0558/1247] Remove NodeId parameter from hsaKmtAllocQueueGWS The NodeId parameter is redundant and can be retrieved from QueueId parameter. Change-Id: I12853849b868b304bd27633fa7653ba644d69026 Signed-off-by: Oak Zeng --- include/hsakmt.h | 1 - include/linux/kfd_ioctl.h | 3 +-- src/queues.c | 10 ---------- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 6d666a931c..78ef5827b1 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -578,7 +578,6 @@ hsaKmtUnmapGraphicHandle( HSAKMT_STATUS HSAKMTAPI hsaKmtAllocQueueGWS( - HSAuint32 NodeId, //IN HSA_QUEUEID QueueId, //IN HSAuint32 nGWS, //IN HSAuint32 *firstGWS //OUT diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index fe355090ee..76d8a96a02 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -470,17 +470,16 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { /* Allocate GWS for specific queue * - * @gpu_id: device identifier * @queue_id: queue's id that GWS is allocated for * @num_gws: how many GWS to allocate * @first_gws: index of the first GWS allocated. * only support contiguous GWS allocation */ struct kfd_ioctl_alloc_queue_gws_args { - __u32 gpu_id; /* to KFD */ __u32 queue_id; /* to KFD */ __u32 num_gws; /* to KFD */ __u32 first_gws; /* from KFD */ + __u32 pad; /* to KFD */ }; struct kfd_ioctl_get_dmabuf_info_args { diff --git a/src/queues.c b/src/queues.c index be3df8f733..f006d5ec2c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -834,25 +834,15 @@ uint32_t *convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues) HSAKMT_STATUS HSAKMTAPI hsaKmtAllocQueueGWS( - HSAuint32 NodeId, HSA_QUEUEID QueueId, HSAuint32 nGWS, HSAuint32 *firstGWS) { struct kfd_ioctl_alloc_queue_gws_args args = {0}; struct queue *q = PORT_UINT64_TO_VPTR(QueueId); - HSAKMT_STATUS result; - uint32_t gpu_id; CHECK_KFD_OPEN(); - result = validate_nodeid(NodeId, &gpu_id); - if (result != HSAKMT_STATUS_SUCCESS) { - pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); - return result; - } - - args.gpu_id = gpu_id; args.queue_id = (HSAuint32)q->queue_id; args.num_gws = nGWS; From 632ad3a749d81f4e9ab21f36f309679bf6673a38 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 6 Aug 2019 15:15:03 -0400 Subject: [PATCH 0559/1247] hsaKmtCloseKFD destroy objects in all apertures Otherwise the parent call hsaKmtCloseKFD and then fork child process, child process will duplicate the vm_objects from the parent. Change-Id: Ia6ffc51cbae983b6a7cdc58ccf3b11ebe4087d97 Signed-off-by: Philip Yang --- src/fmm.c | 54 +++++++++++++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 660ff234b3..5336af6973 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2345,14 +2345,37 @@ sysfs_parse_failed: return ret; } +static void fmm_clear_aperture(manageable_aperture_t *app) +{ + rbtree_node_t *n; + + while ((n = rbtree_node_any(&app->tree, MID))) + vm_remove_object(app, vm_object_entry(n, 0)); + + while (app->vm_ranges) + vm_remove_area(app, app->vm_ranges); +} + void fmm_destroy_process_apertures(void) { + unsigned int i; + release_mmio(); + + fmm_clear_aperture(&cpuvm_aperture); + fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); + fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); + if (gpu_mem) { + for (i = 0; i < gpu_mem_count; i++) { + fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); + fmm_clear_aperture(&gpu_mem[i].scratch_physical); + } + free(gpu_mem); gpu_mem = NULL; + gpu_mem_count = 0; } - gpu_mem_count = 0; } HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, @@ -3531,18 +3554,6 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) return HSAKMT_STATUS_SUCCESS; } -static void fmm_clear_aperture(manageable_aperture_t *app) -{ - rbtree_node_t *n; - - while ((n = rbtree_node_any(&app->tree, MID))) - vm_remove_object(app, vm_object_entry(n, 0)); - - while (app->vm_ranges) - vm_remove_area(app, app->vm_ranges); - -} - /* This is a special funcion that should be called only from the child process * after a fork(). This will clear all vm_objects and mmaps duplicated from * the parent. @@ -3559,19 +3570,7 @@ void fmm_clear_all_mem(void) drm_render_fds[i] = 0; } - /* Nothing is initialized. */ - if (!gpu_mem) - return; - - fmm_clear_aperture(&cpuvm_aperture); - - for (i = 0; i < gpu_mem_count; i++) { - fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); - fmm_clear_aperture(&gpu_mem[i].scratch_physical); - } - - fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); - fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); + fmm_destroy_process_apertures(); if (dgpu_shared_aperture_limit) { /* Use the same dgpu range as the parent. If failed, then set @@ -3597,7 +3596,4 @@ void fmm_clear_all_mem(void) all_gpu_id_array_size = 0; all_gpu_id_array = NULL; - - gpu_mem_count = 0; - free(gpu_mem); } From 836dfd07526fa2d6b28b1387aef24a1ec0ed6d65 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 11 Jul 2019 10:48:54 -0400 Subject: [PATCH 0560/1247] libhsakmt: update dbg enable trap and add query debug events Add data out for enable trap to return poll fd to user space. Add query debug events interface. Change-Id: Ia4afde1cf167e6aa61d502380a8b329ee89d5f44 Signed-off-by: Jonathan Kim --- include/hsakmt.h | 42 ++++++++++++++++- include/hsakmttypes.h | 12 +++++ include/linux/kfd_ioctl.h | 18 +++++++- src/debug.c | 96 ++++++++++++++++++++++++++++++++++----- src/libhsakmt.ver | 2 + 5 files changed, 155 insertions(+), 15 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 78ef5827b1..7c62ee69f1 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -721,6 +721,8 @@ hsaKmtQueueResume( /** Enable debug trap for NodeId. If QueueId is INVALID_QUEUEID then enable for all queues on NodeId, otherwise enable only for QueueId. + Return file descriptor PollFd where on poll wake, fd has readable + FIFO data for pending debug events. When debug trap is enabled the trap handler behavior changes depending on architecture of the node and can include the following: @@ -766,8 +768,18 @@ hsaKmtQueueResume( HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap( - HSAuint32 NodeId, //IN - HSA_QUEUEID QueueId //IN + HSAuint32 NodeId, //IN + HSA_QUEUEID QueueId //IN + ); + + +/* Similar to EnableDebugTrap with polling fd return*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtEnableDebugTrapWithPollFd( + HSAuint32 NodeId, //IN + HSA_QUEUEID QueueId, //IN + HSAint32 *PollFd //OUT ); /** @@ -787,6 +799,32 @@ hsaKmtDisableDebugTrap( HSAuint32 NodeId //IN ); + +/** + Query pending debug event set by ptrace. + + Can query by target QueueId. If QueueId is INVALID_QUEUEID, return the + first queue id that has a pending event. Option to clear pending event + after query is used by the ClearEvents parameter. + + Pending debug event type will be returned in EventsReceived parameter and is + defined by HSA_DEBUG_EVENT_TYPE. Suspended state of queue is returned in + IsSuspended. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful + */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueryDebugEvent( + HSAuint32 NodeId, // IN + HSAuint32 Pid, // IN + HSAuint32 *QueueId, // IN/OUT + bool ClearEvents, // IN + HSA_DEBUG_EVENT_TYPE *EventsReceived, // OUT + bool *IsSuspended // OUT + ); + /** Set the value to use to initialize the TrapData used when initializing trap temp registers for NodeId when debug trap is enabled. diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index cebf5ac699..03f4ea147c 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -866,6 +866,18 @@ typedef enum _HSA_EVENTTYPE HSA_EVENTTYPE_TYPE_SIZE = 0xFFFFFFFF } HSA_EVENTTYPE; + +// +// Definitions for types of pending debug events +// +typedef enum _HSA_DEBUG_EVENT_TYPE +{ + HSA_DEBUG_EVENT_TYPE_NONE = 0, + HSA_DEBUG_EVENT_TYPE_TRAP = 1, + HSA_DEBUG_EVENT_TYPE_VMFAULT = 2, + HSA_DEBUG_EVENT_TYPE_TRAP_VMFAULT = 3 +} HSA_DEBUG_EVENT_TYPE; + typedef HSAuint32 HSA_EVENTID; // diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 76d8a96a02..7a1185c823 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -187,11 +187,19 @@ struct kfd_ioctl_dbg_wave_control_args { __u32 buf_size_in_bytes; /*including gpu_id and buf_size */ }; +/* mapping event types to API spec */ +#define KFD_DBG_EV_STATUS_TRAP 1 +#define KFD_DBG_EV_STATUS_VMFAULT 2 +#define KFD_DBG_EV_STATUS_SUSPENDED 4 +#define KFD_DBG_EV_FLAG_CLEAR_STATUS 1 + +#define KFD_INVALID_QUEUEID 0xFFFFFFFFFFFFFFFFULL + /* KFD_IOC_DBG_TRAP_ENABLE: * ptr: unused * data1: 0=disable, 1=enable * data2: queue ID (for future use) - * data3: unused + * data3: return value for fd */ #define KFD_IOC_DBG_TRAP_ENABLE 0 @@ -235,6 +243,14 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_NODE_RESUME 5 +/* KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT: + * ptr: unused + * data1: queue id (IN/OUT) + * data2: flags (IN) + * data3: suspend[2:2], event type [1:0] (OUT) + */ +#define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 6 + struct kfd_ioctl_dbg_trap_args { __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ __u32 pid; /* to KFD */ diff --git a/src/debug.c b/src/debug.c index 3457da61de..ac7e3c641d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -275,8 +275,8 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, HSAuint32 data2, HSAuint32 data3, HSAuint32 pid, - HSAuint64 pointer - ) + HSAuint64 pointer, + struct kfd_ioctl_dbg_trap_args *argout) { uint32_t gpu_id; HSAKMT_STATUS result; @@ -320,6 +320,9 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); + if (argout) + *argout = args; + if (err == 0) result = HSAKMT_STATUS_SUCCESS; else @@ -328,21 +331,45 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, return result; } -HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32 NodeId, - HSA_QUEUEID QueueId) +HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrapWithPollFd(HSAuint32 NodeId, + HSA_QUEUEID QueueId, + HSAint32 *PollFd) //OUT { + int result; + struct kfd_ioctl_dbg_trap_args argout = {0}; + if (QueueId != INVALID_QUEUEID) return HSAKMT_STATUS_NOT_SUPPORTED; - return debug_trap(NodeId, + result = debug_trap(NodeId, KFD_IOC_DBG_TRAP_ENABLE, 1, QueueId, 0, INVALID_PID, - 0); + 0, + &argout); + + *PollFd = argout.data3; + + return result; } +HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32 NodeId, + HSA_QUEUEID QueueId) +{ + HSAint32 PollFd = 0; + HSAKMT_STATUS status = hsaKmtEnableDebugTrapWithPollFd(NodeId, + QueueId, + &PollFd); + + if (status == HSAKMT_STATUS_SUCCESS) + close(PollFd); + return status; +} + + + HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId) { return debug_trap(NodeId, @@ -351,7 +378,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId) 0, 0, INVALID_PID, - 0); + 0, + NULL); } HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, @@ -364,7 +392,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, TrapData1, 0, INVALID_PID, - 0); + 0, + NULL); } HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( @@ -381,7 +410,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( TrapMask, 0, INVALID_PID, - 0); + 0, + NULL); } HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode( @@ -394,7 +424,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode( 0, 0, INVALID_PID, - 0); + 0, + NULL); } /** @@ -455,7 +486,8 @@ hsaKmtQueueSuspend( NumQueues, GracePeriod, Pid, - (HSAuint64)queue_ids_ptr); + (HSAuint64)queue_ids_ptr, + NULL); free(queue_ids_ptr); return result; @@ -518,7 +550,47 @@ hsaKmtQueueResume( NumQueues, 0, Pid, - (HSAuint64)queue_ids_ptr); + (HSAuint64)queue_ids_ptr, + NULL); free(queue_ids_ptr); return result; } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtQueryDebugEvent( + HSAuint32 NodeId, //IN + HSAuint32 Pid, // IN + HSAuint32 *QueueId, // IN/OUT + bool ClearEvents, // IN + HSA_DEBUG_EVENT_TYPE *EventsReceived, // OUT + bool *IsSuspended // OUT + ) +{ + HSAKMT_STATUS result; + struct kfd_ioctl_dbg_trap_args argout = {0}; + uint32_t flags = 0; + + if (ClearEvents) + flags |= KFD_DBG_EV_FLAG_CLEAR_STATUS; + + result = debug_trap(NodeId, + KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT, + *QueueId, + flags, + 0, + Pid, + 0, + &argout); + + if (result) + return result; + + *QueueId = argout.data1; + *EventsReceived = argout.data3 & + (KFD_DBG_EV_STATUS_TRAP | KFD_DBG_EV_STATUS_VMFAULT); + *IsSuspended = (argout.data3 & KFD_DBG_EV_STATUS_SUSPENDED) >> 2; + + return result; +} + diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 54d79b440d..482a7aeaf8 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -58,7 +58,9 @@ hsaKmtQueryPointerInfo; hsaKmtSetMemoryUserData; hsaKmtGetQueueInfo; hsaKmtEnableDebugTrap; +hsaKmtEnableDebugTrapWithPollFd; hsaKmtDisableDebugTrap; +hsaKmtQueryDebugEvent; hsaKmtSetDebugTrapData2; hsaKmtSetWaveLaunchTrapOverride; hsaKmtSetWaveLaunchMode; From 3e4c42ef13a966eb99756f0488aec9af2d9c78ee Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 30 Jul 2019 20:19:47 -0400 Subject: [PATCH 0561/1247] kfdtest: Improve the printing message for CuMasking tests Decimal is better than hex in this case. Change-Id: Ic15a9373e99160880b98d3dcd6827d551c87b77a Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- tests/kfdtest/src/KFDQMTest.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 9008339a3e..c688faf498 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -5,7 +5,7 @@ # stresses memory management in unusual ways by having lots of memory allocated but # not mapped, which is not relevant to compute applications over ROCr. # -# CU Masking Linear/Split are not working correctly due to how the HW distributes work over CUs. +# CU Masking Linear are not working correctly due to how the HW distributes work over CUs. # They are available for testing but are not currently expected to pass on CI/VI/AI. # # The CheckZeroInitializationVram test is no longer expected to pass as KFD no longer diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 48e4fbb098..9b036193d0 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -655,10 +655,10 @@ TEST_F(KFDQMTest, BasicCuMaskingLinear) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); uint32_t ActiveCU = (pNodeProperties->NumFComputeCores / pNodeProperties->NumSIMDPerCU); uint32_t numSEs = pNodeProperties->NumShaderBanks; - LOG() << std::hex << "# Compute cores: 0x" << pNodeProperties->NumFComputeCores << std::endl; - LOG() << std::hex << "# SIMDs per CPU: 0x" << pNodeProperties->NumSIMDPerCU << std::endl; - LOG() << std::hex << "# Shader engines: 0x" << numSEs << std::endl; - LOG() << std::hex << "# Active CUs: 0x" << ActiveCU << std::endl; + LOG() << std::dec << "# Compute cores: " << pNodeProperties->NumFComputeCores << std::endl; + LOG() << std::dec << "# SIMDs per CU: " << pNodeProperties->NumSIMDPerCU << std::endl; + LOG() << std::dec << "# Shader engines: " << numSEs << std::endl; + LOG() << std::dec << "# Active CUs: " << ActiveCU << std::endl; HSAint64 TimewithCU1, TimewithCU; uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */ uint32_t maskNumBits = maskNumDwords * 32; @@ -718,10 +718,10 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { return; } - LOG() << std::hex << "# Compute cores: 0x" << pNodeProperties->NumFComputeCores << std::endl; - LOG() << std::hex << "# SIMDs per CPU: 0x" << pNodeProperties->NumSIMDPerCU << std::endl; - LOG() << std::hex << "# Shader engines: 0x" << numShaderEngines << std::endl; - LOG() << std::hex << "# Active CUs: 0x" << ActiveCU << std::endl; + LOG() << std::dec << "# Compute cores: " << pNodeProperties->NumFComputeCores << std::endl; + LOG() << std::dec << "# SIMDs per CU: " << pNodeProperties->NumSIMDPerCU << std::endl; + LOG() << std::dec << "# Shader engines: " << numShaderEngines << std::endl; + LOG() << std::dec << "# Active CUs: " << ActiveCU << std::endl; HSAint64 TimewithCU1, TimewithCU; uint32_t maskNumDwords = (ActiveCU + 31) / 32; /* Round up to the nearest multiple of 32 */ uint32_t maskNumBits = maskNumDwords * 32; From cdc10991a92f994cae510545e07f0031ba412fa3 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Tue, 6 Aug 2019 15:24:05 -0400 Subject: [PATCH 0562/1247] kfdtest: avoid TTM eviction in KFDMemoryTest.BigBufferStressTest Reserve half of dma32 zone for non-NUMA system. Change-Id: Id7aea7b6ff6cc1cc7983ecd95f8078b7f1be630c Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDMemoryTest.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index acaa224007..0212917faa 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -905,6 +905,7 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { TEST_START(TESTPROFILE_RUNALL); HSAuint64 AlternateVAGPU; + HSAuint64 Available_size; HsaMemMapFlags mapFlags = {0}; int ret; @@ -920,7 +921,7 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { */ NumaNodeBind("!0"); - BigBufferSystemMemory(defaultGPUNode, granularityMB, NULL); + BigBufferSystemMemory(defaultGPUNode, granularityMB, &Available_size); BigBufferVRAM(defaultGPUNode, granularityMB, NULL); @@ -935,6 +936,13 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { HSAuint64 block_size = block_size_mb * 1024 * 1024; PM4Queue queue; + /* In non-numa system to avoid TTM eviction, + * we have to keep half of dma32 zone (2GB) out of allocation. + */ + if (((numa_available() == -1) || (numa_num_task_nodes() < 2)) && + (Available_size > 0x80000000)) + Available_size -= 0x80000000; + /* Test 4 times to see if there is any memory leak.*/ for (int repeat = 1; repeat < 5; repeat++) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); @@ -951,6 +959,9 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); break; } + + if ((i + 2) * block_size > Available_size) + break; } LOG() << "Allocated system buffers time " << std::dec << repeat << ": " << i << "x" From c04ebb56b99072a6a56057d3e396e1c752d780e0 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 26 Jul 2019 09:21:51 -0400 Subject: [PATCH 0563/1247] kfdtest: add query debug pending events test cases adding test cases to query debugger pending events Change-Id: I089754c508e476ce7b19e1cbd84235e4474b30c4 Signed-off-by: Jonathan Kim Reviewed-by: Kent Russell --- tests/kfdtest/src/KFDDBGTest.cpp | 399 ++++++++++++++++++++++++++++++- 1 file changed, 397 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index e8ab3df780..3fd83c2e52 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -23,11 +23,12 @@ #include "KFDDBGTest.hpp" #include +#include #include "KFDQMTest.hpp" #include "PM4Queue.hpp" #include "PM4Packet.hpp" #include "Dispatch.hpp" - +#include static const char* loop_inc_isa = \ "\ @@ -80,7 +81,8 @@ type(CS)\n\ LOOP:\n\ v_add_co_u32 v5, vcc, 1, v5\n\ s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ + /*compare the result value (v5) to iteration value (v4),*/\n\ + /*and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ v_cmp_lt_u32 vcc, v5, v4\n\ s_cbranch_vccnz LOOP\n\ flat_store_dword v[2,3], v5\n\ @@ -89,6 +91,82 @@ LOOP:\n\ end\n\ "; +static const char* jump_to_trap_gfx9 = \ +"\ +shader jump_to_trap\n\ +asic(GFX9)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + s_trap 1\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ + flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + v_mov_b32 v5, 0\n\ +LOOP:\n\ + v_add_co_u32 v5, vcc, 1, v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + /*compare the result value (v5) to iteration value (v4),*/\n\ + /*and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ + v_cmp_lt_u32 vcc, v5, v4\n\ + s_cbranch_vccnz LOOP\n\ + flat_store_dword v[2,3], v5\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + s_endpgm\n\ + end\n\ +"; + +static const char* trap_handler_gfx9 = \ +"\ +shader trap_handler\n\ +asic(GFX9)\n\ +type(CS)\n\ +CHECK_VMFAULT:\n\ + /*if trap jumped to by vmfault, restore skip m0 signalling*/\n\ + s_getreg_b32 ttmp14, hwreg(HW_REG_TRAPSTS)\n\ + s_and_b32 ttmp2, ttmp14, 0x800\n\ + s_cbranch_scc1 RESTORE_AND_EXIT\n\ +GET_DOORBELL:\n\ + s_mov_b32 ttmp2, exec_lo\n\ + s_mov_b32 ttmp3, exec_hi\n\ + s_mov_b32 exec_lo, 0x80000000\n\ + s_sendmsg 10\n\ +WAIT_SENDMSG:\n\ + /*wait until msb is cleared (i.e. doorbell fetched)*/\n\ + s_nop 7\n\ + s_bitcmp0_b32 exec_lo, 0x1F\n\ + s_cbranch_scc0 WAIT_SENDMSG\n\ +SEND_INTERRUPT:\n\ + /* set context bit and doorbell and restore exec*/\n\ + s_mov_b32 exec_hi, ttmp3\n\ + s_and_b32 exec_lo, exec_lo, 0xfff\n\ + s_mov_b32 ttmp3, exec_lo\n\ + s_bitset1_b32 ttmp3, 23\n\ + s_mov_b32 exec_lo, ttmp2\n\ + s_mov_b32 ttmp2, m0\n\ + /* set m0, send interrupt and restore m0 and exit trap*/\n\ + s_mov_b32 m0, ttmp3\n\ + s_nop 0x0\n\ + s_sendmsg sendmsg(MSG_INTERRUPT)\n\ + s_mov_b32 m0, ttmp2\n\ +RESTORE_AND_EXIT:\n\ + /* restore and increment program counter to skip shader trap jump*/\n\ + s_add_u32 ttmp0, ttmp0, 4\n\ + s_addc_u32 ttmp1, ttmp1, 0\n\ + s_and_b32 ttmp1, ttmp1, 0xffff\n\ + /* restore SQ_WAVE_IB_STS */\n\ + s_lshr_b32 ttmp2, ttmp11, (26 - 15)\n\ + s_and_b32 ttmp2, ttmp2, (0x8000 | 0x1F0000)\n\ + s_setreg_b32 hwreg(HW_REG_IB_STS), ttmp2\n\ + /* restore SQ_WAVE_STATUS */\n\ + s_and_b64 exec, exec, exec\n\ + s_and_b64 vcc, vcc, vcc\n\ + s_setreg_b32 hwreg(HW_REG_STATUS), ttmp12\n\ + s_rfe_b64 [ttmp0, ttmp1]\n\ + end\n\ +"; void KFDDBGTest::SetUp() { ROUTINE_START @@ -292,3 +370,320 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { TEST_END } +TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + HSAint32 PollFd; + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + // enable debug trap and check poll fd creation + ASSERT_SUCCESS(hsaKmtEnableDebugTrapWithPollFd(defaultGPUNode, + INVALID_QUEUEID, + &PollFd)); + ASSERT_GT(PollFd, 0) << "failed to create polling file descriptor"; + + // create shader and trap bufs then enable 2nd level trap + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, true); + HsaMemoryBuffer iterBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + HsaMemoryBuffer resBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + + HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, true); + HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); + + ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, + trap.As(), + 0x1000, + tmaBuf.As(), + 0x1000)); + + // compile and dispatch shader + m_pIsaGen->CompileShader(jump_to_trap_gfx9, "jump_to_trap", isaBuf); + m_pIsaGen->CompileShader(trap_handler_gfx9, "trap_handler", trap); + + PM4Queue queue; + HsaQueueResource *qResources; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + unsigned int* iter = iterBuf.As(); + unsigned int* result = resBuf.As(); + int suspendTimeout = 500; + int syncStatus; + iter[0] = 150000000; + Dispatch *dispatch; + dispatch = new Dispatch(isaBuf); + dispatch->SetArgs(&iter[0], &result[0]); + dispatch->SetDim(1, 1, 1); + + dispatch->Submit(queue); + qResources = queue.GetResource(); + + // poll, read and query for pending trap event + struct pollfd fds[1]; + fds[0].fd = PollFd; + fds[0].events = POLLIN | POLLRDNORM; + ASSERT_GT(poll(fds, 1, 5000), 0); + + char trapChar; + ASSERT_GT(read(PollFd, &trapChar, 1), 0); + ASSERT_EQ('t', trapChar); + + HSAuint32 invalidQid = 0xffffffff; + HSAuint32 qid = invalidQid; + HSA_QUEUEID queueIds[1] = { qResources->QueueId}; + HSA_DEBUG_EVENT_TYPE EventReceived; + bool IsSuspended = false; + + ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, + false, &EventReceived, + &IsSuspended)); + ASSERT_NE(qid, invalidQid); + ASSERT_EQ(IsSuspended, false); + ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_TRAP); + + // suspend queue, query suspended queue and clear pending event + ASSERT_SUCCESS(hsaKmtQueueSuspend(INVALID_PID, 1, queueIds, 10, 0)); + + syncStatus = dispatch->SyncWithStatus(suspendTimeout); + ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); + ASSERT_NE(iter[0], result[0]); + + ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, + &qid, true, &EventReceived, + &IsSuspended)); + ASSERT_EQ(IsSuspended, true); + + ASSERT_SUCCESS(hsaKmtQueueResume(INVALID_PID, 1, queueIds, 0)); + + ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, + false, &EventReceived, + &IsSuspended)); + + ASSERT_EQ(IsSuspended, false); + ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_NONE); + + dispatch->Sync(); + ASSERT_EQ(iter[0], result[0]); + EXPECT_SUCCESS(queue.Destroy()); + ASSERT_SUCCESS(hsaKmtDisableDebugTrap(defaultGPUNode)); + ASSERT_EQ(close(PollFd), 0); + + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } + TEST_END +} + +// clean up routine +static void ExitVMFaultQueryChild(std::string errMsg, + int exitStatus, + HSAint32 pollFd, + PM4Queue *queue1, + PM4Queue *queue2, + HsaEvent *event) { + if (queue1) + queue1->Destroy(); + + if (queue2) + queue2->Destroy(); + + if (event) { + int ret = hsaKmtDestroyEvent(event); + if (ret) { + exitStatus = 1; + errMsg = "event failed to be destroyed"; + } + } + + if (pollFd >= 0) + close(pollFd); + + if (!errMsg.empty()) + WARN() << errMsg << std::endl; + + exit(exitStatus); +} + +TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + pid_t childPid = fork(); + ASSERT_GE(childPid, 0); + + // fork to child since vm faults halt all queues + if (childPid == 0) { + HSAint32 PollFd; + HSAKMT_STATUS ret; + bool childStatus; + + ret = hsaKmtOpenKFD(); + if (ret != HSAKMT_STATUS_SUCCESS) + ExitVMFaultQueryChild("KFD open failed", + 1, -1, NULL, NULL, NULL); + + // enable debug trap and check poll fd creation + ret = hsaKmtEnableDebugTrapWithPollFd(defaultGPUNode, + INVALID_QUEUEID, + &PollFd); + + if (ret != HSAKMT_STATUS_SUCCESS || PollFd <= 0) + ExitVMFaultQueryChild("enable debug trap with poll fd failed", + 1, -1, NULL, NULL, NULL); + + // create shader, vmfault and trap bufs then enable 2nd level trap + HsaMemoryBuffer vmFaultBuf(PAGE_SIZE, defaultGPUNode, true, false, + true); + HsaMemoryBuffer srcBuf(PAGE_SIZE, defaultGPUNode, false); + srcBuf.Fill(0xABCDABCD); + + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, + true); + HsaMemoryBuffer iterBuf(PAGE_SIZE, defaultGPUNode, true, false, + false); + HsaMemoryBuffer resBuf(PAGE_SIZE, defaultGPUNode, true, false, + false); + + HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, + true); + HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, + false); + + ret = hsaKmtSetTrapHandler(defaultGPUNode, + trap.As(), 0x1000, + tmaBuf.As(), 0x1000); + + if (ret != HSAKMT_STATUS_SUCCESS) + ExitVMFaultQueryChild("setting trap handler failed", + 1, PollFd, NULL, NULL, NULL); + + // compile and dispatch shader + m_pIsaGen->CompileShader(jump_to_trap_gfx9, "jump_to_trap", + isaBuf); + m_pIsaGen->CompileShader(trap_handler_gfx9, "trap_handler", trap); + + PM4Queue queue1, queue2; + HSAuint32 qid1; + if (queue1.Create(defaultGPUNode) != HSAKMT_STATUS_SUCCESS) + ExitVMFaultQueryChild("queue 1 creation failed", + 1, PollFd, NULL, NULL, NULL); + + if (queue2.Create(defaultGPUNode) != HSAKMT_STATUS_SUCCESS) + ExitVMFaultQueryChild("queue 2 creation failed", + 1, PollFd, &queue1, NULL, NULL); + + unsigned int* iter = iterBuf.As(); + unsigned int* result = resBuf.As(); + int suspendTimeout = 500; + iter[0] = 150000000; + Dispatch *dispatch1; + dispatch1 = new Dispatch(isaBuf); + dispatch1->SetArgs(&iter[0], &result[0]); + dispatch1->SetDim(1, 1, 1); + dispatch1->Submit(queue1); + + // poll, read and query pending trap event + struct pollfd fds[1]; + fds[0].fd = PollFd; + fds[0].events = POLLIN | POLLRDNORM; + if (poll(fds, 1, 5000) <= 0) + ExitVMFaultQueryChild("poll wake on pending trap event failed", + 1, PollFd, &queue1, &queue2, NULL); + + int kMaxSize = 4096; + char fifoBuf[kMaxSize]; + childStatus = read(PollFd, fifoBuf, 1) == -1\ + || strchr(fifoBuf, 't') == NULL; + if (childStatus) + ExitVMFaultQueryChild("read on pending trap event failed", + 1, PollFd, &queue1, &queue2, NULL); + + memset(fifoBuf, 0, sizeof(fifoBuf)); + + HSA_DEBUG_EVENT_TYPE EventReceived; + bool IsSuspended; + HSAuint32 invalidQid = 0xffffffff; + qid1 = invalidQid; + + ret = hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid1, + false, &EventReceived, &IsSuspended); + + childStatus = ret != HSAKMT_STATUS_SUCCESS + || EventReceived != HSA_DEBUG_EVENT_TYPE_TRAP; + if (childStatus) + ExitVMFaultQueryChild("query on pending trap event failed", + 1, PollFd, &queue1, &queue2, NULL); + + // create and wait on pending vmfault event + HsaEvent *vmFaultEvent; + HsaEventDescriptor eventDesc; + eventDesc.EventType = HSA_EVENTTYPE_MEMORY; + eventDesc.NodeId = defaultGPUNode; + eventDesc.SyncVar.SyncVar.UserData = NULL; + eventDesc.SyncVar.SyncVarSize = 0; + ret = hsaKmtCreateEvent(&eventDesc, true, false, + &vmFaultEvent); + if (ret != HSAKMT_STATUS_SUCCESS) + ExitVMFaultQueryChild("create vmfault event failed", + 1, PollFd, &queue1, &queue2, NULL); + + Dispatch dispatch2(vmFaultBuf, false); + dispatch2.SetArgs( + reinterpret_cast(srcBuf.As()), + reinterpret_cast(0xABBAABBAULL)); + dispatch2.SetDim(1, 1, 1); + dispatch2.Submit(queue2); + + ret = hsaKmtWaitOnEvent(vmFaultEvent, g_TestTimeOut); + if (ret != HSAKMT_STATUS_SUCCESS) + ExitVMFaultQueryChild("wait on vmfault event failed", + 1, PollFd, &queue1, &queue2, + vmFaultEvent); + + // poll, read and query on pending vmfault event + if (poll(fds, 1, 5000) <= 0) + ExitVMFaultQueryChild("poll wake on vmfault event failed", + 1, PollFd, &queue1, &queue2, + vmFaultEvent); + + childStatus = read(PollFd, fifoBuf, kMaxSize) == -1 + || strchr(fifoBuf, 'v') == NULL + || strchr(fifoBuf, 't'); + + if (childStatus) + ExitVMFaultQueryChild("read on vmfault event failed", + 1, PollFd, &queue1, &queue2, + vmFaultEvent); + + ret = hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, + &qid1, true, &EventReceived, + &IsSuspended); + + childStatus = ret != HSAKMT_STATUS_SUCCESS + || EventReceived != + HSA_DEBUG_EVENT_TYPE_TRAP_VMFAULT; + if (childStatus) + ExitVMFaultQueryChild("query on vmfault event failed", + 1, PollFd, &queue1, &queue2, + vmFaultEvent); + + ExitVMFaultQueryChild("", 0, PollFd, &queue1, &queue2, + vmFaultEvent); + + } else { + int childStatus; + ASSERT_EQ(childPid, waitpid(childPid, &childStatus, 0)); + ASSERT_NE(0, WIFEXITED(childStatus)); + ASSERT_EQ(0, WEXITSTATUS(childStatus)); + } + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } + TEST_END +} From 1fddfd316a5993b7ae930bd392c26085132da73e Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Wed, 3 Apr 2019 12:07:11 -0400 Subject: [PATCH 0564/1247] libhsakmt: reduce /proc/cpuinfo opens /proc/cpuinfo are opened, read, and closed multiple times. Once for vendor name and multiple times for model name -- each node opens once. For example in a 2 CPUs + 4 GPUs system, it'll be opened 7 times. This patch reads it one time and stores it in a cpuinfo buffer. This cpuinfo buffer is freed when the snapshot is done. Also replace returns with gotos inside the snapshot to avoid possible memory leak. Change-Id: Iaf26a6c7e7323a8651d137c3706179449b9e3c80 Signed-off-by: Amber Lin --- src/topology.c | 233 +++++++++++++++++++++++++------------------------ 1 file changed, 121 insertions(+), 112 deletions(-) diff --git a/src/topology.c b/src/topology.c index da95c01dfa..54c112b3ae 100644 --- a/src/topology.c +++ b/src/topology.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "libhsakmt.h" #include "fmm.h" @@ -51,7 +52,6 @@ #define KFD_SYSFS_PATH_GENERATION_ID "/sys/devices/virtual/kfd/kfd/topology/generation_id" #define KFD_SYSFS_PATH_SYSTEM_PROPERTIES "/sys/devices/virtual/kfd/kfd/topology/system_properties" #define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" -#define PROC_CPUINFO_PATH "/proc/cpuinfo" typedef struct { uint32_t gpu_id; @@ -72,7 +72,7 @@ static uint32_t *map_user_to_sysfs_node_id; static uint32_t map_user_to_sysfs_node_id_size; static uint32_t num_sysfs_nodes; -static int processor_vendor; +static int processor_vendor = -1; /* Supported System Vendors */ enum SUPPORTED_PROCESSOR_VENDORS { GENUINE_INTEL = 0, @@ -232,6 +232,11 @@ static struct hsa_gfxip_table { { 0x731F, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, }; +struct proc_cpuinfo { + uint32_t apicid; + char model_name[HSA_PUBLIC_NAME_SIZE]; +}; + enum cache_type { CACHE_TYPE_NULL = 0, CACHE_TYPE_DATA = 1, @@ -782,70 +787,27 @@ bool topology_is_svm_needed(uint16_t device_id) } static HSAKMT_STATUS topology_get_cpu_model_name(HsaNodeProperties *props, - bool is_apu) + struct proc_cpuinfo *cpuinfo, int num_procs) { - FILE *fd; - char read_buf[256], cpu_model_name[HSA_PUBLIC_NAME_SIZE]; - const char *p; - uint32_t i = 0, apic_id = 0; + int i, j; - if (!props) + if (!props) { + pr_err("Invalid props to get cpu model name\n"); return HSAKMT_STATUS_INVALID_PARAMETER; - - fd = fopen(PROC_CPUINFO_PATH, "r"); - if (!fd) { - pr_err("Failed to open [%s]. Unable to get CPU Model Name", - PROC_CPUINFO_PATH); - return HSAKMT_STATUS_ERROR; } - while (fgets(read_buf, sizeof(read_buf), fd)) { - /* Get the model name first, in case matching - * apic IDs are also present in the file - */ - if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { - p = strrchr(read_buf, ':'); - if (!p) - goto err; - - p++; // remove separator ':' - for (; isspace(*p); p++) - ; /* remove white space */ - - /* Extract model name from string */ - for (i = 0; i < sizeof(cpu_model_name) - 1 && p[i] != '\n'; i++) - cpu_model_name[i] = p[i]; - cpu_model_name[i] = '\0'; - } - - if (!strncmp("apicid", read_buf, sizeof("apicid") - 1)) { - p = strrchr(read_buf, ':'); - if (!p) - goto err; - - p++; // remove separator ':' - for (; isspace(*p); p++) - ; /* remove white space */ - - /* Extract apic_id from remaining chars */ - apic_id = atoi(p); - - /* Set CPU model name only if corresponding apic id */ - if (props->CComputeIdLo == apic_id) { - /* Retrieve the CAL name of CPU node */ - if (!is_apu) - strncpy((char *)props->AMDName, cpu_model_name, sizeof(props->AMDName)); - /* Convert from UTF8 to UTF16 */ - for (i = 0; cpu_model_name[i] != '\0' && i < HSA_PUBLIC_NAME_SIZE - 1; i++) - props->MarketingName[i] = cpu_model_name[i]; - props->MarketingName[i] = '\0'; - } + for (i = 0; i < num_procs; i++, cpuinfo++) { + if (props->CComputeIdLo == cpuinfo->apicid) { + if (!props->DeviceId) /* CPU-only node */ + strncpy((char *)props->AMDName, cpuinfo->model_name, sizeof(props->AMDName)); + /* Convert from UTF8 to UTF16 */ + for (j = 0; cpuinfo->model_name[j] != '\0' && j < HSA_PUBLIC_NAME_SIZE - 1; j++) + props->MarketingName[j] = cpuinfo->model_name[j]; + props->MarketingName[j] = '\0'; + return HSAKMT_STATUS_SUCCESS; } } - fclose(fd); - return HSAKMT_STATUS_SUCCESS; -err: - fclose(fd); + return HSAKMT_STATUS_ERROR; } @@ -860,44 +822,93 @@ static int topology_search_processor_vendor(const char *processor_name) return -1; } -/* topology_set_processor_vendor - Parse /proc/cpuinfo and - * to find processor vendor and set global variable processor_vendor - * - * cat /proc/cpuinfo format is - "token : Value" - * where token = "vendor_id" and - * Value = indicates System Vendor +/* topology_parse_cpuinfo - Parse /proc/cpuinfo and fill up required + * topology information + * cpuinfo [OUT]: output buffer to hold cpu information + * num_procs: number of processors the output buffer can hold */ -static void topology_set_processor_vendor(void) +static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, + uint32_t num_procs) { + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; FILE *fd; char read_buf[256]; - const char *p; + char *p; + uint32_t proc = 0; + const char *proc_cpuinfo_path = "/proc/cpuinfo"; - fd = fopen(PROC_CPUINFO_PATH, "r"); - if (!fd) { - pr_err("Failed to open [%s]. Setting Processor Vendor to %s", - PROC_CPUINFO_PATH, supported_processor_vendor_name[GENUINE_INTEL]); - processor_vendor = GENUINE_INTEL; - return; + if (!cpuinfo) { + pr_err("CPU information will be missing\n"); + return HSAKMT_STATUS_INVALID_PARAMETER; } + fd = fopen(proc_cpuinfo_path, "r"); + if (!fd) { + pr_err("Failed to open [%s]. Unable to get CPU information", + proc_cpuinfo_path); + return HSAKMT_STATUS_ERROR; + } + + /* Each line in /proc/cpuinfo that read_buf is constructed, the format + * is like this: + * "token : value\n" + * where token is our target like vendor_id, model name, apicid ... + * and value is the answer + */ while (fgets(read_buf, sizeof(read_buf), fd)) { - if (!strncmp("vendor_id", read_buf, sizeof("vendor_id") - 1)) { - p = strrchr(read_buf, ':'); - p++; // remove separator ':' - for (; *p && isspace(*p); p++) - ; /* remove white space */ - processor_vendor = topology_search_processor_vendor(p); - if (processor_vendor != -1) { - fclose(fd); - return; + /* processor number */ + if (!strncmp("processor", read_buf, sizeof("processor") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + proc = atoi(p); + if (proc >= num_procs) { + pr_warn("cpuinfo contains processor %d lager than %u\n", + proc, num_procs); + ret = HSAKMT_STATUS_NO_MEMORY; + goto exit; } + continue; + } + + /* vendor name */ + if (!strncmp("vendor_id", read_buf, sizeof("vendor_id") - 1) && + (processor_vendor == -1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + processor_vendor = topology_search_processor_vendor(p); + continue; + } + + /* model name */ + if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + if (strlen(p) < HSA_PUBLIC_NAME_SIZE) { + /* -1 to remove \n from p */ + strncpy(cpuinfo[proc].model_name, p, strlen(p) - 1); + cpuinfo[proc].model_name[strlen(p) - 1] = '\0'; + } else + strncpy(cpuinfo[proc].model_name, p, HSA_PUBLIC_NAME_SIZE); + continue; + } + + /* apicid */ + if (!strncmp("apicid", read_buf, sizeof("apicid") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + cpuinfo[proc].apicid = atoi(p); } } + + if (processor_vendor < 0) { + pr_err("Failed to get Processor Vendor. Setting to %s", + supported_processor_vendor_name[GENUINE_INTEL]); + processor_vendor = GENUINE_INTEL; + } + +exit: fclose(fd); - pr_err("Failed to get Processor Vendor. Setting to %s", - supported_processor_vendor_name[GENUINE_INTEL]); - processor_vendor = GENUINE_INTEL; + return ret; } HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, @@ -1048,15 +1059,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, /* Retrieve the CAL name of the node */ strncpy((char *)props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName)-1); - if (props->NumCPUCores) { - /* Is APU node */ - ret = topology_get_cpu_model_name(props, true); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("Failed to get APU Model Name from %s\n", PROC_CPUINFO_PATH); - ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ - } - } else { - /* Is dGPU Node + if (!props->NumCPUCores) { + /* Is dGPU Node, not APU * Retrieve the marketing name of the node using pcilib, * convert UTF8 to UTF16 */ @@ -1066,18 +1070,6 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->MarketingName[i] = name[i]; props->MarketingName[i] = '\0'; } - } else { - /* Is CPU Node */ - if (!props->NumFComputeCores || !props->DeviceId) { - ret = topology_get_cpu_model_name(props, false); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("Failed to get CPU Model Name from %s\n", PROC_CPUINFO_PATH); - ret = HSAKMT_STATUS_SUCCESS; /* No hard error, continue regardless */ - } - } else { - ret = HSAKMT_STATUS_ERROR; - goto err; - } } if (props->NumFComputeCores) assert(props->EngineId.ui32.Major); @@ -1808,8 +1800,16 @@ HSAKMT_STATUS topology_take_snapshot(void) HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; struct pci_access *pacc; char *envvar; + struct proc_cpuinfo *cpuinfo; + const uint32_t num_procs = get_nprocs(); + + cpuinfo = calloc(num_procs, sizeof(struct proc_cpuinfo)); + if (!cpuinfo) { + pr_err("Fail to allocate memory for CPU info\n"); + return HSAKMT_STATUS_NO_MEMORY; + } + topology_parse_cpuinfo(cpuinfo, num_procs); - topology_set_processor_vendor(); envvar = getenv("HSA_RUNNING_UNDER_VALGRIND"); if (envvar && !strcmp(envvar, "1")) is_valgrind = 1; @@ -1819,15 +1819,17 @@ HSAKMT_STATUS topology_take_snapshot(void) retry: ret = topology_sysfs_get_generation(&gen_start); if (ret != HSAKMT_STATUS_SUCCESS) - return ret; + goto err; ret = topology_sysfs_get_system_props(&sys_props); if (ret != HSAKMT_STATUS_SUCCESS) - return ret; + goto err; if (sys_props.NumNodes > 0) { topology_create_temp_cpu_cache_list(&cpu_ci_list); temp_props = calloc(sys_props.NumNodes * sizeof(node_props_t), 1); - if (!temp_props) - return HSAKMT_STATUS_NO_MEMORY; + if (!temp_props) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto err; + } pacc = pci_alloc(); pci_init(pacc); for (i = 0; i < sys_props.NumNodes; i++) { @@ -1838,6 +1840,11 @@ retry: free_properties(temp_props, i); goto err; } + + if (temp_props[i].node.NumCPUCores) + topology_get_cpu_model_name(&temp_props[i].node, + cpuinfo, num_procs); + if (temp_props[i].node.NumMemoryBanks) { temp_props[i].mem = calloc(temp_props[i].node.NumMemoryBanks * sizeof(HsaMemoryProperties), 1); if (!temp_props[i].mem) { @@ -1934,7 +1941,8 @@ retry: g_system = malloc(sizeof(HsaSystemProperties)); if (!g_system) { free_properties(temp_props, sys_props.NumNodes); - return HSAKMT_STATUS_NO_MEMORY; + ret = HSAKMT_STATUS_NO_MEMORY; + goto err; } } @@ -1943,6 +1951,7 @@ retry: free(g_props); g_props = temp_props; err: + free(cpuinfo); topology_destroy_temp_cpu_cache_list(cpu_ci_list); return ret; } From 61069a41d32012e94d2b7dc2ba4900ef38a636c9 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 16 Aug 2019 19:30:31 -0400 Subject: [PATCH 0565/1247] kfdtest: Update gfx908 test blacklist Change-Id: Ia83789fca28dc85fe7af4075382dfecbf8972d05 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index c688faf498..cb6b311da2 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -158,12 +158,10 @@ RAVEN_TESTS_BLACKLIST=\ "KFDQMTest.QueueLatency:"\ "KFDQMTest.SdmaEventInterrupt" -# KFDPNPTest.DisableWithActiveProcess (KFD-432) -# KFDQMTest.BasicCuMaskingEven (KFD-433) +# KFDExceptionTest.* (KFD-435) ARCT_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDPNPTest.DisableWithActiveProcess" +"KFDExceptionTest.FaultStorm" # Core tests, used in scenarios like bringup CORE_TESTS=\ From 7fc6a9f7c213792300d49b1f6cbde0fea155d74a Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 19 Aug 2019 17:59:40 -0400 Subject: [PATCH 0566/1247] Revert "hsaKmtCloseKFD destroy objects in all apertures" This reverts commit 632ad3a749d81f4e9ab21f36f309679bf6673a38. This change causes KFDTest failed on gfx803. The first hsaKmtCreateEvent call allocate system memory for events_page because global variable events_page is NULL. And this events page vm address should not be freed until the process exit. The change to destrory objects in hsaKmtCloseKFD removes events page. As a result, KFDTest call hsaKmtOpenKFD again and then allocate memory will get same events_page vm address on gfx803, and map this vm failed because the vm conflict with events_page mapping. KFDTest passed on VG10, gfx906 because allocate memory get different vm address. hsaKmtCreateEvent still works fine as the driver keeps the events page mapping of the process. We should only destroy objects in fork cloned child process regardless if gpu_vm is NULL or not. Change-Id: I174ef65321cbd6074c855c2021318fe961c8c72c Signed-off-by: Philip Yang --- src/fmm.c | 54 +++++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 5336af6973..660ff234b3 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2345,37 +2345,14 @@ sysfs_parse_failed: return ret; } -static void fmm_clear_aperture(manageable_aperture_t *app) -{ - rbtree_node_t *n; - - while ((n = rbtree_node_any(&app->tree, MID))) - vm_remove_object(app, vm_object_entry(n, 0)); - - while (app->vm_ranges) - vm_remove_area(app, app->vm_ranges); -} - void fmm_destroy_process_apertures(void) { - unsigned int i; - release_mmio(); - - fmm_clear_aperture(&cpuvm_aperture); - fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); - fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); - if (gpu_mem) { - for (i = 0; i < gpu_mem_count; i++) { - fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); - fmm_clear_aperture(&gpu_mem[i].scratch_physical); - } - free(gpu_mem); gpu_mem = NULL; - gpu_mem_count = 0; } + gpu_mem_count = 0; } HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSAuint32 gpu_id, @@ -3554,6 +3531,18 @@ HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) return HSAKMT_STATUS_SUCCESS; } +static void fmm_clear_aperture(manageable_aperture_t *app) +{ + rbtree_node_t *n; + + while ((n = rbtree_node_any(&app->tree, MID))) + vm_remove_object(app, vm_object_entry(n, 0)); + + while (app->vm_ranges) + vm_remove_area(app, app->vm_ranges); + +} + /* This is a special funcion that should be called only from the child process * after a fork(). This will clear all vm_objects and mmaps duplicated from * the parent. @@ -3570,7 +3559,19 @@ void fmm_clear_all_mem(void) drm_render_fds[i] = 0; } - fmm_destroy_process_apertures(); + /* Nothing is initialized. */ + if (!gpu_mem) + return; + + fmm_clear_aperture(&cpuvm_aperture); + + for (i = 0; i < gpu_mem_count; i++) { + fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); + fmm_clear_aperture(&gpu_mem[i].scratch_physical); + } + + fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); + fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); if (dgpu_shared_aperture_limit) { /* Use the same dgpu range as the parent. If failed, then set @@ -3596,4 +3597,7 @@ void fmm_clear_all_mem(void) all_gpu_id_array_size = 0; all_gpu_id_array = NULL; + + gpu_mem_count = 0; + free(gpu_mem); } From a11cb2a63361a8efe92c2254694f050cf2caef42 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 20 Aug 2019 10:55:14 -0400 Subject: [PATCH 0567/1247] libhsakmt: child process destroy vm objects in all apertures child process clone vm objects from svm->apertures if parent process doesn't free memory before fork. fmm_clear_all_mem suppose to clear the apertures in forked child process but this only works if gpu_vm is not NULL. parent process call hsaKmtCloseKFD reset gpu_vm to NULL and then fork, then child process will not clear svm->apertures. As a result, the child process will allocate vm object with same address and add to aperture, there are duplicate vm objects with same address in aperture. Then mapping to GPU will find the wrong vm object and create incorrect GPU mapping cause rocrtst IPC test VM fault. The issue happened with HSA_USERPTR_FOR_PAGED_MEM=1. The fix is to clear vm objects in all apertures in clear_after_fork. Change-Id: I92e42a967075a634a3f475b915c8242d82077ecb Signed-off-by: Philip Yang --- src/fmm.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 660ff234b3..dc24b61bbc 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3559,17 +3559,7 @@ void fmm_clear_all_mem(void) drm_render_fds[i] = 0; } - /* Nothing is initialized. */ - if (!gpu_mem) - return; - fmm_clear_aperture(&cpuvm_aperture); - - for (i = 0; i < gpu_mem_count; i++) { - fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); - fmm_clear_aperture(&gpu_mem[i].scratch_physical); - } - fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); @@ -3598,6 +3588,16 @@ void fmm_clear_all_mem(void) all_gpu_id_array_size = 0; all_gpu_id_array = NULL; + /* Nothing is initialized. */ + if (!gpu_mem) + return; + + for (i = 0; i < gpu_mem_count; i++) { + fmm_clear_aperture(&gpu_mem[i].gpuvm_aperture); + fmm_clear_aperture(&gpu_mem[i].scratch_physical); + } + gpu_mem_count = 0; free(gpu_mem); + gpu_mem = NULL; } From 626957a2631718526b788f12ae9478ea7be89c3d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 16 Apr 2019 23:31:28 -0400 Subject: [PATCH 0568/1247] libhsakmt: Enable HSA_USERPTR_FOR_PAGED_MEM by default By using user-allocated pages instead of kernel-allocated pages from TTM, we're not subject to TTM's self imposed limits on kernel memory usage. This also paves the way for for more wide-spread use of HMM on upstream kernels. Change-Id: Iac82964c98a441e29b7f1986d1be1bb5ccb1e569 Signed-off-by: Felix Kuehling --- src/fmm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index dc24b61bbc..0bdb6f3c96 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2093,11 +2093,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) disableCache = getenv("HSA_DISABLE_CACHE"); svm.disable_cache = (disableCache && strcmp(disableCache, "0")); - /* If HSA_USERPTR_FOR_PAGED_MEM is set to a non-0 value, - * enable userptr for all paged memory allocations + /* If HSA_USERPTR_FOR_PAGED_MEM is not set or set to a non-0 + * value, enable userptr for all paged memory allocations */ pagedUserptr = getenv("HSA_USERPTR_FOR_PAGED_MEM"); - svm.userptr_for_paged_mem = (pagedUserptr && strcmp(pagedUserptr, "0")); + svm.userptr_for_paged_mem = (!pagedUserptr || strcmp(pagedUserptr, "0")); /* If HSA_CHECK_USERPTR is set to a non-0 value, check all userptrs * when they are registered From f1c0bc8e35169f69290a0371ea429c9480def882 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 17 Jul 2019 18:15:06 -0400 Subject: [PATCH 0569/1247] kfdtest: Query asic name directly from topology node This saves us from maintaining device ID to Asic mapping in the scripts. Moreover, stop using abbrevation asic names to avoid confusion. Change-Id: I7ce583b26b09b627c142aae41932483b28c545d8 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 10 +--- tests/kfdtest/scripts/run_kfdtest.sh | 71 ++++++--------------------- 2 files changed, 15 insertions(+), 66 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index cb6b311da2..ae1a8070d0 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -117,15 +117,7 @@ FIJI_TESTS_BLACKLIST=\ "KFDQMTest.PM4EventInterrupt:"\ "$SDMA_BLACKLIST" -ELLESMERE_TESTS_BLACKLIST=\ -"$BLACKLIST_ALL_ASICS:"\ -"$SDMA_BLACKLIST" - -BAFFIN_TESTS_BLACKLIST=\ -"$BLACKLIST_ALL_ASICS:"\ -"$SDMA_BLACKLIST" - -LEXA_TESTS_BLACKLIST=\ +POLARIS_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST" diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index f81e3e2687..55e70a0b3d 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -51,7 +51,7 @@ printUsage() { echo echo "Gtest arguments will be forwarded to the app" echo - echo "Valid platform options: cz, kv, tg, fj, hi, pl10/el, pl11/bf, pl12/lx, vg10, vg12, vg20, all" + echo "Valid platform options: polaris10, vega10, vega20, all, and so on" echo "'all' option runs all tests" return 0 @@ -63,22 +63,20 @@ printUsage() { getFilter() { local platform=$1; case "$platform" in - cz ) FILTER="--gtest_filter=$CZ_TESTS_BLACKLIST" ;; - hi ) FILTER="--gtest_filter=$HI_TESTS_BLACKLIST" ;; - kv ) FILTER="--gtest_filter=$KV_TESTS_BLACKLIST" ;; - tg ) FILTER="--gtest_filter=$TONGA_TESTS_BLACKLIST" ;; - fj ) FILTER="--gtest_filter=$FIJI_TESTS_BLACKLIST" ;; - pl10 | el ) FILTER="--gtest_filter=$ELLESMERE_TESTS_BLACKLIST" ;; - pl11 | bf ) FILTER="--gtest_filter=$BAFFIN_TESTS_BLACKLIST" ;; - pl12 | lx ) FILTER="--gtest_filter=$LEXA_TESTS_BLACKLIST" ;; - vg10 ) FILTER="--gtest_filter=$VEGA10_TESTS_BLACKLIST" ;; - vg12 ) FILTER="--gtest_filter=$VEGA12_TESTS_BLACKLIST" ;; - vg20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; - rv ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; - arct ) FILTER="--gtest_filter=$ARCT_TESTS_BLACKLIST" ;; + carrizo ) FILTER="--gtest_filter=$CZ_TESTS_BLACKLIST" ;; + hawaii ) FILTER="--gtest_filter=$HI_TESTS_BLACKLIST" ;; + kaveri ) FILTER="--gtest_filter=$KV_TESTS_BLACKLIST" ;; + tonga ) FILTER="--gtest_filter=$TONGA_TESTS_BLACKLIST" ;; + fiji ) FILTER="--gtest_filter=$FIJI_TESTS_BLACKLIST" ;; + polaris10 | polaris11 | polaris12 ) FILTER="--gtest_filter=$POLARIS_TESTS_BLACKLIST" ;; + vega10 ) FILTER="--gtest_filter=$VEGA10_TESTS_BLACKLIST" ;; + vega12 ) FILTER="--gtest_filter=$VEGA12_TESTS_BLACKLIST" ;; + vega20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; + raven ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; + arcturus ) FILTER="--gtest_filter=$ARCT_TESTS_BLACKLIST" ;; core ) FILTER="--gtest_filter=$CORE_TESTS" ;; all ) FILTER="" ;; - *) die "Unsupported platform $PLATFORM or node $NODE. Exiting" ;; + *) die "Unsupported platform $platform. Exiting" ;; esac echo "$FILTER" } @@ -98,52 +96,11 @@ getHsaNodes() { } -# Prints GPU name for a given Device ID -# param - Device ID. -deviceIdToGpuName() { - local deviceId=$1; shift; - case $deviceId in - 1304 | 1305 | 1306 | 1307 | 1309 | 130a | 130b | 130c | 130d | 130e | 130f | \ - 1310 | 1311 | 1312 | 1313 | 1315 | 1316 | 1317 | 1318 | 131b | 131c | 131d ) - platformName="kv" ;; - 67a0 | 67a1 | 67a2 | 67a8 | 67a9 | 67aa | 67b0 | 67b1 | 67b8 | 67b9 | 67ba | 67be ) - platformName="hi" ;; - 9870 | 9874 | 9875 | 9876 | 9877 ) - platformName="cz" ;; - 6920 | 6921 | 6928 | 6929 | 692b | 692f | 6930 | 6938 | 6939 ) - platformName="tg" ;; - 7300 | 730f) - platformName="fj" ;; - 67c0 | 67c1 | 67c2 | 67c4 | 67c7 | 67c8 | 67c9 | 67ca | 67cc | 67cf | 67d0 | 67df | 6fdf ) - platformName="pl10" ;; - 67e0 | 67e1 | 67e3 | 67e7 | 67e8 | 67e9 | 67eb | 67ef | 67ff ) - platformName="pl11" ;; - 6980 | 6981 | 6985 | 6986 | 6987 | 6995 | 6997 | 699f) - platformName="pl12" ;; - 6860 | 6861 | 6862 | 6863 | 6864 | 6867 | 6868 | 6869 | 686a | 686b | 686c | 687d | 687e | 687f) - platformName="vg10" ;; - 69a0 | 69a1 | 69a2 | 69a3 | 69af) - platformName="vg12" ;; - 66a0 | 66a1 | 66a2 | 66a3 | 66a4 | 66a7 | 66af) - platformName="vg20" ;; - 15dd ) - platformName="rv" ;; - 7380 | 46 | 47 | 48 | 738c | 7388 | 738e) - platformName="arct" ;; - * ) - return ;; - esac - echo "$platformName" -} - - # Prints GPU Name for the given Node ID # param - Node ID getNodeName() { local nodeId=$1; shift; - local gpuIdInDec=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep device_id | awk '{print $2}') - printf -v gpuIdInHex "%x" "$gpuIdInDec" - local gpuName=$(deviceIdToGpuName $gpuIdInHex) + local gpuName=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/name) echo "$gpuName" } From 814e0f0bdcc7a5733a00ebe300730093bccd3e08 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Tue, 27 Nov 2018 15:08:17 -0600 Subject: [PATCH 0570/1247] Reserve 128 SGPRs per wave in context save area Originally reserved 100 SGPRs per wave. Pre-gfx10 needs 102 SGPRs and gfx10 needs 128 SGPRs. Reserve 128 SGPRs per wave for all ASICs to simplify calculation. Also double VGPR register size for gfx908 family Change-Id: I98b741cbfa051f49ed37ff25d99f851f124be7b6 Signed-off-by: Jay Cornwall Signed-off-by: Oak Zeng --- src/queues.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/queues.c b/src/queues.c index f006d5ec2c..5f7384ff4a 100644 --- a/src/queues.c +++ b/src/queues.c @@ -42,8 +42,13 @@ #define DOORBELL_SIZE_GFX9 8 #define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) -#define WG_CONTEXT_DATA_SIZE_PER_CU_VI 344576 -#define WAVES_PER_CU_VI 32 +#define VGPR_SIZE_PER_CU(asic_family) (asic_family == CHIP_ARCTURUS ? 0x80000 : 0x40000) +#define SGPR_SIZE_PER_CU 0x4000 +#define LDS_SIZE_PER_CU 0x10000 +#define HWREG_SIZE_PER_CU 0x1000 +#define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) +#define WAVES_PER_CU 32 +#define CNTL_STACK_BYTES_PER_WAVE 8 struct device_info { enum asic_family_type asic_family; @@ -385,9 +390,8 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) uint32_t ctl_stack_size, wg_data_size; uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; - ctl_stack_size = cu_num * WAVES_PER_CU_VI * 8 + 8; - wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU_VI; - + ctl_stack_size = cu_num * WAVES_PER_CU * CNTL_STACK_BYTES_PER_WAVE + 8; + wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->dev_info->asic_family); q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size + sizeof(HsaUserContextSaveAreaHeader)); From 1ff5cb33b237ed12ff8ec7613fded3f3b52ada48 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 13 Aug 2019 10:47:51 -0400 Subject: [PATCH 0571/1247] add new queue bit test on clear event enable thunk query api to report if queue is newly created test new queue bit test on clear events. also fixup cleanup to disable debug trap. Change-Id: I3ebe2d85da66f28b8c82f0e68461ee7d32ec0b0d Signed-off-by: Jonathan Kim Reviewed-by: Philip Cox Reviewed-by: Felix Kuehling --- include/hsakmt.h | 3 +- include/linux/kfd_ioctl.h | 3 +- src/debug.c | 6 ++-- tests/kfdtest/src/KFDDBGTest.cpp | 61 +++++++++++++++++++++----------- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 7c62ee69f1..49d255a868 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -822,7 +822,8 @@ hsaKmtQueryDebugEvent( HSAuint32 *QueueId, // IN/OUT bool ClearEvents, // IN HSA_DEBUG_EVENT_TYPE *EventsReceived, // OUT - bool *IsSuspended // OUT + bool *IsSuspended, // OUT + bool *IsNew //OUT ); /** diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 7a1185c823..89a18cda15 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -191,9 +191,10 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_DBG_EV_STATUS_TRAP 1 #define KFD_DBG_EV_STATUS_VMFAULT 2 #define KFD_DBG_EV_STATUS_SUSPENDED 4 +#define KFD_DBG_EV_STATUS_NEW_QUEUE 8 #define KFD_DBG_EV_FLAG_CLEAR_STATUS 1 -#define KFD_INVALID_QUEUEID 0xFFFFFFFFFFFFFFFFULL +#define KFD_INVALID_QUEUEID 0xffffffff /* KFD_IOC_DBG_TRAP_ENABLE: * ptr: unused diff --git a/src/debug.c b/src/debug.c index ac7e3c641d..f111d7ff75 100644 --- a/src/debug.c +++ b/src/debug.c @@ -564,7 +564,8 @@ hsaKmtQueryDebugEvent( HSAuint32 *QueueId, // IN/OUT bool ClearEvents, // IN HSA_DEBUG_EVENT_TYPE *EventsReceived, // OUT - bool *IsSuspended // OUT + bool *IsSuspended, // OUT + bool *IsNew // OUT ) { HSAKMT_STATUS result; @@ -589,7 +590,8 @@ hsaKmtQueryDebugEvent( *QueueId = argout.data1; *EventsReceived = argout.data3 & (KFD_DBG_EV_STATUS_TRAP | KFD_DBG_EV_STATUS_VMFAULT); - *IsSuspended = (argout.data3 & KFD_DBG_EV_STATUS_SUSPENDED) >> 2; + *IsSuspended = argout.data3 & KFD_DBG_EV_STATUS_SUSPENDED; + *IsNew = argout.data3 & KFD_DBG_EV_STATUS_NEW_QUEUE; return result; } diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 3fd83c2e52..edd0993474 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -434,12 +434,14 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { HSA_QUEUEID queueIds[1] = { qResources->QueueId}; HSA_DEBUG_EVENT_TYPE EventReceived; bool IsSuspended = false; + bool IsNew = false; ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, false, &EventReceived, - &IsSuspended)); + &IsSuspended, &IsNew)); ASSERT_NE(qid, invalidQid); ASSERT_EQ(IsSuspended, false); + ASSERT_EQ(IsNew, true); ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_TRAP); // suspend queue, query suspended queue and clear pending event @@ -451,16 +453,17 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, true, &EventReceived, - &IsSuspended)); + &IsSuspended, &IsNew)); ASSERT_EQ(IsSuspended, true); ASSERT_SUCCESS(hsaKmtQueueResume(INVALID_PID, 1, queueIds, 0)); ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, false, &EventReceived, - &IsSuspended)); + &IsSuspended, &IsNew)); ASSERT_EQ(IsSuspended, false); + ASSERT_EQ(IsNew, false); ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_NONE); dispatch->Sync(); @@ -482,7 +485,8 @@ static void ExitVMFaultQueryChild(std::string errMsg, HSAint32 pollFd, PM4Queue *queue1, PM4Queue *queue2, - HsaEvent *event) { + HsaEvent *event, + int gpuNode) { if (queue1) queue1->Destroy(); @@ -500,6 +504,14 @@ static void ExitVMFaultQueryChild(std::string errMsg, if (pollFd >= 0) close(pollFd); + if (gpuNode >= 0) { + int ret = hsaKmtDisableDebugTrap(gpuNode); + if (ret) { + exitStatus = 1; + errMsg = "debug trap failed to disable"; + } + } + if (!errMsg.empty()) WARN() << errMsg << std::endl; @@ -525,7 +537,7 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { ret = hsaKmtOpenKFD(); if (ret != HSAKMT_STATUS_SUCCESS) ExitVMFaultQueryChild("KFD open failed", - 1, -1, NULL, NULL, NULL); + 1, -1, NULL, NULL, NULL, -1); // enable debug trap and check poll fd creation ret = hsaKmtEnableDebugTrapWithPollFd(defaultGPUNode, @@ -534,7 +546,7 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { if (ret != HSAKMT_STATUS_SUCCESS || PollFd <= 0) ExitVMFaultQueryChild("enable debug trap with poll fd failed", - 1, -1, NULL, NULL, NULL); + 1, -1, NULL, NULL, NULL, defaultGPUNode); // create shader, vmfault and trap bufs then enable 2nd level trap HsaMemoryBuffer vmFaultBuf(PAGE_SIZE, defaultGPUNode, true, false, @@ -560,7 +572,8 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { if (ret != HSAKMT_STATUS_SUCCESS) ExitVMFaultQueryChild("setting trap handler failed", - 1, PollFd, NULL, NULL, NULL); + 1, PollFd, NULL, NULL, NULL, + defaultGPUNode); // compile and dispatch shader m_pIsaGen->CompileShader(jump_to_trap_gfx9, "jump_to_trap", @@ -571,11 +584,13 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { HSAuint32 qid1; if (queue1.Create(defaultGPUNode) != HSAKMT_STATUS_SUCCESS) ExitVMFaultQueryChild("queue 1 creation failed", - 1, PollFd, NULL, NULL, NULL); + 1, PollFd, NULL, NULL, NULL, + defaultGPUNode); if (queue2.Create(defaultGPUNode) != HSAKMT_STATUS_SUCCESS) ExitVMFaultQueryChild("queue 2 creation failed", - 1, PollFd, &queue1, NULL, NULL); + 1, PollFd, &queue1, NULL, NULL, + defaultGPUNode); unsigned int* iter = iterBuf.As(); unsigned int* result = resBuf.As(); @@ -593,7 +608,8 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { fds[0].events = POLLIN | POLLRDNORM; if (poll(fds, 1, 5000) <= 0) ExitVMFaultQueryChild("poll wake on pending trap event failed", - 1, PollFd, &queue1, &queue2, NULL); + 1, PollFd, &queue1, &queue2, NULL, + defaultGPUNode); int kMaxSize = 4096; char fifoBuf[kMaxSize]; @@ -601,23 +617,27 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { || strchr(fifoBuf, 't') == NULL; if (childStatus) ExitVMFaultQueryChild("read on pending trap event failed", - 1, PollFd, &queue1, &queue2, NULL); + 1, PollFd, &queue1, &queue2, NULL, + defaultGPUNode); memset(fifoBuf, 0, sizeof(fifoBuf)); HSA_DEBUG_EVENT_TYPE EventReceived; bool IsSuspended; + bool IsNew; HSAuint32 invalidQid = 0xffffffff; qid1 = invalidQid; ret = hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid1, - false, &EventReceived, &IsSuspended); + false, &EventReceived, &IsSuspended, + &IsNew); childStatus = ret != HSAKMT_STATUS_SUCCESS || EventReceived != HSA_DEBUG_EVENT_TYPE_TRAP; if (childStatus) ExitVMFaultQueryChild("query on pending trap event failed", - 1, PollFd, &queue1, &queue2, NULL); + 1, PollFd, &queue1, &queue2, NULL, + defaultGPUNode); // create and wait on pending vmfault event HsaEvent *vmFaultEvent; @@ -630,7 +650,8 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { &vmFaultEvent); if (ret != HSAKMT_STATUS_SUCCESS) ExitVMFaultQueryChild("create vmfault event failed", - 1, PollFd, &queue1, &queue2, NULL); + 1, PollFd, &queue1, &queue2, NULL, + defaultGPUNode); Dispatch dispatch2(vmFaultBuf, false); dispatch2.SetArgs( @@ -643,13 +664,13 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { if (ret != HSAKMT_STATUS_SUCCESS) ExitVMFaultQueryChild("wait on vmfault event failed", 1, PollFd, &queue1, &queue2, - vmFaultEvent); + vmFaultEvent, defaultGPUNode); // poll, read and query on pending vmfault event if (poll(fds, 1, 5000) <= 0) ExitVMFaultQueryChild("poll wake on vmfault event failed", 1, PollFd, &queue1, &queue2, - vmFaultEvent); + vmFaultEvent, defaultGPUNode); childStatus = read(PollFd, fifoBuf, kMaxSize) == -1 || strchr(fifoBuf, 'v') == NULL @@ -658,11 +679,11 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { if (childStatus) ExitVMFaultQueryChild("read on vmfault event failed", 1, PollFd, &queue1, &queue2, - vmFaultEvent); + vmFaultEvent, defaultGPUNode); ret = hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid1, true, &EventReceived, - &IsSuspended); + &IsSuspended, &IsNew); childStatus = ret != HSAKMT_STATUS_SUCCESS || EventReceived != @@ -670,10 +691,10 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { if (childStatus) ExitVMFaultQueryChild("query on vmfault event failed", 1, PollFd, &queue1, &queue2, - vmFaultEvent); + vmFaultEvent, defaultGPUNode); ExitVMFaultQueryChild("", 0, PollFd, &queue1, &queue2, - vmFaultEvent); + vmFaultEvent, defaultGPUNode); } else { int childStatus; From f2173254e4cc3819005b8f403ed8f64ba9aa47c5 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Thu, 22 Aug 2019 11:21:30 -0400 Subject: [PATCH 0572/1247] Report domain with HsaNodeProperties PCI domain has moved to 32-bits to accommodate virtualization, so a 32-bit integer is exposed for domain to reflect this change. Change-Id: I0d767acadcdc8e4277db203b5865dd67dd001cef Signed-off-by: Ori Messinger --- include/hsakmttypes.h | 3 ++- src/topology.c | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 03f4ea147c..5d6df38f95 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -302,7 +302,8 @@ typedef struct _HsaNodeProperties HSAuint32 NumSdmaEngines; // number of PCIe optimized SDMA engines HSAuint32 NumSdmaXgmiEngines;// number of XGMI optimized SDMA engines HSAuint32 NumGws; // number of GWS barriers - HSAuint8 Reserved[32]; + HSAuint32 Domain; // PCI domain of the GPU + HSAuint8 Reserved[28]; } HsaNodeProperties; diff --git a/src/topology.c b/src/topology.c index 54c112b3ae..0e2a653dff 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1010,6 +1010,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->DeviceId = (uint32_t)prop_val; else if (strcmp(prop_name, "location_id") == 0) props->LocationId = (uint32_t)prop_val; + else if (strcmp(prop_name, "domain") == 0) + props->Domain = (uint32_t)prop_val; else if (strcmp(prop_name, "max_engine_clk_fcompute") == 0) props->MaxEngineClockMhzFCompute = (uint32_t)prop_val; else if (strcmp(prop_name, "max_engine_clk_ccompute") == 0) From 4fa930af5a3e1eed1d3aff063cc8490bf9674c20 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 20 Aug 2019 12:21:28 -0400 Subject: [PATCH 0573/1247] libhsakmt: get cpu cache info from sysfs Replace cpuid call with sysfs data to get CPU cache information. With this change, x86 check is also removed since sysfs applies to other platforms. CPU cache information can be retrieved from /sys/devices/system/node/nodeX/cpuY/cache where Y is processor number represented in /proc/cpuinfo at "processor" entry. Change-Id: Ic47df6d5dafaf1aae5b46b1fdee42691c697e49e Signed-off-by: Amber Lin --- src/topology.c | 669 ++++++++++++++++++++----------------------------- 1 file changed, 276 insertions(+), 393 deletions(-) diff --git a/src/topology.c b/src/topology.c index 0e2a653dff..b008d338d9 100644 --- a/src/topology.c +++ b/src/topology.c @@ -23,7 +23,6 @@ * DEALINGS IN THE SOFTWARE. */ -#define _GNU_SOURCE #include #include #include @@ -32,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -63,7 +61,6 @@ typedef struct { static HsaSystemProperties *g_system; static node_props_t *g_props; -static int is_valgrind; /* This array caches sysfs based node IDs of CPU nodes + all supported GPU nodes. * It will be used to map user-node IDs to sysfs-node IDs. @@ -232,58 +229,23 @@ static struct hsa_gfxip_table { { 0x731F, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, }; +/* information from /proc/cpuinfo */ struct proc_cpuinfo { - uint32_t apicid; - char model_name[HSA_PUBLIC_NAME_SIZE]; + uint32_t proc_num; /* processor */ + uint32_t apicid; /* apicid */ + char model_name[HSA_PUBLIC_NAME_SIZE]; /* model name */ }; -enum cache_type { - CACHE_TYPE_NULL = 0, - CACHE_TYPE_DATA = 1, - CACHE_TYPE_INST = 2, - CACHE_TYPE_UNIFIED = 3 -}; - -typedef struct cacheinfo { - HsaCacheProperties hsa_cache_prop; - uint32_t num_threads_sharing; /* how many CPUs share this cache */ -} cacheinfo_t; - /* CPU cache table for all CPUs on the system. Each entry has the relative CPU * info and caches connected to that CPU. */ typedef struct cpu_cacheinfo { - uint32_t len; /* length of the table -> number of online procs */ - uint32_t num_caches; /* number of caches connected to this cpu */ - uint32_t num_duplicated_caches; /* to count caches being shared */ - uint32_t apicid; /* this cpu's apic id */ - uint32_t max_num_apicid; /* max number of addressable IDs */ - cacheinfo_t *cache_info; /* an array for cache information */ + uint32_t len; /* length of the table = number of online procs */ + int32_t proc_num; /* this cpu's processor number */ + uint32_t num_caches; /* number of caches reported by this cpu */ + HsaCacheProperties *cache_prop; /* a list of cache properties */ } cpu_cacheinfo_t; -/* Deterministic Cache Parameters Leaf in cpuid */ -union _cpuid_leaf_eax { /* Register EAX */ - struct { - enum cache_type type:5; - uint32_t level:3; - uint32_t is_self_initializing:1; - uint32_t is_fully_associative:1; - uint32_t reserved:4; - uint32_t num_threads_sharing:12; - uint32_t num_cores_on_die:6; - } split; - uint32_t full; -}; - -union _cpuid_leaf_ebx { /* Register EBX */ - struct { - uint32_t coherency_line_size:12; - uint32_t physical_line_partition:10; - uint32_t ways_of_associativity:10; - } split; - uint32_t full; -}; - static void free_properties(node_props_t *props, int size) { if (props) { @@ -328,181 +290,203 @@ static int num_subdirs(char *dirpath, char *prefix) return count; } -#if defined(__x86_64__) || defined(__i386__) -/* cpuid instruction returns processor identification and feature information - * to the EAX, EBX, ECX, and EDX registers, as determined by input entered in - * EAX (in some cases, ECX as well). +/* fscanf_dec - read a file whose content is a decimal number + * @file [IN ] file to read + * @num [OUT] number in the file */ -static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, - uint32_t *edx) +static HSAKMT_STATUS fscanf_dec(char *file, uint32_t *num) { - __asm__ __volatile__( - "cpuid;" - : "=a" (*eax), - "=b" (*ebx), - "=c" (*ecx), - "=d" (*edx) - : "0" (*eax), "2" (*ecx) - : "memory" - ); -} + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; -/* In cases ECX is also used as an input for cpuid, i.e. cache leaf */ -static void cpuid_count(uint32_t op, int count, uint32_t *eax, uint32_t *ebx, - uint32_t *ecx, uint32_t *edx) -{ - *eax = op; - *ecx = count; - cpuid(eax, ebx, ecx, edx); -} - -/* Lock current process to the specified processor */ -static int lock_to_processor(int processor) -{ - cpu_set_t cpuset; - - memset(&cpuset, 0, sizeof(cpu_set_t)); - CPU_SET(processor, &cpuset); - /* 0: this process */ - return sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); -} - -/* Get count's order of 2. In other words, 2^rtn_value = count - * When count is not an order of 2, round it up to the closest. - */ -static int get_count_order(unsigned int count) -{ - int bit; - uint32_t num; - - for (bit = 31; bit >= 0; bit--) { - num = 1 << bit; - if (count >= num) - break; + fd = fopen(file, "r"); + if (!fd) { + pr_err("Failed to open %s\n", file); + return HSAKMT_STATUS_INVALID_PARAMETER; } - if (count & (count - 1)) - ++bit; - - return bit; -} - -/* cpuid_find_num_cache_leaves - Use cpuid instruction to find out how many - * cache leaves the CPU has. - * @op - cpuid opcode to get cache information - * Return - the number of cache leaves - */ -static int cpuid_find_num_cache_leaves(uint32_t op) -{ - union _cpuid_leaf_eax eax; - union _cpuid_leaf_ebx ebx; - unsigned int ecx; - unsigned int edx; - int idx = -1; - - do { - ++idx; - cpuid_count(op, idx, &eax.full, &ebx.full, &ecx, &edx); - /* Modern systems have cache levels up to 3. */ - } while (eax.split.type != CACHE_TYPE_NULL && idx < 4); - return idx; -} - -/* cpuid_get_cpu_cache_info - Use cpuid instruction to get cache information - * @op - cpuid opcode to get cache information - * @cpu_ci - this parameter is an input and also an output. - * [IN] cpu_ci->num_caches: the number of caches of this cpu - * [OUT] cpu_ci->cache_info: to store cache info collected - */ -static void cpuid_get_cpu_cache_info(uint32_t op, cpu_cacheinfo_t *cpu_ci) -{ - union _cpuid_leaf_eax eax; - union _cpuid_leaf_ebx ebx; - uint32_t ecx; - uint32_t edx; - uint32_t index; - cacheinfo_t *this_leaf; - - for (index = 0; index < cpu_ci->num_caches; index++) { - cpuid_count(op, index, &eax.full, &ebx.full, &ecx, &edx); - this_leaf = cpu_ci->cache_info + index; - this_leaf->hsa_cache_prop.ProcessorIdLow = cpu_ci->apicid; - this_leaf->num_threads_sharing = - eax.split.num_threads_sharing + 1; - this_leaf->hsa_cache_prop.CacheLevel = eax.split.level; - this_leaf->hsa_cache_prop.CacheType.ui32.CPU = 1; - if (eax.split.type & CACHE_TYPE_DATA) - this_leaf->hsa_cache_prop.CacheType.ui32.Data = 1; - if (eax.split.type & CACHE_TYPE_INST) - this_leaf->hsa_cache_prop.CacheType.ui32.Instruction = 1; - this_leaf->hsa_cache_prop.CacheLineSize = - ebx.split.coherency_line_size + 1; - this_leaf->hsa_cache_prop.CacheAssociativity = - ebx.split.ways_of_associativity + 1; - this_leaf->hsa_cache_prop.CacheLinesPerTag = - ebx.split.physical_line_partition + 1; - this_leaf->hsa_cache_prop.CacheSize = (ecx + 1) * - (ebx.split.coherency_line_size + 1) * - (ebx.split.physical_line_partition + 1) * - (ebx.split.ways_of_associativity + 1); + if (fscanf(fd, "%u", num) != 1) { + pr_err("Failed to parse %s as a decimal.\n", file); + ret = HSAKMT_STATUS_ERROR; } + + fclose(fd); + return ret; } -/* find_cpu_cache_siblings - In the cache list, some caches may be listed more - * than once if they are shared by multiple CPUs. Identify the cache's CPU - * siblings, record it to SiblingMap[], then remove the duplicated cache by - * changing the cache size to 0. +/* fscanf_str - read a file whose content is a string + * @file [IN ] file to read + * @str [OUT] string in the file */ -static void find_cpu_cache_siblings(cpu_cacheinfo_t *cpu_ci_list) +static HSAKMT_STATUS fscanf_str(char *file, char *str) { - cacheinfo_t *this_leaf, *leaf2; - uint32_t n, j, idx_msb, apicid1, apicid2; - cpu_cacheinfo_t *this_cpu, *cpu2; - uint32_t index; + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - /* FixMe: cpuid under Valgrind doesn't return data from the processor we set - * affinity to. We can't use that data to calculate siblings. - */ - if (is_valgrind) - return; + fd = fopen(file, "r"); + if (!fd) { + pr_err("Failed to open %s\n", file); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + if (fscanf(fd, "%s", str) != 1) { + pr_err("Failed to parse %s as a string.\n", file); + ret = HSAKMT_STATUS_ERROR; + } - for (n = 0; n < cpu_ci_list->len; n++) { - this_cpu = cpu_ci_list + n; - for (index = 0; index < this_cpu->num_caches; index++) { - this_leaf = this_cpu->cache_info + index; - /* CacheSize 0 means an invalid cache */ - if (!this_leaf->hsa_cache_prop.CacheSize) - continue; - if (this_leaf->num_threads_sharing == 1) // no siblings - continue; - idx_msb = get_count_order(this_leaf->num_threads_sharing); - for (j = n + 1; j < cpu_ci_list->len; j++) { - cpu2 = cpu_ci_list + j; - leaf2 = cpu2->cache_info + index; - apicid1 = this_leaf->hsa_cache_prop.ProcessorIdLow; - apicid2 = leaf2->hsa_cache_prop.ProcessorIdLow; - if ((apicid2 >> idx_msb) != (apicid1 >> idx_msb)) - continue; - /* A sibling leaf is found. Cache properties - * use ProcIdLow as offset to represent siblings - * in SiblingMap, so keep the lower apicid and - * delete the other by changing CacheSize to 0. - */ - if (apicid1 < apicid2) { - this_leaf->hsa_cache_prop.SiblingMap[0] = 1; - this_leaf->hsa_cache_prop.SiblingMap[apicid2 - apicid1] = 1; - leaf2->hsa_cache_prop.CacheSize = 0; - cpu2->num_duplicated_caches++; - } else { - leaf2->hsa_cache_prop.SiblingMap[0] = 1; - leaf2->hsa_cache_prop.SiblingMap[apicid1 - apicid2] = 1; - this_leaf->hsa_cache_prop.CacheSize = 0; - this_cpu->num_duplicated_caches++; - } - } + fclose(fd); + return ret; +} + +/* fscanf_size - read a file whose content represents size as a string + * @file [IN ] file to read + * @bytes [OUT] sizes in bytes + */ +static HSAKMT_STATUS fscanf_size(char *file, uint32_t *bytes) +{ + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + char unit; + int n; + + fd = fopen(file, "r"); + if (!fd) { + pr_err("Failed to open %s\n", file); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + n = fscanf(fd, "%u%c", bytes, &unit); + if (n < 1) { + pr_err("Failed to parse %s\n", file); + ret = HSAKMT_STATUS_ERROR; + } + + if (n == 2) { + switch (unit) { + case 'K': + *bytes <<= 10; break; + case 'M': + *bytes <<= 20; break; + case 'G': + *bytes <<= 30; break; + default: + ret = HSAKMT_STATUS_ERROR; break; } } + + fclose(fd); + return ret; +} + +/* cpumap_to_cpu_ci - translate shared_cpu_map string + cpuinfo->apicid into + * SiblingMap in cache + * @shared_cpu_map [IN ] shared_cpu_map string + * @cpuinfo [IN ] cpuinfo to get apicid + * @this_cache [OUT] CPU cache to fill in SiblingMap + */ +static void cpumap_to_cpu_ci(char *shared_cpu_map, + struct proc_cpuinfo *cpuinfo, + HsaCacheProperties *this_cache) +{ + int num_hexs, bit; + uint32_t proc, apicid, mask; + char *ch_ptr; + + /* shared_cpu_map is shown as ...X3,X2,X1 Each X is a hex without 0x + * and it's up to 8 characters(32 bits). For the first 32 CPUs(actually + * procs), it's presented in X1. The next 32 is in X2, and so on. + */ + num_hexs = (strlen(shared_cpu_map) + 8) / 9; /* 8 characters + "," */ + ch_ptr = strtok(shared_cpu_map, ","); + while (num_hexs-- > 0) { + mask = strtol(ch_ptr, NULL, 16); /* each X */ + for (bit = 0; bit < 32; bit++) { + if (!((1 << bit) & mask)) + continue; + proc = num_hexs * 32 + bit; + apicid = cpuinfo[proc].apicid; + if (apicid >= HSA_CPU_SIBLINGS) { + pr_warn("SiblingMap buffer %d is too small\n", + HSA_CPU_SIBLINGS); + continue; + } + this_cache->SiblingMap[apicid] = 1; + } + ch_ptr = strtok(NULL, ","); + } +} + +/* get_cpu_cache_info - get specified CPU's cache information from sysfs + * @prefix [IN] sysfs path for target cpu cache, + * /sys/devices/system/node/nodeX/cpuY/cache + * @cpuinfo [IN] /proc/cpuinfo data to get apicid + * @cpu_ci: CPU specified. This parameter is an input and also an output. + * [IN] cpu_ci->num_caches: number of index dirs + * [OUT] cpu_ci->cache_info: to store cache info collected + * [OUT] cpu_ci->num_caches: reduces when shared with other cpu(s) + * Return: number of cache reported from this cpu + */ +static int get_cpu_cache_info(const char *prefix, struct proc_cpuinfo *cpuinfo, + cpu_cacheinfo_t *cpu_ci) +{ + int idx, num_idx, n; + HsaCacheProperties *this_cache; + char path[256], str[256]; + + this_cache = cpu_ci->cache_prop; + num_idx = cpu_ci->num_caches; + for (idx = 0; idx < num_idx; idx++) { + /* If this cache is shared by multiple CPUs, we only need + * to list it in the first CPU. + */ + snprintf(path, 256, "%s/index%d/shared_cpu_list", prefix, idx); + /* shared_cpu_list is shown as n1,n2... or n1-n2,n3-n4... + * For both cases, this cache is listed to proc n1 only. + */ + fscanf_dec(path, (uint32_t *)&n); + if (cpu_ci->proc_num != n) { + /* proc is not n1. Skip and reduce the cache count. */ + --cpu_ci->num_caches; + continue; + } + + this_cache->ProcessorIdLow = cpuinfo[cpu_ci->proc_num].apicid; + + /* CacheLevel */ + snprintf(path, 256, "%s/index%d/level", prefix, idx); + fscanf_dec(path, &this_cache->CacheLevel); + /* CacheType */ + snprintf(path, 256, "%s/index%d/type", prefix, idx); + fscanf_str(path, str); + if (!strcmp(str, "Data")) + this_cache->CacheType.ui32.Data = 1; + if (!strcmp(str, "Instruction")) + this_cache->CacheType.ui32.Instruction = 1; + if (!strcmp(str, "Unified")) { + this_cache->CacheType.ui32.Data = 1; + this_cache->CacheType.ui32.Instruction = 1; + } + this_cache->CacheType.ui32.CPU = 1; + /* CacheSize */ + snprintf(path, 256, "%s/index%d/size", prefix, idx); + fscanf_size(path, &this_cache->CacheSize); + /* CacheLineSize */ + snprintf(path, 256, "%s/index%d/coherency_line_size", prefix, idx); + fscanf_dec(path, &this_cache->CacheLineSize); + /* CacheAssociativity */ + snprintf(path, 256, "%s/index%d/ways_of_associativity", prefix, idx); + fscanf_dec(path, &this_cache->CacheAssociativity); + /* CacheLinesPerTag */ + snprintf(path, 256, "%s/index%d/physical_line_partition", prefix, idx); + fscanf_dec(path, &this_cache->CacheLinesPerTag); + /* CacheSiblings */ + snprintf(path, 256, "%s/index%d/shared_cpu_map", prefix, idx); + fscanf_str(path, str); + cpumap_to_cpu_ci(str, cpuinfo, this_cache); + + ++this_cache; + } + + return cpu_ci->num_caches; } -#endif /* X86 platform */ static HSAKMT_STATUS topology_sysfs_get_generation(uint32_t *gen) { @@ -1144,22 +1128,19 @@ err1: return ret; } -#if defined(__x86_64__) || defined(__i386__) -/* topology_destroy_temp_cpu_cache_list - Free the memory allocated in - * topology_create_temp_cpu_cache_list(). +/* topology_destroy_temp_cpu_cache_list - + * Free the memory allocated in topology_create_temp_cpu_cache_list(). */ -static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) +static void topology_destroy_temp_cpu_cache_list( + cpu_cacheinfo_t *temp_cpu_ci_list) { uint32_t n; - cpu_cacheinfo_t *p_temp_cpu_ci_list = (cpu_cacheinfo_t *)temp_cpu_ci_list; - cpu_cacheinfo_t *this_cpu; + cpu_cacheinfo_t *p_temp_cpu_ci_list = temp_cpu_ci_list; + cpu_cacheinfo_t *cpu_ci = p_temp_cpu_ci_list; if (p_temp_cpu_ci_list) { - for (n = 0; n < p_temp_cpu_ci_list->len; n++) { - this_cpu = p_temp_cpu_ci_list + n; - if (this_cpu->cache_info) - free(this_cpu->cache_info); - } + for (n = 0; n < p_temp_cpu_ci_list->len; n++, cpu_ci++) + free(cpu_ci->cache_prop); free(p_temp_cpu_ci_list); } @@ -1168,176 +1149,103 @@ static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) /* topology_create_temp_cpu_cache_list - Create a temporary cpu-cache list to * store cpu cache information. This list will be used to copy - * cache information to each CPU node. Must call - * topology_destroy_temp_cpu_cache_list to free the memory after - * the information is copied. - * @temp_cpu_ci_list - [OUT] temporary cpu-cache-info list to store data - * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure + * HsaCacheProperties in the CPU node. Two buffers are allocated + * inside this function: cpu_ci list and cache_prop under each + * cpu_ci. Must call topology_destroy_temp_cpu_cache_list to free + * the memory after the information is copied. + * @node [IN] CPU node number + * @cpuinfo [IN] /proc/cpuinfo data + * @temp_cpu_ci_list [OUT] cpu-cache-info list with data filled + * Return: total number of caches under this CPU node */ -static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) +static int topology_create_temp_cpu_cache_list(int node, + struct proc_cpuinfo *cpuinfo, cpu_cacheinfo_t **temp_cpu_ci_list) { - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - void *p_temp_cpu_ci_list; - int procs_online; - cpu_set_t orig_cpuset; - int i, j = 0; - uint32_t cpuid_op_cache; - uint32_t eax, ebx, ecx = 0, edx; /* cpuid registers */ - cpu_cacheinfo_t *cpu_ci_list, *this_cpu; - bool x2apic = false; + cpu_cacheinfo_t *p_temp_cpu_ci_list; /* a list of cpu_ci */ + char path[256], node_dir[256]; + int max_cpus; + cpu_cacheinfo_t *this_cpu; /* one cpu_ci in cpu_ci_list */ + int cache_cnt = 0; + DIR *dirp; + struct dirent *dir; + char *p; if (!temp_cpu_ci_list) { - ret = HSAKMT_STATUS_ERROR; + pr_err("Invalid temp_cpu_ci_list\n"); goto exit; } *temp_cpu_ci_list = NULL; - /* Find the number of processors available to the current process, based - * on a system call that returns the "affinity" between the current process - * and each CPU. For getting each CPU cache information the process affinity - * is modified. Save the original affinity mask so that it can be restored - * after finding the cache information. - */ - CPU_ZERO(&orig_cpuset); - if (sched_getaffinity(0, sizeof(cpu_set_t), &orig_cpuset)) { - pr_warn("Failed to get CPU affinity %s\n", strerror(errno)); - ret = HSAKMT_STATUS_ERROR; - goto exit; - } - procs_online = CPU_COUNT(&orig_cpuset); - if (procs_online <= 0) { - pr_warn("Invalid number of online processors %d\n", procs_online); - ret = HSAKMT_STATUS_ERROR; + /* Get info from /sys/devices/system/node/nodeX/cpuY/cache */ + snprintf(node_dir, 256, "/sys/devices/system/node/node%d", node); + /* Other than cpuY folders, this dir also has cpulist and cpumap */ + max_cpus = num_subdirs(node_dir, "cpu"); + if (max_cpus <= 0) { + pr_err("Fail to get cpu* dirs under %s\n", path); goto exit; } - p_temp_cpu_ci_list = calloc(sizeof(cpu_cacheinfo_t) * procs_online, 1); + p_temp_cpu_ci_list = calloc(max_cpus, sizeof(cpu_cacheinfo_t)); if (!p_temp_cpu_ci_list) { - ret = HSAKMT_STATUS_NO_MEMORY; + pr_err("Fail to allocate p_temp_cpu_ci_list\n"); goto exit; } + p_temp_cpu_ci_list->len = 0; - cpu_ci_list = (cpu_cacheinfo_t *)p_temp_cpu_ci_list; - cpu_ci_list->len = procs_online; - - if (processor_vendor == AUTHENTIC_AMD) - cpuid_op_cache = 0x8000001d; - else - cpuid_op_cache = 0x4; - - for (i = 0; i < CPU_SETSIZE && j < procs_online; i++) { - /* Ignore the CPUs that is not available for the current process */ - if (!CPU_ISSET(i, &orig_cpuset)) + this_cpu = p_temp_cpu_ci_list; + dirp = opendir(node_dir); + while ((dir = readdir(dirp)) != 0) { + if (strncmp(dir->d_name, "cpu", 3)) continue; - - this_cpu = cpu_ci_list + j++; - /* so cpuid is executed in correct cpu */ - if (lock_to_processor(i)) { - pr_err("Failed to set CPU affinity %s\n", strerror(errno)); - free(p_temp_cpu_ci_list); - ret = HSAKMT_STATUS_ERROR; + if (!isdigit(dir->d_name[3])) /* ignore files like cpulist */ + continue; + snprintf(path, 256, "/sys/devices/system/node/node%d/%s/cache", + node, dir->d_name); + this_cpu->num_caches = num_subdirs(path, "index"); + this_cpu->cache_prop = calloc(this_cpu->num_caches, + sizeof(HsaCacheProperties)); + if (!this_cpu->cache_prop) { + pr_err("Fail to allocate cache_info\n"); goto exit; } - - /* Detect the availability of the extended topology leaf */ - eax = 0x0; - cpuid(&eax, &ebx, &ecx, &edx); - if (eax >= 11) { - eax = 0xb; - ecx = 0x0; - cpuid(&eax, &ebx, &ecx, &edx); - if (ebx) - x2apic = true; - } - - if (x2apic) { - eax = 0xb; - cpuid(&eax, &ebx, &ecx, &edx); - this_cpu->apicid = edx; - cpuid_count(4, 0, &eax, &ebx, &ecx, &edx); - this_cpu->max_num_apicid = (eax >> 26) + 1; - } else { - eax = 0x1; - cpuid(&eax, &ebx, &ecx, &edx); - this_cpu->apicid = (ebx >> 24) & 0xff; - this_cpu->max_num_apicid = (ebx >> 16) & 0x0FF; - } - this_cpu->num_caches = cpuid_find_num_cache_leaves(cpuid_op_cache); - this_cpu->num_duplicated_caches = 0; - this_cpu->cache_info = calloc( - sizeof(cacheinfo_t) * this_cpu->num_caches, 1); - if (!this_cpu->cache_info) { - free(p_temp_cpu_ci_list); - ret = HSAKMT_STATUS_NO_MEMORY; - goto err; - } - cpuid_get_cpu_cache_info(cpuid_op_cache, this_cpu); + p = &dir->d_name[3]; + this_cpu->proc_num = atoi(p); + cache_cnt += get_cpu_cache_info(path, cpuinfo, this_cpu); + ++p_temp_cpu_ci_list->len; + ++this_cpu; } - - if (j < procs_online) { - pr_warn("CPU cache information missing\n"); - cpu_ci_list->len = j; - } - - find_cpu_cache_siblings(cpu_ci_list); *temp_cpu_ci_list = p_temp_cpu_ci_list; -err: - /* restore affinity to original */ - sched_setaffinity(0, sizeof(cpu_set_t), &orig_cpuset); exit: - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_warn("Topology fails to create cpu cache list\n"); - topology_destroy_temp_cpu_cache_list(*temp_cpu_ci_list); - } - return ret; + if (dirp) + closedir(dirp); + return cache_cnt; } -/* topology_get_cpu_cache_props - Read CPU cache information from the temporary - * cache list and put them to the node's cache properties entry. - * @tbl - the node table to fill up - * @cpu_ci_list - the cpu cache information list to look up cache info - * Return - HSAKMT_STATUS_SUCCESS in success or error number in failure +/* topology_get_cpu_cache_props - Read CPU cache information from sysfs + * @node [IN] CPU node number + * @cpuinfo [IN] /proc/cpuinfo data + * @tbl [OUT] the node table to fill up + * Return: HSAKMT_STATUS_SUCCESS in success or error number in failure */ -static HSAKMT_STATUS topology_get_cpu_cache_props(node_props_t *tbl, - cpu_cacheinfo_t *cpu_ci_list) +static HSAKMT_STATUS topology_get_cpu_cache_props(int node, + struct proc_cpuinfo *cpuinfo, node_props_t *tbl) { HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - uint32_t apicid_low = tbl->node.CComputeIdLo, apicid_max = 0; - uint32_t n, cache_cnt, idx; - cpu_cacheinfo_t *this_cpu; - cacheinfo_t *this_leaf; + cpu_cacheinfo_t *cpu_ci_list = NULL; + uint32_t n, cache_cnt, i; + cpu_cacheinfo_t *cpu_ci; + HsaCacheProperties *this_cache; - /* CPU cache info list contains all CPUs. Find out CPUs belonging to - * this node and number of caches under, so we can allocate the cache - * properties in the node. - */ - tbl->node.NumCaches = 0; - for (n = 0; n < cpu_ci_list->len; n++) { - this_cpu = cpu_ci_list + n; - if (this_cpu->apicid == apicid_low) - /* found the first cpu in the node */ - apicid_max = apicid_low + this_cpu->max_num_apicid - 1; - - if ((this_cpu->apicid < apicid_low) || - (this_cpu->apicid > apicid_max)) - continue; /* this cpu doesn't belong to the node */ - tbl->node.NumCaches += - this_cpu->num_caches - this_cpu->num_duplicated_caches; + tbl->node.NumCaches = topology_create_temp_cpu_cache_list( + node, cpuinfo, &cpu_ci_list); + if (!tbl->node.NumCaches) { + pr_err("Fail to get cache info for node %d\n", node); + ret = HSAKMT_STATUS_ERROR; + goto exit; } - /* FixMe: cpuid under Valgrind doesn't return data from the processor we set - * affinity to. All the data come from one specific processor. We'll report - * this one processor's cache and ignore others. - */ - if (is_valgrind) { - this_cpu = cpu_ci_list; - tbl->node.NumCaches = this_cpu->num_caches; - apicid_low = apicid_max = this_cpu->apicid; - } - - tbl->cache = calloc( - sizeof(HsaCacheProperties) * tbl->node.NumCaches, 1); + tbl->cache = calloc(tbl->node.NumCaches, sizeof(HsaCacheProperties)); if (!tbl->cache) { ret = HSAKMT_STATUS_NO_MEMORY; goto exit; @@ -1345,38 +1253,23 @@ static HSAKMT_STATUS topology_get_cpu_cache_props(node_props_t *tbl, /* Now fill in the information to cache properties. */ cache_cnt = 0; - for (n = 0; n < cpu_ci_list->len; n++) { - this_cpu = cpu_ci_list + n; - if ((this_cpu->apicid < apicid_low) || this_cpu->apicid > apicid_max) - continue; /* this cpu doesn't belong to the node */ - for (idx = 0; idx < this_cpu->num_caches; idx++) { - this_leaf = this_cpu->cache_info + idx; - if (this_leaf->hsa_cache_prop.CacheSize > 0) - memcpy(&tbl->cache[cache_cnt++], &this_leaf->hsa_cache_prop, sizeof(HsaCacheProperties)); + cpu_ci = cpu_ci_list; + for (n = 0; n < cpu_ci_list->len; n++, cpu_ci++) { + this_cache = cpu_ci->cache_prop; + for (i = 0; i < cpu_ci->num_caches; i++, this_cache++) { + memcpy(&tbl->cache[cache_cnt++], + this_cache, + sizeof(HsaCacheProperties)); if (cache_cnt >= tbl->node.NumCaches) goto exit; } } exit: + topology_destroy_temp_cpu_cache_list(cpu_ci_list); + return ret; } -#else /* not X86 */ -static void topology_destroy_temp_cpu_cache_list(void *temp_cpu_ci_list) -{ -} - -static HSAKMT_STATUS topology_create_temp_cpu_cache_list(void **temp_cpu_ci_list) -{ - return HSAKMT_STATUS_SUCCESS; -} - -static HSAKMT_STATUS topology_get_cpu_cache_props(node_props_t *tbl, - cpu_cacheinfo_t *cpu_ci_list) -{ - return HSAKMT_STATUS_SUCCESS; -} -#endif static HSAKMT_STATUS topology_sysfs_get_cache_props(uint32_t node_id, uint32_t cache_id, @@ -1798,10 +1691,8 @@ HSAKMT_STATUS topology_take_snapshot(void) uint32_t gen_start, gen_end, i, mem_id, cache_id; HsaSystemProperties sys_props; node_props_t *temp_props = 0; - void *cpu_ci_list = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; struct pci_access *pacc; - char *envvar; struct proc_cpuinfo *cpuinfo; const uint32_t num_procs = get_nprocs(); @@ -1812,12 +1703,6 @@ HSAKMT_STATUS topology_take_snapshot(void) } topology_parse_cpuinfo(cpuinfo, num_procs); - envvar = getenv("HSA_RUNNING_UNDER_VALGRIND"); - if (envvar && !strcmp(envvar, "1")) - is_valgrind = 1; - else - is_valgrind = 0; - retry: ret = topology_sysfs_get_generation(&gen_start); if (ret != HSAKMT_STATUS_SUCCESS) @@ -1826,7 +1711,6 @@ retry: if (ret != HSAKMT_STATUS_SUCCESS) goto err; if (sys_props.NumNodes > 0) { - topology_create_temp_cpu_cache_list(&cpu_ci_list); temp_props = calloc(sys_props.NumNodes * sizeof(node_props_t), 1); if (!temp_props) { ret = HSAKMT_STATUS_NO_MEMORY; @@ -1879,7 +1763,7 @@ retry: } } else if (!temp_props[i].gpu_id) { /* a CPU node */ ret = topology_get_cpu_cache_props( - &temp_props[i], cpu_ci_list); + i, cpuinfo, &temp_props[i]); if (ret != HSAKMT_STATUS_SUCCESS) { free_properties(temp_props, i + 1); goto err; @@ -1954,7 +1838,6 @@ retry: g_props = temp_props; err: free(cpuinfo); - topology_destroy_temp_cpu_cache_list(cpu_ci_list); return ret; } From 8ab8b149021d10549eb21ab411f03a4a0ed06999 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Tue, 27 Aug 2019 00:37:05 -0500 Subject: [PATCH 0574/1247] Initialize dirp in topology_create_temp_cpu_cache_list to NULL. Avoids uninitialized use in early exit (error) paths. Change-Id: I5fb24863f0a5da48776608d47f25e1c8d8aafe35 --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index b008d338d9..1538cfae83 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1166,7 +1166,7 @@ static int topology_create_temp_cpu_cache_list(int node, int max_cpus; cpu_cacheinfo_t *this_cpu; /* one cpu_ci in cpu_ci_list */ int cache_cnt = 0; - DIR *dirp; + DIR *dirp = NULL; struct dirent *dir; char *p; From ccac07cb143b3af4629fdd07095e50cb2d81b25e Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 27 Aug 2019 07:56:18 -0400 Subject: [PATCH 0575/1247] topology: Fix compile warnings regarding snprintf and path snprintf throws a warning from -Wformat-truncation where the string could be truncated. We address this by referencing the maximum size that can be returned from a file according to MAXNAMLEN . This should safely guard us from truncating the path value. Change-Id: If1d208990d8775e9494835b0deb890d2616fd15b Signed-off-by: Kent Russell --- src/topology.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/topology.c b/src/topology.c index 1538cfae83..633afcd392 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1161,8 +1161,14 @@ static void topology_destroy_temp_cpu_cache_list( static int topology_create_temp_cpu_cache_list(int node, struct proc_cpuinfo *cpuinfo, cpu_cacheinfo_t **temp_cpu_ci_list) { + /* Get max path size from /sys/devices/system/node/node%d/%s/cache + * below, which will max out according to the largest filename, + * which can be present twice in the string above. 29 is for the prefix + * and the +6 is for the cache suffix + */ + const uint32_t MAXPATHSIZE = 29 + MAXNAMLEN + (MAXNAMLEN + 6); cpu_cacheinfo_t *p_temp_cpu_ci_list; /* a list of cpu_ci */ - char path[256], node_dir[256]; + char path[MAXPATHSIZE], node_dir[MAXPATHSIZE]; int max_cpus; cpu_cacheinfo_t *this_cpu; /* one cpu_ci in cpu_ci_list */ int cache_cnt = 0; @@ -1177,7 +1183,7 @@ static int topology_create_temp_cpu_cache_list(int node, *temp_cpu_ci_list = NULL; /* Get info from /sys/devices/system/node/nodeX/cpuY/cache */ - snprintf(node_dir, 256, "/sys/devices/system/node/node%d", node); + snprintf(node_dir, MAXPATHSIZE, "/sys/devices/system/node/node%d", node); /* Other than cpuY folders, this dir also has cpulist and cpumap */ max_cpus = num_subdirs(node_dir, "cpu"); if (max_cpus <= 0) { @@ -1199,7 +1205,7 @@ static int topology_create_temp_cpu_cache_list(int node, continue; if (!isdigit(dir->d_name[3])) /* ignore files like cpulist */ continue; - snprintf(path, 256, "/sys/devices/system/node/node%d/%s/cache", + snprintf(path, MAXPATHSIZE, "/sys/devices/system/node/node%d/%s/cache", node, dir->d_name); this_cpu->num_caches = num_subdirs(path, "index"); this_cpu->cache_prop = calloc(this_cpu->num_caches, From dbe9af7777dd5196581708961b41b687b9f384ed Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 28 Aug 2019 18:39:09 -0400 Subject: [PATCH 0576/1247] libhsakmt: Improve the confusing code The code tends to confuse readers. Improve it. Change-Id: I5c6cbf7a114b6e7d26ce3b9f54350a153032267d Signed-off-by: Yong Zhao --- src/openclose.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/openclose.c b/src/openclose.c index d4eb264783..b9754fe57c 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -126,14 +126,14 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); - if (fd != -1) { - kfd_fd = fd; - kfd_open_count = 1; - } else { + if (fd == -1) { result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; goto open_failed; } + kfd_fd = fd; + kfd_open_count = 1; + init_page_size(); result = topology_sysfs_get_system_props(&sys_props); From 545ca6263f642431692412900cdd263a78a94ad2 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Thu, 29 Aug 2019 15:26:34 +0800 Subject: [PATCH 0577/1247] add device ID for gfx908 VF SRIOV: Fix issue that kfdtest cannot detect gfx908 VF inside VM. Change-Id: Ie05fd66d4e14b47818fddbe404df1059567b76a2 Signed-off-by: Jack Zhang --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index 633afcd392..9fe8db4c7d 100644 --- a/src/topology.c +++ b/src/topology.c @@ -221,6 +221,7 @@ static struct hsa_gfxip_table { { 0x7388, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, { 0x738C, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, { 0x738E, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, + { 0x7390, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, /* Navi10 */ { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x7312, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, From cabebaa0918c12c6ea12354eb9f3a469553334f9 Mon Sep 17 00:00:00 2001 From: "Lad, Aditya" Date: Tue, 27 Aug 2019 20:32:32 -0400 Subject: [PATCH 0578/1247] Support for SLES on thunk Made changes in the CMakeLists.txt 1) Detects the OS of the system 2) Accordingly setsup the runtime dependencies Change-Id: Ief9a0217caae77d4be4850167e2a9b8387f639e4 Signed-off-by: Lad, Aditya --- CMakeLists.txt | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f75e55d142..55cfa743dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -168,13 +168,28 @@ set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT library for AMD KFD support" ) set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) +#checking for OS-version +if( EXISTS "/etc/os-release" ) + file( STRINGS "/etc/os-release" DISTRO_ID REGEX "^ID=" ) + file( STRINGS "/etc/os-release" DISTRO_RELEASE REGEX "^VERSION_ID=" ) + string( REPLACE "ID=" "" DISTRO_ID "${DISTRO_ID}" ) + string( REPLACE "VERSION_ID=" "" DISTRO_RELEASE "${DISTRO_RELEASE}" ) + message( STATUS "Detected distribution: ${DISTRO_ID}:${DISTRO_RELEASE}" ) +else() + message( STATUS "Not able to detect OS" ) +endif() + # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm" ) set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libnuma1, libpci3, zlib1g, libudev1" ) # RPM package specific variables -set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl-libs, pciutils-libs, libgcc" ) +if( DISTRO_ID MATCHES "sles" ) + set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, libnuma-devel, pciutils, libgcc_s1") +else() + set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl-libs, pciutils-libs, libgcc" ) +endif() set ( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) From c7a946dd5db9dfdb7c6c66171cb3a1a550d59edc Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Thu, 25 Jul 2019 15:08:52 -0400 Subject: [PATCH 0579/1247] KFDTest: Add gfx1010 support in KFDTest script Change-Id: I3c7132c73c68139677259aa57ea0b443ed664c14 Signed-off-by: shaoyunl --- tests/kfdtest/scripts/kfdtest.exclude | 9 +++++++++ tests/kfdtest/scripts/run_kfdtest.sh | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index ae1a8070d0..b0fa0c0d0e 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -155,6 +155,15 @@ ARCT_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm" +NAVI10_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ +"KFDEvictTest.BurstyTest:"\ +"KFDEvictTest.QueueTest:"\ +"KFDExceptionTest.InvalidBadAddress:"\ +"KFDExceptionTest.InvalidWriteAddress:"\ +"KFDQMTest.GPUDoorbellWrite" + # Core tests, used in scenarios like bringup CORE_TESTS=\ "KFDQMTest.CreateDestroyCpQueue:"\ diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 55e70a0b3d..9694b7415e 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -74,7 +74,8 @@ getFilter() { vega20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; raven ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; arcturus ) FILTER="--gtest_filter=$ARCT_TESTS_BLACKLIST" ;; - core ) FILTER="--gtest_filter=$CORE_TESTS" ;; + navi10 ) FILTER="--gtest_filter=$NAVI10_TESTS_BLACKLIST" ;; + core ) FILTER="--gtest_filter=$CORE_TESTS" ;; all ) FILTER="" ;; *) die "Unsupported platform $platform. Exiting" ;; esac From 8f9f9a47fff667bf2849335ca0a86ab39ed7dade Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 20 Aug 2019 17:21:30 -0400 Subject: [PATCH 0580/1247] kfdtest: Use hsakmt header files from source tree Instead of the installed hsakmt header files, use the ones from the source tree, since they are in the same git repository. This allows using kfd_ioctl.h even when we don't install this file with an upcoming change. Change-Id: I9a30abd5445806d2141bdb1ccd88d3794a74ed20 Signed-off-by: Felix Kuehling --- tests/kfdtest/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 523d70356f..3d56abf132 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -60,7 +60,6 @@ else() endif() if( DEFINED LIBHSAKMT_PATH ) - set ( HSAKMT_INCLUDE_DIRS ${LIBHSAKMT_PATH}/include ) set ( HSAKMT_LIBRARY_DIRS ${LIBHSAKMT_PATH}/lib ) set ( HSAKMT_LIBRARIES hsakmt ) endif() @@ -71,9 +70,9 @@ set ( SP3_DIR ${PROJECT_SOURCE_DIR}/sp3 ) include_directories(${PROJECT_SOURCE_DIR}/gtest-1.6.0) include_directories(${PROJECT_SOURCE_DIR}/include) +include_directories(${PROJECT_SOURCE_DIR}/../../include) include_directories(${SP3_DIR}) -include_directories(${HSAKMT_INCLUDE_DIRS}) include_directories(${DRM_INCLUDE_DIRS}) aux_source_directory(${PROJECT_SOURCE_DIR}/gtest-1.6.0 SRC_FILES) From 6c9a49076aad071a34690da2573ea7d1acf54f5a Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 22 Jul 2019 16:18:08 -0400 Subject: [PATCH 0581/1247] Install LICENSE.md to DOCDIR LICENSE.md should be installed to the DOCDIR, not /usr/libhsakmt Change-Id: I2020547b3174b9d91c1f800d9db2d73f627a6ce3 Signed-off-by: Craig Andrews Signed-off-by: Felix Kuehling --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 55cfa743dd..1e644e9911 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,7 +156,7 @@ add_custom_target ( package-dev DEPENDS build-dev ## Add the install directives for the runtime library. install ( TARGETS ${HSAKMT_TARGET} DESTINATION ${CMAKE_INSTALL_LIBDIR} ) -install ( FILES ${SOURCE_DIR}/LICENSE.md DESTINATION libhsakmt ) +install ( FILES ${SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} ) ## Add the packaging directives for the runtime library. set ( CPACK_PACKAGE_NAME ${HSAKMT_PACKAGE} ) From eebbb5a0ea7accda7ee07f01f92aa2fca82f389e Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 22 Jul 2019 17:06:16 -0400 Subject: [PATCH 0582/1247] Install libhsakmt.pc to the standard location pkgconfig files should be installed to /usr/share/pkgconfig/, not /usr/libhsakmt/ Change-Id: Ifd08f612addb375de1d00282ee9e7c257528bf74 Signed-off-by: Craig Andrews Signed-off-by: Felix Kuehling --- hsakmt-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index eb0d3e2acb..195cc6c963 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -43,7 +43,7 @@ set ( BUILD_VERSION_PATCH @BUILD_VERSION_PATCH@ ) set ( CMAKE_VERBOSE_MAKEFILE on ) ## Set the install targets -install ( FILES libhsakmt.pc DESTINATION libhsakmt ) +install ( FILES libhsakmt.pc DESTINATION ${CMAKE_INSTALL_PREFIX}/share/pkgconfig ) install ( DIRECTORY ${SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_COMPONENT} ) From ec1461b84bc0a011096a1fae2b406a6c06038dc2 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Thu, 8 Aug 2019 14:01:04 -0400 Subject: [PATCH 0583/1247] hsakmt-dev should not install linux kernel headers hsakmt-dev should not install include/linux/* (currently just kfd_ioctl.h) as those are linux kernel headers provided by the linux kernel header packages (`linux-headers-*` on Debian/Ubuntu or `kernel-headers-*` on Red Hat / Fedora) Change-Id: Ib6e62ca2f3582c5ad7351225f5827081bf8e05c0 Signed-off-by: Craig Andrews Signed-off-by: Felix Kuehling --- hsakmt-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index 195cc6c963..b3de70f7dc 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -44,7 +44,7 @@ set ( CMAKE_VERBOSE_MAKEFILE on ) ## Set the install targets install ( FILES libhsakmt.pc DESTINATION ${CMAKE_INSTALL_PREFIX}/share/pkgconfig ) -install ( DIRECTORY ${SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_COMPONENT} ) +install ( DIRECTORY ${SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_COMPONENT} PATTERN "linux" EXCLUDE ) ## Set the default generator types for the devel package. From e320913e9e9e6c0f3e831ee02e3c4734fce1fbf6 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 30 Aug 2019 00:14:26 -0400 Subject: [PATCH 0584/1247] libhsakmt: Fix userptr mappings on gfx802 The memory size alignment workaround for a TLB bug on gfx802 was breaking userptrs because it would attempt to get_user_pages beyond the end of a VMA. Refine this workaround based on our understanding of the HW bug. It only affects L2 cacheline allocation, which is decided by the last page in the cache line (8 entries = 32KB of address space). Thus aligning memory allocation so that the last page falls on the end of a 8 entry TLB cache line allows caching to work correctly. Imported images require specific alignments. If their size is not naturally aligned with 8 cache lines, it may have bad TLB cache performance. This patch will only have the desired effect if redundant size padding in KFD is also removed. Change-Id: I984cbe7fa61fec04d70fa387aaf9aab370eabeb9 Signed-off-by: Felix Kuehling --- src/fmm.c | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 0bdb6f3c96..ea798d1ae2 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -413,8 +413,6 @@ static vm_object_t *vm_find_object_by_address_userptr(manageable_aperture_t *app { vm_object_t *cur = NULL; - if (is_userptr == 0) - size = ALIGN_UP(size, app->align); rbtree_t *tree = vm_object_tree(app, is_userptr); rbtree_key_t key = rbtree_key((unsigned long)address, size); void *start; @@ -561,8 +559,7 @@ static bool aperture_is_valid(void *app_base, void *app_limit) */ static uint64_t vm_align_area_size(manageable_aperture_t *app, uint64_t size) { - return ALIGN_UP(ALIGN_UP(size, app->align) + (uint64_t)app->guard_pages * PAGE_SIZE, - app->align); + return size + (uint64_t)app->guard_pages * PAGE_SIZE; } /* @@ -632,6 +629,7 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, uint64_t MemorySizeInBytes, uint64_t align) { + uint64_t offset = 0, orig_align = align; vm_area_t *cur, *next; void *start; @@ -644,12 +642,22 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, while (align < GPU_HUGE_PAGE_SIZE && MemorySizeInBytes >= (align << 1)) align <<= 1; + /* If no specific alignment was requested, align the end of + * buffers instead of the start. For fragment optimizations, + * aligning the start or the end achieves the same effective + * optimization. End alignment to the TLB cache line size is + * needed as a workaround for TLB issues on some older GPUs. + */ + if (orig_align <= (uint64_t)PAGE_SIZE) + offset = align - (MemorySizeInBytes & (align - 1)); + MemorySizeInBytes = vm_align_area_size(app, MemorySizeInBytes); /* Find a big enough "hole" in the address space */ cur = NULL; next = app->vm_ranges; - start = address ? address : (void *)ALIGN_UP((uint64_t)app->base, align); + start = address ? address : + (void *)(ALIGN_UP((uint64_t)app->base, align) + offset); while (next) { if (next->start > start && VOID_PTRS_SUB(next->start, start) >= MemorySizeInBytes) @@ -658,7 +666,7 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, cur = next; next = next->next; if (!address) - start = (void *)ALIGN_UP((uint64_t)cur->end + 1, align); + start = (void *)(ALIGN_UP((uint64_t)cur->end + 1, align) + offset); } if (!next && VOID_PTRS_SUB(app->limit, start) + 1 < MemorySizeInBytes) /* No hole found and not enough space after the last area */ @@ -706,18 +714,12 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, return NULL; } - if (align < aper->align) - align = aper->align; - /* Align big buffers to the next power-of-2 up to huge page * size for flexible fragment size TLB optimizations */ while (align < GPU_HUGE_PAGE_SIZE && size >= (align << 1)) align <<= 1; - /* Align memory size to match aperture requirements */ - size = ALIGN_UP(size, aper->align); - /* Add padding to guarantee proper alignment and leave guard * pages on both sides */ @@ -763,9 +765,6 @@ static void mmap_aperture_release(manageable_aperture_t *aper, return; } - /* Align memory size to match aperture requirements */ - size = ALIGN_UP(size, aper->align); - /* Reset NUMA policy */ mbind(addr, size, MPOL_DEFAULT, NULL, 0, 0); @@ -801,8 +800,6 @@ static vm_object_t *aperture_allocate_object(manageable_aperture_t *app, { vm_object_t *new_object; - MemorySizeInBytes = ALIGN_UP(MemorySizeInBytes, app->align); - /* Allocate new object */ new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, @@ -929,7 +926,7 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, /* Allocate memory from amdkfd */ args.gpu_id = gpu_id; - args.size = ALIGN_UP(MemorySizeInBytes, aperture->align); + args.size = MemorySizeInBytes; args.flags = flags | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; @@ -1579,7 +1576,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, } if (flags.ui32.AQLQueueMemory) { - uint64_t my_buf_size = ALIGN_UP(size, aperture->align) / 2; + uint64_t my_buf_size = size / 2; memset(ret, 0, MemorySizeInBytes); mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, @@ -1796,6 +1793,15 @@ static HSAKMT_STATUS init_mmap_apertures(HSAuint64 base, HSAuint64 limit, { void *addr; + if (align > (HSAuint32)PAGE_SIZE) { + /* This should never happen. Alignment constraints + * only apply to old GPUs that don't support 48-bit + * virtual addresses. + */ + pr_info("Falling back to reserved SVM apertures due to alignment contraints.\n"); + return HSAKMT_STATUS_ERROR; + } + /* Set up one SVM aperture */ svm.apertures[SVM_DEFAULT].base = (void *)base; svm.apertures[SVM_DEFAULT].limit = (void *)limit; @@ -3080,7 +3086,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, goto error_free_metadata; pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area_aligned(aperture, NULL, infoArgs.size, - MAX(aperture->align, IMAGE_ALIGN)); + IMAGE_ALIGN); pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) goto error_free_metadata; From 8f91d6a222a5ffb5f21c95709e6a6b5622ff7aa6 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 30 Aug 2019 00:20:21 -0400 Subject: [PATCH 0585/1247] kfdtest: Enable more tests on gfx802 A number of tests are no longer broken on gfx802. Change-Id: If70c77423f8f14de59490ab8ca156b0c4e7b5cf1 Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/kfdtest.exclude | 8 +------- tests/kfdtest/src/KFDMemoryTest.cpp | 5 ----- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index b0fa0c0d0e..9559cd549a 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -102,14 +102,8 @@ TONGA_TESTS_BLACKLIST=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDCWSRTest.BasicTest:"\ -"KFDEventTest.*:"\ -"KFDLocalMemoryTest.*:"\ -"KFDMemoryTest.FlatScratchAccess:"\ -"KFDMemoryTest.MapMemoryToGPU:"\ -"KFDMemoryTest.MemoryRegister*:"\ -"KFDMemoryTest.QueryPointerInfo:"\ "KFDPerfCountersTest.*:"\ -"KFDQMTest.*" +"KFDQMTest.OverSubscribeCpQueues" # Since Navi10 was merged, the PM4Event test takes 6min to run FIJI_TESTS_BLACKLIST=\ diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 0212917faa..cd73a0f073 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -432,11 +432,6 @@ TEST_F(KFDMemoryTest, AccessPPRMem) { // Linux OS-specific Test for registering OS allocated memory TEST_F(KFDMemoryTest, MemoryRegister) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); - if (isTonga(pNodeProperties)) { - LOG() << "Skipping test: Workaround in thunk for Tonga causes failure." << std::endl; - return; - } - TEST_START(TESTPROFILE_RUNALL) int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); From 01957d4da388736575cfc9de92f15aef40c0f30d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 29 Aug 2019 14:28:08 -0400 Subject: [PATCH 0586/1247] kfdtest: Use CoarseGrain for VRAM buffers Change-Id: I320099e61fc8f1012184fc68c248da4e224bc3ca Signed-off-by: Felix Kuehling --- tests/kfdtest/src/KFDTestUtil.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 3f5c687a0b..d1e398fa1b 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -184,9 +184,11 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b if (isLocal) { m_Flags.ui32.HostAccess = 0; m_Flags.ui32.NonPaged = 1; + m_Flags.ui32.CoarseGrain = 1; } else { m_Flags.ui32.HostAccess = 1; m_Flags.ui32.NonPaged = 0; + m_Flags.ui32.CoarseGrain = 0; } if (isExec) From 9048fb5a1a22514bd75801636f04f8536766bd7e Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 14 Aug 2019 20:34:44 -0400 Subject: [PATCH 0587/1247] kfdtest: Implement SuspendAndWakeUp() Change-Id: I16384cd0ca35f15841344ded4d4237a0a862a934 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 14 ++++++++++---- tests/kfdtest/scripts/run_kfdtest.sh | 3 ++- tests/kfdtest/src/LinuxOSWrapper.cpp | 18 ++++++++++++++++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 9559cd549a..e694b1a9d4 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -1,5 +1,14 @@ + +# Power management tests +PM_TESTS=\ +"KFDPMTest.SuspendWithActiveProcess:"\ +"KFDPMTest.SuspendWithIdleQueue:"\ +"KFDPMTest.SuspendWithIdleQueueAfterWork:"\ + # Permanent exclusions # These tests are included for debugging, but are not executed in normal execution on any ASIC: +# PM_TESTS need human intervention, so put it here. Developers can run them +# manually through "-p pm" option. # # Fragmentation test fails intermittently on different systems (see KFD-344). It # stresses memory management in unusual ways by having lots of memory allocated but @@ -12,6 +21,7 @@ # clears memory at allocation time. PERMANENT_BLACKLIST_ALL_ASICS=\ "-KFDEventTest.MeasureInterruptConsumption:"\ +"$PM_TESTS:"\ "KFDLocalMemoryTest.Fragmentation:"\ "KFDQMTest.BasicCuMaskingLinear:"\ "RDMATest.GPUDirect:"\ @@ -24,15 +34,11 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # failures can be found # NOTE: If you update this alphabetical listing, add the corresponding JIRA ticket for reference # -# KFDPMTest.SuspendWith[ActiveProcess|IdleQueue|IdleQueueAfterWork] (KFD-326|327|328) # KFDPNPTest.DisableAndCreateQueue (KFD-329) # KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) # KFDQMTest.mGPUShareBO (KFD-334) # KFDHWSTest.* (SWDEV-193035) TEMPORARY_BLACKLIST_ALL_ASICS=\ -"KFDPMTest.SuspendWithActiveProcess:"\ -"KFDPMTest.SuspendWithIdleQueue:"\ -"KFDPMTest.SuspendWithIdleQueueAfterWork:"\ "KFDPNPTest.DisableAndCreateQueue:"\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 9694b7415e..46f3ffaee1 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -51,7 +51,7 @@ printUsage() { echo echo "Gtest arguments will be forwarded to the app" echo - echo "Valid platform options: polaris10, vega10, vega20, all, and so on" + echo "Valid platform options: pm, core, polaris10, vega10, vega20, all, and so on" echo "'all' option runs all tests" return 0 @@ -76,6 +76,7 @@ getFilter() { arcturus ) FILTER="--gtest_filter=$ARCT_TESTS_BLACKLIST" ;; navi10 ) FILTER="--gtest_filter=$NAVI10_TESTS_BLACKLIST" ;; core ) FILTER="--gtest_filter=$CORE_TESTS" ;; + pm ) FILTER="--gtest_filter=$PM_TESTS" ;; all ) FILTER="" ;; *) die "Unsupported platform $platform. Exiting" ;; esac diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/tests/kfdtest/src/LinuxOSWrapper.cpp index fd1e7b8aeb..a8728944c1 100644 --- a/tests/kfdtest/src/LinuxOSWrapper.cpp +++ b/tests/kfdtest/src/LinuxOSWrapper.cpp @@ -87,8 +87,22 @@ HSAuint64 GetSystemTickCountInMicroSec() { } bool SuspendAndWakeUp() { - // TODO: Implement - return false; + printf("Please press any key after the system suspends....\n"); + + // Use "sudo apt install pm-utils" to install the "pm-suspend" + int ret = system("sudo pm-suspend"); + + if (ret == -1) { + printf("The system linux command could not be run!\n"); + return false; + } else { + if (WEXITSTATUS(ret)) { + printf("Use 'sudo apt install pm-utils' to install 'pm-suspend' on Ubuntu\n"); + return false; + } + } + + return true; } void AcquirePrivilege(OS_PRIVILEGE priv) { From 09d192802ab259123f15e7b1e894b3ad45724fb1 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 16 Aug 2019 19:43:35 -0400 Subject: [PATCH 0588/1247] kfdtest: Remove useless test cases Those tests are actually did not function up to its expectation because some underlying functions such as suspend/resume and disable/enable KFD were not implemented. Those interfaces would never be implemented, so delete them. Change-Id: Ib5872ba2f35e307221e43791cda1782c6b6bb4d1 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDPMTest.cpp | 18 ----- tests/kfdtest/src/KFDPMTest.hpp | 6 -- tests/kfdtest/src/KFDPNPTest.cpp | 100 --------------------------- tests/kfdtest/src/KFDPNPTest.hpp | 47 ------------- tests/kfdtest/src/LinuxOSWrapper.cpp | 12 ---- tests/kfdtest/src/OSWrapper.hpp | 5 -- 6 files changed, 188 deletions(-) delete mode 100644 tests/kfdtest/src/KFDPNPTest.cpp delete mode 100644 tests/kfdtest/src/KFDPNPTest.hpp diff --git a/tests/kfdtest/src/KFDPMTest.cpp b/tests/kfdtest/src/KFDPMTest.cpp index 734b32ba51..79b385cf72 100644 --- a/tests/kfdtest/src/KFDPMTest.cpp +++ b/tests/kfdtest/src/KFDPMTest.cpp @@ -27,24 +27,6 @@ #include "PM4Queue.hpp" #include "hsakmt.h" -bool KFDPMTest::m_SetupSuccess = false; - -void KFDPMTest::SetUpTestCase() { - ROUTINE_START - - m_SetupSuccess = false; - AcquirePrivilege(OS_SUSPEND); - - // If AcquirePrivilege fails, it will throw and we will not reach here. - m_SetupSuccess = true; - - ROUTINE_END -} - -void KFDPMTest::TearDownTestCase() { -} - - void KFDPMTest::SetUp() { ROUTINE_START diff --git a/tests/kfdtest/src/KFDPMTest.hpp b/tests/kfdtest/src/KFDPMTest.hpp index 7cc7c85ad6..b8960afefe 100644 --- a/tests/kfdtest/src/KFDPMTest.hpp +++ b/tests/kfdtest/src/KFDPMTest.hpp @@ -33,14 +33,8 @@ class KFDPMTest : public KFDBaseComponentTest { ~KFDPMTest() {} protected: - static void SetUpTestCase(); - static void TearDownTestCase(); - virtual void SetUp(); virtual void TearDown(); - - protected: - static bool m_SetupSuccess; }; #endif // __KFDPMTEST_HPP__ diff --git a/tests/kfdtest/src/KFDPNPTest.cpp b/tests/kfdtest/src/KFDPNPTest.cpp deleted file mode 100644 index fc53f891ab..0000000000 --- a/tests/kfdtest/src/KFDPNPTest.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include "KFDPNPTest.hpp" -#include "KFDTestUtil.hpp" -#include "PM4Queue.hpp" -#include "PM4Packet.hpp" -#include "hsakmt.h" - -bool KFDPNPTest::m_SetupSuccess = false; - -void KFDPNPTest::SetUpTestCase() { - ROUTINE_START - - AcquirePrivilege(OS_DRIVER_OPERATIONS); - - // If AcquirePrivilege fails, it will throw and we will not reach here. - m_SetupSuccess = true; - - ROUTINE_END -} - -void KFDPNPTest::TearDownTestCase() { -} - - -void KFDPNPTest::SetUp() { - ROUTINE_START - - ASSERT_TRUE(m_SetupSuccess); - - KFDBaseComponentTest::SetUp(); - - ROUTINE_END -} - -void KFDPNPTest::TearDown() { - ROUTINE_START - - KFDBaseComponentTest::TearDown(); - - ROUTINE_END -} - -TEST_F(KFDPNPTest, DisableWithActiveProcess) { - TEST_START(TESTPROFILE_RUNALL); - - DisableKfd(); - EnableKfd(); - - TEST_END -} - -TEST_F(KFDPNPTest, DisableAndCreateQueue) { - TEST_START(TESTPROFILE_RUNALL); - - PM4Queue queue; - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - HsaMemoryBuffer destBuffer(PAGE_SIZE, defaultGPUNode); - - ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - - queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuffer.As(), 0x1, 0x2)); - queue.Wait4PacketConsumption(); - - WaitOnValue(&(destBuffer.As()[0]), 0x1); - WaitOnValue(&(destBuffer.As()[1]), 0x2); - - EXPECT_SUCCESS(queue.Destroy()); - - DisableKfd(); - EnableKfd(); - - EXPECT_NE(HSAKMT_STATUS_SUCCESS, queue.Create(defaultGPUNode)) - << "Queue creation should fail after a topology change."; - - TEST_END -} diff --git a/tests/kfdtest/src/KFDPNPTest.hpp b/tests/kfdtest/src/KFDPNPTest.hpp deleted file mode 100644 index e9ce4d619a..0000000000 --- a/tests/kfdtest/src/KFDPNPTest.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef __KFD_PNP_TEST__H__ -#define __KFD_PNP_TEST__H__ - -#include -#include "KFDBaseComponentTest.hpp" - - -class KFDPNPTest : public KFDBaseComponentTest { - public: - KFDPNPTest() {} - ~KFDPNPTest() {} - - protected: - static void SetUpTestCase(); - static void TearDownTestCase(); - - virtual void SetUp(); - virtual void TearDown(); - - protected: - static bool m_SetupSuccess; -}; - -#endif // __KFD_PNP_TEST__H__ diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/tests/kfdtest/src/LinuxOSWrapper.cpp index a8728944c1..e529c2b6dd 100644 --- a/tests/kfdtest/src/LinuxOSWrapper.cpp +++ b/tests/kfdtest/src/LinuxOSWrapper.cpp @@ -105,18 +105,6 @@ bool SuspendAndWakeUp() { return true; } -void AcquirePrivilege(OS_PRIVILEGE priv) { - // TODO: Implement -} - -void DisableKfd() { - // TODO: Implement -} - -void EnableKfd() { - // TODO: Implement -} - bool ReadDriverConfigValue(CONFIG_VALUE config, unsigned int& rValue) { return false; } diff --git a/tests/kfdtest/src/OSWrapper.hpp b/tests/kfdtest/src/OSWrapper.hpp index 480b0c91de..99a7ff5829 100644 --- a/tests/kfdtest/src/OSWrapper.hpp +++ b/tests/kfdtest/src/OSWrapper.hpp @@ -102,11 +102,6 @@ HSAuint64 GetSystemTickCountInMicroSec(); */ bool SuspendAndWakeUp(); -void AcquirePrivilege(OS_PRIVILEGE priv); - -void DisableKfd(); -void EnableKfd(); - bool ReadDriverConfigValue(CONFIG_VALUE config, unsigned int& rValue); bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs); From d6539ddc240b5b33c7dd5230aa9b1caf02c5f6b0 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 4 Sep 2019 19:03:11 -0400 Subject: [PATCH 0589/1247] libhsakmt: Implement HSA_FORCE_ASIC_TYPE to overwrite asic type Force all the GPUs to a certain type, use the below command: HSA_FORCE_ASIC_TYPE="10.1.0 1 gfx1010 14" meaning major.minor.step dgpu asic_name asic_id This will faciliate the cooperation across the teams for bringing up ASICs which reuse existing device IDs. Change-Id: I40fe4c9b46d3ccb3e38ea52250e80e82fb50fb0f Signed-off-by: Yong Zhao --- src/libhsakmt.h | 43 +++++++++++++++++++++++++++++-------------- src/openclose.c | 34 ++++++++++++++++++++++++++++++++-- src/topology.c | 19 ++++++++++--------- 3 files changed, 71 insertions(+), 25 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 98ebadd6e8..2366821415 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -37,6 +37,10 @@ extern unsigned long kfd_open_count; extern pthread_mutex_t hsakmt_mutex; extern bool is_dgpu; +extern int force_asic; +extern char force_asic_name[HSA_PUBLIC_NAME_SIZE]; +extern struct hsa_gfxip_table force_asic_entry; + #undef HSAKMTAPI #define HSAKMTAPI __attribute__((visibility ("default"))) @@ -90,20 +94,31 @@ extern int hsakmt_debug_level; enum asic_family_type { CHIP_KAVERI = 0, - CHIP_HAWAII, - CHIP_CARRIZO, - CHIP_TONGA, - CHIP_FIJI, - CHIP_POLARIS10, - CHIP_POLARIS11, - CHIP_POLARIS12, - CHIP_VEGAM, - CHIP_VEGA10, - CHIP_VEGA12, - CHIP_VEGA20, - CHIP_RAVEN, - CHIP_ARCTURUS, - CHIP_NAVI10, + CHIP_HAWAII, /* 1 */ + CHIP_CARRIZO, /* 2 */ + CHIP_TONGA, /* 3 */ + CHIP_FIJI, /* 4 */ + CHIP_POLARIS10, /* 5 */ + CHIP_POLARIS11, /* 6 */ + CHIP_POLARIS12, /* 7 */ + CHIP_VEGAM, /* 8 */ + CHIP_VEGA10, /* 9 */ + CHIP_VEGA12, /* 10 */ + CHIP_VEGA20, /* 11 */ + CHIP_RAVEN, /* 12 */ + CHIP_ARCTURUS, /* 13 */ + CHIP_NAVI10, /* 14 */ + CHIP_LAST +}; + +struct hsa_gfxip_table { + uint16_t device_id; // Device ID + unsigned char major; // GFXIP Major engine version + unsigned char minor; // GFXIP Minor engine version + unsigned char stepping; // GFXIP Stepping info + unsigned char is_dgpu; // Predicate for dGPU devices + const char *amd_name; // CALName of the device + enum asic_family_type asic_family; // Device family id }; #define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) diff --git a/src/openclose.c b/src/openclose.c index b9754fe57c..b25ded37d9 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -82,7 +82,7 @@ static inline void init_page_size(void) PAGE_SHIFT = ffs(PAGE_SIZE) - 1; } -static void init_vars_from_env(void) +static HSAKMT_STATUS init_vars_from_env(void) { char *envvar; int debug_level; @@ -104,6 +104,34 @@ static void init_vars_from_env(void) envvar = getenv("HSA_ZFB"); if (envvar) zfb_support = atoi(envvar); + + /* Force all the GPUs to a certain type, use the below command: + * export HSA_FORCE_ASIC_TYPE="10.1.0 1 Navi10 14" + * meaning major.minor.step dgpu asic_name asic_id + */ + envvar = getenv("HSA_FORCE_ASIC_TYPE"); + if (envvar) { + uint32_t major, minor, step, dgpu, asic_family; + + if ((sscanf(envvar, "%u.%u.%u %u %63s %u", &major, &minor, &step, + &dgpu, force_asic_name, &asic_family) != 6) + || (major > 63 || minor > 255 || step > 255) + || dgpu > 1 || asic_family >= CHIP_LAST) { + pr_err("HSA_FORCE_ASIC_TYPE %s is invalid\n", envvar); + return HSAKMT_STATUS_ERROR; + } + + force_asic_entry.major = major; + force_asic_entry.minor = minor; + force_asic_entry.stepping = step; + force_asic_entry.is_dgpu = dgpu; + + force_asic_entry.asic_family = asic_family; + + force_asic = 1; + } + + return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) @@ -122,7 +150,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) clear_after_fork(); if (kfd_open_count == 0) { - init_vars_from_env(); + result = init_vars_from_env(); + if (result != HSAKMT_STATUS_SUCCESS) + goto open_failed; fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); diff --git a/src/topology.c b/src/topology.c index 9fe8db4c7d..c686480625 100644 --- a/src/topology.c +++ b/src/topology.c @@ -84,15 +84,13 @@ static const char *supported_processor_vendor_name[] = { static HSAKMT_STATUS topology_take_snapshot(void); static HSAKMT_STATUS topology_drop_snapshot(void); -static struct hsa_gfxip_table { - uint16_t device_id; // Device ID - unsigned char major; // GFXIP Major engine version - unsigned char minor; // GFXIP Minor engine version - unsigned char stepping; // GFXIP Stepping info - unsigned char is_dgpu; // Predicate for dGPU devices - const char *amd_name; // CALName of the device - enum asic_family_type asic_family; -} gfxip_lookup_table[] = { +int force_asic; +char force_asic_name[HSA_PUBLIC_NAME_SIZE]; +struct hsa_gfxip_table force_asic_entry = { + .amd_name = force_asic_name, +}; + +static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Kaveri Family */ { 0x1304, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, { 0x1305, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, @@ -722,6 +720,9 @@ static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id) { uint32_t i, table_size; + if (force_asic) + return &force_asic_entry; + table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); for (i = 0; i < table_size; i++) { if (gfxip_lookup_table[i].device_id == device_id) From 3ecd83e52d6b3e8968baea0c877174238341cabc Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 4 Sep 2019 19:13:34 -0400 Subject: [PATCH 0590/1247] libhsakmt: Support gfx1012 The gfx version item is yet to be added. Change-Id: Ia6c487447e5a5df80c0c12fe150939175068024b Signed-off-by: Yong Zhao --- src/libhsakmt.h | 1 + src/queues.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 2366821415..7f2c0670ea 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -108,6 +108,7 @@ enum asic_family_type { CHIP_RAVEN, /* 12 */ CHIP_ARCTURUS, /* 13 */ CHIP_NAVI10, /* 14 */ + CHIP_NAVI14, /* 15 */ CHIP_LAST }; diff --git a/src/queues.c b/src/queues.c index 5f7384ff4a..e0cca33c58 100644 --- a/src/queues.c +++ b/src/queues.c @@ -146,6 +146,12 @@ const struct device_info navi10_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info navi14_device_info = { + .asic_family = CHIP_NAVI14, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -162,6 +168,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_RAVEN] = &raven_device_info, [CHIP_ARCTURUS] = &arcturus_device_info, [CHIP_NAVI10] = &navi10_device_info, + [CHIP_NAVI14] = &navi14_device_info, }; struct queue { From baa2957895419a4cdb227c8f10dd33a0576d7686 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 4 Sep 2019 19:14:33 -0400 Subject: [PATCH 0591/1247] kfdtest: Add gfx1012 initial test coverage Change-Id: I983d4dd6c17b2eaadd5abf4b71e090a14cace303 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 9 +++++++++ tests/kfdtest/scripts/run_kfdtest.sh | 1 + 2 files changed, 10 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index e694b1a9d4..0ec7160abc 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -164,6 +164,15 @@ NAVI10_TESTS_BLACKLIST=\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.GPUDoorbellWrite" +NAVI14_TESTS_BLACKLIST=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDDBGTest.*:"\ +"KFDExceptionTest.*:"\ +"KFDPerfCountersTest.*:"\ +"KFDQMTest.CreateQueueStressSingleThreaded:"\ +"KFDPNPTest.DisableWithActiveProcess" + # Core tests, used in scenarios like bringup CORE_TESTS=\ "KFDQMTest.CreateDestroyCpQueue:"\ diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 46f3ffaee1..1277225fa6 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -75,6 +75,7 @@ getFilter() { raven ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; arcturus ) FILTER="--gtest_filter=$ARCT_TESTS_BLACKLIST" ;; navi10 ) FILTER="--gtest_filter=$NAVI10_TESTS_BLACKLIST" ;; + navi14 ) FILTER="--gtest_filter=$NAVI14_TESTS_BLACKLIST" ;; core ) FILTER="--gtest_filter=$CORE_TESTS" ;; pm ) FILTER="--gtest_filter=$PM_TESTS" ;; all ) FILTER="" ;; From ec1375ac66d6b9ed2fbbe761fd87bd06f3536572 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 9 Sep 2019 15:57:22 -0400 Subject: [PATCH 0592/1247] kfdtest: Automate the gtest filter name generation This will emilinate the need of updating the run_kfdtest.sh every time a new platform is added. Change-Id: I584d65b462de36a685fa2d29d43962078ba511dc Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 67 ++++++++++++++++----------- tests/kfdtest/scripts/run_kfdtest.sh | 35 ++++++-------- 2 files changed, 54 insertions(+), 48 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 0ec7160abc..360cbc92e4 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -1,13 +1,27 @@ +declare -A FILTER # Power management tests -PM_TESTS=\ +FILTER[pm]=\ "KFDPMTest.SuspendWithActiveProcess:"\ "KFDPMTest.SuspendWithIdleQueue:"\ -"KFDPMTest.SuspendWithIdleQueueAfterWork:"\ +"KFDPMTest.SuspendWithIdleQueueAfterWork" + + +# Core tests, used in scenarios like bringup +FILTER[core]=\ +"KFDQMTest.CreateDestroyCpQueue:"\ +"KFDQMTest.SubmitPacketCpQueue:"\ +"KFDQMTest.MultipleCpQueues:"\ +"KFDQMTest.CreateDestroySdmaQueue:"\ +"KFDQMTest.SubmitPacketSdmaQueue:"\ +"KFDQMTest.MultipleSdmaQueues:"\ +"KFDCWSRTest.BasicTest:"\ +"KFDEventTest.SignalEvent" + # Permanent exclusions # These tests are included for debugging, but are not executed in normal execution on any ASIC: -# PM_TESTS need human intervention, so put it here. Developers can run them +# FILTER[pm] need human intervention, so put it here. Developers can run them # manually through "-p pm" option. # # Fragmentation test fails intermittently on different systems (see KFD-344). It @@ -21,7 +35,7 @@ PM_TESTS=\ # clears memory at allocation time. PERMANENT_BLACKLIST_ALL_ASICS=\ "-KFDEventTest.MeasureInterruptConsumption:"\ -"$PM_TESTS:"\ +"${FILTER[pm]}:"\ "KFDLocalMemoryTest.Fragmentation:"\ "KFDQMTest.BasicCuMaskingLinear:"\ "RDMATest.GPUDirect:"\ @@ -80,7 +94,7 @@ KV_QUEUE_BLACKLIST=\ # KFDQMTest.CpuWriteCoherence fails. 0 dwordsAvailable (KFD-338) # KFDMemoryTest.MemoryRegister fails on SDMA queue creation (KFD-337) -KV_TESTS_BLACKLIST=\ +FILTER[kaveri]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "$KV_QUEUE_BLACKLIST:"\ @@ -90,13 +104,13 @@ KV_TESTS_BLACKLIST=\ # KFDLocalMemoryTest.BasicTest is failing intermittently (KFD-368) # KFDMemoryTest.BigBufferStressTest was failing intermittently on 4.9 and hangs when executed twice (KFD-312) # KFDQMTest.GPUDoorbellWrite fails on Hawaii. Could be HW-related (KFD-342) -HI_TESTS_BLACKLIST=\ +FILTER[hawaii]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDLocalMemoryTest.BasicTest:"\ "KFDMemoryTest.BigBufferStressTest:"\ "KFDQMTest.GPUDoorbellWrite" -CZ_TESTS_BLACKLIST=\ +FILTER[carrizo]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDExceptionTest.InvalidWriteAddress" @@ -104,7 +118,7 @@ CZ_TESTS_BLACKLIST=\ # KFDPerfCountersTest.*Trace fail (KFD-339) # KFDMemoryTest.QueryPointerInfo/MemoryRegister* (KFD-341) # The remaining tests listed here fail on map memory to GPU with a VA conflict (KFD-340) -TONGA_TESTS_BLACKLIST=\ +FILTER[tonga]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDCWSRTest.BasicTest:"\ @@ -112,19 +126,27 @@ TONGA_TESTS_BLACKLIST=\ "KFDQMTest.OverSubscribeCpQueues" # Since Navi10 was merged, the PM4Event test takes 6min to run -FIJI_TESTS_BLACKLIST=\ +FILTER[fiji]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.PM4EventInterrupt:"\ "$SDMA_BLACKLIST" -POLARIS_TESTS_BLACKLIST=\ +FILTER[polaris10]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST" + +FILTER[polaris11]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST" + +FILTER[polaris12]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST" # KFDIPCTest.BasicTest (ROCMOPS-459) .CMABasicTest (ROCMOPS-460) .CrossMemoryAttachTest (ROCMOPS-461) # KFDEvictTest.BurstyTest (ROCMOPS-464) # KFDQMTest.MultipleSdmaQueues (ROCMOPS-463) -VEGA10_TESTS_BLACKLIST=\ +FILTER[vega10]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDIPCTest.BasicTest:"\ "KFDEvictTest.BurstyTest:"\ @@ -132,12 +154,12 @@ VEGA10_TESTS_BLACKLIST=\ "KFDIPCTest.CrossMemoryAttachTest:"\ "KFDQMTest.MultipleSdmaQueues" -VEGA12_TESTS_BLACKLIST=\ +FILTER[vega12]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST"\ # KFDEvictTest.BurstyTest (ROCMOPS-464) -VEGA20_TESTS_BLACKLIST=\ +FILTER[vega20]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDEvictTest.BurstyTest:"\ @@ -145,17 +167,17 @@ VEGA20_TESTS_BLACKLIST=\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.GPUDoorbellWrite" -RAVEN_TESTS_BLACKLIST=\ +FILTER[raven]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.QueueLatency:"\ "KFDQMTest.SdmaEventInterrupt" # KFDExceptionTest.* (KFD-435) -ARCT_TESTS_BLACKLIST=\ +FILTER[arcturus]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm" -NAVI10_TESTS_BLACKLIST=\ +FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDEvictTest.BurstyTest:"\ @@ -164,7 +186,7 @@ NAVI10_TESTS_BLACKLIST=\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.GPUDoorbellWrite" -NAVI14_TESTS_BLACKLIST=\ +FILTER[navi14]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ @@ -172,14 +194,3 @@ NAVI14_TESTS_BLACKLIST=\ "KFDPerfCountersTest.*:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ "KFDPNPTest.DisableWithActiveProcess" - -# Core tests, used in scenarios like bringup -CORE_TESTS=\ -"KFDQMTest.CreateDestroyCpQueue:"\ -"KFDQMTest.SubmitPacketCpQueue:"\ -"KFDQMTest.MultipleCpQueues:"\ -"KFDQMTest.CreateDestroySdmaQueue:"\ -"KFDQMTest.SubmitPacketSdmaQueue:"\ -"KFDQMTest.MultipleSdmaQueues:"\ -"KFDCWSRTest.BasicTest:"\ -"KFDEventTest.SignalEvent" diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 1277225fa6..7493dc00f1 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -40,7 +40,8 @@ printUsage() { echo echo "Options:" echo " -p , --platform Only run tests that"\ - "pass on the specified platform" + "pass on the specified platform. Usually you"\ + "don't need this option" echo " -g , --gdb Run in debugger" echo " -n , --node NodeId to test. If"\ "not specified test will be run on all nodes" @@ -61,27 +62,21 @@ printUsage() { # pass in Multi GPU environment, this flag can be removed # param - Platform. getFilter() { +# For regular platforms such as vega10, this will automatically generate +# the valid variable BLACKLIST based on the variable platform. local platform=$1; + case "$platform" in - carrizo ) FILTER="--gtest_filter=$CZ_TESTS_BLACKLIST" ;; - hawaii ) FILTER="--gtest_filter=$HI_TESTS_BLACKLIST" ;; - kaveri ) FILTER="--gtest_filter=$KV_TESTS_BLACKLIST" ;; - tonga ) FILTER="--gtest_filter=$TONGA_TESTS_BLACKLIST" ;; - fiji ) FILTER="--gtest_filter=$FIJI_TESTS_BLACKLIST" ;; - polaris10 | polaris11 | polaris12 ) FILTER="--gtest_filter=$POLARIS_TESTS_BLACKLIST" ;; - vega10 ) FILTER="--gtest_filter=$VEGA10_TESTS_BLACKLIST" ;; - vega12 ) FILTER="--gtest_filter=$VEGA12_TESTS_BLACKLIST" ;; - vega20 ) FILTER="--gtest_filter=$VEGA20_TESTS_BLACKLIST" ;; - raven ) FILTER="--gtest_filter=$RAVEN_TESTS_BLACKLIST" ;; - arcturus ) FILTER="--gtest_filter=$ARCT_TESTS_BLACKLIST" ;; - navi10 ) FILTER="--gtest_filter=$NAVI10_TESTS_BLACKLIST" ;; - navi14 ) FILTER="--gtest_filter=$NAVI14_TESTS_BLACKLIST" ;; - core ) FILTER="--gtest_filter=$CORE_TESTS" ;; - pm ) FILTER="--gtest_filter=$PM_TESTS" ;; - all ) FILTER="" ;; - *) die "Unsupported platform $platform. Exiting" ;; + all ) gtestFilter="" ;; + * ) + if [ -z "${FILTER[$platform]}" ]; then + echo "Unsupported platform $platform. Exiting" + exit 1 + fi + + gtestFilter="--gtest_filter=${FILTER[$platform]}" + ;; esac - echo "$FILTER" } TOPOLOGY_SYSFS_DIR=/sys/devices/virtual/kfd/kfd/topology/nodes @@ -136,7 +131,7 @@ runKfdTest() { nodeName="$PLATFORM" fi - gtestFilter=$(getFilter $nodeName) + getFilter $nodeName if [ "$RUN_IN_DOCKER" == "true" ]; then if [ "$NODE" == "" ]; then From 4da09813a31529c80eedce92dd333b6fe716ffb5 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 10 Sep 2019 11:49:56 -0400 Subject: [PATCH 0593/1247] libhsakmt: fix mbind failed on docker Docker seccomp by default blocks mbind system call, so mbind return failed on docker. thunk should not fail this otherwise application cannot allocate system memory on docker. Use pr_warn_once and pr_err_once to avoid duplicate same error messages Change-Id: I61a7c0e4abaa3dcfe7abf2ea48db90f669f9638a Signed-off-by: Philip Yang --- src/fmm.c | 38 +++++++++++++++++++++++++------------- src/libhsakmt.h | 16 ++++++++++++++++ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ea798d1ae2..e7c83d3fd3 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1455,11 +1455,12 @@ static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, } static int bind_mem_to_numa(uint32_t node_id, void *mem, - uint64_t MemorySizeInBytes, HsaMemFlags flags) + uint64_t SizeInBytes, HsaMemFlags flags) { int mode = MPOL_F_STATIC_NODES; struct bitmask *node_mask; int num_node; + long r; if (numa_available() == -1) return 0; @@ -1472,22 +1473,33 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, return 0; } - if (num_node > 1) { - node_mask = numa_bitmask_alloc(num_node); - if (!node_mask) - return -ENOMEM; + if (num_node <= 1) + return 0; - numa_bitmask_setbit(node_mask, node_id); - mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; - if (mbind(mem, MemorySizeInBytes, mode, node_mask->maskp, - num_node + 1, 0)) { - pr_warn("Failed to set NUMA policy for %p\n", mem); + node_mask = numa_bitmask_alloc(num_node); + if (!node_mask) + return -ENOMEM; - numa_bitmask_free(node_mask); - return -EFAULT; + numa_bitmask_setbit(node_mask, node_id); + mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; + r = mbind(mem, SizeInBytes, mode, node_mask->maskp, num_node + 1, 0); + numa_bitmask_free(node_mask); + + if (r) { + pr_warn_once("Failed to set NUMA policy for %p: %s\n", mem, + strerror(errno)); + + /* If applcation is running inside docker, still return + * ok because docker seccomp blocks mbind by default, + * otherwise application cannot allocate system memory. + */ + if (errno == EPERM) { + pr_err_once("mbind is blocked by seccomp\n"); + + return 0; } - numa_bitmask_free(node_mask); + return -EFAULT; } return 0; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 7f2c0670ea..63e3efa0f3 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -91,6 +91,22 @@ extern int hsakmt_debug_level; hsakmt_print(HSAKMT_DEBUG_LEVEL_INFO, fmt, ##__VA_ARGS__) #define pr_debug(fmt, ...) \ hsakmt_print(HSAKMT_DEBUG_LEVEL_DEBUG, fmt, ##__VA_ARGS__) +#define pr_err_once(fmt, ...) \ +({ \ + static bool __print_once; \ + if (!__print_once) { \ + __print_once = true; \ + pr_err(fmt, ##__VA_ARGS__); \ + } \ +}) +#define pr_warn_once(fmt, ...) \ +({ \ + static bool __print_once; \ + if (!__print_once) { \ + __print_once = true; \ + pr_warn(fmt, ##__VA_ARGS__); \ + } \ +}) enum asic_family_type { CHIP_KAVERI = 0, From 42392f093ff6ec885ba003289e8bf37befde8341 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 13 Sep 2019 16:04:36 -0400 Subject: [PATCH 0594/1247] libhsakmt: handle NUMA system with no memory on node 0 on NUMA system, node 0 may have no memory, application pass node id 0 to hsaKmtAllocMemory will fail because mbind to specify the allocation from node 0 return EINVAL. Add new flag NoNUMABind for application to pass it to hsaKmtAllocMemory to skip mbind. hsaKmtCreateEvent and hsaKmtCreateQueue specify the new flag NoNUMABind to allocate system memory for event page and CWSR area, don't bind the system memory to a specific NUMA node. Change-Id: I854e5a57502c7807c4c5ff2e441d499ae515c309 Signed-off-by: Philip Yang --- include/hsakmttypes.h | 3 ++- src/fmm.c | 7 +++++-- src/queues.c | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 5d6df38f95..5ac51c2208 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -533,7 +533,8 @@ typedef struct _HsaMemFlags // The KFD will ensure that the memory returned is allocated in the optimal memory location // and optimal alignment requirements unsigned int FixedAddress : 1; // Allocate memory at specified virtual address. Fail if address is not free. - unsigned int Reserved : 16; + unsigned int NoNUMABind: 1; // Don't bind system memory to a specific NUMA node + unsigned int Reserved : 15; } ui32; HSAuint32 Value; diff --git a/src/fmm.c b/src/fmm.c index e7c83d3fd3..a3f6e7b6a2 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1397,7 +1397,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, flags.Value = 0; flags.ui32.NonPaged = 1; flags.ui32.HostAccess = 1; - flags.ui32.Reserved = 0xBe11; + flags.ui32.Reserved = 0xBe1; pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->flags = flags.Value; @@ -1462,10 +1462,13 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, int num_node; long r; + if (flags.ui32.NoNUMABind) + return 0; + if (numa_available() == -1) return 0; - num_node = numa_num_task_nodes(); + num_node = numa_max_node(); /* Ignore binding requests to invalid nodes IDs */ if (node_id >= (unsigned)num_node) { diff --git a/src/queues.c b/src/queues.c index e0cca33c58..8505bdbb1a 100644 --- a/src/queues.c +++ b/src/queues.c @@ -424,6 +424,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; flags.ui32.CoarseGrain = DeviceLocal; + flags.ui32.NoNUMABind = 1; size = ALIGN_UP(size, align); From 69d8f2d734052937776eff4199919e0bed24334f Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 13 Sep 2019 16:18:00 -0400 Subject: [PATCH 0595/1247] kfdtest: use flag NoNUMABind to allocate system memory Allocate system memory from node id 0 will fail on NUMA system which has no memory on node 0. Change to use new flag NoNUMABind to allocate system memory from NUMA nodes which have free memory. Change-Id: I8ef9ca28fc2ab5dd31d07a2d3eaf1d5886e798a0 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDMemoryTest.cpp | 3 +++ tests/kfdtest/src/KFDQMTest.cpp | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index cd73a0f073..c261accca0 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -364,6 +364,7 @@ TEST_F(KFDMemoryTest , MapMemoryToGPU) { TEST_F(KFDMemoryTest, InvalidMemoryPointerAlloc) { TEST_START(TESTPROFILE_RUNALL) + m_MemoryFlags.ui32.NoNUMABind = 1; EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtAllocMemory(0 /* system */, PAGE_SIZE, m_MemoryFlags, NULL)); TEST_END @@ -1685,6 +1686,8 @@ TEST_F(KFDMemoryTest, CheckZeroInitializationSysMem) { unsigned int offset = 257; // a constant offset, should be smaller than 512. unsigned int size = sysBufSize / sizeof(*pDb); + m_MemoryFlags.ui32.NoNUMABind = 1; + while (count--) { ret = hsaKmtAllocMemory(0 /* system */, sysBufSize, m_MemoryFlags, reinterpret_cast(&pDb)); diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 9b036193d0..20dbf598e0 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1486,6 +1486,7 @@ TEST_F(KFDQMTest, P2PTest) { memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 1; memFlags.ui32.NonPaged = 1; + memFlags.ui32.NoNUMABind = 1; unsigned int end = size / sizeof(HSAuint32) - 1; /* 1. Allocate a system buffer and allow the access to GPUs */ From 71cf3cf5d397ced49bf4eb0536906e3fcbc2eab5 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 16 Sep 2019 16:22:55 -0400 Subject: [PATCH 0596/1247] libhsakmt: correct number of NUMA nodes calculation numa_max_node() return the highest node number available on the current system, number of NUMA nodes should be numa_max_node() + 1. Change-Id: I20a6c17af071e73e853cb5ea6d0304c8aca52681 Signed-off-by: Philip Yang --- src/fmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index a3f6e7b6a2..e98bf01c36 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1468,7 +1468,7 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, if (numa_available() == -1) return 0; - num_node = numa_max_node(); + num_node = numa_max_node() + 1; /* Ignore binding requests to invalid nodes IDs */ if (node_id >= (unsigned)num_node) { From a1e399a3ff068d8c2630a297c75c478b4c3d72a1 Mon Sep 17 00:00:00 2001 From: shaoyunl Date: Tue, 24 Sep 2019 18:47:23 -0400 Subject: [PATCH 0597/1247] Thunk : Add gfx1011 support from thunk side Change-Id: I6b202b75fc1ad0e69576a35a6a3e499818137e04 Signed-off-by: shaoyunl --- src/libhsakmt.h | 3 ++- src/queues.c | 7 +++++++ src/topology.c | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 63e3efa0f3..d0e95ae170 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -124,7 +124,8 @@ enum asic_family_type { CHIP_RAVEN, /* 12 */ CHIP_ARCTURUS, /* 13 */ CHIP_NAVI10, /* 14 */ - CHIP_NAVI14, /* 15 */ + CHIP_NAVI12, /* 15 */ + CHIP_NAVI14, /* 16 */ CHIP_LAST }; diff --git a/src/queues.c b/src/queues.c index 8505bdbb1a..8ee165cf0c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -146,6 +146,12 @@ const struct device_info navi10_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info navi12_device_info = { + .asic_family = CHIP_NAVI12, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + const struct device_info navi14_device_info = { .asic_family = CHIP_NAVI14, .eop_buffer_size = 4096, @@ -168,6 +174,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_RAVEN] = &raven_device_info, [CHIP_ARCTURUS] = &arcturus_device_info, [CHIP_NAVI10] = &navi10_device_info, + [CHIP_NAVI12] = &navi12_device_info, [CHIP_NAVI14] = &navi14_device_info, }; diff --git a/src/topology.c b/src/topology.c index c686480625..6f1e490a78 100644 --- a/src/topology.c +++ b/src/topology.c @@ -226,6 +226,10 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x7318, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x731A, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x731F, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, + /* Navi12 */ + { 0x7360, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, + { 0x7362, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, + }; /* information from /proc/cpuinfo */ From 5a09880620c0d974273011d3b7d6add697320e20 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 30 Sep 2019 14:29:39 -0400 Subject: [PATCH 0598/1247] libhsakmt: fix typo in error message When fail to get CPU dirs from //sys/devices/system/node/nodeX directory, the error message should print node_dir, not path. Change-Id: If76a51918c8dd55fa6605a62f3d29f9efc6fadb3 Signed-off-by: Amber Lin --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 6f1e490a78..327315429a 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1193,7 +1193,7 @@ static int topology_create_temp_cpu_cache_list(int node, /* Other than cpuY folders, this dir also has cpulist and cpumap */ max_cpus = num_subdirs(node_dir, "cpu"); if (max_cpus <= 0) { - pr_err("Fail to get cpu* dirs under %s\n", path); + pr_err("Fail to get cpu* dirs under %s\n", node_dir); goto exit; } From 63eb3948beefcb41c6d28470bb5811c667896de1 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 9 Sep 2019 11:02:56 -0500 Subject: [PATCH 0599/1247] More parameter check in HsaMemoryBuffer constructor if parameter "zero" is set, check buffer host access. Change-Id: I9893062726fc240777405167a638cbea18fdf559 Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 2 +- tests/kfdtest/src/KFDTestUtil.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index 0af52e9d8e..4c86594ab9 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -128,7 +128,7 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer SysBufferA(BufferSize, defaultGPUNode, false); HsaMemoryBuffer SysBufferB(BufferSize, defaultGPUNode, true); - HsaMemoryBuffer LocalBuffer(BufferSize, defaultGPUNode, true, true); + HsaMemoryBuffer LocalBuffer(BufferSize, defaultGPUNode, false, true); SysBufferA.Fill(0x01010101); diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index d1e398fa1b..9383c31002 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -197,6 +197,9 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b if (isReadOnly) m_Flags.ui32.ReadOnly = 1; + if (zero) + EXPECT_EQ(m_Flags.ui32.HostAccess, 1); + EXPECT_SUCCESS(hsaKmtAllocMemory(m_Node, m_Size, m_Flags, &m_pBuf)); if (is_dgpu()) { if (map_specific_gpu) From 1ac8b5538cf707ab8a5276ae6fe8a316b0855ba9 Mon Sep 17 00:00:00 2001 From: Cole Nelson Date: Wed, 2 Oct 2019 14:06:03 -0700 Subject: [PATCH 0600/1247] kfdtest.exclude: blacklist KFDEvictTest.BurstyTest for gfx908 (gfx908) Change-Id: I5e1ac25c066ee20e34e102043d27eeab73313c6f Signed-off-by: Cole Nelson --- tests/kfdtest/scripts/kfdtest.exclude | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 360cbc92e4..3851d52fd1 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -173,9 +173,11 @@ FILTER[raven]=\ "KFDQMTest.SdmaEventInterrupt" # KFDExceptionTest.* (KFD-435) +# KFDEvictTest.BurstyTest (KFD-425) FILTER[arcturus]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDExceptionTest.FaultStorm" +"KFDExceptionTest.FaultStorm:"\ +"KFDEvictTest.BurstyTest" FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From 51388973a1bd0f4ad5cb29ec27c76eef380e0094 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 5 Sep 2019 12:47:16 -0500 Subject: [PATCH 0601/1247] Add gfx908 to asic family Change-Id: I838aa4be45ddfb34c5d36c519e28b4218fc32ba4 Signed-off-by: Oak Zeng --- tests/kfdtest/src/IsaGenerator.cpp | 1 + tests/kfdtest/src/KFDTestFlags.hpp | 1 + tests/kfdtest/src/KFDTestUtil.cpp | 2 ++ 3 files changed, 4 insertions(+) diff --git a/tests/kfdtest/src/IsaGenerator.cpp b/tests/kfdtest/src/IsaGenerator.cpp index 10cd994028..9c7376a0a4 100644 --- a/tests/kfdtest/src/IsaGenerator.cpp +++ b/tests/kfdtest/src/IsaGenerator.cpp @@ -91,6 +91,7 @@ IsaGenerator* IsaGenerator::Create(unsigned int familyId) { return new IsaGenerator_Gfx8; case FAMILY_AI: case FAMILY_RV: + case FAMILY_AR: return new IsaGenerator_Gfx9; case FAMILY_NV: return new IsaGenerator_Gfx10; diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index 1514695890..921b8bc832 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -58,6 +58,7 @@ enum KfdFamilyId { FAMILY_CZ, // Carrizo, Nolan, Amur FAMILY_AI, // Arctic Islands FAMILY_RV, // Raven + FAMILY_AR, // Arcturus FAMILY_NV, // Navi10 }; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 9383c31002..41d13f9e9a 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -115,6 +115,8 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { familyId = FAMILY_AI; if (props->EngineId.ui32.Stepping == 2) familyId = FAMILY_RV; + if (props->EngineId.ui32.Stepping == 8) + familyId = FAMILY_AR; break; case 10: familyId = FAMILY_NV; From d7c53bb1faec596d4a9964aa8491684cad5a718b Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 5 Sep 2019 10:40:27 -0500 Subject: [PATCH 0602/1247] Test new RW mtype for gfx908 Change-Id: Ia859c8f2e3c486f119772231a2d887f6783caf36 Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDMemoryTest.cpp | 178 ++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index c261accca0..47eca580e3 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2065,3 +2065,181 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { TEST_END } + +/* Test should only run on Arcturus series which has the new RW mtype + * Map a local VRAM with RW mtype (coarse grain for upper layer), + * read it locally to cache it and write with local SDMA, remote devices( + * CPU or Remote GPU shader connected with PCIe or XGMI), + * then read again. The second read should get back what SDMA wrote, + * since the cache should be invalidated on write and second read + * should go to physical VRAM instead of cache. + */ +TEST_F(KFDMemoryTest, CacheInvalidateOnSdmaWrite) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); + volatile HSAuint32 *tmp = tmpBuffer.As(); + const int dwLocation = 100; + + if (m_FamilyId != FAMILY_AR) { + LOG() << "Skipping test: Test requires arcturus series asics." << std::endl; + return; + } + + HsaMemoryBuffer buffer(PAGE_SIZE, defaultGPUNode, false/*zero*/, true/*local*/, false/*exec*/); + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + buffer.Fill(0, sdmaQueue, 0, PAGE_SIZE); + sdmaQueue.PlacePacket(SDMAWriteDataPacket(sdmaQueue.GetFamilyId(), buffer.As(), 0x5678)); + + /* Read buffer from shader to fill cache */ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); + dispatch.Submit(queue); + + /* Delay 100ms to make sure shader executed*/ + Delay(100); + + /* SDMA writes to buffer. Shader should get what sdma writes and quits*/ + sdmaQueue.SubmitPacket(); + sdmaQueue.Wait4PacketConsumption(); + + /* Check test result*/ + dispatch.Sync(); + EXPECT_EQ(buffer.IsPattern(dwLocation*sizeof(int), 0x5678, sdmaQueue, tmp), true); + + // Clean up + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); + + TEST_END +} + +TEST_F(KFDMemoryTest, CacheInvalidateOnCPUWrite) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + if (m_FamilyId != FAMILY_AR) { + LOG() << "Skipping test: Test requires arcturus series asics." << std::endl; + return; + } + + if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPUNode)) { + LOG() << "Skipping test: Test requires a large bar GPU." << std::endl; + return; + } + + int *buffer; + HsaMemFlags memFlags = {0}; + /* Host accessible vram */ + memFlags.ui32.HostAccess = 1; + memFlags.ui32.NonPaged = 1; + memFlags.ui32.CoarseGrain = 1; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, memFlags, reinterpret_cast(&buffer))); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(buffer, PAGE_SIZE, NULL)); + *buffer = 0; + + /* Read buffer from shader to fill cache */ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(buffer, buffer+100); + dispatch.Submit(queue); + + /* Delay 100ms to make sure shader executed*/ + Delay(100); + + /* CPU writes to buffer. Shader should get what CPU writes and quits*/ + *buffer = 0x5678; + + /* Check test result*/ + dispatch.Sync(); + EXPECT_EQ(buffer[100], 0x5678); + + // Clean up + EXPECT_SUCCESS(hsaKmtFreeMemory(buffer, PAGE_SIZE)); + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDMemoryTest, CacheInvalidateOnRemoteWrite) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); + volatile HSAuint32 *tmp = tmpBuffer.As(); + const int dwLocation = 100; + const int dwLocation1 = 50; + + if (m_FamilyId != FAMILY_AR) { + LOG() << "Skipping test: Test requires arcturus series asics." << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: At least two GPUs are required." << std::endl; + return; + } + + HSAuint32 nondefaultNode; + for (unsigned i = 0; i < gpuNodes.size(); i++) { + if (gpuNodes.at(i) != defaultGPUNode) { + nondefaultNode = gpuNodes.at(i); + break; + } + } + + HsaMemoryBuffer buffer(PAGE_SIZE, defaultGPUNode, false/*zero*/, true/*local*/, false/*exec*/); + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + buffer.Fill(0, sdmaQueue, 0, PAGE_SIZE); + + /* Read buffer from shader to fill cache */ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); + dispatch.Submit(queue); + + /* Delay 100ms to make sure shader executed*/ + Delay(100); + + /* Using a remote shader to copy data from dwLocation1 to the beginning of the buffer. + * Local shader should get what remote writes and quits + */ + PM4Queue queue1; + ASSERT_SUCCESS(queue1.Create(nondefaultNode)); + buffer.Fill(0x5678, sdmaQueue, dwLocation1*sizeof(int), 4); + HsaMemoryBuffer isaBuffer1(PAGE_SIZE, nondefaultNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->GetCopyDwordIsa(isaBuffer1); + Dispatch dispatch1(isaBuffer1); + dispatch1.SetArgs(buffer.As()+dwLocation1, buffer.As()); + dispatch1.Submit(queue1); + dispatch1.Sync(g_TestTimeOut); + + /* Check test result*/ + dispatch.Sync(); + EXPECT_EQ(buffer.IsPattern(dwLocation*sizeof(int), 0x5678, sdmaQueue, tmp), true); + + // Clean up + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue1.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); + + TEST_END +} From dadbbbb03c6065180e19372005c8f468a137a078 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Wed, 2 Oct 2019 15:43:41 -0500 Subject: [PATCH 0603/1247] Disable KFDMemoryTest.CacheInvalidateOnRemoteWrite temporarily This is some data fabric/vbios issue that causing system hard hang while running this test. Will enable it after the HW/vibos fix. Change-Id: Ic0753c2d92e9e4863c310da9a595b2af302f17f8 Signed-off-by: Oak Zeng --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 3851d52fd1..286a4962e8 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -57,6 +57,7 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ "KFDQMTest.SdmaEventInterrupt:"\ +"KFDMemoryTest.CacheInvalidateOnRemoteWrite:"\ "KFDDBGTest.BasicDebuggerSuspendResume:"\ "KFDHWSTest.*" From da789a2584079a112628861a21456d7f2424cd50 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 8 Oct 2019 16:33:51 -0500 Subject: [PATCH 0604/1247] Fix memory map issue in KFDMemoryTest.CacheInvalidationOnRemoteWrite The memory need to be mapped for both local and remote GPU access Change-Id: I4aeaffc0851b6107fc91e9eaa6150764b06f5ca9 Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDMemoryTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 47eca580e3..10426392ed 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2203,6 +2203,7 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnRemoteWrite) { } HsaMemoryBuffer buffer(PAGE_SIZE, defaultGPUNode, false/*zero*/, true/*local*/, false/*exec*/); + buffer.MapMemToNodes(&nondefaultNode, 1); SDMAQueue sdmaQueue; ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); buffer.Fill(0, sdmaQueue, 0, PAGE_SIZE); From b48f7d6ea3b8da08a2b236f3f987dc0d4948d858 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Thu, 10 Oct 2019 12:48:30 -0400 Subject: [PATCH 0605/1247] kfdtest: Disable kfd debugger tests on gfx10 The KFD debugger is only supported on gfx9 platforms, so we need to restrict it from running on gfx10 platforms until it is supported. Change-Id: I500f0e20fda71021f2cce70a67fc8d9d042209fe Signed-off-by: Philip Cox --- tests/kfdtest/src/KFDDBGTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index edd0993474..ec3a134a80 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -284,7 +284,7 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI) { + if (m_FamilyId >= FAMILY_AI && m_FamilyId <= FAMILY_AR) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -372,7 +372,7 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI) { + if (m_FamilyId >= FAMILY_AI && m_FamilyId <= FAMILY_AR) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); HSAint32 PollFd; @@ -520,7 +520,7 @@ static void ExitVMFaultQueryChild(std::string errMsg, TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI) { + if (m_FamilyId >= FAMILY_AI && m_FamilyId <= FAMILY_AR) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; From 35d56297d3709415c71a85aef16a9a9c3c5f8069 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Mon, 7 Oct 2019 10:56:09 -0400 Subject: [PATCH 0606/1247] Add kfd debugger version support To check the KFD debugger API support, we need to be able to check the major/minor version of the kfd debugger version, so we need to expose this function from the kernel. Change-Id: I8a3dc617607e2efa9e65306d08b8583b8b1a2172 Signed-off-by: Philip Cox --- include/linux/kfd_ioctl.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 89a18cda15..66c72c9f8d 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -28,6 +28,8 @@ #define KFD_IOCTL_MAJOR_VERSION 1 #define KFD_IOCTL_MINOR_VERSION 2 +#define KFD_IOCTL_DBG_MAJOR_VERSION 0 +#define KFD_IOCTL_DBG_MINOR_VERSION 2 struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ @@ -252,6 +254,14 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 6 +/* KFD_IOC_DBG_TRAP_GET_VERSION: + * prt: unsused + * data1: major version (OUT) + * data2: minor version (OUT) + * data3: unused + */ +#define KFD_IOC_DBG_TRAP_GET_VERSION 8 + struct kfd_ioctl_dbg_trap_args { __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ __u32 pid; /* to KFD */ From dbbd189b331b236fa560d25a2133e02ce57bde16 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Mon, 7 Oct 2019 11:25:24 -0400 Subject: [PATCH 0607/1247] Add functions to get the kfd debugger version info To support adding new features to the kfd debugger, and not break functionality, we need to be able to check the kfd debugger support version info from the kernel. Change-Id: Icd88e4edab8430c35eaed588e62d892c1b5c62ec Signed-off-by: Philip Cox --- include/hsakmt.h | 30 ++++++++++++++++++++++++++++ src/debug.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++- src/libhsakmt.ver | 2 ++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 49d255a868..d5af173b30 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -917,6 +917,36 @@ hsaKmtSetWaveLaunchMode( HSA_DBG_WAVE_LAUNCH_MODE WaveLaunchMode //IN ); +/** + * Get the major and minor version of the kernel debugger support. + * + * Returns: + * - HSAKMT_STATUS_SUCCESS if successful. + * + * - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. + * + * - HSAKMT_STATUS_NOT_SUPPORTED if debug trap not supported for NodeId. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetKernelDebugTrapVersionInfo( + HSAuint32 *Major, //Out + HSAuint32 *Minor //Out + ); + +/** + * Get the major and minor version of the thunk debugger support. +*/ +void +HSAKMTAPI +hsaKmtGetThunkDebugTrapVersionInfo( + HSAuint32 *Major, //Out + HSAuint32 *Minor //Out + ); + + + + /** Set a debug memory access watch point. A memory access of the kind specified by WatchMode to an matching address will cause the trap diff --git a/src/debug.c b/src/debug.c index f111d7ff75..6d2f7b8773 100644 --- a/src/debug.c +++ b/src/debug.c @@ -286,7 +286,8 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, CHECK_KFD_OPEN(); if (op == KFD_IOC_DBG_TRAP_NODE_SUSPEND || - op == KFD_IOC_DBG_TRAP_NODE_RESUME) { + op == KFD_IOC_DBG_TRAP_NODE_RESUME || + op == KFD_IOC_DBG_TRAP_GET_VERSION) { if (NodeId != INVALID_NODEID) return HSAKMT_STATUS_INVALID_HANDLE; @@ -596,3 +597,50 @@ hsaKmtQueryDebugEvent( return result; } +/** + * Get the major and minor version of the kernel debugger support. + * + * Returns: + * - HSAKMT_STATUS_SUCCESS if successful. + * + * - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. + * + * - HSAKMT_STATUS_NOT_SUPPORTED if debug trap not supported for NodeId. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetKernelDebugTrapVersionInfo( + HSAuint32 *Major, //Out + HSAuint32 *Minor //Out +) +{ + HSAKMT_STATUS result; + struct kfd_ioctl_dbg_trap_args argout = {0}; + + result = debug_trap(INVALID_NODEID, + KFD_IOC_DBG_TRAP_GET_VERSION, + 0, + 0, + 0, + INVALID_PID, + 0, + &argout); + + *Major = argout.data1; + *Minor = argout.data2; + return result; +} + +/** + * Get the major and minor version of the Thunk debugger support. +*/ +void +HSAKMTAPI +hsaKmtGetThunkDebugTrapVersionInfo( + HSAuint32 *Major, //Out + HSAuint32 *Minor //Out +) +{ + *Major = KFD_IOCTL_DBG_MAJOR_VERSION; + *Minor = KFD_IOCTL_DBG_MINOR_VERSION; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 482a7aeaf8..3f2e787ac5 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -67,6 +67,8 @@ hsaKmtSetWaveLaunchMode; hsaKmtQueueSuspend; hsaKmtQueueResume; hsaKmtAllocQueueGWS; +hsaKmtGetKernelDebugTrapVersionInfo; +hsaKmtGetThunkDebugTrapVersionInfo; local: *; }; From efe37698359b3fe13cee3f9922199a8e9c86f27a Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Mon, 7 Oct 2019 13:10:46 -0400 Subject: [PATCH 0608/1247] kfdtest: Check kfd debugger version in tests Need to check the kfd debugger version of the kernel before calling kfd debugger tests in kfdtest. If they are out of sync, the tests may fail. Change-Id: I1df5e89fb1199304e6fbe8973c60b76062514c03 Signed-off-by: Philip Cox --- tests/kfdtest/src/KFDDBGTest.cpp | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index ec3a134a80..78b38e4c35 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -282,6 +282,55 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { } #endif +/** + * checkDebugVersion: + * Inputs: + * HSAuint32 requiredMajor + * -- Required major version number + * HSAuint32 requiredMinor + * -- Required minor version number + * Output: + * bool: + * i) false if major version of thunk and kernel are not the same + * ii) false if kernel minor version is less than the required + * version if the major version is the required version. + * iii) false if thunk minor version is less than the required + * version if the major version is the required version. + * iv) false if hsaKmtGetKernelDebugTrapVersionInfo() call fails. + * v) true otherwise. + * +*/ +static bool checkDebugVersion(HSAuint32 requiredMajor, HSAuint32 requiredMinor) +{ + HSAuint32 kernelMajorNumber = 0; + HSAuint32 kernelMinorNumber = 0; + HSAuint32 thunkMajorNumber = 0; + HSAuint32 thunkMinorNumber = 0; + + hsaKmtGetThunkDebugTrapVersionInfo(&thunkMajorNumber, &thunkMinorNumber); + + if (hsaKmtGetKernelDebugTrapVersionInfo(&kernelMajorNumber, + &kernelMinorNumber)) { + LOG() << "Failed to get kernel debugger version!" << std::endl; + return false; + } + + if (kernelMajorNumber != thunkMajorNumber) + return false; + + if (kernelMajorNumber < requiredMajor || + (kernelMajorNumber == requiredMajor && + kernelMinorNumber < requiredMinor)) + return false; + + if (thunkMajorNumber < requiredMajor || + (thunkMajorNumber == requiredMajor && + thunkMinorNumber < requiredMinor)) + return false; + + return true; +} + TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId >= FAMILY_AI && m_FamilyId <= FAMILY_AR) { @@ -290,6 +339,12 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HSAuint32 Flags = 0; + + if(!checkDebugVersion(0, 2)) { + LOG() << "Test disabled due to debug API version mismatch"; + goto exit; + } + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer iterateBuf(PAGE_SIZE, defaultGPUNode, true, false, false); HsaMemoryBuffer resultBuf(PAGE_SIZE, defaultGPUNode, true, false, false); @@ -367,6 +422,8 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { } else { LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; } +exit: + LOG() << std::endl; TEST_END } @@ -378,6 +435,11 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + if(!checkDebugVersion(0, 2)) { + LOG() << "Test disabled due to debug API version mismatch"; + goto exit; + } + // enable debug trap and check poll fd creation ASSERT_SUCCESS(hsaKmtEnableDebugTrapWithPollFd(defaultGPUNode, INVALID_QUEUEID, @@ -476,6 +538,8 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; } +exit: + LOG() << std::endl; TEST_END } @@ -525,6 +589,11 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + if(!checkDebugVersion(0, 2)) { + LOG() << "Test disabled due to debug API version mismatch"; + goto exit; + } + pid_t childPid = fork(); ASSERT_GE(childPid, 0); @@ -706,5 +775,7 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; } +exit: + LOG() << std::endl; TEST_END } From 6933540c819910d93dc5e94e32ba05f7eb471fdf Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Tue, 13 Aug 2019 11:03:43 -0400 Subject: [PATCH 0609/1247] Remove debugger data reg accesses The debug trap accesses the data0/data1 registers, so we do not want the userspace to write values to it. We remove the calls to set the data0/data1 register values. Change-Id: Iaba842a4c445f339f16a39fe1994526ff78a2f3c Signed-off-by: Philip Cox --- include/hsakmt.h | 33 --------------------------------- include/linux/kfd_ioctl.h | 24 ++++++++---------------- src/debug.c | 14 -------------- 3 files changed, 8 insertions(+), 63 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index d5af173b30..79eb5e3446 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -735,9 +735,6 @@ hsaKmtQueueResume( - X, Y, Z grid and work-group position of the wave within the dispatch. - - The value of TrapData registers. hsaKmtEnableDebugTrap() sets - these to 0 and they can be changed by hsaKmtSetDebugTrapData2(). - - The scratch backing memory address. - Enable wave launch trap override. hsaKmtEnableDebugTrap() sets the @@ -826,36 +823,6 @@ hsaKmtQueryDebugEvent( bool *IsNew //OUT ); -/** - Set the value to use to initialize the TrapData used when - initializing trap temp registers for NodeId when debug trap is enabled. - - An error is returned if debug trap is not currently enabled for - NodeId. Debug trap is enabled by hsaKmtEnableDebugTrap() which - initializes TrapData to 0. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_NOT_SUPPORTED if debug trap data is not supported - by NodeId. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. - - - HSAKMT_STATUS_INVALID_PARAMETER if TrapDataIndex is larger than - trap-data-count - 1. - - - HSAKMT_STATUS_ERROR if debug trap is not currently enabled by - hsaKmtEnableDebugTrap() for NodeId. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetDebugTrapData2( - HSAuint32 NodeId, //IN - HSAuint32 TrapData0, //IN - HSAuint32 TrapData1 //IN - ); - /** Set the trap override mask. When debug trap is enabled by hsaKmtEnableDebugTrap() each wave launched has its initial diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 66c72c9f8d..4e02aa8158 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -28,8 +28,8 @@ #define KFD_IOCTL_MAJOR_VERSION 1 #define KFD_IOCTL_MINOR_VERSION 2 -#define KFD_IOCTL_DBG_MAJOR_VERSION 0 -#define KFD_IOCTL_DBG_MINOR_VERSION 2 +#define KFD_IOCTL_DBG_MAJOR_VERSION 1 +#define KFD_IOCTL_DBG_MINOR_VERSION 0 struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ @@ -206,21 +206,13 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_ENABLE 0 -/* KFD_IOC_DBG_TRAP_SET_TRAP_DATA: - * ptr: unused - * data1: SPI_GDBG_TRAP_DATA0 - * data2: SPI_GDBG_TRAP_DATA1 - * data3: unused - */ -#define KFD_IOC_DBG_TRAP_SET_TRAP_DATA 1 - /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE: * ptr: unused * data1: override mode: 0=OR, 1=REPLACE * data2: mask * data3: unused */ -#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 2 +#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 1 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE: * ptr: unused @@ -228,7 +220,7 @@ struct kfd_ioctl_dbg_wave_control_args { * data2: unused * data3: unused */ -#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 3 +#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 2 /* KFD_IOC_DBG_TRAP_NODE_SUSPEND: * ptr: pointer to an array of Queues IDs @@ -236,7 +228,7 @@ struct kfd_ioctl_dbg_wave_control_args { * data2: number of queues * data3: grace period */ -#define KFD_IOC_DBG_TRAP_NODE_SUSPEND 4 +#define KFD_IOC_DBG_TRAP_NODE_SUSPEND 3 /* KFD_IOC_DBG_TRAP_NODE_RESUME: * ptr: pointer to an array of Queues IDs @@ -244,7 +236,7 @@ struct kfd_ioctl_dbg_wave_control_args { * data2: number of queues * data3: unused */ -#define KFD_IOC_DBG_TRAP_NODE_RESUME 5 +#define KFD_IOC_DBG_TRAP_NODE_RESUME 4 /* KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT: * ptr: unused @@ -252,7 +244,7 @@ struct kfd_ioctl_dbg_wave_control_args { * data2: flags (IN) * data3: suspend[2:2], event type [1:0] (OUT) */ -#define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 6 +#define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 5 /* KFD_IOC_DBG_TRAP_GET_VERSION: * prt: unsused @@ -260,7 +252,7 @@ struct kfd_ioctl_dbg_wave_control_args { * data2: minor version (OUT) * data3: unused */ -#define KFD_IOC_DBG_TRAP_GET_VERSION 8 +#define KFD_IOC_DBG_TRAP_GET_VERSION 7 struct kfd_ioctl_dbg_trap_args { __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ diff --git a/src/debug.c b/src/debug.c index 6d2f7b8773..8aa67446ad 100644 --- a/src/debug.c +++ b/src/debug.c @@ -383,20 +383,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId) NULL); } -HSAKMT_STATUS HSAKMTAPI hsaKmtSetDebugTrapData2(HSAuint32 NodeId, - HSAuint32 TrapData0, - HSAuint32 TrapData1) -{ - return debug_trap(NodeId, - KFD_IOC_DBG_TRAP_SET_TRAP_DATA, - TrapData0, - TrapData1, - 0, - INVALID_PID, - 0, - NULL); -} - HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( HSAuint32 NodeId, HSA_DBG_TRAP_OVERRIDE TrapOverride, From 0174377351d7124de2d57366ddabd9fb11ffe550 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Fri, 4 Oct 2019 16:49:38 -0400 Subject: [PATCH 0610/1247] kfdtest: add xgmi path for p2p tests When large bar is not available, we can use xgmi to do p2p tests. Change-Id: Ib7b59fb8a4d41f605739a0428973f6b2f1a3450f Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDMemoryTest.cpp | 20 ++++++------- tests/kfdtest/src/KFDPerformanceTest.cpp | 32 +++++++++++++-------- tests/kfdtest/src/KFDQMTest.cpp | 13 +++++---- tests/kfdtest/src/KFDTestUtil.cpp | 36 ++++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 3 ++ 5 files changed, 75 insertions(+), 29 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 10426392ed..33ef74862c 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1967,6 +1967,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { HSAuint32 *mmioBase = NULL; unsigned int *nullPtr = NULL; std::vector nodes; + int numPeers; const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); if (gpuNodes.size() < 2) { @@ -1978,26 +1979,23 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[0])) { - LOG() << "Skipping test: first GPU specified is not a large bar GPU." << std::endl; + + if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && + !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + LOG() << "Skipping test: first GPU specified is not peer-accessible." << std::endl; return; } + if (nodes[0] == nodes[1]) { LOG() << "Skipping test: Different GPUs must be specified (2 GPUs required)." << std::endl; return; } } else { HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - if (!m_NodeInfo.IsGPUNodeLargeBar(defaultGPU)) { - LOG() << "Skipping test: Default GPUs must be large bar." << std::endl; - return; - } - nodes.push_back(defaultGPU); - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (gpuNodes.at(i) != defaultGPU) - nodes.push_back(gpuNodes.at(i)); + m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, false); if (nodes.size() < 2) { - LOG() << "Skipping test: At least 2 GPUs required." << std::endl; + LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; + LOG() << " or two GPUs are XGMI connected." << std::endl; return; } } diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index e4f5af2b51..50cff1544d 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -147,15 +147,24 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); std::vector nodes; const bool isSpecified = g_TestDstNodeId != -1 && g_TestNodeId != -1; + int numPeers = 0; - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && - /* Users can use "--node=gpu1 --dst_node=gpu2" to specify devices */ - (!isSpecified || gpuNodes.at(i) == g_TestDstNodeId || gpuNodes.at(i) == g_TestNodeId)) - nodes.push_back(gpuNodes.at(i)); + if (isSpecified) { + if (g_TestNodeId != g_TestDstNodeId) { + nodes.push_back(g_TestNodeId); + nodes.push_back(g_TestDstNodeId); + if ((m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && + m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId)) || + m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) + numPeers = 2; + } + } else { + HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); + numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + } - if (nodes.size() < 2) { - LOG() << "Skipping test: Need at least two large bar GPU." << std::endl; + if (numPeers < 2) { + LOG() << "Skipping test: Need at least two large bar GPU or XGMI connected." << std::endl; return; } @@ -302,12 +311,11 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) { const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); std::vector nodes; - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i))) - nodes.push_back(gpuNodes.at(i)); + HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); + int numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); - if (nodes.size() < 2) { - LOG() << "Skipping test: Need at least two large bar GPU." << std::endl; + if (numPeers < 2) { + LOG() << "Skipping test: Need at least two large bar GPU or XGMI connected." << std::endl; return; } diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 20dbf598e0..c5abb9ff5e 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -23,6 +23,7 @@ #include #include +#include #include "KFDQMTest.hpp" #include "PM4Queue.hpp" #include "PM4Packet.hpp" @@ -1459,8 +1460,10 @@ TEST_F(KFDQMTest, P2PTest) { if (g_TestDstNodeId != -1 && g_TestNodeId != -1) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[1])) { - LOG() << "Skipping test: Dst GPU is not a large bar GPU." << std::endl; + + if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId) && + !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + LOG() << "Skipping test: Dst GPU specified is not peer-accessible." << std::endl; return; } if (nodes[0] == nodes[1]) { @@ -1469,12 +1472,10 @@ TEST_F(KFDQMTest, P2PTest) { } } else { HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - nodes.push_back(defaultGPU); - for (unsigned i = 0; i < gpuNodes.size(); i++) - if (m_NodeInfo.IsGPUNodeLargeBar(gpuNodes.at(i)) && gpuNodes.at(i) != defaultGPU) - nodes.push_back(gpuNodes.at(i)); + m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); if (nodes.size() < 2) { LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; + LOG() << " or two GPUs are XGMI connected." << std::endl; return; } } diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 41d13f9e9a..991bea2e11 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -589,3 +589,39 @@ const int HsaNodeInfo::FindLargeBarGPUNode() const { return -1; } + +const bool HsaNodeInfo::AreGPUNodesXGMI(int node0, int node1) const { + const HsaNodeProperties *pNodeProperties0 = GetNodeProperties(node0); + const HsaNodeProperties *pNodeProperties1 = GetNodeProperties(node1); + + if ((pNodeProperties0->HiveID != 0) && (pNodeProperties1->HiveID != 0) && + (pNodeProperties0->HiveID == pNodeProperties1->HiveID)) + return true; + + return false; +} + +int HsaNodeInfo::FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, + bool bidirectional) const { + peers->push_back(dstNode); + if (IsGPUNodeLargeBar(dstNode)) { + for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { + if (m_NodesWithGPU.at(i) == dstNode) + continue; + + if (!bidirectional || IsGPUNodeLargeBar(m_NodesWithGPU.at(i)) || + AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) + peers->push_back(m_NodesWithGPU.at(i)); + } + } else { + for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { + if (m_NodesWithGPU.at(i) == dstNode) + continue; + + if (AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) + peers->push_back(m_NodesWithGPU.at(i)); + } + } + + return peers->size(); +} diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 1e40e8b4e4..4cbaa395e7 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -188,6 +188,9 @@ class HsaNodeInfo { // @brief Find the first available Large-BAR GPU node // @return: Node ID if successful or -1 const int FindLargeBarGPUNode() const; + const bool AreGPUNodesXGMI(int node0, int node1) const; + int FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, + bool bidirectional) const; }; #endif // __KFD__TEST__UTIL__H__ From 23541e0289f38f060ca542990301fc3fdd972965 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 15 Oct 2019 13:38:13 -0400 Subject: [PATCH 0611/1247] libhsakmt: handle CPU cache info on non-NUMA sys When CONFIG_NUMA is not enabled in the kernel config, only one CPU node presents on the system and /sys/devices/system/node/nodeX directories don't exist. Read CPU cache information from /sys/devices/system/cpu in this situation. Change-Id: I017ff17dd72678a0551edcc77446664501aa42ca Signed-off-by: Amber Lin --- src/topology.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/topology.c b/src/topology.c index 327315429a..5bbc100318 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1193,8 +1193,20 @@ static int topology_create_temp_cpu_cache_list(int node, /* Other than cpuY folders, this dir also has cpulist and cpumap */ max_cpus = num_subdirs(node_dir, "cpu"); if (max_cpus <= 0) { - pr_err("Fail to get cpu* dirs under %s\n", node_dir); - goto exit; + /* If CONFIG_NUMA is not enabled in the kernel, + * /sys/devices/system/node doesn't exist. + */ + if (node) { /* CPU node must be 0 or something is wrong */ + pr_err("Fail to get cpu* dirs under %s.", node_dir); + goto exit; + } + /* Fall back to use /sys/devices/system/cpu */ + snprintf(node_dir, MAXPATHSIZE, "/sys/devices/system/cpu"); + max_cpus = num_subdirs(node_dir, "cpu"); + if (max_cpus <= 0) { + pr_err("Fail to get cpu* dirs under %s\n", node_dir); + goto exit; + } } p_temp_cpu_ci_list = calloc(max_cpus, sizeof(cpu_cacheinfo_t)); @@ -1211,8 +1223,7 @@ static int topology_create_temp_cpu_cache_list(int node, continue; if (!isdigit(dir->d_name[3])) /* ignore files like cpulist */ continue; - snprintf(path, MAXPATHSIZE, "/sys/devices/system/node/node%d/%s/cache", - node, dir->d_name); + snprintf(path, MAXPATHSIZE, "%s/%s/cache", node_dir, dir->d_name); this_cpu->num_caches = num_subdirs(path, "index"); this_cpu->cache_prop = calloc(this_cpu->num_caches, sizeof(HsaCacheProperties)); From 16fa78b1349c4eb23103099bf7188db2d0eb72c6 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 2 Oct 2019 18:30:20 -0400 Subject: [PATCH 0612/1247] libhsakmt: Print an error message when map_mmio failes Without this change, the failure was hard to notice when it happened. Change-Id: I99c3e8cea0d0cbd3bcfe79069410e6e870e225bf Signed-off-by: Yong Zhao --- src/fmm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index e98bf01c36..43d735af54 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2348,6 +2348,9 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_id].mmio_aperture.limit = (void *) ((char *)gpu_mem[gpu_mem_id].mmio_aperture.base + PAGE_SIZE - 1); + else + pr_err("Failed to map remapped mmio page on gpu_mem %d\n", + gpu_mem_id); } free(process_apertures); From 1c7755d2da3eb73d70df45d1dd9bed6a0eaa26a7 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 16 Oct 2019 18:28:55 -0400 Subject: [PATCH 0613/1247] libhsakmt: Add gfx1012 device IDs Now the gfx1012 device IDs are okay to reveal. Change-Id: I9da2a036b74ec7b6b8b1fb7587597a5847f02205 Signed-off-by: Yong Zhao --- src/topology.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/topology.c b/src/topology.c index 5bbc100318..570d16e57e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -226,6 +226,10 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x7318, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x731A, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x731F, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, + /* Navi14 */ + { 0x7340, 10, 1, 2, 1, "Navi14", CHIP_NAVI14 }, + { 0x7341, 10, 1, 2, 1, "Navi14", CHIP_NAVI14 }, + { 0x7347, 10, 1, 2, 1, "Navi14", CHIP_NAVI14 }, /* Navi12 */ { 0x7360, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, { 0x7362, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, From ab2daf653887aa3f0c7452b52e6a0c3b8d63bfdf Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 2 Oct 2019 18:30:20 -0400 Subject: [PATCH 0614/1247] libhsakmt: Add a message when a device is not supported This helps to quickly triage problems. Change-Id: Iad2b4b74209ab972be0c2f6311eeb3aaf098d29f Signed-off-by: Yong Zhao --- src/topology.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 570d16e57e..b73753ff6e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1066,7 +1066,11 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->MarketingName[i] = name[i]; props->MarketingName[i] = '\0'; } - } + } else if (props->DeviceId) + /* still return success */ + pr_err("device ID 0x%x is not supported in libhsakmt\n", + props->DeviceId); + if (props->NumFComputeCores) assert(props->EngineId.ui32.Major); From bee47ab0134b744a73a855c5833510b5b3099f36 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 21 Oct 2019 14:52:07 -0400 Subject: [PATCH 0615/1247] libhsakmt: Fix installation path for pkg-config file CMAKE_INSTALL_PREFIX would give incorrect result for packages build by CPack. Change-Id: Iaeb4d30eb44080b7924ecf892de011ed6e365f5f Signed-off-by: Felix Kuehling --- hsakmt-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index b3de70f7dc..ba9406d17d 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -43,7 +43,7 @@ set ( BUILD_VERSION_PATCH @BUILD_VERSION_PATCH@ ) set ( CMAKE_VERBOSE_MAKEFILE on ) ## Set the install targets -install ( FILES libhsakmt.pc DESTINATION ${CMAKE_INSTALL_PREFIX}/share/pkgconfig ) +install ( FILES libhsakmt.pc DESTINATION ${CMAKE_INSTALL_DATADIR}/pkgconfig ) install ( DIRECTORY ${SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_COMPONENT} PATTERN "linux" EXCLUDE ) From ffbdb726ac3b9302dc55090b594bd64267cea460 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 17 Oct 2019 17:51:28 -0400 Subject: [PATCH 0616/1247] kfdtest: Temporarily disable performance counter tests for gfx1010 We are still working on those tests for gfx1010, so disable them temporarily. Change-Id: I5d51b4b02bc753137014684859cc033f759b2899 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 286a4962e8..52f65b08bd 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -185,6 +185,7 @@ FILTER[navi10]=\ "$SDMA_BLACKLIST:"\ "KFDEvictTest.BurstyTest:"\ "KFDEvictTest.QueueTest:"\ +"KFDPerfCountersTest.*:"\ "KFDExceptionTest.InvalidBadAddress:"\ "KFDExceptionTest.InvalidWriteAddress:"\ "KFDQMTest.GPUDoorbellWrite" From 98b0652917efedd9a5e34fa400ed9ff9906b4588 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Sun, 22 Sep 2019 17:10:14 -0400 Subject: [PATCH 0617/1247] kfdtest: Add a core test filter for software scheduler mode The new filter can be used by "./run_kfdtest.sh -p core_sws". Change-Id: I1c43669cfc07c09ccafb9fa2e2851932ac59307d Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 9 +++++++-- tests/kfdtest/scripts/run_kfdtest.sh | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 52f65b08bd..e133df1852 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -8,16 +8,21 @@ FILTER[pm]=\ # Core tests, used in scenarios like bringup -FILTER[core]=\ +# Software scheduler mode, i. e. non HWS mode +FILTER[core_sws]=\ "KFDQMTest.CreateDestroyCpQueue:"\ "KFDQMTest.SubmitPacketCpQueue:"\ "KFDQMTest.MultipleCpQueues:"\ "KFDQMTest.CreateDestroySdmaQueue:"\ "KFDQMTest.SubmitPacketSdmaQueue:"\ "KFDQMTest.MultipleSdmaQueues:"\ -"KFDCWSRTest.BasicTest:"\ "KFDEventTest.SignalEvent" +# HWS mode +FILTER[core]=\ +"${FILTER[core_sws]}:"\ +"KFDCWSRTest.BasicTest" + # Permanent exclusions # These tests are included for debugging, but are not executed in normal execution on any ASIC: diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 7493dc00f1..52a1dd20a0 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -52,7 +52,7 @@ printUsage() { echo echo "Gtest arguments will be forwarded to the app" echo - echo "Valid platform options: pm, core, polaris10, vega10, vega20, all, and so on" + echo "Valid platform options: core_sws, core, polaris10, vega10, vega20, pm, all, and so on" echo "'all' option runs all tests" return 0 From 75c654cfea0dcec1ab905f17a0cd9f707f67609a Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Sun, 22 Sep 2019 17:13:17 -0400 Subject: [PATCH 0618/1247] kfdtest: Add SDMANopPacket class The class is very useful for triaging complex SDMA issues. Change-Id: Ib5de729f7fc62f41e894ef98d3967e7e1745d454 Signed-off-by: Yong Zhao --- tests/kfdtest/include/sdma_pkt_struct.h | 30 +++++++++++++++++++++++++ tests/kfdtest/src/SDMAPacket.cpp | 13 +++++++++++ tests/kfdtest/src/SDMAPacket.hpp | 15 +++++++++++++ 3 files changed, 58 insertions(+) diff --git a/tests/kfdtest/include/sdma_pkt_struct.h b/tests/kfdtest/include/sdma_pkt_struct.h index 127c44f43d..e87f789eb7 100644 --- a/tests/kfdtest/include/sdma_pkt_struct.h +++ b/tests/kfdtest/include/sdma_pkt_struct.h @@ -25,6 +25,7 @@ #define __SDMA_PKT_STRUCT_H__ +const unsigned int SDMA_OP_NOP = 0; const unsigned int SDMA_OP_COPY = 1; const unsigned int SDMA_OP_WRITE = 2; @@ -354,4 +355,33 @@ typedef struct SDMA_PKT_TIMESTAMP_TAG } ADDR_HI_UNION; } SDMA_PKT_TIMESTAMP, *PSDMA_PKT_TIMESTAMP; + +/* +** Definitions for SDMA_PKT_NOP packet +*/ + +typedef struct SDMA_PKT_NOP_TAG +{ + union + { + struct + { + unsigned int op:8; + unsigned int sub_op:8; + unsigned int count:14; + unsigned int reserved_0:2; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union + { + struct + { + unsigned int data0:32; + }; + unsigned int DW_1_DATA; + } DATA0_UNION; +} SDMA_PKT_NOP, *PSDMA_PKT_NOP; + #endif // __SDMA_PKT_STRUCT_H__ diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index df5eb6b18f..251e019d7c 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -246,3 +246,16 @@ void SDMATimePacket::InitPacket(void *destaddr) { packetData.ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ packetData.ADDR_HI_UNION.DW_2_DATA); /*dst_addr_63_32*/ } + +SDMANopPacket::SDMANopPacket(unsigned int count) { + packetSize = count; + packetData = reinterpret_cast(calloc(count, sizeof(unsigned int))); + + packetData->HEADER_UNION.op = SDMA_OP_NOP; + packetData->HEADER_UNION.sub_op = 0; + packetData->HEADER_UNION.count = count - 1; +} + +SDMANopPacket::~SDMANopPacket() { + free(packetData); +} diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index 03f837806b..009ddea251 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -158,5 +158,20 @@ class SDMATimePacket : public SDMAPacket { SDMA_PKT_TIMESTAMP packetData; }; +class SDMANopPacket : public SDMAPacket { + public: + SDMANopPacket(unsigned int count = 1); + virtual ~SDMANopPacket(void); + + // @returns Pointer to the packet + virtual const void *GetPacket() const { return packetData; } + // @returns Packet size in bytes + virtual unsigned int SizeInBytes() const { return packetSize; } + + private: + SDMA_PKT_NOP *packetData; + unsigned int packetSize; +}; + #endif // __KFD_SDMA_PACKET__H__ From ac5c4334201006c5663f7e641d6df552c5a895f9 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Sun, 22 Sep 2019 17:55:14 -0400 Subject: [PATCH 0619/1247] kfdtest: Add a Nop packet submission test for CP and SDMA queue The tests are useful to triage the fundamental queue submission functionality by excluding the packet format variable from the equation. Change-Id: I2c7fcda811f93bdefc1b62396233559416be44e7 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 2 ++ tests/kfdtest/src/KFDQMTest.cpp | 38 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index e133df1852..b0ea7a7947 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -11,9 +11,11 @@ FILTER[pm]=\ # Software scheduler mode, i. e. non HWS mode FILTER[core_sws]=\ "KFDQMTest.CreateDestroyCpQueue:"\ +"KFDQMTest.SubmitNopCpQueue:"\ "KFDQMTest.SubmitPacketCpQueue:"\ "KFDQMTest.MultipleCpQueues:"\ "KFDQMTest.CreateDestroySdmaQueue:"\ +"KFDQMTest.SubmitNopSdmaQueue:"\ "KFDQMTest.SubmitPacketSdmaQueue:"\ "KFDQMTest.MultipleSdmaQueues:"\ "KFDEventTest.SignalEvent" diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index c5abb9ff5e..13cfb96fb4 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -71,6 +71,25 @@ TEST_F(KFDQMTest, CreateDestroyCpQueue) { TEST_END } +TEST_F(KFDQMTest, SubmitNopCpQueue) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4NopPacket()); + + queue.Wait4PacketConsumption(); + + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + TEST_F(KFDQMTest, SubmitPacketCpQueue) { TEST_START(TESTPROFILE_RUNALL) @@ -142,6 +161,25 @@ TEST_F(KFDQMTest, CreateDestroySdmaQueue) { TEST_END } +TEST_F(KFDQMTest, SubmitNopSdmaQueue) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + SDMAQueue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(SDMANopPacket()); + + queue.Wait4PacketConsumption(); + + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + TEST_F(KFDQMTest, SubmitPacketSdmaQueue) { TEST_START(TESTPROFILE_RUNALL) From da9a404ac3ebbcebdce5bd46fbc7ce8ffc130f12 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Sun, 22 Sep 2019 17:48:10 -0400 Subject: [PATCH 0620/1247] kfdtest: Add a function to wait indefinitely until user input This will facilitate the need of halting the execution during debugging. Change-Id: I058c81bbc9f655bbc477b2b542d6b43aa423324c Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDTestUtil.cpp | 8 ++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 1 + 2 files changed, 9 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 991bea2e11..c949259439 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -30,6 +30,14 @@ #include "Dispatch.hpp" #include "SDMAPacket.hpp" +void WaitUntilInput() { + char dummy; + printf("Press enter to continue: "); + do { + scanf("%c", &dummy); + } while (dummy != 10); // enter key's ascii value is 10 +} + uint64_t RoundToPowerOf2(uint64_t val) { int bytes = sizeof(uint64_t); diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 4cbaa395e7..7e2fe7e16d 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -35,6 +35,7 @@ class BaseQueue; #define ALIGN_UP(x, align) (((uint64_t)(x) + (align) - 1) & ~(uint64_t)((align)-1)) #define CounterToNanoSec(x) ((x) * 1000 / (is_dgpu() ? 27 : 100)) +void WaitUntilInput(); uint64_t RoundToPowerOf2(uint64_t val); // @brief: waits until the value is written to the buffer or until time out if received through args From cb4d4107ed820dd454a98dac898984cd25eca73f Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Fri, 25 Oct 2019 15:01:58 -0400 Subject: [PATCH 0621/1247] libhsakmt: default packaging prefix to install If CPACK_PACKAGING_INSTALL_PREFIX is not set, it's safer to assume it's the same as CMAKE_INSTALL_PREFIX, incase only the latter is set. Change-Id: I70727ebbc50f21f8d6e3add10d7f9f2d5e747dee Signed-off-by: Jeremy Newton --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e644e9911..1bf4e8cf15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,7 +131,7 @@ target_link_libraries ( ${HSAKMT_TARGET} ## Define default variable and variables for the optional build target hsakmt-dev set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) -set ( CPACK_PACKAGING_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default packaging prefix." ) +set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "Default packaging prefix." ) set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) ## Specify build, install and package targets hsakmt-dev From 7593b415758317530236b258129a8c173bad264a Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 28 Oct 2019 16:16:35 -0500 Subject: [PATCH 0622/1247] Unmap memory from GPU before free Change-Id: Ic33b17cbaee5de7908d37527254f4f146e6b71e3 Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDMemoryTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 33ef74862c..747b1daa5a 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2165,6 +2165,7 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnCPUWrite) { EXPECT_EQ(buffer[100], 0x5678); // Clean up + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(buffer)); EXPECT_SUCCESS(hsaKmtFreeMemory(buffer, PAGE_SIZE)); EXPECT_SUCCESS(queue.Destroy()); From fa0cb9ebeb174784d5bfbbd208f07004deec511b Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 28 Oct 2019 19:16:40 -0500 Subject: [PATCH 0623/1247] Handle IOCTL failure in fmm_release FREE_MEMORY_OF_GPU ioctl could fail, e.g., if memory is still mapped to GPU. Handle this failure by return error in fmm_release/HsaKmtFreeMemory Change-Id: I5461db39964f733cf97376d50e44906a9b4c0f13 Signed-off-by: Oak Zeng --- src/fmm.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 43d735af54..fd69522aa0 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -286,7 +286,7 @@ static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( } extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); -static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture); +static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address); @@ -1628,12 +1628,12 @@ void *fmm_allocate_host(uint32_t node_id, void *address, return fmm_allocate_host_cpu(address, MemorySizeInBytes, flags); } -static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) +static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) { struct kfd_ioctl_free_memory_of_gpu_args args = {0}; if (!object) - return; + return -EINVAL; pthread_mutex_lock(&aperture->fmm_mutex); @@ -1643,12 +1643,16 @@ static void __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) * free the BO before unmapping the pages. */ args.handle = object->handle; - kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return -errno; + } aperture_release_area(aperture, object->start, object->size); vm_remove_object(aperture, object); pthread_mutex_unlock(&aperture->fmm_mutex); + return 0; } HSAKMT_STATUS fmm_release(void *address) @@ -1682,7 +1686,9 @@ HSAKMT_STATUS fmm_release(void *address) } else { pthread_mutex_unlock(&aperture->fmm_mutex); - __fmm_release(object, aperture); + if (__fmm_release(object, aperture)) + return HSAKMT_STATUS_ERROR; + if (!aperture->is_cpu_accessible) fmm_print(gpu_mem[i].gpu_id); } @@ -2843,9 +2849,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, pthread_mutex_unlock(&aperture->fmm_mutex); /* free object in scratch backing aperture */ - __fmm_release(object, aperture); - - return 0; + return __fmm_release(object, aperture); err: pthread_mutex_unlock(&aperture->fmm_mutex); From 12e4ea94f1d8f59b283b6cd58d63946d9623e5f2 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 30 Oct 2019 22:11:22 -0400 Subject: [PATCH 0624/1247] kfdtest: Fix SDMA NOP packet size packetSize is used as the size in bytes. Change-Id: I900f9a4f37b840cbb957184705db04a4c64d1654 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/SDMAPacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index 251e019d7c..09b061268b 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -248,7 +248,7 @@ void SDMATimePacket::InitPacket(void *destaddr) { } SDMANopPacket::SDMANopPacket(unsigned int count) { - packetSize = count; + packetSize = count * sizeof(unsigned int); packetData = reinterpret_cast(calloc(count, sizeof(unsigned int))); packetData->HEADER_UNION.op = SDMA_OP_NOP; From ea7619c9293d11d95e57f71ac357402c3ee98bd1 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 30 Oct 2019 20:20:05 -0400 Subject: [PATCH 0625/1247] kfdtest: Re-factor dynamic packet allocation into BasePacket Avoids unnecessary and error-prone duplication of dynamic packet allocation in many packet classes. Change-Id: Icec981ae2cd7a3d88cdf9213298940d85627f853 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/BasePacket.cpp | 20 ++++++++++++++- tests/kfdtest/src/BasePacket.hpp | 7 ++++-- tests/kfdtest/src/PM4Packet.cpp | 42 ++++++++------------------------ tests/kfdtest/src/PM4Packet.hpp | 9 +++---- tests/kfdtest/src/SDMAPacket.cpp | 25 +++---------------- tests/kfdtest/src/SDMAPacket.hpp | 8 +++--- 6 files changed, 46 insertions(+), 65 deletions(-) diff --git a/tests/kfdtest/src/BasePacket.cpp b/tests/kfdtest/src/BasePacket.cpp index daae6691ae..5ecbbe0e4e 100644 --- a/tests/kfdtest/src/BasePacket.cpp +++ b/tests/kfdtest/src/BasePacket.cpp @@ -25,10 +25,15 @@ #include "KFDTestUtil.hpp" #include "KFDBaseComponentTest.hpp" -BasePacket::BasePacket(void) { +BasePacket::BasePacket(void): m_packetAllocation(NULL) { m_FamilyId = g_baseTest->GetFamilyIdFromDefaultNode(); } +BasePacket::~BasePacket(void) { + if (m_packetAllocation) + free(m_packetAllocation); +} + void BasePacket::Dump() const { unsigned int size = SizeInDWords(); const HSAuint32 *packet = (const HSAuint32 *)GetPacket(); @@ -40,3 +45,16 @@ void BasePacket::Dump() const { log << " " << std::setw(8) << std::setfill('0') << packet[i]; log << std::endl; } + +void *BasePacket::AllocPacket(void) { + unsigned int size = SizeInBytes(); + + EXPECT_NE(0, size); + if (!size) + return NULL; + + m_packetAllocation = calloc(1, size); + EXPECT_NOTNULL(m_packetAllocation); + + return m_packetAllocation; +} diff --git a/tests/kfdtest/src/BasePacket.hpp b/tests/kfdtest/src/BasePacket.hpp index efbebc88f1..cc638a1316 100644 --- a/tests/kfdtest/src/BasePacket.hpp +++ b/tests/kfdtest/src/BasePacket.hpp @@ -38,7 +38,7 @@ enum PACKETTYPE { class BasePacket { public: BasePacket(void); - virtual ~BasePacket(void) {} + virtual ~BasePacket(void); // @returns Packet type virtual PACKETTYPE PacketType() const = 0; @@ -52,7 +52,10 @@ class BasePacket { void Dump() const; protected: - unsigned int m_FamilyId; + unsigned int m_FamilyId; + void *m_packetAllocation; + + void *AllocPacket(void); }; #endif // __KFD_BASE_PACKET__H__ diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index 195256fbaf..df13ca30d2 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -43,19 +43,12 @@ void PM4Packet::InitPM4Header(PM4_TYPE_3_HEADER &header, it_opcode_type opCode) header.reserved1 = 0; } -PM4WriteDataPacket::~PM4WriteDataPacket(void) { - if (m_pPacketData) - free(m_pPacketData); -} - unsigned int PM4WriteDataPacket::SizeInBytes() const { return (offsetof(PM4WRITE_DATA_CI, data) + m_ndw*sizeof(uint32_t)); } void PM4WriteDataPacket::InitPacket(unsigned int *destBuf, void *data) { - m_pPacketData = reinterpret_cast(calloc(1, SizeInBytes())); - // Verify that the memory is allocated successfully, cannot use assert here - EXPECT_NOTNULL(m_pPacketData); + m_pPacketData = reinterpret_cast(AllocPacket()); InitPM4Header(m_pPacketData->header, IT_WRITE_DATA); @@ -90,9 +83,8 @@ void PM4ReleaseMemoryPacket::InitPacketCI(bool isPolling, uint64_t address, PM4_RELEASE_MEM_CI *pkt; m_packetSize = sizeof(PM4_RELEASE_MEM_CI); - pkt = reinterpret_cast(calloc(1, m_packetSize)); + pkt = reinterpret_cast(AllocPacket()); m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); InitPM4Header(pkt->header, IT_RELEASE_MEM); @@ -155,9 +147,8 @@ void PM4ReleaseMemoryPacket::InitPacketAI(bool isPolling, uint64_t address, PM4MEC_RELEASE_MEM_AI *pkt; m_packetSize = sizeof(PM4MEC_RELEASE_MEM_AI); - pkt = reinterpret_cast(calloc(1, m_packetSize)); + pkt = reinterpret_cast(AllocPacket()); m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); InitPM4Header(pkt->header, IT_RELEASE_MEM); @@ -201,9 +192,8 @@ void PM4ReleaseMemoryPacket::InitPacketNV(bool isPolling, uint64_t address, PM4MEC_RELEASE_MEM_NV *pkt; m_packetSize = sizeof(PM4_MEC_RELEASE_MEM_NV); - pkt = reinterpret_cast(calloc(1, m_packetSize)); + pkt = reinterpret_cast(AllocPacket()); m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); InitPM4Header(pkt->header, IT_RELEASE_MEM); @@ -277,9 +267,8 @@ void PM4AcquireMemoryPacket::InitPacketAI(void) { PM4ACQUIRE_MEM *pkt; m_packetSize = sizeof(PM4ACQUIRE_MEM); - pkt = reinterpret_cast(calloc(1, m_packetSize)); + pkt = reinterpret_cast(AllocPacket()); m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); InitPM4Header(pkt->header, IT_ACQUIRE_MEM); pkt->bitfields2.coher_cntl = 0x28c00000; // copied from the way the HSART does this. @@ -293,9 +282,9 @@ void PM4AcquireMemoryPacket::InitPacketAI(void) { void PM4AcquireMemoryPacket::InitPacketNV(void) { PM4ACQUIRE_MEM_NV *pkt; m_packetSize = sizeof(PM4ACQUIRE_MEM_NV); - pkt = reinterpret_cast(calloc(1, m_packetSize)); + pkt = reinterpret_cast(AllocPacket()); m_pPacketData = pkt; - EXPECT_NOTNULL(m_pPacketData); + InitPM4Header(pkt->header, IT_ACQUIRE_MEM); pkt->coher_size = 0xFFFFFFFF; pkt->bitfields3.coher_size_hi = 0; @@ -309,21 +298,14 @@ void PM4AcquireMemoryPacket::InitPacketNV(void) { pkt->bitfields6.gcr_cntl = (1<<14|1<<9|1<<8|1<<7|1); } -PM4SetShaderRegPacket::PM4SetShaderRegPacket(void) - : m_packetDataAllocated(false) { +PM4SetShaderRegPacket::PM4SetShaderRegPacket(void) { } PM4SetShaderRegPacket::PM4SetShaderRegPacket(unsigned int baseOffset, const unsigned int regValues[], - unsigned int numRegs) - : m_packetDataAllocated(false) { + unsigned int numRegs) { InitPacket(baseOffset, regValues, numRegs); } -PM4SetShaderRegPacket::~PM4SetShaderRegPacket(void) { - if (m_packetDataAllocated) - free(m_pPacketData); -} - void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs) { // 1st register is a part of the packet struct. @@ -332,11 +314,7 @@ void PM4SetShaderRegPacket::InitPacket(unsigned int baseOffset, const unsigned i /* Allocating the size of the packet, since the packet is assembled from a struct * followed by an additional dword data */ - m_pPacketData = reinterpret_cast(malloc(m_packetSize)); - - EXPECT_NOTNULL(m_pPacketData); - - m_packetDataAllocated = true; + m_pPacketData = reinterpret_cast(AllocPacket()); memset(m_pPacketData, 0, m_packetSize); diff --git a/tests/kfdtest/src/PM4Packet.hpp b/tests/kfdtest/src/PM4Packet.hpp index 8624db0dd4..aa5acba391 100644 --- a/tests/kfdtest/src/PM4Packet.hpp +++ b/tests/kfdtest/src/PM4Packet.hpp @@ -59,7 +59,7 @@ class PM4WriteDataPacket : public PM4Packet { InitPacket(destBuf, data); } - virtual ~PM4WriteDataPacket(void); + virtual ~PM4WriteDataPacket(void) {} // @returns Packet size in bytes virtual unsigned int SizeInBytes() const; // @returns Pointer to the packet @@ -91,7 +91,7 @@ class PM4ReleaseMemoryPacket : public PM4Packet { PM4ReleaseMemoryPacket(unsigned int familyId, bool isPolling, uint64_t address, uint64_t data, bool is64bit = false, bool isTimeStamp = false); - virtual ~PM4ReleaseMemoryPacket(void) {if (m_pPacketData)free(m_pPacketData);} + virtual ~PM4ReleaseMemoryPacket(void) {} // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return m_packetSize; } // @returns Pointer to the packet @@ -135,7 +135,7 @@ class PM4IndirectBufPacket : public PM4Packet { class PM4AcquireMemoryPacket : public PM4Packet { public: PM4AcquireMemoryPacket(unsigned int familyId); - virtual ~PM4AcquireMemoryPacket(void) {if (m_pPacketData)free(m_pPacketData);} + virtual ~PM4AcquireMemoryPacket(void) {} // @returns the packet size in bytes virtual unsigned int SizeInBytes() const { return m_packetSize; } @@ -156,7 +156,7 @@ class PM4SetShaderRegPacket : public PM4Packet { PM4SetShaderRegPacket(unsigned int baseOffset, const unsigned int regValues[], unsigned int numRegs); - virtual ~PM4SetShaderRegPacket(void); + virtual ~PM4SetShaderRegPacket(void) {} // @returns Packet size in bytes virtual unsigned int SizeInBytes() const { return m_packetSize; } @@ -167,7 +167,6 @@ class PM4SetShaderRegPacket : public PM4Packet { private: unsigned int m_packetSize; - bool m_packetDataAllocated; // PM4SET_SH_REG struct contains all the packet's data PM4SET_SH_REG *m_pPacketData; }; diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index 09b061268b..ee12e358b2 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -45,16 +45,11 @@ SDMAWriteDataPacket::SDMAWriteDataPacket(unsigned int familyId, void* destAddr, InitPacket(destAddr, ndw, data); } -SDMAWriteDataPacket::~SDMAWriteDataPacket(void) { - if (packetData) - free(packetData); -} - void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int ndw, void *data) { packetSize = sizeof(SDMA_PKT_WRITE_UNTILED) + (ndw - 1) * sizeof(unsigned int); - packetData = reinterpret_cast(calloc(1, packetSize)); + packetData = reinterpret_cast(AllocPacket()); packetData->HEADER_UNION.op = SDMA_OP_WRITE; packetData->HEADER_UNION.sub_op = SDMA_SUBOP_WRITE_LINEAR; @@ -69,10 +64,6 @@ void SDMAWriteDataPacket::InitPacket(void* destAddr, unsigned int ndw, #define BITS (21) #define TWO_MEG (1 << BITS) -SDMACopyDataPacket::~SDMACopyDataPacket(void) { - free(packetData); -} - SDMACopyDataPacket::SDMACopyDataPacket(unsigned int familyId, void *const dsts[], void *src, int n, unsigned int surfsize) { int32_t size = 0, i; @@ -88,7 +79,7 @@ SDMACopyDataPacket::SDMACopyDataPacket(unsigned int familyId, packetSize = ((surfsize + TWO_MEG - 1) >> BITS) * singlePacketSize; - SDMA_PKT_COPY_LINEAR *pSDMA = reinterpret_cast(malloc(packetSize)); + SDMA_PKT_COPY_LINEAR *pSDMA = reinterpret_cast(AllocPacket()); packetData = pSDMA; while (surfsize > 0) { @@ -125,10 +116,6 @@ SDMACopyDataPacket::SDMACopyDataPacket(unsigned int familyId, void* dst, void *s new (this)SDMACopyDataPacket(familyId, &dst, src, 1, surfsize); } -SDMAFillDataPacket::~SDMAFillDataPacket() { - free(m_PacketData); -} - SDMAFillDataPacket::SDMAFillDataPacket(unsigned int familyId, void *dst, unsigned int data, unsigned int size) { unsigned int copy_size; SDMA_PKT_CONSTANT_FILL *pSDMA; @@ -136,7 +123,7 @@ SDMAFillDataPacket::SDMAFillDataPacket(unsigned int familyId, void *dst, unsigne m_FamilyId = familyId; /* SDMA support maximum 0x3fffe0 byte in one copy. Use 2M copy_size */ m_PacketSize = ((size + TWO_MEG - 1) >> BITS) * sizeof(SDMA_PKT_CONSTANT_FILL); - pSDMA = reinterpret_cast(calloc(1, m_PacketSize)); + pSDMA = reinterpret_cast(AllocPacket()); m_PacketData = pSDMA; while (size > 0) { @@ -249,13 +236,9 @@ void SDMATimePacket::InitPacket(void *destaddr) { SDMANopPacket::SDMANopPacket(unsigned int count) { packetSize = count * sizeof(unsigned int); - packetData = reinterpret_cast(calloc(count, sizeof(unsigned int))); + packetData = reinterpret_cast(AllocPacket()); packetData->HEADER_UNION.op = SDMA_OP_NOP; packetData->HEADER_UNION.sub_op = 0; packetData->HEADER_UNION.count = count - 1; } - -SDMANopPacket::~SDMANopPacket() { - free(packetData); -} diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/tests/kfdtest/src/SDMAPacket.hpp index 009ddea251..4630236236 100644 --- a/tests/kfdtest/src/SDMAPacket.hpp +++ b/tests/kfdtest/src/SDMAPacket.hpp @@ -42,7 +42,7 @@ class SDMAWriteDataPacket : public SDMAPacket { SDMAWriteDataPacket(unsigned int familyId, void* destAddr, unsigned int data); SDMAWriteDataPacket(unsigned int familyId, void* destAddr, unsigned int ndw, void *data); - virtual ~SDMAWriteDataPacket(void); + virtual ~SDMAWriteDataPacket(void) {} // @returns Pointer to the packet virtual const void *GetPacket() const { return packetData; } @@ -63,7 +63,7 @@ class SDMACopyDataPacket : public SDMAPacket { SDMACopyDataPacket(unsigned int familyId, void *dest, void *src, unsigned int size); SDMACopyDataPacket(unsigned int familyId, void *const dst[], void *src, int n, unsigned int surfsize); - virtual ~SDMACopyDataPacket(void); + virtual ~SDMACopyDataPacket(void) {} // @returns Pointer to the packet virtual const void *GetPacket() const { return packetData; } @@ -83,7 +83,7 @@ class SDMAFillDataPacket : public SDMAPacket { // This contructor will also init the packet, no need for additional calls SDMAFillDataPacket(unsigned int familyId, void *dest, unsigned int data, unsigned int size); - virtual ~SDMAFillDataPacket(void); + virtual ~SDMAFillDataPacket(void) {} // @returns Pointer to the packet virtual const void *GetPacket() const { return m_PacketData; } @@ -161,7 +161,7 @@ class SDMATimePacket : public SDMAPacket { class SDMANopPacket : public SDMAPacket { public: SDMANopPacket(unsigned int count = 1); - virtual ~SDMANopPacket(void); + virtual ~SDMANopPacket(void) {} // @returns Pointer to the packet virtual const void *GetPacket() const { return packetData; } From 664bb13bc49aeea4ae3c58717946761e7ecd0ed9 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 30 Oct 2019 20:30:03 -0400 Subject: [PATCH 0626/1247] kfdtest: Add PM4 NOP packet payload capability Useful for allocating space for write-back data in a queue or IB. Change-Id: I47bd2dcb8537a91410e9a91413979a8a2c1c5f21 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/PM4Packet.cpp | 9 +++------ tests/kfdtest/src/PM4Packet.hpp | 9 +++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index df13ca30d2..3571c3ee81 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -358,10 +358,7 @@ unsigned int PM4PartialFlushPacket::SizeInBytes() const { return sizeof(PM4EVENT_WRITE) - sizeof(uint32_t)*2; } -PM4NopPacket::PM4NopPacket(void) { - InitPM4Header(m_packetData, IT_NOP); -} - -unsigned int PM4NopPacket::SizeInBytes() const { - return sizeof(m_packetData); +PM4NopPacket::PM4NopPacket(unsigned int count): m_packetSize(count * 4) { + m_packetData = reinterpret_cast(AllocPacket()); + InitPM4Header(*m_packetData, IT_NOP); } diff --git a/tests/kfdtest/src/PM4Packet.hpp b/tests/kfdtest/src/PM4Packet.hpp index aa5acba391..c02ef290ad 100644 --- a/tests/kfdtest/src/PM4Packet.hpp +++ b/tests/kfdtest/src/PM4Packet.hpp @@ -211,16 +211,17 @@ class PM4PartialFlushPacket : public PM4Packet { // @class PM4NopPacket class PM4NopPacket : public PM4Packet { public: - PM4NopPacket(void); + PM4NopPacket(unsigned int count = 1); virtual ~PM4NopPacket(void) {} // @returns Packet size in bytes - virtual unsigned int SizeInBytes() const; + virtual unsigned int SizeInBytes() const { return m_packetSize; } // @returns Pointer to the packet - virtual const void *GetPacket() const { return &m_packetData; } + virtual const void *GetPacket() const { return m_packetData; } private: - PM4_TYPE_3_HEADER m_packetData; + unsigned int m_packetSize; + PM4_TYPE_3_HEADER *m_packetData; }; #endif // __KFD_PM4_PACKET__H__ From 07b8c30ce8f79860e815aeb80ed5d39386193b29 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 30 Oct 2019 19:21:37 -0400 Subject: [PATCH 0627/1247] kfdtest: Return address of packet from IndirectBuffer::AddPacket This can be used for allocating space in the IB for write-back data in a NOP-packet with a payload. Change-Id: I6202b89a455a65326366a15291789004dfdcc0b9 Signed-off-by: Felix Kuehling --- tests/kfdtest/src/IndirectBuffer.cpp | 10 +++++----- tests/kfdtest/src/IndirectBuffer.hpp | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/src/IndirectBuffer.cpp b/tests/kfdtest/src/IndirectBuffer.cpp index 3fa70f27f6..b820230b0d 100644 --- a/tests/kfdtest/src/IndirectBuffer.cpp +++ b/tests/kfdtest/src/IndirectBuffer.cpp @@ -37,16 +37,16 @@ IndirectBuffer::~IndirectBuffer(void) { delete m_IndirectBuf; } -void IndirectBuffer::AddPacket(const BasePacket &packet) { - ASSERT_EQ(packet.PacketType(), m_PacketTypeAllowed) << "Cannot add a packet since packet type doesn't match queue"; +uint32_t *IndirectBuffer::AddPacket(const BasePacket &packet) { + EXPECT_EQ(packet.PacketType(), m_PacketTypeAllowed) << "Cannot add a packet since packet type doesn't match queue"; unsigned int writePtr = m_ActualSize; - ASSERT_GE(m_MaxSize, packet.SizeInDWords() + writePtr) << "Cannot add a packet, not enough room"; + EXPECT_GE(m_MaxSize, packet.SizeInDWords() + writePtr) << "Cannot add a packet, not enough room"; memcpy(m_IndirectBuf->As() + writePtr , packet.GetPacket(), packet.SizeInBytes()); m_ActualSize += packet.SizeInDWords(); m_NumOfPackets++; + + return m_IndirectBuf->As() + writePtr; } - - diff --git a/tests/kfdtest/src/IndirectBuffer.hpp b/tests/kfdtest/src/IndirectBuffer.hpp index c049402558..debf96c62a 100644 --- a/tests/kfdtest/src/IndirectBuffer.hpp +++ b/tests/kfdtest/src/IndirectBuffer.hpp @@ -39,7 +39,7 @@ class IndirectBuffer { ~IndirectBuffer(void); // @brief Add packet to queue, all validations are done with gtest ASSERT and EXPECT - void AddPacket(const BasePacket &packet); + uint32_t *AddPacket(const BasePacket &packet); // @returns Actual size of the indirect queue in DWords, equivalent to write pointer unsigned int SizeInDWord() { return m_ActualSize; } // @returns Indirect queue address From e7f45fae8aacd90e7b1cba69bfe0bd2c37235dc9 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Tue, 5 Nov 2019 13:07:13 -0500 Subject: [PATCH 0628/1247] Add non-priv PMC blocks to GFX10 This patch adds the non-privileged PMC blocks for GFX10/gfx1010. Change-Id: I4b98cb2159d71113c12920ca7fd10e45096b4e2c Signed-off-by: Ori Messinger --- src/pmc_table.c | 379 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 379 insertions(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index dd4c69b618..f53f5d1a56 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -105,6 +105,37 @@ static uint32_t gfx9_cb_counter_ids[] = { 437 }; +static uint32_t gfx10_cb_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, +405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, +421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, +437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, +453, 454, 455, 456, 457, 458, 459, 460 +}; + /****** CPF ******/ static uint32_t gfx7_cpf_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 @@ -119,6 +150,11 @@ static uint32_t gfx9_cpf_counter_ids[] = { 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, }; +static uint32_t gfx10_cpf_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39 +}; + /****** CPG ******/ static uint32_t gfx7_cpg_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -138,6 +174,13 @@ static uint32_t gfx9_cpg_counter_ids[] = { 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58 }; +static uint32_t gfx10_cpg_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81 +}; + /****** DB ******/ static uint32_t gfx7_db_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -181,6 +224,31 @@ static uint32_t gfx9_db_counter_ids[] = { 325, 326, 327 }; +static uint32_t gfx10_db_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369 +}; + /****** GDS ******/ static uint32_t gfx7_gds_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -194,6 +262,16 @@ static uint32_t gfx7_gds_counter_ids[] = { /* gfx8_gds_counter_ids are the same as gfx7_gds_counter_ids */ /* gfx9_gds_counter_ids are the same as gfx7_gds_counter_ids */ +static uint32_t gfx10_gds_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122 +}; + /****** GRBM ******/ static uint32_t gfx7_grbm_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -206,6 +284,12 @@ static uint32_t gfx9_grbm_counter_ids[] = { 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37 }; +static uint32_t gfx10_grbm_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46 +}; + /****** GRBMSE ******/ static uint32_t gfx7_grbmse_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 @@ -216,6 +300,10 @@ static uint32_t gfx9_grbmse_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; +static uint32_t gfx10_grbmse_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 +}; + /****** IA ******/ static uint32_t gfx7_ia_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 @@ -230,6 +318,7 @@ static uint32_t gfx9_ia_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 }; +/* gfx10 doesn't have IA */ /****** PASC ******/ static uint32_t gfx7_pasc_counter_ids[] = { @@ -319,6 +408,43 @@ static uint32_t gfx9_pasc_counter_ids[] = { 485, 486, 487, 488, 489, 490 }; +static uint32_t gfx10_pasc_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, +405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, +421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, +437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, +453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, +469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, +485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, +501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, +517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, +533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, +549, 550, 551 +}; + /****** PASU ******/ static uint32_t gfx7_pasu_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -353,6 +479,25 @@ static uint32_t gfx9_pasu_counter_ids[] = { 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291 }; +static uint32_t gfx10_pasu_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265 +}; + /****** SPI ******/ static uint32_t gfx7_spi_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -396,6 +541,29 @@ static uint32_t gfx9_spi_counter_ids[] = { 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195 }; +static uint32_t gfx10_spi_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328 +}; + /****** SQ ******/ /* Unused counters - 163-167 */ static uint32_t gfx7_sq_counter_ids[] = { @@ -482,6 +650,40 @@ static uint32_t gfx9_sq_counter_ids[] = { 364, 365, 366, 367, 368, 369, 370, 371, 372 }; +static uint32_t gfx10_sq_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, +229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, +245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, +261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, +277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, +293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, +309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, +325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, +341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, +357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, +373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, +389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, +405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, +421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, +437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, +453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, +469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, +485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, +501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511 +}; + /****** SRBM ******/ static uint32_t gfx7_srbm_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18 @@ -492,6 +694,7 @@ static uint32_t gfx8_srbm_counter_ids[] = { 22, 23, 24, 25, 26, 27 }; /* gfx9 doesn't have SRBM */ +/* gfx10 doesn't have SRBM */ /****** SX ******/ static uint32_t gfx7_sx_counter_ids[] = { @@ -515,6 +718,22 @@ static uint32_t gfx9_sx_counter_ids[] = { 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207 }; +static uint32_t gfx10_sx_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224 +}; + /****** TA ******/ static uint32_t gfx7_ta_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -536,6 +755,22 @@ static uint32_t gfx8_ta_counter_ids[] = { }; /* gfx9_ta_counter_ids is same as gfx8_ta_counter_ids */ +static uint32_t gfx10_ta_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, +82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, +101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, +117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, +133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, +149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, +165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, +181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, +197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, +213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225 +}; + /****** TCA ******/ static uint32_t gfx7_tca_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -547,6 +782,7 @@ static uint32_t gfx8_tca_counter_ids[] = { 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34 }; /* gfx9_tca_counter_ids is same as gfx8_tca_counter_ids */ +/* gfx10 doesn't have TCA */ /****** TCC ******/ static uint32_t gfx7_tcc_counter_ids[] = { @@ -590,6 +826,7 @@ static uint32_t gfx8_cz_tcc_counter_ids[] = { 185, 186, 187, 188, 189, 190, 191 }; /* gfx9_tcc_counter_ids is same as gfx8_tcc_counter_ids */ +/* gfx10 doesn't have TCC */ /****** TCP ******/ static uint32_t gfx7_tcp_counter_ids[] = { @@ -626,6 +863,13 @@ static uint32_t gfx9_tcp_counter_ids[] = { 82, 83, 84 }; +static uint32_t gfx10_tcp_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, +62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76 +}; + /****** TCS ******/ static uint32_t gfx7_tcs_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 64, @@ -636,6 +880,7 @@ static uint32_t gfx7_tcs_counter_ids[] = { }; /* gfx8 doesn't have TCS */ /* gfx9 doesn't have TCS */ +/* gfx10 doesn't have TCS */ /****** TD ******/ static uint32_t gfx7_td_counter_ids[] = { @@ -651,6 +896,12 @@ static uint32_t gfx9_td_counter_ids[] = { 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56 }; +static uint32_t gfx10_td_counter_ids[] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, +22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, +42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60 +}; + /****** VGT ******/ static uint32_t gfx7_vgt_counter_ids[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, @@ -695,6 +946,7 @@ static uint32_t gfx9_vgt_counter_ids[] = { 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147 }; +/* gfx10 doesn't have VGT */ /****** WD ******/ static uint32_t gfx7_wd_counter_ids[] = { @@ -711,6 +963,7 @@ static uint32_t gfx9_wd_counter_ids[] = { 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57 }; +/* gfx10 doesn't have WD */ static struct perf_counter_block kaveri_blocks[PERFCOUNTER_BLOCKID__MAX] = { [PERFCOUNTER_BLOCKID__SQ] = { @@ -1582,6 +1835,129 @@ static struct perf_counter_block vega_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; +static struct perf_counter_block navi_blocks[PERFCOUNTER_BLOCKID__MAX] = { + [PERFCOUNTER_BLOCKID__CB] = { + .num_of_slots = 7, + .num_of_counters = sizeof(gfx10_cb_counter_ids) / + sizeof(*gfx10_cb_counter_ids), + .counter_ids = gfx10_cb_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPF] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx10_cpf_counter_ids) / + sizeof(*gfx10_cpf_counter_ids), + .counter_ids = gfx10_cpf_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__CPG] = { + .num_of_slots = 6, + .num_of_counters = sizeof(gfx10_cpg_counter_ids) / + sizeof(*gfx10_cpg_counter_ids), + .counter_ids = gfx10_cpg_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__DB] = { + .num_of_slots = 12, + .num_of_counters = sizeof(gfx10_db_counter_ids) / + sizeof(*gfx10_db_counter_ids), + .counter_ids = gfx10_db_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GDS] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx10_gds_counter_ids) / + sizeof(*gfx10_gds_counter_ids), + .counter_ids = gfx10_gds_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBM] = { + .num_of_slots = 2, + .num_of_counters = sizeof(gfx10_grbm_counter_ids) / + sizeof(*gfx10_grbm_counter_ids), + .counter_ids = gfx10_grbm_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__GRBMSE] = { + .num_of_slots = 1, + .num_of_counters = sizeof(gfx10_grbmse_counter_ids) / + sizeof(*gfx10_grbmse_counter_ids), + .counter_ids = gfx10_grbmse_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASC] = { + .num_of_slots = 11, + .num_of_counters = sizeof(gfx10_pasc_counter_ids) / + sizeof(*gfx10_pasc_counter_ids), + .counter_ids = gfx10_pasc_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__PASU] = { + .num_of_slots = 16, + .num_of_counters = sizeof(gfx10_pasu_counter_ids) / + sizeof(*gfx10_pasu_counter_ids), + .counter_ids = gfx10_pasu_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SPI] = { + .num_of_slots = 18, + .num_of_counters = sizeof(gfx10_spi_counter_ids) / + sizeof(*gfx10_spi_counter_ids), + .counter_ids = gfx10_spi_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SQ] = { + .num_of_slots = 16, + .num_of_counters = sizeof(gfx10_sq_counter_ids) / + sizeof(*gfx10_sq_counter_ids), + .counter_ids = gfx10_sq_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__SX] = { + .num_of_slots = 4, + .num_of_counters = sizeof(gfx10_sx_counter_ids) / + sizeof(*gfx10_sx_counter_ids), + .counter_ids = gfx10_sx_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TA] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx10_ta_counter_ids) / + sizeof(*gfx10_ta_counter_ids), + .counter_ids = gfx10_ta_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TCP] = { + .num_of_slots = 10, + .num_of_counters = sizeof(gfx10_tcp_counter_ids) / + sizeof(*gfx10_tcp_counter_ids), + .counter_ids = gfx10_tcp_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, + [PERFCOUNTER_BLOCKID__TD] = { + .num_of_slots = 5, + .num_of_counters = sizeof(gfx10_td_counter_ids) / + sizeof(*gfx10_td_counter_ids), + .counter_ids = gfx10_td_counter_ids, + .counter_size_in_bits = 64, + .counter_mask = BITMASK(64) + }, +}; + /* Current APUs only have one IOMMU. If NUMA is introduced to APUs, we'll need * to expand the struct here to an array. */ @@ -1753,6 +2129,9 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, case CHIP_ARCTURUS: *block = vega_blocks[block_id]; break; + case CHIP_NAVI10: + *block = navi_blocks[block_id]; + break; default: return HSAKMT_STATUS_INVALID_PARAMETER; } From 59c857476fa39447e176d32c6f19c66a446795d2 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 6 Nov 2019 14:32:25 -0500 Subject: [PATCH 0629/1247] libhsakmt: use the closest NUMA node to allocate queue ctx area On NUMA system, allocate queue ctx save restore area on the closest NUMA node to the GPU which the queue is going to run. This will improve performance on NUMA system generally by reducing schedule latency and fix the multi-node rccl-tests unstable performance issue. If the closest NUMA node has no memory available, set flags NoNUMABind=1 to bypass mbind, to use default NUMA memory policy to allocate system memory. Change-Id: Ic62bfa5bb2efbf4f6ae79ff403e9610ddf18d45c Signed-off-by: Philip Yang --- src/fmm.c | 3 +++ src/libhsakmt.h | 1 + src/queues.c | 15 +++++++++++++-- src/topology.c | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index fd69522aa0..37e543a099 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1462,6 +1462,9 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, int num_node; long r; + pr_debug("%s mem %p flags 0x%x size 0x%lx node_id %d\n", __func__, + mem, flags.Value, SizeInBytes, node_id); + if (flags.ui32.NoNUMABind) return 0; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index d0e95ae170..40f5e97762 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -147,6 +147,7 @@ bool prefer_ats(HSAuint32 node_id); uint16_t get_device_id_by_node_id(HSAuint32 node_id); bool is_kaveri(HSAuint32 node_id); uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); +uint32_t get_direct_link_cpu(uint32_t gpu_node); int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); diff --git a/src/queues.c b/src/queues.c index 8ee165cf0c..7d0e67d0a1 100644 --- a/src/queues.c +++ b/src/queues.c @@ -424,6 +424,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, HSAuint64 gpu_va; HsaMemFlags flags; HSAKMT_STATUS ret; + HSAuint32 cpu_id = 0; flags.Value = 0; flags.ui32.HostAccess = !DeviceLocal; @@ -431,11 +432,21 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; flags.ui32.CoarseGrain = DeviceLocal; - flags.ui32.NoNUMABind = 1; + + /* Get the closest cpu_id to GPU NodeId for system memory allocation + * nonPaged=1 system memory allocation uses GTT path + */ + if (!DeviceLocal && !nonPaged) { + cpu_id = get_direct_link_cpu(NodeId); + if (cpu_id == INVALID_NODEID) { + flags.ui32.NoNUMABind = 1; + cpu_id = 0; + } + } size = ALIGN_UP(size, align); - ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : 0, size, flags, &mem); + ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : cpu_id, size, flags, &mem); if (ret != HSAKMT_STATUS_SUCCESS) return NULL; diff --git a/src/topology.c b/src/topology.c index b73753ff6e..742252d5df 100644 --- a/src/topology.c +++ b/src/topology.c @@ -2224,6 +2224,25 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) return 0; } +uint32_t get_direct_link_cpu(uint32_t gpu_node) +{ + HSAuint64 size = 0; + int32_t cpu_id; + HSAuint32 i; + + cpu_id = gpu_get_direct_link_cpu(gpu_node, g_props); + if (cpu_id == -1) + return INVALID_NODEID; + + assert(g_props[cpu_id].mem); + + for (i = 0; i < g_props[cpu_id].node.NumMemoryBanks; i++) + size += g_props[cpu_id].mem[i].SizeInBytes; + + return size ? (uint32_t)cpu_id : INVALID_NODEID; +} + + HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray) { From 2fa7d23a82b18cdd148276e05af3b53d8a6ff6b8 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 6 Nov 2019 18:33:46 -0500 Subject: [PATCH 0630/1247] kfdtest: use flag NoNUMABind for more test cases If NUMA system no available memory on node 0, mbind will fail on node 0, so set flag NoNUMABind=1 to bypass mbind for all test cases which use node 0 and allocate system memory. Change-Id: I7962938ad2bed5a293ca5e6a8500c7f7e15ff453 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDMemoryTest.cpp | 6 +++++- tests/kfdtest/src/KFDTestUtil.cpp | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 747b1daa5a..297f8c85b9 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -385,6 +385,7 @@ TEST_F(KFDMemoryTest, MemoryAlloc) { TEST_START(TESTPROFILE_RUNALL) unsigned int* pDb = NULL; + m_MemoryFlags.ui32.NoNUMABind = 1; EXPECT_SUCCESS(hsaKmtAllocMemory(0 /* system */, PAGE_SIZE, m_MemoryFlags, reinterpret_cast(&pDb))); TEST_END @@ -1094,6 +1095,7 @@ TEST_F(KFDMemoryTest, MMBench) { memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 1; memFlags.ui32.NonPaged = 0; + memFlags.ui32.NoNUMABind = 1; } else { allocNode = defaultGPUNode; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; @@ -1328,6 +1330,7 @@ TEST_F(KFDMemoryTest, PtraceAccess) { // Alloc system memory from node 0 and initialize it memFlags.ui32.NonPaged = 0; + memFlags.ui32.NoNUMABind = 1; ASSERT_SUCCESS(hsaKmtAllocMemory(0, PAGE_SIZE*2, memFlags, &mem[0])); for (i = 0; i < 4*sizeof(HSAint64) + 4; i++) { (reinterpret_cast(mem[0]))[i] = i; // source @@ -1624,7 +1627,7 @@ TEST_F(KFDMemoryTest, SignalHandling) { * Try to allocate 1/4th System RAM */ size = (sysMemSize >> 2) & ~(HSAuint64)(PAGE_SIZE - 1); - + m_MemoryFlags.ui32.NoNUMABind = 1; ASSERT_SUCCESS(hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, reinterpret_cast(&pDb))); // Verify that pDb is not null before it's being used EXPECT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; @@ -1794,6 +1797,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 1; memFlags.ui32.NonPaged = 0; + memFlags.ui32.NoNUMABind = 1; } else { /* Alloc visible vram*/ allocNode = defaultGPUNode; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index c949259439..d2914a7bc2 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -199,6 +199,7 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b m_Flags.ui32.HostAccess = 1; m_Flags.ui32.NonPaged = 0; m_Flags.ui32.CoarseGrain = 0; + m_Flags.ui32.NoNUMABind = 1; } if (isExec) From 174484aac39e4eedf757860b887429ab65eaf9c3 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 6 Nov 2019 18:57:36 -0500 Subject: [PATCH 0631/1247] kfdtest: Use GetSystemTickCountInMicroSec() as much as possible GetSystemTickCountInMicroSec() wraps the function gettimeofday(). Change-Id: I7b767a6efdd1db491fc8113313945b578ac69382 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDQMTest.cpp | 27 +++++++++++++-------------- tests/kfdtest/src/KFDTestUtil.cpp | 7 +++++++ tests/kfdtest/src/KFDTestUtil.hpp | 2 ++ tests/kfdtest/src/LinuxOSWrapper.cpp | 7 ------- tests/kfdtest/src/OSWrapper.hpp | 2 -- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 13cfb96fb4..0c543e6202 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -640,14 +640,13 @@ HSAint64 KFDQMTest::TimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t ma EXPECT_SUCCESS(queue.SetCUMask(mask, mask_count)); queue.SetSkipWaitConsump(true); - struct timeval start, now; - gettimeofday(&start, NULL); + HSAuint64 startTime = GetSystemTickCountInMicroSec(); dispatch.Submit(queue); dispatch.Sync(); - gettimeofday(&now, NULL); + HSAuint64 endTime = GetSystemTickCountInMicroSec(); EXPECT_SUCCESS(queue.Destroy()); - return (now.tv_sec *1000LL + now.tv_usec/1000LL - start.tv_sec * 1000LL - start.tv_usec/1000LL); + return endTime - startTime; } /* To cover for outliers, allow us to get the Average time based on a specified number of iterations */ @@ -831,7 +830,7 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { }; int activeTaskBitmap = 0x3; - struct timeval start, end[2]; + HSAuint64 startTime, endTime[2]; HsaEvent *pHsaEvent[2]; int numEvent = 2; PM4Queue queue[2]; @@ -850,7 +849,7 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { dispatch[i].SetDim(1024, 16, 16); } - gettimeofday(&start, NULL); + startTime = GetSystemTickCountInMicroSec(); for (i = 0; i < 2; i++) dispatch[i].Submit(queue[i]); @@ -858,7 +857,7 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { hsaKmtWaitOnMultipleEvents(pHsaEvent, numEvent, false, g_TestTimeOut); for (i = 0; i < 2; i++) { if ((activeTaskBitmap & (1 << i)) && (syncBuffer[i] == pHsaEvent[i]->EventId)) { - gettimeofday(&end[i], NULL); + endTime[i] = GetSystemTickCountInMicroSec(); activeTaskBitmap &= ~(1 << i); } } @@ -866,9 +865,9 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { for (i = 0; i < 2; i++) { EXPECT_SUCCESS(queue[i].Destroy()); - int ms = end[i].tv_sec *1000LL + end[i].tv_usec/1000LL - start.tv_sec * 1000LL - start.tv_usec/1000LL; + int usecs = endTime[i] - startTime; LOG() << "Task priority: " << std::dec << priority[i] << "\t"; - LOG() << "Task duration: " << std::dec << ms << "ms" << std::endl; + LOG() << "Task duration: " << std::dec << usecs << "usecs" << std::endl; } TEST_END @@ -896,7 +895,7 @@ TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { }; int activeTaskBitmap = 0x3; - struct timeval start, end[2]; + HSAuint64 startTime, endTime[2]; HsaEvent *pHsaEvent[2]; int numEvent = 2; PM4Queue queue[13]; @@ -924,7 +923,7 @@ TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { dispatch[i].SetDim(1024, 16, 16); } - gettimeofday(&start, NULL); + startTime = GetSystemTickCountInMicroSec(); for (i = 0; i < 2; i++) dispatch[i].Submit(queue[i]); @@ -932,16 +931,16 @@ TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { hsaKmtWaitOnMultipleEvents(pHsaEvent, numEvent, false, g_TestTimeOut); for (i = 0; i < 2; i++) { if ((activeTaskBitmap & (1 << i)) && (syncBuffer[i] == pHsaEvent[i]->EventId)) { - gettimeofday(&end[i], NULL); + endTime[i] = GetSystemTickCountInMicroSec(); activeTaskBitmap &= ~(1 << i); } } } for (i = 0; i < 2; i++) { - int ms = end[i].tv_sec *1000LL + end[i].tv_usec/1000LL - start.tv_sec * 1000LL - start.tv_usec/1000LL; + int usecs = endTime[i] - startTime; LOG() << "Task priority: " << std::dec << priority[i] << "\t"; - LOG() << "Task duration: " << std::dec << ms << "ms" << std::endl; + LOG() << "Task duration: " << std::dec << usecs << "usecs" << std::endl; } for (i = 0; i <= 12; i++) { diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index d2914a7bc2..105cc00679 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -24,6 +24,7 @@ #include "KFDTestUtil.hpp" #include +#include #include #include #include "BaseQueue.hpp" @@ -171,6 +172,12 @@ bool isTonga(const HsaNodeProperties *props) { return false; } +HSAuint64 GetSystemTickCountInMicroSec() { + struct timeval t; + gettimeofday(&t, 0); + return t.tv_sec * 1000000ULL + t.tv_usec; +} + const HsaMemoryBuffer HsaMemoryBuffer::Null; HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 7e2fe7e16d..5fefff0516 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -56,6 +56,8 @@ void GetSdmaInfo(const HsaNodeProperties *props, unsigned int *p_num_sdma_xgmi_engines, unsigned int *p_num_sdma_queues_per_engine); +HSAuint64 GetSystemTickCountInMicroSec(); + class HsaMemoryBuffer { public: static const HsaMemoryBuffer Null; diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/tests/kfdtest/src/LinuxOSWrapper.cpp index e529c2b6dd..f5308249bf 100644 --- a/tests/kfdtest/src/LinuxOSWrapper.cpp +++ b/tests/kfdtest/src/LinuxOSWrapper.cpp @@ -39,7 +39,6 @@ #include #include #include -#include static int protection_flags[8] = {PROT_NONE, PROT_READ, PROT_WRITE, PROT_READ | PROT_WRITE, PROT_EXEC, PROT_EXEC | PROT_READ, PROT_EXEC | PROT_WRITE, @@ -80,12 +79,6 @@ bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProc return false; } -HSAuint64 GetSystemTickCountInMicroSec() { - struct timeval t; - gettimeofday(&t, 0); - return t.tv_sec * 1000000ULL + t.tv_usec; -} - bool SuspendAndWakeUp() { printf("Please press any key after the system suspends....\n"); diff --git a/tests/kfdtest/src/OSWrapper.hpp b/tests/kfdtest/src/OSWrapper.hpp index 99a7ff5829..4ded18e0e0 100644 --- a/tests/kfdtest/src/OSWrapper.hpp +++ b/tests/kfdtest/src/OSWrapper.hpp @@ -95,8 +95,6 @@ void MemoryBarrier(); */ bool MultiProcessTest(const char *testToRun, int numOfProcesses, int runsPerProcess = 1); -HSAuint64 GetSystemTickCountInMicroSec(); - /* Put the system to S3/S4 power state and bring it back to S0. * @return 'true' on success, 'false' on failure. */ From 4b36a1e728118a94620f52d9b0dfd5e63792f480 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 7 Nov 2019 18:12:15 -0500 Subject: [PATCH 0632/1247] kfdtest: Rename KFDQMTest.MultipleSdmaQueues to AllSdmaQueues The test actually tested all available SDMA queues, so change the name to reflect the fact. Change-Id: Ia23df3e5ac79b692b0b60194b05603ba8dd897a4 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 6 +++--- tests/kfdtest/src/KFDQMTest.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index b0ea7a7947..bfc15f9b62 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -17,7 +17,7 @@ FILTER[core_sws]=\ "KFDQMTest.CreateDestroySdmaQueue:"\ "KFDQMTest.SubmitNopSdmaQueue:"\ "KFDQMTest.SubmitPacketSdmaQueue:"\ -"KFDQMTest.MultipleSdmaQueues:"\ +"KFDQMTest.AllSdmaQueues:"\ "KFDEventTest.SignalEvent" # HWS mode @@ -153,14 +153,14 @@ FILTER[polaris12]=\ # KFDIPCTest.BasicTest (ROCMOPS-459) .CMABasicTest (ROCMOPS-460) .CrossMemoryAttachTest (ROCMOPS-461) # KFDEvictTest.BurstyTest (ROCMOPS-464) -# KFDQMTest.MultipleSdmaQueues (ROCMOPS-463) +# KFDQMTest.AllSdmaQueues (ROCMOPS-463) FILTER[vega10]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDIPCTest.BasicTest:"\ "KFDEvictTest.BurstyTest:"\ "KFDIPCTest.CMABasicTest:"\ "KFDIPCTest.CrossMemoryAttachTest:"\ -"KFDQMTest.MultipleSdmaQueues" +"KFDQMTest.AllSdmaQueues" FILTER[vega12]=\ "$BLACKLIST_ALL_ASICS:"\ diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 0c543e6202..e75537ef5a 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -205,7 +205,7 @@ TEST_F(KFDQMTest, SubmitPacketSdmaQueue) { TEST_END } -TEST_F(KFDQMTest, MultipleSdmaQueues) { +TEST_F(KFDQMTest, AllSdmaQueues) { TEST_START(TESTPROFILE_RUNALL) int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); From 6742e585be741ef35048557985ec2195d8e8006f Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Wed, 16 Oct 2019 11:07:20 -0400 Subject: [PATCH 0633/1247] kfdtest: Enable kfd debugger tests on gfx10 Change-Id: If3f86624c76805a6bc0e154d31b957eb63120418 Signed-off-by: Philip Cox --- tests/kfdtest/src/KFDDBGTest.cpp | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 78b38e4c35..70b7558c4e 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -30,6 +30,7 @@ #include "Dispatch.hpp" #include +#if 0 static const char* loop_inc_isa = \ "\ shader loop_inc_isa\n\ @@ -64,11 +65,12 @@ trap_present(1)\n\ \n\ end\n\ "; +#endif -static const char* iterate_isa_gfx9 = \ +static const char* iterate_isa_gfx = \ "\ shader iterate_isa\n\ -asic(GFX9)\n\ +wave_size(32) \n\ type(CS)\n\ /*copy the parameters from scalar registers to vector registers*/\n\ v_mov_b32 v0, s0\n\ @@ -91,10 +93,10 @@ LOOP:\n\ end\n\ "; -static const char* jump_to_trap_gfx9 = \ +static const char* jump_to_trap_gfx = \ "\ shader jump_to_trap\n\ -asic(GFX9)\n\ +wave_size(32) \n\ type(CS)\n\ /*copy the parameters from scalar registers to vector registers*/\n\ s_trap 1\n\ @@ -118,10 +120,10 @@ LOOP:\n\ end\n\ "; -static const char* trap_handler_gfx9 = \ +static const char* trap_handler_gfx = \ "\ shader trap_handler\n\ -asic(GFX9)\n\ +wave_size(32) \n\ type(CS)\n\ CHECK_VMFAULT:\n\ /*if trap jumped to by vmfault, restore skip m0 signalling*/\n\ @@ -333,7 +335,7 @@ static bool checkDebugVersion(HSAuint32 requiredMajor, HSAuint32 requiredMinor) TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI && m_FamilyId <= FAMILY_AR) { + if (m_FamilyId >= FAMILY_AI) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -355,7 +357,9 @@ TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { int suspendTimeout = 500; int syncStatus; - m_pIsaGen->CompileShader(iterate_isa_gfx9, "iterate_isa", isaBuffer); + m_pIsaGen->CompileShader(iterate_isa_gfx, + "iterate_isa", + isaBuffer); PM4Queue queue1; HsaQueueResource *qResources; @@ -429,7 +433,7 @@ exit: TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI && m_FamilyId <= FAMILY_AR) { + if (m_FamilyId >= FAMILY_AI) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); HSAint32 PollFd; @@ -461,8 +465,8 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { 0x1000)); // compile and dispatch shader - m_pIsaGen->CompileShader(jump_to_trap_gfx9, "jump_to_trap", isaBuf); - m_pIsaGen->CompileShader(trap_handler_gfx9, "trap_handler", trap); + m_pIsaGen->CompileShader(jump_to_trap_gfx, "jump_to_trap", isaBuf); + m_pIsaGen->CompileShader(trap_handler_gfx, "trap_handler", trap); PM4Queue queue; HsaQueueResource *qResources; @@ -584,7 +588,7 @@ static void ExitVMFaultQueryChild(std::string errMsg, TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI && m_FamilyId <= FAMILY_AR) { + if (m_FamilyId >= FAMILY_AI) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -645,9 +649,9 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { defaultGPUNode); // compile and dispatch shader - m_pIsaGen->CompileShader(jump_to_trap_gfx9, "jump_to_trap", + m_pIsaGen->CompileShader(jump_to_trap_gfx, "jump_to_trap", isaBuf); - m_pIsaGen->CompileShader(trap_handler_gfx9, "trap_handler", trap); + m_pIsaGen->CompileShader(trap_handler_gfx, "trap_handler", trap); PM4Queue queue1, queue2; HSAuint32 qid1; From fdba74c2fb7ab1408d6cf21dc86dbe9d838ecba6 Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Tue, 12 Nov 2019 14:32:19 +0800 Subject: [PATCH 0634/1247] libhsakmt: add gfx90c support for thunk This patch adds the support for gfx90c apu. So far we treat it as "dgpu" and gfx900. Will update hsa gfxip table while the isa/llvm is implemented on gfx90c. Change-Id: I6ef164bf3e751fe6dd6287cac212a500dce84b1a Signed-off-by: Huang Rui --- src/libhsakmt.h | 9 +++++---- src/pmc_table.c | 1 + src/queues.c | 7 +++++++ src/topology.c | 2 ++ tests/kfdtest/scripts/kfdtest.exclude | 5 +++++ 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 40f5e97762..6cb49c59c5 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -122,10 +122,11 @@ enum asic_family_type { CHIP_VEGA12, /* 10 */ CHIP_VEGA20, /* 11 */ CHIP_RAVEN, /* 12 */ - CHIP_ARCTURUS, /* 13 */ - CHIP_NAVI10, /* 14 */ - CHIP_NAVI12, /* 15 */ - CHIP_NAVI14, /* 16 */ + CHIP_RENOIR, /* 13 */ + CHIP_ARCTURUS, /* 14 */ + CHIP_NAVI10, /* 15 */ + CHIP_NAVI12, /* 16 */ + CHIP_NAVI14, /* 17 */ CHIP_LAST }; diff --git a/src/pmc_table.c b/src/pmc_table.c index f53f5d1a56..ccc22a3965 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -2126,6 +2126,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, case CHIP_VEGA12: case CHIP_VEGA20: case CHIP_RAVEN: + case CHIP_RENOIR: case CHIP_ARCTURUS: *block = vega_blocks[block_id]; break; diff --git a/src/queues.c b/src/queues.c index 7d0e67d0a1..85d1f549ba 100644 --- a/src/queues.c +++ b/src/queues.c @@ -128,6 +128,12 @@ const struct device_info raven_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info renoir_device_info = { + .asic_family = CHIP_RENOIR, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + const struct device_info vega20_device_info = { .asic_family = CHIP_VEGA20, .eop_buffer_size = 4096, @@ -172,6 +178,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_VEGA12] = &vega12_device_info, [CHIP_VEGA20] = &vega20_device_info, [CHIP_RAVEN] = &raven_device_info, + [CHIP_RENOIR] = &renoir_device_info, [CHIP_ARCTURUS] = &arcturus_device_info, [CHIP_NAVI10] = &navi10_device_info, [CHIP_NAVI12] = &navi12_device_info, diff --git a/src/topology.c b/src/topology.c index 742252d5df..6aab6323d5 100644 --- a/src/topology.c +++ b/src/topology.c @@ -207,6 +207,8 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Raven */ { 0x15DD, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, { 0x15D8, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, + /* Renoir */ + { 0x1636, 9, 0, 0, 1, "Renoir", CHIP_RENOIR }, /* Vega20 */ { 0x66A0, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, { 0x66A1, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index bfc15f9b62..de268ac323 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -180,6 +180,11 @@ FILTER[raven]=\ "KFDQMTest.QueueLatency:"\ "KFDQMTest.SdmaEventInterrupt" +FILTER[renoir]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.QueueLatency:"\ +"KFDQMTest.SdmaEventInterrupt" + # KFDExceptionTest.* (KFD-435) # KFDEvictTest.BurstyTest (KFD-425) FILTER[arcturus]=\ From a4d570fa2b0cd457147622c6ed7fa6a8cbe90149 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 6 Nov 2019 19:10:58 -0500 Subject: [PATCH 0635/1247] kfdtest: Expand KFDQMTest.MultipleCpQueues to cover all CP queues Because of that, rename the test to AllCpQueues. Change-Id: I57105f863db2558e850c703d151ffebcce2c7a17 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- tests/kfdtest/src/KFDBaseComponentTest.cpp | 4 ++-- tests/kfdtest/src/KFDBaseComponentTest.hpp | 1 + tests/kfdtest/src/KFDQMTest.cpp | 14 ++++++-------- tests/kfdtest/src/KFDTestUtil.cpp | 11 ++++++++--- tests/kfdtest/src/KFDTestUtil.hpp | 3 ++- 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index de268ac323..4d1b3b34e4 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -13,7 +13,7 @@ FILTER[core_sws]=\ "KFDQMTest.CreateDestroyCpQueue:"\ "KFDQMTest.SubmitNopCpQueue:"\ "KFDQMTest.SubmitPacketCpQueue:"\ -"KFDQMTest.MultipleCpQueues:"\ +"KFDQMTest.AllCpQueues:"\ "KFDQMTest.CreateDestroySdmaQueue:"\ "KFDQMTest.SubmitNopSdmaQueue:"\ "KFDQMTest.SubmitPacketSdmaQueue:"\ diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 27f3db1cda..f950a7a1f1 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -63,8 +63,8 @@ void KFDBaseComponentTest::SetUp() { m_FamilyId = FamilyIdFromNode(nodeProperties); - GetSdmaInfo(nodeProperties, &m_numSdmaEngines, &m_numSdmaXgmiEngines, - &m_numSdmaQueuesPerEngine); + GetHwQueueInfo(nodeProperties, &m_numCpQueues, &m_numSdmaEngines, + &m_numSdmaXgmiEngines, &m_numSdmaQueuesPerEngine); g_baseTest = this; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index fce5c0d884..8a99e405e4 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -62,6 +62,7 @@ class KFDBaseComponentTest : public testing::Test { HsaVersionInfo m_VersionInfo; HsaSystemProperties m_SystemProperties; unsigned int m_FamilyId; + unsigned int m_numCpQueues; unsigned int m_numSdmaEngines; unsigned int m_numSdmaXgmiEngines; unsigned int m_numSdmaQueuesPerEngine; diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index e75537ef5a..b5ab04bc63 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -115,32 +115,30 @@ TEST_F(KFDQMTest, SubmitPacketCpQueue) { TEST_END } -TEST_F(KFDQMTest, MultipleCpQueues) { +TEST_F(KFDQMTest, AllCpQueues) { TEST_START(TESTPROFILE_RUNALL) int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - static const unsigned int MAX_CP_QUEUES = 16; - HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); destBuf.Fill(0xFF); - PM4Queue queues[MAX_CP_QUEUES]; + std::vector queues(m_numCpQueues); - for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + for (unsigned int qidx = 0; qidx < m_numCpQueues; ++qidx) ASSERT_SUCCESS(queues[qidx].Create(defaultGPUNode)) << " QueueId=" << qidx; - for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) { + for (unsigned int qidx = 0; qidx < m_numCpQueues; ++qidx) { queues[qidx].PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As()+qidx*2, qidx, qidx)); queues[qidx].Wait4PacketConsumption(); - WaitOnValue(destBuf.As()+qidx*2, qidx); + EXPECT_TRUE(WaitOnValue(destBuf.As()+qidx*2, qidx)); } - for (unsigned int qidx = 0; qidx < MAX_CP_QUEUES; ++qidx) + for (unsigned int qidx = 0; qidx < m_numCpQueues; ++qidx) EXPECT_SUCCESS(queues[qidx].Destroy()); TEST_END diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 105cc00679..327d4f6702 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -140,17 +140,19 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { return familyId; } -void GetSdmaInfo(const HsaNodeProperties *props, +void GetHwQueueInfo(const HsaNodeProperties *props, + unsigned int *p_num_cp_queues, unsigned int *p_num_sdma_engines, unsigned int *p_num_sdma_xgmi_engines, unsigned int *p_num_sdma_queues_per_engine) { + int num_cp_queues = 24; int num_sdma_queues_per_engine = 2; if (props->EngineId.ui32.Major == 9) { if (props->EngineId.ui32.Stepping == 6) // VEGA20 num_sdma_queues_per_engine = 8; - } else if (props->EngineId.ui32.Major == 10) { //NAVIi - num_sdma_queues_per_engine = 8; + } else if (props->EngineId.ui32.Major == 10) { // NAVI + num_sdma_queues_per_engine = 8; } if (p_num_sdma_engines) @@ -161,6 +163,9 @@ void GetSdmaInfo(const HsaNodeProperties *props, if (p_num_sdma_queues_per_engine) *p_num_sdma_queues_per_engine = num_sdma_queues_per_engine; + + if (p_num_cp_queues) + *p_num_cp_queues = num_cp_queues; } bool isTonga(const HsaNodeProperties *props) { diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 5fefff0516..139fbd1513 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -51,7 +51,8 @@ bool is_dgpu(); bool isTonga(const HsaNodeProperties *props); unsigned int FamilyIdFromNode(const HsaNodeProperties *props); -void GetSdmaInfo(const HsaNodeProperties *props, +void GetHwQueueInfo(const HsaNodeProperties *props, + unsigned int *p_num_cp_queues, unsigned int *p_num_sdma_engines, unsigned int *p_num_sdma_xgmi_engines, unsigned int *p_num_sdma_queues_per_engine); From 0b7fcadc639b9efe9a0dfb80ac65044477f6739e Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 30 Oct 2019 21:11:04 -0400 Subject: [PATCH 0636/1247] kfdtest: Add PM4WaitRegMemPacket v2: optimize_ace_offload_mode=1, recommended by firmware team Change-Id: Ia54e37242b4eaaf631c35e61a59f03ee0f85ca35 Signed-off-by: Felix Kuehling --- tests/kfdtest/include/pm4_pkt_struct_common.h | 91 +++++++++++++++++++ tests/kfdtest/src/PM4Packet.cpp | 43 +++++++++ tests/kfdtest/src/PM4Packet.hpp | 21 +++++ 3 files changed, 155 insertions(+) diff --git a/tests/kfdtest/include/pm4_pkt_struct_common.h b/tests/kfdtest/include/pm4_pkt_struct_common.h index d3b78427b9..44950a7760 100644 --- a/tests/kfdtest/include/pm4_pkt_struct_common.h +++ b/tests/kfdtest/include/pm4_pkt_struct_common.h @@ -255,6 +255,97 @@ typedef struct _PM4_MEC_INDIRECT_BUFFER } PM4MEC_INDIRECT_BUFFER, *PPM4MEC_INDIRECT_BUFFER; +//--------------------MEC_WAIT_REG_MEM-------------------- + +enum MEC_WAIT_REG_MEM_function_enum { + function__mec_wait_reg_mem__always_pass = 0, + function__mec_wait_reg_mem__less_than_ref_value = 1, + function__mec_wait_reg_mem__less_than_equal_to_the_ref_value = 2, + function__mec_wait_reg_mem__equal_to_the_reference_value = 3, + function__mec_wait_reg_mem__not_equal_reference_value = 4, + function__mec_wait_reg_mem__greater_than_or_equal_reference_value = 5, + function__mec_wait_reg_mem__greater_than_reference_value = 6 }; + +enum MEC_WAIT_REG_MEM_mem_space_enum { + mem_space__mec_wait_reg_mem__register_space = 0, + mem_space__mec_wait_reg_mem__memory_space = 1 }; + +enum MEC_WAIT_REG_MEM_operation_enum { + operation__mec_wait_reg_mem__wait_reg_mem = 0, + operation__mec_wait_reg_mem__wr_wait_wr_reg = 1, + operation__mec_wait_reg_mem__wait_mem_preemptable = 3 }; + + +typedef struct PM4_MEC_WAIT_REG_MEM +{ + union + { + PM4_TYPE_3_HEADER header; ///header + uint32_t ordinal1; + }; + + union + { + struct + { + MEC_WAIT_REG_MEM_function_enum function:3; + uint32_t reserved1:1; + MEC_WAIT_REG_MEM_mem_space_enum mem_space:2; + MEC_WAIT_REG_MEM_operation_enum operation:2; + uint32_t reserved2:24; + } bitfields2; + uint32_t ordinal2; + }; + + union + { + struct + { + uint32_t reserved3:2; + uint32_t mem_poll_addr_lo:30; + } bitfields3a; + struct + { + uint32_t reg_poll_addr:18; + uint32_t reserved4:14; + } bitfields3b; + struct + { + uint32_t reg_write_addr1:18; + uint32_t reserved5:14; + } bitfields3c; + uint32_t ordinal3; + }; + + union + { + uint32_t mem_poll_addr_hi; + + struct + { + uint32_t reg_write_addr2:18; + uint32_t reserved6:14; + } bitfields4b; + uint32_t ordinal4; + }; + + uint32_t reference; + + uint32_t mask; + + union + { + struct + { + uint32_t poll_interval:16; + uint32_t reserved7:15; + uint32_t optimize_ace_offload_mode:1; + } bitfields7; + uint32_t ordinal7; + }; + +} PM4MEC_WAIT_REG_MEM, *PPM4MEC_WAIT_REG_MEM; + //--------------------MEC_WRITE_DATA-------------------- enum MEC_WRITE_DATA_dst_sel_enum { dst_sel_mec_write_data_MEM_MAPPED_REGISTER_0 = 0, dst_sel_mec_write_data_TC_L2_2 = 2, dst_sel_mec_write_data_GDS_3 = 3, dst_sel_mec_write_data_MEMORY_5 = 5 }; diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index 3571c3ee81..3d26ef3fe6 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -362,3 +362,46 @@ PM4NopPacket::PM4NopPacket(unsigned int count): m_packetSize(count * 4) { m_packetData = reinterpret_cast(AllocPacket()); InitPM4Header(*m_packetData, IT_NOP); } + +PM4WaitRegMemPacket::PM4WaitRegMemPacket(bool memory, uint64_t addr, + uint32_t ref, uint16_t pollInterval) { + InitPacket(function__mec_wait_reg_mem__equal_to_the_reference_value, + memory ? + mem_space__mec_wait_reg_mem__memory_space : + mem_space__mec_wait_reg_mem__register_space, + operation__mec_wait_reg_mem__wait_reg_mem, + addr, ref, 0xffffffff, pollInterval); +} +PM4WaitRegMemPacket::PM4WaitRegMemPacket(unsigned int function, + unsigned int space, + unsigned int operation, + uint64_t addr, uint32_t ref, + uint32_t mask, uint16_t pollInterval) { + InitPacket(function, space, operation, addr, ref, mask, pollInterval); +} + +void PM4WaitRegMemPacket::InitPacket(unsigned int function, + unsigned int space, + unsigned int operation, + uint64_t addr, uint32_t ref, + uint32_t mask, uint16_t pollInterval) { + memset(&m_packetData, 0, SizeInBytes()); + InitPM4Header(m_packetData.header, IT_WAIT_REG_MEM); + + m_packetData.bitfields2.function = (MEC_WAIT_REG_MEM_function_enum)function; + m_packetData.bitfields2.mem_space = (MEC_WAIT_REG_MEM_mem_space_enum)space; + m_packetData.bitfields2.operation = (MEC_WAIT_REG_MEM_operation_enum)operation; + + m_packetData.ordinal3 = addr; + m_packetData.mem_poll_addr_hi = addr >> 32; + + m_packetData.reference = ref; + m_packetData.mask = mask; + + m_packetData.bitfields7.poll_interval = pollInterval; + m_packetData.bitfields7.optimize_ace_offload_mode = 1; +} + +unsigned int PM4WaitRegMemPacket::SizeInBytes() const { + return sizeof(m_packetData); +} diff --git a/tests/kfdtest/src/PM4Packet.hpp b/tests/kfdtest/src/PM4Packet.hpp index c02ef290ad..e7049c3fa8 100644 --- a/tests/kfdtest/src/PM4Packet.hpp +++ b/tests/kfdtest/src/PM4Packet.hpp @@ -224,4 +224,25 @@ class PM4NopPacket : public PM4Packet { PM4_TYPE_3_HEADER *m_packetData; }; +// @class PM4WaitRegMemPacket +class PM4WaitRegMemPacket : public PM4Packet { + public: + PM4WaitRegMemPacket(void) {} + PM4WaitRegMemPacket(bool memory, uint64_t addr, uint32_t ref, uint16_t pollInterval); + PM4WaitRegMemPacket(unsigned int function, unsigned int space, unsigned int operation, + uint64_t addr, uint32_t ref, uint32_t mask, uint16_t pollInterval); + virtual ~PM4WaitRegMemPacket(void) {} + + // @returns Packet size in bytes + virtual unsigned int SizeInBytes() const; + // @returns Pointer to the packet + virtual const void *GetPacket() const { return &m_packetData; } + + void InitPacket(unsigned int function, unsigned int space, unsigned int operation, + uint64_t addr, uint32_t ref, uint32_t mask, uint16_t pollInterval); + + private: + PM4MEC_WAIT_REG_MEM m_packetData; +}; + #endif // __KFD_PM4_PACKET__H__ From 8afdf520015b8d721dfb937032a5695740bf3237 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 30 Oct 2019 21:49:44 -0400 Subject: [PATCH 0637/1247] kfdtest: Change PM4 dispatch to workaround GWS-related FW problem v2: Remove useless AcquireMem packet after the dispatch v3: Minimum poll_interval is 4 Change-Id: I352eb21c781ed9e03d62c0febd532da6a9854afa Signed-off-by: Felix Kuehling --- tests/kfdtest/src/Dispatch.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index dc75c4c671..6ed67ce39b 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -230,7 +230,15 @@ void Dispatch::BuildIb() { m_IndirectBuf.AddPacket(PM4DispatchDirectPacket(m_DimX, m_DimY, m_DimZ, DISPATCH_INIT_VALUE)); - m_IndirectBuf.AddPacket(PM4PartialFlushPacket()); - - m_IndirectBuf.AddPacket(PM4AcquireMemoryPacket(m_FamilyId)); + // EVENT_WRITE.partial_flush causes problems with preemptions in + // GWS testing. Since this is specific to this PM4 command and + // doesn't affect AQL, it's easier to fix KFDTest than the + // firmware. + // + // Replace PartialFlush with an ReleaseMem (with no interrupt) + WaitRegMem + // + // Original: m_IndirectBuf.AddPacket(PM4PartialFlushPacket()); + uint32_t *nop = m_IndirectBuf.AddPacket(PM4NopPacket(2)); // NOP packet with one dword payload for the release-mem fence + m_IndirectBuf.AddPacket(PM4ReleaseMemoryPacket(m_FamilyId, true, (uint64_t)&nop[1], 0xdeadbeef)); + m_IndirectBuf.AddPacket(PM4WaitRegMemPacket(true, (uint64_t)&nop[1], 0xdeadbeef, 4)); } From cbe21fa2613b14a87fd68faa74acfcbffd4e978b Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 22 Nov 2019 15:58:17 -0500 Subject: [PATCH 0638/1247] kfdtest: Remove the queue submission in BigBufferStressTest In order to accommodate the flaky queue submission under memory shortage scenarios, BigBufferStressTest has become very much a hack, undermining its purpose of testing the basic memory related operations. Therefore, remove the queue submission part. The EvictTest should serve the purpose of testing the memory and queue submission functionalities when memory eviction happens. Change-Id: I3c3603a0e834267eccb72f46efeabe1e053c8fc5 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDMemoryTest.cpp | 55 +---------------------------- tests/kfdtest/src/KFDMemoryTest.hpp | 1 - 2 files changed, 1 insertion(+), 55 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 297f8c85b9..05a3dbb318 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -862,24 +862,6 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, << vramSizeMB * 15 / 16 << "MB" << std::endl; } -void KFDMemoryTest::NumaNodeBind(const char *nodeStr) { - if (numa_available() != -1) { - int num_node = numa_num_task_nodes(); - - if (num_node > 1) { - struct bitmask *nodemask; - - LOG() << "NUMA total nodes " << num_node << ", bind to " << nodeStr << std::endl; - - nodemask = numa_parse_nodestring(nodeStr); - if (nodemask) { - numa_bind(nodemask); - numa_free_nodemask(nodemask); - } - } - } -} - /* BigBufferStressTest allocs, maps/unmaps, and frees the biggest possible system * buffers. Its size is found using binary search in the range (0, RAM SIZE) with * a granularity of 128M. Repeat the similar logic on local buffers (VRAM). @@ -902,7 +884,6 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { TEST_START(TESTPROFILE_RUNALL); HSAuint64 AlternateVAGPU; - HSAuint64 Available_size; HsaMemMapFlags mapFlags = {0}; int ret; @@ -911,14 +892,7 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - /* Don't run on node 0 on multiple NUMA node machine because dma32 zone is on node 0, - * Use all memory including dma32 zone on node 0 will cause TTM eviction to free dma32 - * zone for other devices which supports 32bit physical address. The eviction and - * restore may retry if busy and cause queue timeout and test failure. - */ - NumaNodeBind("!0"); - - BigBufferSystemMemory(defaultGPUNode, granularityMB, &Available_size); + BigBufferSystemMemory(defaultGPUNode, granularityMB, NULL); BigBufferVRAM(defaultGPUNode, granularityMB, NULL); @@ -931,18 +905,9 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { unsigned int* pDb_array[ARRAY_ENTRIES]; HSAuint64 block_size_mb = 128; HSAuint64 block_size = block_size_mb * 1024 * 1024; - PM4Queue queue; - - /* In non-numa system to avoid TTM eviction, - * we have to keep half of dma32 zone (2GB) out of allocation. - */ - if (((numa_available() == -1) || (numa_num_task_nodes() < 2)) && - (Available_size > 0x80000000)) - Available_size -= 0x80000000; /* Test 4 times to see if there is any memory leak.*/ for (int repeat = 1; repeat < 5; repeat++) { - ASSERT_SUCCESS(queue.Create(defaultGPUNode)); for (i = 0; i < ARRAY_ENTRIES; i++) { ret = hsaKmtAllocMemory(0 /* system */, block_size, m_MemoryFlags, @@ -956,9 +921,6 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[i], block_size)); break; } - - if ((i + 2) * block_size > Available_size) - break; } LOG() << "Allocated system buffers time " << std::dec << repeat << ": " << i << "x" @@ -968,27 +930,12 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { allocationCount = i; EXPECT_GE(i, allocationCount) << "There might be memory leak!" << std::endl; - for (int j = 0; j < i; j++) { - /* To see if GPU can access the memory correctly*/ - unsigned int *begin = pDb_array[j]; - *begin = 0; - queue.PlaceAndSubmitPacket( - PM4WriteDataPacket(begin, 0xdeadbeaf)); - queue.Wait4PacketConsumption(NULL, 300000); - EXPECT_TRUE(WaitOnValue(begin, 0xdeadbeaf)); - } - - EXPECT_SUCCESS(queue.Destroy()); - for (int j = 0; j < i; j++) { EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb_array[j])); EXPECT_SUCCESS(hsaKmtFreeMemory(pDb_array[j], block_size)); } } - /* Reset to run on all task nodes */ - NumaNodeBind("all"); - TEST_END } diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp index 8f1f78b9ac..947968999e 100644 --- a/tests/kfdtest/src/KFDMemoryTest.hpp +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -44,7 +44,6 @@ class KFDMemoryTest : public KFDBaseComponentTest { void BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); void BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); - void NumaNodeBind(const char *nodeStr); }; #endif // __KFD_MEMORY_TEST__H__ From b6cefa7bda45325e84a2249750705ed4f384b893 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 4 Jun 2019 15:36:42 -0400 Subject: [PATCH 0639/1247] kfdtest: Split BigBufferStressTest into two smaller tests The previous BigBufferStressTest has too much stuff and takes a long time to run. By separating largest*BufferTest out into other tests, we dramatically reduce the time to run BigBufferStressTest and therefore make reproducing issues much easier. Meanwhile, rename the test to BigSysBufferStressTest to express more information. Change-Id: I5911f113c0bd50627ee6d84bbb4f2972cbed8886 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 5 +- tests/kfdtest/src/KFDMemoryTest.cpp | 69 ++++++++++++++++++++------- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 4d1b3b34e4..590cd3aa2b 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -110,12 +110,13 @@ FILTER[kaveri]=\ "KFDQMTest.CpuWriteCoherence" # KFDLocalMemoryTest.BasicTest is failing intermittently (KFD-368) -# KFDMemoryTest.BigBufferStressTest was failing intermittently on 4.9 and hangs when executed twice (KFD-312) +# KFDMemoryTest.BigSysBufferStressTest was failing intermittently on 4.9 +# and hangs when executed twice (KFD-312) # KFDQMTest.GPUDoorbellWrite fails on Hawaii. Could be HW-related (KFD-342) FILTER[hawaii]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDLocalMemoryTest.BasicTest:"\ -"KFDMemoryTest.BigBufferStressTest:"\ +"KFDMemoryTest.BigSysBufferStressTest:"\ "KFDQMTest.GPUDoorbellWrite" FILTER[carrizo]=\ diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 05a3dbb318..9be43aa53f 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -862,20 +862,61 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, << vramSizeMB * 15 / 16 << "MB" << std::endl; } -/* BigBufferStressTest allocs, maps/unmaps, and frees the biggest possible system - * buffers. Its size is found using binary search in the range (0, RAM SIZE) with - * a granularity of 128M. Repeat the similar logic on local buffers (VRAM). - * Finally, it allocs and maps 128M system buffers in a loop until it - * fails, then unmaps and frees them afterwards. - * Please note we limit the biggest possible system buffer to be smaller than +/* + * Largest*BufferTest allocates, maps/unmaps, and frees the largest possible + * buffers. Its size is found using binary search in the range + * (0, RAM SIZE) with a granularity of 128M. Also, the similar logic is + * repeated on local buffers (VRAM). + * Please note we limit the largest possible system buffer to be smaller than * the RAM size. The reason is that the system buffer can make use of virtual * memory so that a system buffer could be very large even though the RAM size - * is small. For example, on a typical Carrizo platform, the biggest allocated + * is small. For example, on a typical Carrizo platform, the largest allocated * system buffer could be more than 14G even though it only has 4G memory. - * In that situation, it will take too much time to finish the test, because of + * In that situation, it will take too much time to finish the test because of * the onerous memory swap operation. So we limit the buffer size that way. */ -TEST_F(KFDMemoryTest, BigBufferStressTest) { +TEST_F(KFDMemoryTest, LargestSysBufferTest) { + if (!is_dgpu()) { + LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl; + return; + } + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint64 granularityMB = 128; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + BigBufferSystemMemory(defaultGPUNode, granularityMB, NULL); + + TEST_END +} + +TEST_F(KFDMemoryTest, LargestVramBufferTest) { + if (!is_dgpu()) { + LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl; + return; + } + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint64 granularityMB = 128; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + BigBufferVRAM(defaultGPUNode, granularityMB, NULL); + + TEST_END +} + +/* + * BigSysBufferStressTest allocates and maps 128M system buffers in a loop until it + * fails, then unmaps and frees them afterwards. Meanwhile, a queue task is + * performed on each buffer. + */ +TEST_F(KFDMemoryTest, BigSysBufferStressTest) { if (!is_dgpu()) { LOG() << "Skipping test: Running on APU fails and locks the system." << std::endl; return; @@ -887,15 +928,9 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { HsaMemMapFlags mapFlags = {0}; int ret; - HSAuint64 granularityMB = 128; - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - BigBufferSystemMemory(defaultGPUNode, granularityMB, NULL); - - BigBufferVRAM(defaultGPUNode, granularityMB, NULL); - /* Repeatedly allocate and map big buffers in system memory until it fails, * then unmap and free them. */ @@ -923,8 +958,8 @@ TEST_F(KFDMemoryTest, BigBufferStressTest) { } } - LOG() << "Allocated system buffers time " << std::dec << repeat << ": " << i << "x" - << block_size_mb << "MB" << std::endl; + LOG() << "Allocated system buffers time " << std::dec << repeat << ": " + << i << " * " << block_size_mb << "MB" << std::endl; if (allocationCount == 0) allocationCount = i; From 4daa25fceb7b8ba0286535dec197766b3d20c475 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 21 Nov 2019 18:19:39 -0500 Subject: [PATCH 0640/1247] kfdtest: Merge the two largest buffer test helper functions This is cleaner. Change-Id: I7740f3e0f93a63b35fefc3cb69712dfad68df552 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDMemoryTest.cpp | 132 ++++++++++------------------ tests/kfdtest/src/KFDMemoryTest.hpp | 5 +- 2 files changed, 49 insertions(+), 88 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 9be43aa53f..abdbfa3b5a 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -749,95 +749,35 @@ TEST_F(KFDMemoryTest, GetTileConfigTest) { TEST_END } -void KFDMemoryTest::BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granularityMB, - HSAuint64 *lastSize) { - HSAuint64 sysMemSizeMB; - HsaMemMapFlags mapFlags = {0}; - HSAuint64 AlternateVAGPU; +void KFDMemoryTest::BinarySearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, + HSAuint64 highMB, int nodeToMap, + HSAuint64 *lastSizeMB) { int ret; - sysMemSizeMB = GetSysMemSize() >> 20; - - LOG() << "Found System Memory of " << std::dec << sysMemSizeMB - << "MB" << std::endl; - - /* Testing big buffers in system memory */ - unsigned int * pDb = NULL; - HSAuint64 lowMB = 0; - HSAuint64 highMB = (sysMemSizeMB + granularityMB - 1) & ~(granularityMB - 1); - - HSAuint64 sizeMB; - HSAuint64 size = 0; - HSAuint64 lastTestedSize = 0; - - while (highMB - lowMB > granularityMB) { - sizeMB = (lowMB + highMB) / 2; - size = sizeMB * 1024 * 1024; - ret = hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, - reinterpret_cast(&pDb)); - if (ret) { - highMB = sizeMB; - continue; - } - - ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, - mapFlags, 1, reinterpret_cast(&defaultGPUNode)); - if (ret) { - EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); - highMB = sizeMB; - continue; - } - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); - EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); - - lowMB = sizeMB; - lastTestedSize = sizeMB; - } - - /* Save the biggest allocated system buffer for signal handling test */ - LOG() << "The biggest allocated system buffer is " << std::dec - << lastTestedSize << "MB" << std::endl; - if (lastSize) - *lastSize = lastTestedSize * 1024 *1024; -} - -void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, - HSAuint64 *lastSize) { - HSAuint64 AlternateVAGPU; - int ret; - HSAuint64 vramSizeMB; - HsaMemFlags memFlags; HsaMemMapFlags mapFlags = {0}; - - vramSizeMB = GetVramSize(defaultGPUNode) >> 20; - - LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; + HSAuint64 granularityMB = 128; /* Testing big buffers in VRAM */ unsigned int * pDb = NULL; HSAuint64 lowMB = 0; - HSAuint64 highMB = (vramSizeMB + granularityMB - 1) & ~(granularityMB - 1); + + highMB = (highMB + granularityMB - 1) & ~(granularityMB - 1); HSAuint64 sizeMB; HSAuint64 size = 0; - HSAuint64 lastTestedSize = 0; - - memset(&memFlags, 0, sizeof(memFlags)); - memFlags.ui32.HostAccess = 0; - memFlags.ui32.NonPaged = 1; while (highMB - lowMB > granularityMB) { sizeMB = (lowMB + highMB) / 2; size = sizeMB * 1024 * 1024; - ret = hsaKmtAllocMemory(defaultGPUNode, size, memFlags, + ret = hsaKmtAllocMemory(allocNode, size, memFlags, reinterpret_cast(&pDb)); if (ret) { highMB = sizeMB; continue; } - ret = hsaKmtMapMemoryToGPUNodes(pDb, size, &AlternateVAGPU, - mapFlags, 1, reinterpret_cast(&defaultGPUNode)); + ret = hsaKmtMapMemoryToGPUNodes(pDb, size, NULL, + mapFlags, 1, reinterpret_cast(&nodeToMap)); if (ret) { EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); highMB = sizeMB; @@ -847,19 +787,10 @@ void KFDMemoryTest::BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); lowMB = sizeMB; - lastTestedSize = sizeMB; } - LOG() << "The biggest allocated VRAM buffer is " << std::dec - << lastTestedSize << "MB" << std::endl; - if (lastSize) - *lastSize = lastTestedSize * 1024 * 1024; - - /* Make sure 3/4 vram can be allocated.*/ - EXPECT_GE(lastTestedSize * 4, vramSizeMB * 3); - if (lastTestedSize * 16 < vramSizeMB * 15) - WARN() << "The biggest allocated VRAM buffer size is smaller than the expected " - << vramSizeMB * 15 / 16 << "MB" << std::endl; + if (lastSizeMB) + *lastSizeMB = lowMB; } /* @@ -883,12 +814,22 @@ TEST_F(KFDMemoryTest, LargestSysBufferTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); - HSAuint64 granularityMB = 128; - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - BigBufferSystemMemory(defaultGPUNode, granularityMB, NULL); + HSAuint64 lastTestedSizeMB = 0; + + HSAuint64 sysMemSizeMB; + sysMemSizeMB = GetSysMemSize() >> 20; + + LOG() << "Found System Memory of " << std::dec << sysMemSizeMB + << "MB" << std::endl; + + BinarySearchLargestBuffer(0, m_MemoryFlags, sysMemSizeMB, defaultGPUNode, + &lastTestedSizeMB); + + LOG() << "The largest allocated system buffer is " << std::dec + << lastTestedSizeMB << "MB" << std::endl; TEST_END } @@ -901,12 +842,31 @@ TEST_F(KFDMemoryTest, LargestVramBufferTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); - HSAuint64 granularityMB = 128; - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - BigBufferVRAM(defaultGPUNode, granularityMB, NULL); + HSAuint64 lastTestedSizeMB = 0; + + HsaMemFlags memFlags = {0}; + memFlags.ui32.HostAccess = 0; + memFlags.ui32.NonPaged = 1; + + HSAuint64 vramSizeMB; + vramSizeMB = GetVramSize(defaultGPUNode) >> 20; + + LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; + + BinarySearchLargestBuffer(defaultGPUNode, memFlags, vramSizeMB, defaultGPUNode, + &lastTestedSizeMB); + + LOG() << "The largest allocated VRAM buffer is " << std::dec + << lastTestedSizeMB << "MB" << std::endl; + + /* Make sure 3/4 vram can be allocated.*/ + EXPECT_GE(lastTestedSizeMB * 4, vramSizeMB * 3); + if (lastTestedSizeMB * 16 < vramSizeMB * 15) + WARN() << "The largest allocated VRAM buffer size is smaller than the expected " + << vramSizeMB * 15 / 16 << "MB" << std::endl; TEST_END } diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp index 947968999e..ea93395f71 100644 --- a/tests/kfdtest/src/KFDMemoryTest.hpp +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -42,8 +42,9 @@ class KFDMemoryTest : public KFDBaseComponentTest { protected: IsaGenerator* m_pIsaGen; - void BigBufferVRAM(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); - void BigBufferSystemMemory(int defaultGPUNode, HSAuint64 granularityMB, HSAuint64 *lastSize); + void BinarySearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, + HSAuint64 highMB, int nodeToMap, + HSAuint64 *lastSizeMB); }; #endif // __KFD_MEMORY_TEST__H__ From 0b71bbb787b609b7069b071a4d518fa22103304a Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 4 Jun 2019 09:25:14 -0500 Subject: [PATCH 0641/1247] Support XGMI optimized SDMA queue Change-Id: I6ad62fc94a9838df505879b1ddaccfeb9881a6e8 Signed-off-by: Oak Zeng --- tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp diff --git a/tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp b/tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp new file mode 100644 index 0000000000..bbd719fed9 --- /dev/null +++ b/tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2014-2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_XGMI_OPTIMIZED_SDMA_QUEUE__H__ +#define __KFD_XGMI_OPTIMIZED_SDMA_QUEUE__H__ + +#include "SDMAQueue.hpp" + +class XgmiOptimizedSDMAQueue : public SDMAQueue { + public: + XgmiOptimizedSDMAQueue(void) {CMD_NOP = 0;} + virtual ~XgmiOptimizedSDMAQueue(void) {} + protected: + virtual _HSA_QUEUE_TYPE GetQueueType() { return HSA_QUEUE_SDMA_XGMI; } +}; + +#endif // __KFD_XGMI_OPTIMIZED_SDMA_QUEUE__H__ From ad76ffd544eaba078d66e6bfdcd9db3590438a6c Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 26 Nov 2019 18:54:22 -0500 Subject: [PATCH 0642/1247] kfdtest: Declare SetUpTestCase() to be public SetUpTestCase() and TearDownTestCase() are declared as protected, preventing us from using TEST_P(). Change-Id: I1d049a475a1b3bd44b5f96305a48751b90d572ce Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDBaseComponentTest.hpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 8a99e405e4..5fc61015e4 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -58,6 +58,11 @@ class KFDBaseComponentTest : public testing::Test { unsigned int GetFamilyIdFromNodeId(unsigned int nodeId); unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; } + // @brief Executed before the first test that uses KFDBaseComponentTest. + static void SetUpTestCase(); + // @brief Executed after the last test from KFDBaseComponentTest. + static void TearDownTestCase(); + protected: HsaVersionInfo m_VersionInfo; HsaSystemProperties m_SystemProperties; @@ -69,13 +74,9 @@ class KFDBaseComponentTest : public testing::Test { HsaMemFlags m_MemoryFlags; HsaNodeInfo m_NodeInfo; - // @brief Executed before the first test that uses KFDOpenCloseKFDTest class and opens KFD. - static void SetUpTestCase(); - // @brief Executed after the last test from KFDOpenCloseKFDTest class and closes KFD. - static void TearDownTestCase(); - // @brief Executed before every test that uses KFDOpenCloseKFDTest class and sets all common settings for the tests. + // @brief Executed before every test that uses KFDBaseComponentTest class and sets all common settings for the tests. virtual void SetUp(); - // @brief Executed after every test that uses KFDOpenCloseKFDTest class. + // @brief Executed after every test that uses KFDBaseComponentTest class. virtual void TearDown(); }; From 8b01a1c4c53408da3c9e87010d460a9218456e3e Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 29 Aug 2019 00:21:52 -0400 Subject: [PATCH 0643/1247] add queue snapshot test adds api and test to get newly create queue snapshot per ptraced process. Change-Id: Ife97123a5b930e837ccaa386801145ef23c2cc2c Signed-off-by: Jonathan Kim --- include/hsakmt.h | 21 ++++++++++++++++++ include/linux/kfd_ioctl.h | 24 ++++++++++++++++++++- src/debug.c | 37 +++++++++++++++++++++++++++++++- src/libhsakmt.ver | 2 +- tests/kfdtest/src/KFDDBGTest.cpp | 34 ++++++++++++++++++++++++++--- 5 files changed, 112 insertions(+), 6 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 79eb5e3446..cc96e492b6 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -823,6 +823,27 @@ hsaKmtQueryDebugEvent( bool *IsNew //OUT ); +/** + Newly created queue snapshot per ptraced process. + + Returns queue snapshot including queue id, gpuid, context save base address, + queue status word, queue address and size, and queue read and write pointer. + + ClearEvents set will clear new queue bit and queue status word bits. + + Returns: + - HSAKMT_STATUS_SUCCESS if successful + */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueSnapshot( + HSAuint32 NodeId, // IN + HSAuint32 Pid, // IN + bool ClearEvents, // IN + void *SnapshotBuf, // IN + HSAuint32 *QssEntries // IN/OUT + ); + /** Set the trap override mask. When debug trap is enabled by hsaKmtEnableDebugTrap() each wave launched has its initial diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 4e02aa8158..fa14bd44a3 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -93,6 +93,19 @@ struct kfd_ioctl_get_queue_wave_state_args { __u32 pad; }; +struct kfd_queue_snapshot_entry { + __u64 ring_base_address; + __u64 write_pointer_address; + __u64 read_pointer_address; + __u64 ctx_save_restore_address; + __u32 queue_id; + __u32 gpu_id; + __u32 ring_size; + __u32 queue_type; + __u32 queue_status; + __u32 reserved[19]; +}; + /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ #define KFD_IOC_CACHE_POLICY_COHERENT 0 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1 @@ -246,6 +259,14 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 5 +/* KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT: + * ptr: user buffer (IN) + * data1: flags (IN) + * data2: number of queue snapshot entries (IN/OUT) + * data3: unused + */ +#define KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT 6 + /* KFD_IOC_DBG_TRAP_GET_VERSION: * prt: unsused * data1: major version (OUT) @@ -254,6 +275,7 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_GET_VERSION 7 + struct kfd_ioctl_dbg_trap_args { __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ __u32 pid; /* to KFD */ @@ -677,7 +699,7 @@ struct kfd_ioctl_cross_memory_copy_args { AMDKFD_IOWR(0x20, struct kfd_ioctl_ipc_export_handle_args) #define AMDKFD_IOC_DBG_TRAP \ - AMDKFD_IOW(0x21, struct kfd_ioctl_dbg_trap_args) + AMDKFD_IOWR(0x21, struct kfd_ioctl_dbg_trap_args) #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x22, struct kfd_ioctl_cross_memory_copy_args) diff --git a/src/debug.c b/src/debug.c index 8aa67446ad..92cbaeb186 100644 --- a/src/debug.c +++ b/src/debug.c @@ -287,7 +287,8 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, if (op == KFD_IOC_DBG_TRAP_NODE_SUSPEND || op == KFD_IOC_DBG_TRAP_NODE_RESUME || - op == KFD_IOC_DBG_TRAP_GET_VERSION) { + op == KFD_IOC_DBG_TRAP_GET_VERSION || + op == KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT) { if (NodeId != INVALID_NODEID) return HSAKMT_STATUS_INVALID_HANDLE; @@ -630,3 +631,37 @@ hsaKmtGetThunkDebugTrapVersionInfo( *Major = KFD_IOCTL_DBG_MAJOR_VERSION; *Minor = KFD_IOCTL_DBG_MINOR_VERSION; } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetQueueSnapshot( + HSAuint32 NodeId, //IN + HSAuint32 Pid, // IN + bool ClearEvents, //IN + void *SnapshotBuf, //IN + HSAuint32 *QssEntries //IN/OUT + ) +{ + HSAKMT_STATUS result; + struct kfd_ioctl_dbg_trap_args argout = {0}; + uint32_t flags = 0; + + if (ClearEvents) + flags |= KFD_DBG_EV_FLAG_CLEAR_STATUS; + + result = debug_trap(NodeId, + KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT, + flags, + *QssEntries, + 0, + Pid, + (HSAuint64)SnapshotBuf, + &argout); + + if (result) + return result; + + *QssEntries = argout.data2; + + return 0; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 3f2e787ac5..fc9564f3ec 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -61,7 +61,7 @@ hsaKmtEnableDebugTrap; hsaKmtEnableDebugTrapWithPollFd; hsaKmtDisableDebugTrap; hsaKmtQueryDebugEvent; -hsaKmtSetDebugTrapData2; +hsaKmtGetQueueSnapshot; hsaKmtSetWaveLaunchTrapOverride; hsaKmtSetWaveLaunchMode; hsaKmtQueueSuspend; diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 70b7558c4e..9e7d93cc4c 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -24,6 +24,7 @@ #include "KFDDBGTest.hpp" #include #include +#include "linux/kfd_ioctl.h" #include "KFDQMTest.hpp" #include "PM4Queue.hpp" #include "PM4Packet.hpp" @@ -439,7 +440,7 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - if(!checkDebugVersion(0, 2)) { + if (!checkDebugVersion(0, 2)) { LOG() << "Test disabled due to debug API version mismatch"; goto exit; } @@ -510,13 +511,40 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { ASSERT_EQ(IsNew, true); ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_TRAP); - // suspend queue, query suspended queue and clear pending event + // suspend queue, get snapshot, query suspended queue + // and clear pending event ASSERT_SUCCESS(hsaKmtQueueSuspend(INVALID_PID, 1, queueIds, 10, 0)); syncStatus = dispatch->SyncWithStatus(suspendTimeout); ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); ASSERT_NE(iter[0], result[0]); + struct kfd_queue_snapshot_entry qssBuf[1] = {}; + HSAuint32 QssEntries = 0; + + // get only number of queues and don't update the snapshot buffer + ASSERT_SUCCESS(hsaKmtGetQueueSnapshot(INVALID_NODEID, INVALID_PID, + false, + reinterpret_cast(qssBuf), + &QssEntries)); + + ASSERT_EQ(QssEntries, 1); + ASSERT_EQ(qssBuf[0].ctx_save_restore_address, 0); + ASSERT_EQ(qssBuf[0].ring_base_address, 0); + ASSERT_EQ(qssBuf[0].ring_size, 0); + + // update the snapshot buffer + QssEntries = 1; + ASSERT_SUCCESS(hsaKmtGetQueueSnapshot(INVALID_NODEID, INVALID_PID, + false, + reinterpret_cast(qssBuf), + &QssEntries)); + + ASSERT_EQ(QssEntries, 1); + ASSERT_NE(qssBuf[0].ctx_save_restore_address, 0); + ASSERT_NE(qssBuf[0].ring_base_address, 0); + ASSERT_NE(qssBuf[0].ring_size, 0); + ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, true, &EventReceived, &IsSuspended, &IsNew)); @@ -593,7 +621,7 @@ TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - if(!checkDebugVersion(0, 2)) { + if (!checkDebugVersion(0, 2)) { LOG() << "Test disabled due to debug API version mismatch"; goto exit; } From 8f140fc03dfc008eb26a41206087f0cc8a028d9f Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 26 Nov 2019 17:34:36 -0500 Subject: [PATCH 0644/1247] kfdtest: Add some logs to Atomics test This will help us triage the unexpected hangs on the farm systems. Meanwhile, simplify the logic. Change-Id: Ie50b97a34cb86891720dce11f2d178bff9aa2cd5 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDQMTest.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index b5ab04bc63..2505cb3ba8 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1331,20 +1331,19 @@ TEST_F(KFDQMTest, SdmaQueueWraparound) { struct AtomicIncThreadParams { HSAint64* pDest; volatile unsigned int count; - volatile bool stop; + volatile bool loop; }; unsigned int AtomicIncThread(void* pCtx) { AtomicIncThreadParams* pArgs = reinterpret_cast(pCtx); - while (pArgs->stop) - {} - - while (!pArgs->stop) { + while (pArgs->loop) { AtomicInc(pArgs->pDest); ++pArgs->count; } + LOG() << "CPU atomic increments finished" << std::endl; + return 0; } @@ -1375,28 +1374,31 @@ TEST_F(KFDQMTest, Atomics) { AtomicIncThreadParams params; params.pDest = destBuf.As(); - params.stop = true; + params.loop = true; params.count = 0; uint64_t threadId; ASSERT_EQ(true, StartThread(&AtomicIncThread, ¶ms, threadId)); - params.stop = false; + LOG() << "Waiting for CPU to atomic increment 1000 times" << std::endl; - while (params.count == 0) + while (params.count < 1000) {} + LOG() << "Submitting the GPU atomic increment shader" << std::endl; + dispatch.Submit(queue); dispatch.Sync(); - params.stop = true; + params.loop = false; WaitForThread(threadId); EXPECT_EQ(destBuf.As()[0], 1024 + params.count); - LOG() << "GPU increments: 1024, CPU increments: " << params.count << std::endl; + LOG() << "GPU increments: 1024, CPU increments: " << std::dec + << params.count << std::endl; queue.Destroy(); From caa744944eb5c0cfa22471139d45041f96f30587 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Thu, 5 Dec 2019 14:24:50 -0500 Subject: [PATCH 0645/1247] kfdtest: change baseline in BasicCuMaskingEven To keep normal performance, we have to use symmetrical cu masks based on shader engines. So change baseline from 1 cu to 1 cu per SE. Change-Id: I9e83a87fb670bb406f7983714fa0d8ab673609eb Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDQMTest.cpp | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 2505cb3ba8..ae76fa1a47 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -765,15 +765,24 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { int numCuPerShader = ActiveCU / numShaderEngines; double ratio; - /* Set Mask to 1 for a single CU */ - mask[0] = 0x1; - for (int i = 1; i < maskNumDwords; i++) - mask[i] = 0x0; + /* In KFD we symmetrically map mask to all SEs: + * mask[0] bit0 -> se0 cu0; + * mask[0] bit1 -> se1 cu0; + * ... (if # SE is 4) + * mask[0] bit4 -> se0 cu1; + * ... + */ + /* Set Mask to 1 CU per SE */ + memset(mask, 0, maskNumDwords * sizeof(uint32_t)); + for (int i = 0; i < numShaderEngines; i++) { + int maskIndex = (i / 32) % maskNumDwords; + mask[maskIndex] |= 1 << (i % 32); + } /* Execute once to get any HW optimizations out of the way */ TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); - LOG() << "Getting baseline performance numbers (1 CU)" << std::endl; + LOG() << "Getting baseline performance numbers (1 CU per SE)" << std::endl; TimewithCU1 = GetAverageTimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits, 3); /* Each loop will add 1 more CU per SE. We use the mod and divide to handle @@ -786,12 +795,12 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { int maskIndex = (offset / 32) % maskNumDwords; mask[maskIndex] |= 1 << (offset % 32); } - int nCUs = numShaderEngines * (x + 1); + int nCUs = x + 1; TimewithCU = TimeConsumedwithCUMask(defaultGPUNode, mask, maskNumBits); ratio = (double)(TimewithCU1) / ((double)(TimewithCU) * nCUs); - LOG() << "Expected performance of " << nCUs << " CUs vs 1 CU:" << std::endl; + LOG() << "Expected performance of " << nCUs << " CU(s)/SE vs 1 CU/SE:" << std::endl; LOG() << std::setprecision(2) << CuNegVariance << " <= " << std::fixed << std::setprecision(8) << ratio << " <= " << std::setprecision(2) << CuPosVariance << std::endl; From 10ffc63d7bdc3e5e94ccfa209d83db66708d6664 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 6 Nov 2019 19:52:03 -0500 Subject: [PATCH 0646/1247] kfdtest: Add AllQueues test This puts all CP and SDMA queues in a single test, which is currently missing. Change-Id: I98bf58df1be65fe9daf6311c016a48569a8ab674 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 1 + tests/kfdtest/src/KFDQMTest.cpp | 61 +++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 590cd3aa2b..42e0df356c 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -18,6 +18,7 @@ FILTER[core_sws]=\ "KFDQMTest.SubmitNopSdmaQueue:"\ "KFDQMTest.SubmitPacketSdmaQueue:"\ "KFDQMTest.AllSdmaQueues:"\ +"KFDQMTest.AllQueues:"\ "KFDEventTest.SignalEvent" # HWS mode diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index ae76fa1a47..0d8a43886f 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -243,6 +243,67 @@ TEST_F(KFDQMTest, AllSdmaQueues) { TEST_END } +TEST_F(KFDQMTest, AllQueues) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + int bufSize = PAGE_SIZE; + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + unsigned int i, j; + + const unsigned int numCpQueues = m_numCpQueues; + const unsigned int numSdmaQueues = m_numSdmaEngines * m_numSdmaQueuesPerEngine; + + HsaMemoryBuffer destBufCp(PAGE_SIZE, defaultGPUNode, false); + destBufCp.Fill(0xFF); + + HsaMemoryBuffer destBuf(bufSize << 1 , defaultGPUNode, false); + HsaMemoryBuffer srcBuf(bufSize, defaultGPUNode, false); + destBuf.Fill(0xFF); + + std::vector cpQueues(numCpQueues); + std::vector sdmaQueues(numSdmaQueues); + + for (i = 0; i < numCpQueues; ++i) + ASSERT_SUCCESS(cpQueues[i].Create(defaultGPUNode)) << " QueueId=" << i; + + for (j = 0; j < numSdmaQueues; ++j) + ASSERT_SUCCESS(sdmaQueues[j].Create(defaultGPUNode)); + + for (i = 0; i < numCpQueues; ++i) { + cpQueues[i].PlaceAndSubmitPacket(PM4WriteDataPacket(destBufCp.As()+i*2, i, i)); + + cpQueues[i].Wait4PacketConsumption(); + + EXPECT_TRUE(WaitOnValue(destBufCp.As()+i*2, i)); + } + + for (j = 0; j < numSdmaQueues; ++j) { + destBuf.Fill(0x0); + srcBuf.Fill(j + 0xa0); + sdmaQueues[j].PlaceAndSubmitPacket( + SDMACopyDataPacket(sdmaQueues[j].GetFamilyId(), destBuf.As(), srcBuf.As(), bufSize)); + sdmaQueues[j].PlaceAndSubmitPacket( + SDMAWriteDataPacket(sdmaQueues[j].GetFamilyId(), destBuf.As() + bufSize/4, 0x02020202)); + + sdmaQueues[j].Wait4PacketConsumption(); + + EXPECT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); + + EXPECT_SUCCESS(memcmp( + destBuf.As(), srcBuf.As(), bufSize)); + } + + for (i = 0; i < numCpQueues; ++i) + EXPECT_SUCCESS(cpQueues[i].Destroy()); + + for (j = 0; j < numSdmaQueues; ++j) + EXPECT_SUCCESS(sdmaQueues[j].Destroy()); + + TEST_END +} + /* The following test is designed to reproduce an intermittent hang on * Fiji and other VI/Polaris GPUs. This test typically hangs in a few * seconds. According to analysis done by HW engineers, the culprit From 1c2c5a7b9ab504edb30630764ad14bff842bbdad Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 12 Dec 2019 21:21:59 -0500 Subject: [PATCH 0647/1247] kfdtest: Delete two useless exception tests The InvalidPPR* tests are only useful for gfx801 right now, on which they won't trigger exceptions. So they are not relevent in the KFDExceptionTest category. In addition, given AccessPPRMem already tests the PPR memory functionality, we can just delete those two tests. Change-Id: Id5c6e23c4c0ce47a4f04e9e1f0fa9083e0a9d0e0 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 1 - tests/kfdtest/src/KFDExceptionTest.cpp | 106 ------------------------- 2 files changed, 107 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 42e0df356c..ca2cbb5c5b 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -92,7 +92,6 @@ SDMA_BLACKLIST=\ KV_QUEUE_BLACKLIST=\ "KFDExceptionTest.InvalidBadAddress:"\ "KFDExceptionTest.InvalidWriteAddress:"\ -"KFDExceptionTest.InvalidPPR*:"\ "KFDLocalMemoryTest.*:"\ "KFDEventTest.Signal*Event*:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 428e4a64d7..1a25e4cad4 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -155,112 +155,6 @@ TEST_F(KFDExceptionTest, InvalidBadAddress) { TEST_END } -/* TODO: Check why this test doesn't behave as expected. - * Expected behaviour: Since pDst is not mapped in GPU VM, an attempt - * to write to pDst should trigger a Memory Fault. - * However, If pDst > 40bits (MC Size) GPU does not create Memory Fault. - * An EOP interrupt is generated however as expected no data is copied - * into pDst. - */ -TEST_F(KFDExceptionTest, InvalidPPRWriteProtection) { - TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); - TEST_START(TESTPROFILE_RUNALL); - - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - if (is_dgpu()) { - LOG() << "Skipping test: Test requires APU." << std::endl; - return; - } - - if (m_FamilyId == FAMILY_RV) { - LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; - return; - } - - m_ChildPid = fork(); - if (m_ChildPid == 0) { - m_ChildStatus = hsaKmtOpenKFD(); - if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { - WARN() << "KFD open failed in child process" << std::endl; - return; - } - - HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode, false); - void* pDst = VirtualAllocMemory(NULL, PAGE_SIZE, MEM_READ); - - LOG() << "PPR dst address is " << pDst << std::endl; - - srcBuffer.Fill(0xAA55AA55); - TestMemoryException(defaultGPUNode, srcBuffer.As(), - (HSAuint64)pDst); - - /* Wait for enough time here to ensure this process got killed by kernel - * due to PPR exception. - */ - sleep(5); - } else { - int childStatus; - - waitpid(m_ChildPid, &childStatus, 0); - EXPECT_EQ(WIFSIGNALED(childStatus), true); - EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); - } - - TEST_END -} - -/* TODO: Same as previous test InvalidPPRWriteProtection */ -TEST_F(KFDExceptionTest, InvalidPPRReadProtection) { - TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); - TEST_START(TESTPROFILE_RUNALL); - - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - if (is_dgpu()) { - LOG() << "Skipping test: Test requires APU." << std::endl; - return; - } - - if (m_FamilyId == FAMILY_RV) { - LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; - return; - } - - m_ChildPid = fork(); - if (m_ChildPid == 0) { - m_ChildStatus = hsaKmtOpenKFD(); - if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { - WARN() << "KFD open failed in child process" << std::endl; - return; - } - - HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode, false); - void* pSrc = VirtualAllocMemory(NULL, PAGE_SIZE, MEM_NONE); - - LOG() << "PPR src address is " << pSrc << std::endl; - - dstBuffer.Fill(0xAA55AA55); - TestMemoryException(defaultGPUNode, (HSAuint64)pSrc, - dstBuffer.As()); - - /* Wait for enough time here to ensure this process got killed by kernel - * due to PPR exception. - */ - sleep(5); - } else { - int childStatus; - - waitpid(m_ChildPid, &childStatus, 0); - EXPECT_EQ(WIFSIGNALED(childStatus), true); - EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); - } - - TEST_END -} - /* Allocate Read Only buffer. Test Memory Exception failure by * attempting to write to that buffer in the child process. */ From f7c0172385a602898f9e8d54e64a991a9f21b735 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 13 Dec 2019 11:17:02 -0500 Subject: [PATCH 0648/1247] kfdtest: Rename two exception test cases The old names are not accurate enough and we rename them according to their corresponding fault types. Change-Id: Icf4d52ba0ab9d49af5d912a0feb82665b1e8d344 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 14 +++++++------- tests/kfdtest/src/KFDExceptionTest.cpp | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index ca2cbb5c5b..ea43c27168 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -90,8 +90,8 @@ SDMA_BLACKLIST=\ # Anything involving CP queue creation is failing on Kaveri. Separate them here for convenience (KFD-336) KV_QUEUE_BLACKLIST=\ -"KFDExceptionTest.InvalidBadAddress:"\ -"KFDExceptionTest.InvalidWriteAddress:"\ +"KFDExceptionTest.AddressFault:"\ +"KFDExceptionTest.PermissionFault:"\ "KFDLocalMemoryTest.*:"\ "KFDEventTest.Signal*Event*:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ @@ -122,7 +122,7 @@ FILTER[hawaii]=\ FILTER[carrizo]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ -"KFDExceptionTest.InvalidWriteAddress" +"KFDExceptionTest.PermissionFault" # KFDPerfCountersTest.*Trace fail (KFD-339) # KFDMemoryTest.QueryPointerInfo/MemoryRegister* (KFD-341) @@ -172,8 +172,8 @@ FILTER[vega20]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDEvictTest.BurstyTest:"\ -"KFDExceptionTest.InvalidBadAddress:"\ -"KFDExceptionTest.InvalidWriteAddress:"\ +"KFDExceptionTest.AddressFault:"\ +"KFDExceptionTest.PermissionFault:"\ "KFDQMTest.GPUDoorbellWrite" FILTER[raven]=\ @@ -199,8 +199,8 @@ FILTER[navi10]=\ "KFDEvictTest.BurstyTest:"\ "KFDEvictTest.QueueTest:"\ "KFDPerfCountersTest.*:"\ -"KFDExceptionTest.InvalidBadAddress:"\ -"KFDExceptionTest.InvalidWriteAddress:"\ +"KFDExceptionTest.AddressFault:"\ +"KFDExceptionTest.PermissionFault:"\ "KFDQMTest.GPUDoorbellWrite" FILTER[navi14]=\ diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 1a25e4cad4..6656317304 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -114,7 +114,7 @@ queuefail: } /* Test Bad Address access in a child process */ -TEST_F(KFDExceptionTest, InvalidBadAddress) { +TEST_F(KFDExceptionTest, AddressFault) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -158,7 +158,7 @@ TEST_F(KFDExceptionTest, InvalidBadAddress) { /* Allocate Read Only buffer. Test Memory Exception failure by * attempting to write to that buffer in the child process. */ -TEST_F(KFDExceptionTest, InvalidWriteAddress) { +TEST_F(KFDExceptionTest, PermissionFault) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL) From 57b42045d027fc40ff386c771158f1e6dba9f7df Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 25 Nov 2019 10:49:30 -0500 Subject: [PATCH 0649/1247] Set package name explicitly Instead of letting CPack generate the filename according to CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_SYSTEM_NAME}, set the package name ourselves. Also take the change to fix the name of hsakmt-dev to hsakmt-devel for RPM, since that's their convention Previous packaging format: hsakmt-roct-1.0.9-229-g2144854-Linux.deb hsakmt-roct-1.0.9-229-g2144854-Linux.rpm hsakmt-roct-dev-1.0.9-229-g2144854-Linux.deb hsakmt-roct-dev-1.0.9-229-g2144854-Linux.rpm New format: hsakmt-roct-1.0.9-292-gc66f8cf.x86_64.deb hsakmt-roct-1.0.9-292-gc66f8cf.x86_64.rpm hsakmt-roct-dev-1.0.9-292-gc66f8cf.x86_64.deb hsakmt-roct-devel-1.0.9-292-gc66f8cf.x86_64.rpm Change-Id: I4fc4e0fd2eafd25669c1cfffb39860e25a0b645c Signed-off-by: Kent Russell --- CMakeLists.txt | 1 + hsakmt-dev.txt | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bf4e8cf15..3e4b5b3b38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,6 +167,7 @@ set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT library for AMD KFD support" ) set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) +set ( CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64" ) #checking for OS-version if( EXISTS "/etc/os-release" ) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index ba9406d17d..5c350b7e1a 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -52,6 +52,7 @@ set ( CPACK_GENERATOR "DEB;RPM;TGZ" CACHE STRING "Default packaging generators. ## Packaging directives set ( CPACK_PACKAGE_NAME ${HSAKMT_DEV_PACKAGE} ) +set ( CPACK_RPM_PACKAGE_NAME "hsakmt-roct-devel" ) set ( CPACK_PACKAGE_VENDOR "AMD" ) set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) @@ -59,6 +60,8 @@ set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT development package." ) set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) +set ( CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64" ) +set ( CPACK_RPM_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) From 8b14ea2e83bdb73999930b01b7cabac693171ffd Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 18 Dec 2019 12:19:28 -0500 Subject: [PATCH 0650/1247] kfdtest.exclude: Remove BasicCuMaskingEven from gfx908 The initial baseline measurements are proving inconsistent, which results in the test failing more often with different variant rates Change-Id: I1f4e04bf7d615cf39de9605bd5141a997b22cdfc --- tests/kfdtest/scripts/kfdtest.exclude | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index ea43c27168..af73cc0e29 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -188,9 +188,11 @@ FILTER[renoir]=\ # KFDExceptionTest.* (KFD-435) # KFDEvictTest.BurstyTest (KFD-425) +# KFQMTest.BasicCuMaskingEven (Issues with baseline measurement) FILTER[arcturus]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm:"\ +"KFDQMTest.BasicCuMaskingEven:"\ "KFDEvictTest.BurstyTest" FILTER[navi10]=\ From 44db5cb011e8ee91bb21c90a27b874e4fe873960 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 17 Dec 2019 20:32:16 -0500 Subject: [PATCH 0651/1247] kfdtest: Enable KFDExceptionTest on gfx906 and gfx1xxx series KFDExceptionTest on those platform is passing. Change-Id: I328ee4fd4ff5b339e560f2f79e754fd34459210a Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index af73cc0e29..92fdc31bbe 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -172,8 +172,6 @@ FILTER[vega20]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDEvictTest.BurstyTest:"\ -"KFDExceptionTest.AddressFault:"\ -"KFDExceptionTest.PermissionFault:"\ "KFDQMTest.GPUDoorbellWrite" FILTER[raven]=\ @@ -201,15 +199,12 @@ FILTER[navi10]=\ "KFDEvictTest.BurstyTest:"\ "KFDEvictTest.QueueTest:"\ "KFDPerfCountersTest.*:"\ -"KFDExceptionTest.AddressFault:"\ -"KFDExceptionTest.PermissionFault:"\ "KFDQMTest.GPUDoorbellWrite" FILTER[navi14]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ -"KFDExceptionTest.*:"\ "KFDPerfCountersTest.*:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ "KFDPNPTest.DisableWithActiveProcess" From 22e9ef73038e01417328ec4903c8adb2706e784f Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 18 Dec 2019 18:43:29 -0500 Subject: [PATCH 0652/1247] libhsakmt: Add the perf counter support for gfx1012 Change-Id: I55d68a77928617edaabd33ae0807bf23f739c8de Signed-off-by: Yong Zhao --- src/pmc_table.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index ccc22a3965..5d30391497 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -2131,6 +2131,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, *block = vega_blocks[block_id]; break; case CHIP_NAVI10: + case CHIP_NAVI14: *block = navi_blocks[block_id]; break; default: From facb6c056d385bf81d5323e78769c09ee30a7de0 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 6 Nov 2019 17:47:27 -0500 Subject: [PATCH 0653/1247] kfdtest: Rework the shader used for CWSR test The shader supports multiple work items. It also eliminates one input buffer. Change-Id: If0596b306065980b74fb92613e95610defc00164 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDCWSRTest.cpp | 129 +++++++++++++++--------------- 1 file changed, 66 insertions(+), 63 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 0fbb425f72..bc9b9698f9 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -24,32 +24,44 @@ #include "KFDCWSRTest.hpp" #include "Dispatch.hpp" + +/* Initial state: + * s[0:1] - 64 bits iteration number; only the lower 32 bits are useful. + * s[2:3] - result buffer base address + * s4 - workgroup id + * v0 - workitem id, always 0 because + * NUM_THREADS_X(number of threads) in workgroup set to 1 + * Registers: + * v0 - calculated workitem = v0 + s4 * NUM_THREADS_X, which is s4 + * v2 - = s0, 32 bits iteration number + * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 + * v6 - counter + */ + static const char* iterate_isa_gfx8 = \ "\ shader iterate_isa\n\ -asic(VI)\n\ +wave_size(32)\n\ type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ -/*v0 stores the workitem ID inside the workgroup, use it to caculate the dest*/\n\ -// v_lshlrev_b32 v0, 2, v0\n\ -// v_add_u32 v2, vcc, v0, v2\n\ - v_mov_b32 v0, s0\n\ - flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - v_mov_b32 v5, 0\n\ -LOOP:\n\ - v_add_u32 v5, vcc, 1, v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ - v_cmp_lt_u32 vcc, v5, v4\n\ - s_cbranch_vccnz LOOP\n\ - flat_store_dword v[2,3], v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - s_endpgm\n\ -end\n\ + // copy the parameters from scalar registers to vector registers\n\ + v_mov_b32 v2, s0 // v[2:3] = s[0:1] \n\ + v_mov_b32 v3, s1 // v[2:3] = s[0:1] \n\ + v_mov_b32 v0, s4 // use workgroup id as index \n\ + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 \n\ + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 \n\ + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 \n\ + v_add_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 \n\ + v_mov_b32 v6, 0 \n\ +LOOP: \n\ + v_add_u32 v6, vcc, 1, v6 \n\ + // compare the result value (v6) to iteration value (v2), and \n\ + // jump if equal (i.e. if VCC is not zero after the comparison) \n\ + v_cmp_lt_u32 vcc, v6, v2 \n\ + s_cbranch_vccnz LOOP \n\ + flat_store_dword v[4:5], v6 \n\ + s_waitcnt vmcnt(0)&lgkmcnt(0) \n\ + s_endpgm \n\ +end \n\ "; //This shader can be used by gfx9 and gfx10 @@ -58,32 +70,27 @@ static const char* iterate_isa_gfx9 = \ shader iterate_isa\n\ wave_size(32)\n\ type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ - flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - v_mov_b32 v5, 0\n\ -LOOP:\n\ - v_add_co_u32 v5, vcc, 1, v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - /*compare the result value (v5) to iteration value (v4), and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ - v_cmp_lt_u32 vcc, v5, v4\n\ - s_cbranch_vccnz LOOP\n\ - flat_store_dword v[2,3], v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - s_endpgm\n\ -end\n\ + // copy the parameters from scalar registers to vector registers\n\ + v_mov_b32 v2, s0 // v[2:3] = s[0:1] \n\ + v_mov_b32 v3, s1 // v[2:3] = s[0:1] \n\ + v_mov_b32 v0, s4 // use workgroup id as index \n\ + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 \n\ + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 \n\ + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 \n\ + v_add_co_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 \n\ + v_mov_b32 v6, 0 \n\ +LOOP: \n\ + v_add_co_u32 v6, vcc, 1, v6 \n\ + // compare the result value (v6) to iteration value (v2), and \n\ + // jump if equal (i.e. if VCC is not zero after the comparison) \n\ + v_cmp_lt_u32 vcc, v6, v2 \n\ + s_cbranch_vccnz LOOP \n\ + flat_store_dword v[4:5], v6 \n\ + s_waitcnt vmcnt(0)&lgkmcnt(0) \n\ + s_endpgm \n\ +end \n\ "; - -/* -v[0:1] = target iteration value -v[2:3] = iterate result -*/ - void KFDCWSRTest::SetUp() { ROUTINE_START @@ -91,10 +98,6 @@ void KFDCWSRTest::SetUp() { m_pIsaGen = IsaGenerator::Create(m_FamilyId); - /* TODO: In the ISA, the workitem_id is not obtained as expected, so the destination cannot - * be set based on workitem_id. Set the wave_num to 1 for now as a workarpound. - * Will set it to 8 or even 256 in the future. - */ wave_number = 1; ROUTINE_END @@ -126,11 +129,14 @@ TEST_F(KFDCWSRTest, BasicTest) { if (m_FamilyId >= FAMILY_VI) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - HsaMemoryBuffer iterateBuf(PAGE_SIZE, defaultGPUNode, true, false, false); - HsaMemoryBuffer resultBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + HsaMemoryBuffer resultBuf1(PAGE_SIZE, defaultGPUNode, true, false, false); + HsaMemoryBuffer resultBuf2(PAGE_SIZE, defaultGPUNode, true, false, false); - unsigned int* iter = iterateBuf.As(); - unsigned int* result = resultBuf.As(); + int count1 = 40000000; + int count2 = 20000000; + + unsigned int* result1 = resultBuf1.As(); + unsigned int* result2 = resultBuf2.As(); const char *pIterateIsa; if (m_FamilyId < FAMILY_AI) @@ -149,14 +155,11 @@ TEST_F(KFDCWSRTest, BasicTest) { dispatch1 = new Dispatch(isaBuffer); dispatch2 = new Dispatch(isaBuffer); - dispatch1->SetArgs(&iter[0], &result[0]); + dispatch1->SetArgs(reinterpret_cast(count1), result1); dispatch1->SetDim(wave_number, 1, 1); - dispatch2->SetArgs(&iter[1], &result[wave_number]); + dispatch2->SetArgs(reinterpret_cast(count2), result2); dispatch2->SetDim(wave_number, 1, 1); - iter[0] = 40000000; - iter[1] = 20000000; - // Submit the shader, queue1 dispatch1->Submit(queue1); // Create queue2 during queue1 still running will trigger the CWSR @@ -168,12 +171,12 @@ TEST_F(KFDCWSRTest, BasicTest) { // Ensure all the waves complete as expected int i; for (i = 0 ; i < wave_number; ++i) { - if (result[i] != iter[0]) { - LOG() << "Dispatch 1, work item " << i << ' ' << result[i] << std::endl; + if (result1[i] != count1) { + LOG() << "Dispatch 1, work item " << i << ' ' << result1[i] << std::endl; break; } - if (result[i + wave_number] != iter[1]) { - LOG() << "Dispatch 2, work item " << i << ' ' << result[i] << std::endl; + if (result2[i] != count2) { + LOG() << "Dispatch 2, work item " << i << ' ' << result2[i] << std::endl; break; } } From af7feb93dbaa1ce7d60fe4ddebae2a43becd7d05 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 4 Jun 2019 14:38:42 -0500 Subject: [PATCH 0654/1247] Add KFD GWS test Change-Id: Ie90d9119da6cee41ddab10deb427d4ae9fd9a16b Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDGWSTest.cpp | 164 +++++++++++++++++++++++++++++++ tests/kfdtest/src/KFDGWSTest.hpp | 45 +++++++++ 2 files changed, 209 insertions(+) create mode 100644 tests/kfdtest/src/KFDGWSTest.cpp create mode 100644 tests/kfdtest/src/KFDGWSTest.hpp diff --git a/tests/kfdtest/src/KFDGWSTest.cpp b/tests/kfdtest/src/KFDGWSTest.cpp new file mode 100644 index 0000000000..bb0097d802 --- /dev/null +++ b/tests/kfdtest/src/KFDGWSTest.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2014-2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDGWSTest.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "Dispatch.hpp" + +/* Shader to initialize gws counter to 1*/ +const char* gfx9_GwsInit = +"\ +shader GwsInit\n\ +asic(GFX9)\n\ +type(CS)\n\ + s_mov_b32 m0, 0\n\ + s_nop 0\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt 0\n\ + v_mov_b32 v0, s16\n\ + s_waitcnt 0\n\ + ds_gws_init v0 gds:1 offset0:0\n\ + s_waitcnt 0\n\ + s_endpgm\n\ + end\n\ +"; + +/* Atomically increase a value in memory + * This is expected to be executed from + * multiple work groups simultaneously. + * GWS semaphore is used to guarantee + * the operation is atomic. + */ +const char* gfx9_AtomicIncrease = +"\ +shader AtomicIncrease\n\ +asic(GFX9)\n\ +type(CS)\n\ +/* Assume src address in s0, s1 */\n\ + s_mov_b32 m0, 0\n\ + s_nop 0\n\ + ds_gws_sema_p gds:1 offset0:0\n\ + s_waitcnt 0\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt 0\n\ + s_add_u32 s16, s16, 1\n\ + s_store_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt lgkmcnt(0)\n\ + ds_gws_sema_v gds:1 offset0:0\n\ + s_waitcnt 0\n\ + s_endpgm\n\ + end\n\ +"; + +void KFDGWSTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDGWSTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDGWSTest, Allocate) { + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 firstGWS; + PM4Queue queue; + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + if (!pNodeProperties || !pNodeProperties->NumGws) { + LOG() << "Skip test: GPU node doesn't support GWS" << std::endl; + return; + } + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + ASSERT_SUCCESS(hsaKmtAllocQueueGWS(defaultGPUNode, + queue.GetResource()->QueueId, + pNodeProperties->NumGws,&firstGWS)); + EXPECT_EQ(0, firstGWS); + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + +TEST_F(KFDGWSTest, Semaphore) { + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + HSAuint32 firstGWS; + HSAuint32 numResources = 1; + PM4Queue queue; + + if (!pNodeProperties || !pNodeProperties->NumGws) { + LOG() << "Skip test: GPU node doesn't support GWS" << std::endl; + return; + } + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer buffer(PAGE_SIZE, defaultGPUNode, true, false, false); + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + ASSERT_SUCCESS(hsaKmtAllocQueueGWS(defaultGPUNode, + queue.GetResource()->QueueId, + pNodeProperties->NumGws,&firstGWS)); + EXPECT_EQ(0, firstGWS); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + m_pIsaGen->CompileShader(gfx9_GwsInit, "GwsInit", isaBuffer); + Dispatch dispatch0(isaBuffer); + buffer.Fill(numResources, 0, 4); + dispatch0.SetArgs(buffer.As(), NULL); + dispatch0.Submit(queue); + dispatch0.Sync(); + + m_pIsaGen->CompileShader(gfx9_AtomicIncrease, "AtomicIncrease", isaBuffer); + + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(buffer.As(), NULL); + dispatch.SetDim(1024, 16, 16); + + dispatch.Submit(queue); + dispatch.Sync(); + + EXPECT_EQ(1024*16*16+1, *buffer.As()); + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + diff --git a/tests/kfdtest/src/KFDGWSTest.hpp b/tests/kfdtest/src/KFDGWSTest.hpp new file mode 100644 index 0000000000..15e61ee235 --- /dev/null +++ b/tests/kfdtest/src/KFDGWSTest.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2014-2019 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_GWS_TEST__H__ +#define __KFD_GWS_TEST__H__ + +#include + +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" + +class KFDGWSTest : public KFDBaseComponentTest { + public: + KFDGWSTest() :m_pIsaGen(NULL) {} + ~KFDGWSTest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: // Members + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_GWS_TEST__H__ From 06464b917d93015f8a1ff32746784683c424b38d Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Mon, 16 Dec 2019 11:43:33 +0800 Subject: [PATCH 0655/1247] libhsakmt: add NumCpQueues and NumSdmaQueuesPerEngine data field (v3) NumCpQueues and NumSdmaQueuesPerEngine should be got by kfd driver not hardcode. So add two data fields in HsaNodeProperties then thunk is able to get it from sysfs that exposed by kfd. v2: change NumCpQueues/NumSdmaQueuesPerEngine to one byte. v3: merge two commits as one to avoid ABI update two times. Change-Id: Ie386e4685f13493e22db6e207a399db6a4c5b9dc Signed-off-by: Huang Rui --- include/hsakmttypes.h | 7 ++++++- src/topology.c | 4 ++++ tests/kfdtest/src/KFDTestUtil.cpp | 14 ++------------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 5ac51c2208..ab55047739 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -301,7 +301,12 @@ typedef struct _HsaNodeProperties // number hash created by the PSP HSAuint32 NumSdmaEngines; // number of PCIe optimized SDMA engines HSAuint32 NumSdmaXgmiEngines;// number of XGMI optimized SDMA engines - HSAuint32 NumGws; // number of GWS barriers + + HSAuint8 NumSdmaQueuesPerEngine;// number of SDMA queue per one engine + HSAuint8 NumCpQueues; // number of Compute queues + HSAuint8 NumGws; // number of GWS barriers + HSAuint8 Reserved2; + HSAuint32 Domain; // PCI domain of the GPU HSAuint8 Reserved[28]; } HsaNodeProperties; diff --git a/src/topology.c b/src/topology.c index 6aab6323d5..9bcf0ad243 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1026,6 +1026,10 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->NumSdmaXgmiEngines = prop_val; else if (strcmp(prop_name, "num_gws") == 0) props->NumGws = prop_val; + else if (strcmp(prop_name, "num_sdma_queues_per_engine") == 0) + props->NumSdmaQueuesPerEngine = prop_val; + else if (strcmp(prop_name, "num_cp_queues") == 0) + props->NumCpQueues = prop_val; } hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 327d4f6702..f8743cb020 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -145,16 +145,6 @@ void GetHwQueueInfo(const HsaNodeProperties *props, unsigned int *p_num_sdma_engines, unsigned int *p_num_sdma_xgmi_engines, unsigned int *p_num_sdma_queues_per_engine) { - int num_cp_queues = 24; - int num_sdma_queues_per_engine = 2; - - if (props->EngineId.ui32.Major == 9) { - if (props->EngineId.ui32.Stepping == 6) // VEGA20 - num_sdma_queues_per_engine = 8; - } else if (props->EngineId.ui32.Major == 10) { // NAVI - num_sdma_queues_per_engine = 8; - } - if (p_num_sdma_engines) *p_num_sdma_engines = props->NumSdmaEngines; @@ -162,10 +152,10 @@ void GetHwQueueInfo(const HsaNodeProperties *props, *p_num_sdma_xgmi_engines = props->NumSdmaXgmiEngines; if (p_num_sdma_queues_per_engine) - *p_num_sdma_queues_per_engine = num_sdma_queues_per_engine; + *p_num_sdma_queues_per_engine = props->NumSdmaQueuesPerEngine; if (p_num_cp_queues) - *p_num_cp_queues = num_cp_queues; + *p_num_cp_queues = props->NumCpQueues; } bool isTonga(const HsaNodeProperties *props) { From 38b429fd52fa95efa5419173bb0ab6c3f7c97351 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Sun, 5 Jan 2020 15:43:38 -0600 Subject: [PATCH 0656/1247] Fix GWS test function call parameter issue Change-Id: I3f4fde9ec8268362cffecb2d95177913b583486f Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDGWSTest.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDGWSTest.cpp b/tests/kfdtest/src/KFDGWSTest.cpp index bb0097d802..111e5e186d 100644 --- a/tests/kfdtest/src/KFDGWSTest.cpp +++ b/tests/kfdtest/src/KFDGWSTest.cpp @@ -107,8 +107,7 @@ TEST_F(KFDGWSTest, Allocate) { } ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - ASSERT_SUCCESS(hsaKmtAllocQueueGWS(defaultGPUNode, - queue.GetResource()->QueueId, + ASSERT_SUCCESS(hsaKmtAllocQueueGWS(queue.GetResource()->QueueId, pNodeProperties->NumGws,&firstGWS)); EXPECT_EQ(0, firstGWS); EXPECT_SUCCESS(queue.Destroy()); @@ -134,8 +133,7 @@ TEST_F(KFDGWSTest, Semaphore) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer buffer(PAGE_SIZE, defaultGPUNode, true, false, false); ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - ASSERT_SUCCESS(hsaKmtAllocQueueGWS(defaultGPUNode, - queue.GetResource()->QueueId, + ASSERT_SUCCESS(hsaKmtAllocQueueGWS(queue.GetResource()->QueueId, pNodeProperties->NumGws,&firstGWS)); EXPECT_EQ(0, firstGWS); From 1589cf5dc57a3f448a7441c50ddeed34c4dc95fc Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 6 Jan 2020 07:29:30 -0500 Subject: [PATCH 0657/1247] Fix package naming without using newer variables CMake 3.6.0 and older don't have CPACK_RPM_FILE_NAME support, so make the packaging simpler by checking the OS. If it's Ubuntu, call the dev package hsakmt-roct-dev, otherwise call it hsakmt-roct-devel, since Ubuntu is the only officially-ROCm-supported deb-based OS, and deb-based dev packages end with -dev (while rpm-based use -devel) Change-Id: If75324f82f507c3b312bb6176c06643d521ccb68 --- hsakmt-dev.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index 5c350b7e1a..8a181e8a52 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -25,7 +25,17 @@ cmake_minimum_required ( VERSION 3.5.0 ) -set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-dev" ) +#checking for OS-version +if( EXISTS "/etc/os-release" ) + file( STRINGS "/etc/os-release" DISTRO_ID REGEX "^ID=" ) + string( REPLACE "ID=" "" DISTRO_ID "${DISTRO_ID}" ) +endif() + +if (DISTRO_ID STREQUAL "ubuntu" ) + set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-dev" ) +else() + set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-devel" ) +endif() project ( ${HSAKMT_DEV_PACKAGE} ) @@ -52,7 +62,6 @@ set ( CPACK_GENERATOR "DEB;RPM;TGZ" CACHE STRING "Default packaging generators. ## Packaging directives set ( CPACK_PACKAGE_NAME ${HSAKMT_DEV_PACKAGE} ) -set ( CPACK_RPM_PACKAGE_NAME "hsakmt-roct-devel" ) set ( CPACK_PACKAGE_VENDOR "AMD" ) set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) @@ -61,8 +70,6 @@ set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT development package." ) set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) set ( CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64" ) -set ( CPACK_RPM_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) - # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) From 2b70d73f680ee9316135051314376863b5135ed6 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 18 Dec 2019 19:19:53 -0500 Subject: [PATCH 0658/1247] kfdtest: Improve the stablility of SignalHandling test On gfx1012, allocating 1/4 of the system memory on a 32G RAM machine could fail, resulting in this test to fail. Limit the maximum buffer to allocate to be smaller than 3G to accommodate this situation. Change-Id: I38b0a0f7da1f0b9ca851e04d2d0a51767858c801 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDMemoryTest.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index abdbfa3b5a..aa816b84e4 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1569,6 +1569,13 @@ TEST_F(KFDMemoryTest, SignalHandling) { * Try to allocate 1/4th System RAM */ size = (sysMemSize >> 2) & ~(HSAuint64)(PAGE_SIZE - 1); + + /* We don't need a too large buffer for this test. If it is too large, + * on some platform, the upcoming hsaKmtAllocMemory() might fail. In + * order to avoid this flaky behavior, limit the size to 3G. + */ + size = size > (3ULL << 30) ? (3ULL << 30) : size; + m_MemoryFlags.ui32.NoNUMABind = 1; ASSERT_SUCCESS(hsaKmtAllocMemory(0 /* system */, size, m_MemoryFlags, reinterpret_cast(&pDb))); // Verify that pDb is not null before it's being used From 3f8a07e4603fdbc4cb57ab23726811fc78bf3501 Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Tue, 17 Sep 2019 18:11:50 -0400 Subject: [PATCH 0659/1247] support installtion of multiple ROCM version Changes to resolve 1) Multiple rocm release installation support 2) Multiple rocm shared lib conflicts Change-Id: I792feb36cdf4516d108f1ef71abe0c87522f018a Signed-off-by: Pruthvi Madugundu Signed-off-by: Freddy Paul --- CMakeLists.txt | 6 +++++- tests/kfdtest/CMakeLists.txt | 13 ++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e4b5b3b38..e324c08a11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,7 +61,11 @@ set ( BUILD_VERSION_PATCH ${VERSION_PATCH} ) set ( LIB_VERSION_MAJOR 1) set ( LIB_VERSION_MINOR 0) -set ( LIB_VERSION_PATCH 6) +if (${ROCM_PATCH_VERSION}) + set ( LIB_VERSION_PATCH ${ROCM_PATCH_VERSION} ) +else () + set ( LIB_VERSION_PATCH 6) +endif () set ( LIB_VERSION_STRING "${LIB_VERSION_MAJOR}.${LIB_VERSION_MINOR}.${LIB_VERSION_PATCH}" ) if ( DEFINED VERSION_BUILD AND NOT ${VERSION_BUILD} STREQUAL "" ) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 3d56abf132..0fddb38617 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -52,10 +52,17 @@ if( DEFINED ENV{LIBHSAKMT_PATH} ) set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} ) message ( "LIBHSAKMT_PATH environment variable is set" ) else() - set ( ENV{PKG_CONFIG_PATH} /opt/rocm/libhsakmt/ ) - pkg_check_modules(HSAKMT libhsakmt) + if ( ${ROCM_INSTALL_PATH} ) + set ( ENV{PKG_CONFIG_PATH} ${ROCM_INSTALL_PATH} ) + pkg_check_modules(HSAKMT libhsakmt) + else() + set ( ENV{PKG_CONFIG_PATH} /opt/rocm/libhsakmt/ ) + pkg_check_modules(HSAKMT libhsakmt) + endif() if( NOT HSAKMT_FOUND ) - set ( LIBHSAKMT_PATH $ENV{OUT_DIR} ) + set ( ENV{PKG_CONFIG_PATH} /opt/rocm/libhsakmt/ ) + pkg_check_modules(HSAKMT libhsakmt) + set ( LIBHSAKMT_PATH $ENV{OUT_DIR} ) endif() endif() From 5c2fc61d95c0b2d1d8095b5bf609ac91ffeceb98 Mon Sep 17 00:00:00 2001 From: Srinivasan Subramanian Date: Sun, 22 Sep 2019 22:28:40 -0700 Subject: [PATCH 0660/1247] Use runpath instead of rpath for libraries enable-new-dtags option is added Change-Id: I1f406b3f30ddc6491aad3ef7a84dfd415917b1aa Signed-off-by: Freddy Paul --- CMakeLists.txt | 3 ++- tests/kfdtest/CMakeLists.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e324c08a11..5a3e969764 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,7 +94,8 @@ endif () set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) ## Linker Flags -set (HSAKMT_LINK_FLAGS "-Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) +## Add --enable-new-dtags to generate DT_RUNPATH +set (HSAKMT_LINK_FLAGS "-Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) ## Source files set ( HSAKMT_SRC "src/debug.c" diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 0fddb38617..ce87fc978f 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -94,7 +94,8 @@ message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) #add_definitions(-Wall -std=c++11) if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") -set ( CMAKE_CXX_FLAGS "-std=gnu++11" ) +## Add --enable-new-dtags to generate DT_RUNPATH +set ( CMAKE_CXX_FLAGS "-std=gnu++11 -Wl,--enable-new-dtags" ) endif() if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2" ) From fe97612800d2bf9afa2c72ccb32f03ea7bf5cee2 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 10 Jan 2020 15:26:35 -0500 Subject: [PATCH 0661/1247] kfdtest: Add basic tests for XGMI SDMA queues After XGMI SDMA queues were separated from regular SDMA queues, they were not covered in the current tests. Add tests for them now. Change-Id: I036e3ca5d583ab7f022a9dc6cda3ef867f4773a0 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 1 + tests/kfdtest/src/KFDQMTest.cpp | 81 ++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 92fdc31bbe..a19b08dda7 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -18,6 +18,7 @@ FILTER[core_sws]=\ "KFDQMTest.SubmitNopSdmaQueue:"\ "KFDQMTest.SubmitPacketSdmaQueue:"\ "KFDQMTest.AllSdmaQueues:"\ +"KFDQMTest.AllXgmiSdmaQueues:"\ "KFDQMTest.AllQueues:"\ "KFDEventTest.SignalEvent" diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 0d8a43886f..6a0d60db46 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -28,7 +28,7 @@ #include "PM4Queue.hpp" #include "PM4Packet.hpp" #include "SDMAPacket.hpp" -#include "SDMAQueue.hpp" +#include "XgmiOptimizedSDMAQueue.hpp" #include "AqlQueue.hpp" #include @@ -212,6 +212,9 @@ TEST_F(KFDQMTest, AllSdmaQueues) { const unsigned int numSdmaQueues = m_numSdmaEngines * m_numSdmaQueuesPerEngine; + LOG() << "Regular SDMA engines number: " << m_numSdmaEngines + << " SDMA queues per engine: " << m_numSdmaQueuesPerEngine << std::endl; + HsaMemoryBuffer destBuf(bufSize << 1 , defaultGPUNode, false); HsaMemoryBuffer srcBuf(bufSize, defaultGPUNode, false); destBuf.Fill(0xFF); @@ -243,6 +246,53 @@ TEST_F(KFDQMTest, AllSdmaQueues) { TEST_END } +TEST_F(KFDQMTest, AllXgmiSdmaQueues) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + int bufSize = PAGE_SIZE; + int j; + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + const unsigned int numXgmiSdmaQueues = + m_numSdmaXgmiEngines * m_numSdmaQueuesPerEngine; + + LOG() << "XGMI SDMA engines number: " << m_numSdmaXgmiEngines + << " SDMA queues per engine: " << m_numSdmaQueuesPerEngine << std::endl; + + HsaMemoryBuffer destBuf(bufSize << 1 , defaultGPUNode, false); + HsaMemoryBuffer srcBuf(bufSize, defaultGPUNode, false); + destBuf.Fill(0xFF); + + std::vector xgmiSdmaQueues(numXgmiSdmaQueues); + + for (j = 0; j < numXgmiSdmaQueues; ++j) + ASSERT_SUCCESS(xgmiSdmaQueues[j].Create(defaultGPUNode)); + + for (j = 0; j < numXgmiSdmaQueues; ++j) { + destBuf.Fill(0x0); + srcBuf.Fill(j + 0xa0); + xgmiSdmaQueues[j].PlaceAndSubmitPacket( + SDMACopyDataPacket(xgmiSdmaQueues[j].GetFamilyId(), + destBuf.As(), srcBuf.As(), bufSize)); + xgmiSdmaQueues[j].PlaceAndSubmitPacket( + SDMAWriteDataPacket(xgmiSdmaQueues[j].GetFamilyId(), + destBuf.As() + bufSize/4, 0x02020202)); + + xgmiSdmaQueues[j].Wait4PacketConsumption(); + + EXPECT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); + + EXPECT_SUCCESS(memcmp( + destBuf.As(), srcBuf.As(), bufSize)); + } + + for (j = 0; j < numXgmiSdmaQueues; ++j) + EXPECT_SUCCESS(xgmiSdmaQueues[j].Destroy()); + + TEST_END +} + TEST_F(KFDQMTest, AllQueues) { TEST_START(TESTPROFILE_RUNALL) @@ -254,6 +304,8 @@ TEST_F(KFDQMTest, AllQueues) { const unsigned int numCpQueues = m_numCpQueues; const unsigned int numSdmaQueues = m_numSdmaEngines * m_numSdmaQueuesPerEngine; + const unsigned int numXgmiSdmaQueues = + m_numSdmaXgmiEngines * m_numSdmaQueuesPerEngine; HsaMemoryBuffer destBufCp(PAGE_SIZE, defaultGPUNode, false); destBufCp.Fill(0xFF); @@ -264,6 +316,7 @@ TEST_F(KFDQMTest, AllQueues) { std::vector cpQueues(numCpQueues); std::vector sdmaQueues(numSdmaQueues); + std::vector xgmiSdmaQueues(numXgmiSdmaQueues); for (i = 0; i < numCpQueues; ++i) ASSERT_SUCCESS(cpQueues[i].Create(defaultGPUNode)) << " QueueId=" << i; @@ -271,6 +324,10 @@ TEST_F(KFDQMTest, AllQueues) { for (j = 0; j < numSdmaQueues; ++j) ASSERT_SUCCESS(sdmaQueues[j].Create(defaultGPUNode)); + for (j = 0; j < numXgmiSdmaQueues; ++j) + ASSERT_SUCCESS(xgmiSdmaQueues[j].Create(defaultGPUNode)); + + for (i = 0; i < numCpQueues; ++i) { cpQueues[i].PlaceAndSubmitPacket(PM4WriteDataPacket(destBufCp.As()+i*2, i, i)); @@ -295,12 +352,34 @@ TEST_F(KFDQMTest, AllQueues) { destBuf.As(), srcBuf.As(), bufSize)); } + for (j = 0; j < numXgmiSdmaQueues; ++j) { + destBuf.Fill(0x0); + srcBuf.Fill(j + 0xa0); + xgmiSdmaQueues[j].PlaceAndSubmitPacket( + SDMACopyDataPacket(xgmiSdmaQueues[j].GetFamilyId(), + destBuf.As(), srcBuf.As(), bufSize)); + xgmiSdmaQueues[j].PlaceAndSubmitPacket( + SDMAWriteDataPacket(xgmiSdmaQueues[j].GetFamilyId(), + destBuf.As() + bufSize/4, 0x02020202)); + + xgmiSdmaQueues[j].Wait4PacketConsumption(); + + EXPECT_TRUE(WaitOnValue(destBuf.As() + bufSize/4, 0x02020202)); + + EXPECT_SUCCESS(memcmp( + destBuf.As(), srcBuf.As(), bufSize)); + } + + for (i = 0; i < numCpQueues; ++i) EXPECT_SUCCESS(cpQueues[i].Destroy()); for (j = 0; j < numSdmaQueues; ++j) EXPECT_SUCCESS(sdmaQueues[j].Destroy()); + for (j = 0; j < numXgmiSdmaQueues; ++j) + EXPECT_SUCCESS(xgmiSdmaQueues[j].Destroy()); + TEST_END } From 87e10cd0b4d2769c4ae73235bfffef134d025503 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 4 Dec 2019 23:06:00 -0500 Subject: [PATCH 0662/1247] libhsakmt: Improve error handling in child process Check for errno == EBADF in kmtIoctl to detect misuse of the kfd_fd in a forked child process. Detect being in a forked child process pro-actively by implementing a pthread_atfork callback. Make sure all mutexes get reinitialized in the child process to avoid deadlocks. Check for being in a forked child process in CHECK_KFD_OPENED so that all hsaKmt functions will return the appropriate status HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED. Update InvalidKFDHandleTest to expect that error code. Change-Id: I0238e5fba344dcaa454e97a35db2e2dcc8d1f607 Signed-off-by: Felix Kuehling --- src/fmm.c | 2 + src/libhsakmt.c | 9 +++++ src/libhsakmt.h | 3 +- src/openclose.c | 48 ++++++++++++++++++++++- tests/kfdtest/src/KFDOpenCloseKFDTest.cpp | 2 +- 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 37e543a099..b48fdc8f61 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3566,6 +3566,8 @@ static void fmm_clear_aperture(manageable_aperture_t *app) { rbtree_node_t *n; + pthread_mutex_init(&app->fmm_mutex, NULL); + while ((n = rbtree_node_any(&app->tree, MID))) vm_remove_object(app, vm_object_entry(n, 0)); diff --git a/src/libhsakmt.c b/src/libhsakmt.c index 4d0775ccbe..9236b8e280 100644 --- a/src/libhsakmt.c +++ b/src/libhsakmt.c @@ -1,3 +1,4 @@ +#include #include #include @@ -12,5 +13,13 @@ int kmtIoctl(int fd, unsigned long request, void *arg) ret = ioctl(fd, request, arg); } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); + if (errno == EBADF) { + /* In case pthread_atfork didn't catch it, this will + * make any subsequent hsaKmt calls fail in CHECK_KFD_OPEN. + */ + pr_err("KFD file descriptor not valid in this process\n"); + hsakmt_forked = true; + } + return ret; } diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 6cb49c59c5..f929833d28 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -34,6 +34,7 @@ extern int kfd_fd; extern unsigned long kfd_open_count; +extern bool hsakmt_forked; extern pthread_mutex_t hsakmt_mutex; extern bool is_dgpu; @@ -51,7 +52,7 @@ extern struct hsa_gfxip_table force_asic_entry; #define PORT_UINT64_TO_VPTR(v) ((void*)(unsigned long)(v)) #define CHECK_KFD_OPEN() \ - do { if (kfd_open_count == 0) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) + do { if (kfd_open_count == 0 || hsakmt_forked) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) extern int PAGE_SIZE; extern int PAGE_SHIFT; diff --git a/src/openclose.c b/src/openclose.c index b25ded37d9..11f930f486 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -38,25 +38,53 @@ static const char kfd_device_name[] = "/dev/kfd"; static pid_t parent_pid = -1; int hsakmt_debug_level; +bool hsakmt_forked; /* zfb is mainly used during emulation */ int zfb_support; +/* is_forked_child detects when the process has forked since the last + * time this function was called. We cannot rely on pthread_atfork + * because the process can fork without calling the fork function in + * libc (using clone or calling the system call directly). + */ static bool is_forked_child(void) { - pid_t cur_pid = getpid(); + pid_t cur_pid; + + if (hsakmt_forked) + return true; + + cur_pid = getpid(); if (parent_pid == -1) { parent_pid = cur_pid; return false; } - if (parent_pid != cur_pid) + if (parent_pid != cur_pid) { + hsakmt_forked = true; return true; + } return false; } +/* Callbacks from pthread_atfork */ +static void prepare_fork_handler(void) +{ + pthread_mutex_lock(&hsakmt_mutex); +} +static void parent_fork_handler(void) +{ + pthread_mutex_unlock(&hsakmt_mutex); +} +static void child_fork_handler(void) +{ + pthread_mutex_init(&hsakmt_mutex, NULL); + hsakmt_forked = true; +} + /* Call this from the child process after fork. This will clear all * data that is duplicated from the parent process, that is not valid * in the child. @@ -74,6 +102,8 @@ static void clear_after_fork(void) kfd_fd = 0; } kfd_open_count = 0; + parent_pid = -1; + hsakmt_forked = false; } static inline void init_page_size(void) @@ -150,6 +180,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) clear_after_fork(); if (kfd_open_count == 0) { + static bool atfork_installed = false; + result = init_vars_from_env(); if (result != HSAKMT_STATUS_SUCCESS) goto open_failed; @@ -182,6 +214,18 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) pr_warn("Insufficient Memory. Debugging unavailable\n"); init_counter_props(sys_props.NumNodes); + + if (!atfork_installed) { + /* Atfork handlers cannot be uninstalled and + * must be installed only once. Otherwise + * prepare will deadlock when trying to take + * the same lock multiple times. + */ + pthread_atfork(prepare_fork_handler, + parent_fork_handler, + child_fork_handler); + atfork_installed = true; + } } else { kfd_open_count++; result = HSAKMT_STATUS_KERNEL_ALREADY_OPENED; diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp b/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp index 86e485218a..06f57d6c1a 100644 --- a/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp +++ b/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp @@ -75,7 +75,7 @@ TEST_F(KFDOpenCloseKFDTest, InvalidKFDHandleTest ) { HsaVersionInfo m_VersionInfo; pid_t m_ChildPid = fork(); if (m_ChildPid == 0) { - EXPECT_EQ(HSAKMT_STATUS_ERROR, hsaKmtGetVersion(&m_VersionInfo)); + EXPECT_EQ(HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED, hsaKmtGetVersion(&m_VersionInfo)); exit(0); } else { int childStatus; From 634b30119afd18f2614606bda3264315676e6820 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Thu, 23 Jan 2020 14:25:14 -0500 Subject: [PATCH 0663/1247] kfdtest: fix infinite loop in sdma multicopy when variable interval become zero, the loop of while (timeConsumption >= interval) will be a infinite loop. The fix is to avoid that case. Change-Id: I8fd07296925300bace5ab7d3da86482b6d8b0d03 Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDTestUtilQueue.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/tests/kfdtest/src/KFDTestUtilQueue.cpp index 92283f71fb..8f9b857a70 100644 --- a/tests/kfdtest/src/KFDTestUtilQueue.cpp +++ b/tests/kfdtest/src/KFDTestUtilQueue.cpp @@ -422,6 +422,9 @@ void sdma_multicopy_report(std::vector &array, HSAuint64 countPe begin = array[base].timeBegin; end = array[base + countPerGroup - 1].timeEnd; + if (begin == 0 && end == 0) + continue; + if (timeConsumptionMax < end - begin) timeConsumptionMax = end - begin; From 5858aa17a9ab7e4ca8e577d2dc7f861caba632bc Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 23 Jan 2020 15:50:10 -0500 Subject: [PATCH 0664/1247] libhsakmt: Ignore mbind failure if flag NoSubstitute = 0 From Thunk spec, flag NoSubstitute = 0, if specific memory type is not available on node, allocation may fall back to other memory that can replace it on that node. mbind return failure if no memory available on the specific node, we should ignore the mbind failure for this case. Change-Id: I651a1bedf1852330604e56965cc17862403ebf87 Signed-off-by: Philip Yang --- src/fmm.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index b48fdc8f61..e2c45b5324 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1492,9 +1492,6 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, numa_bitmask_free(node_mask); if (r) { - pr_warn_once("Failed to set NUMA policy for %p: %s\n", mem, - strerror(errno)); - /* If applcation is running inside docker, still return * ok because docker seccomp blocks mbind by default, * otherwise application cannot allocate system memory. @@ -1505,6 +1502,13 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, return 0; } + /* Ignore mbind failure if no memory available on node */ + if (!flags.ui32.NoSubstitute) + return 0; + + pr_warn_once("Failed to set NUMA policy for %p: %s\n", mem, + strerror(errno)); + return -EFAULT; } From 4f2ff25a3d83f48453d1c18eb3b1581b21d98191 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 18 Dec 2019 17:15:12 -0500 Subject: [PATCH 0665/1247] kfdtest: Enable some tests on gfx1xxx series Those tests are currently all passing. Change-Id: I233afe33e8275d482bab5b5590b856fce49af76d Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index a19b08dda7..9ecbcc1b55 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -57,12 +57,10 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # failures can be found # NOTE: If you update this alphabetical listing, add the corresponding JIRA ticket for reference # -# KFDPNPTest.DisableAndCreateQueue (KFD-329) # KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) # KFDQMTest.mGPUShareBO (KFD-334) # KFDHWSTest.* (SWDEV-193035) TEMPORARY_BLACKLIST_ALL_ASICS=\ -"KFDPNPTest.DisableAndCreateQueue:"\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ "KFDQMTest.SdmaEventInterrupt:"\ @@ -196,16 +194,7 @@ FILTER[arcturus]=\ FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ -"$SDMA_BLACKLIST:"\ -"KFDEvictTest.BurstyTest:"\ -"KFDEvictTest.QueueTest:"\ -"KFDPerfCountersTest.*:"\ -"KFDQMTest.GPUDoorbellWrite" +"KFDMemoryTest.MMBench" FILTER[navi14]=\ -"$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDDBGTest.*:"\ -"KFDPerfCountersTest.*:"\ -"KFDQMTest.CreateQueueStressSingleThreaded:"\ -"KFDPNPTest.DisableWithActiveProcess" +"$BLACKLIST_ALL_ASICS" From f2b8965d7b2ec076afd8c592166387a33b530c0c Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 28 Jan 2020 15:09:03 -0500 Subject: [PATCH 0666/1247] Fix upgrade for new package name in RPM The name changed and thus will end up throwing errors if an upgrade is done, compared to a plain install Change-Id: Ibc7876a66a414034a00f924cdd750e6a08d6c9cc --- hsakmt-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index 8a181e8a52..5286fc0f72 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -76,6 +76,8 @@ set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-T # RPM package specific variables set ( CPACK_RPM_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} = ${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) +# Since we changed the package name to match RPM specs, take care of older builds that had -dev installed +set ( CPACK_RPM_PACKAGE_OBSOLETES "hsakmt-roct-dev" ) # Create cmake configuration files include(CMakePackageConfigHelpers) From 6957202df89780311ec495e12ba25a80adbba90e Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Wed, 5 Feb 2020 01:55:39 -0600 Subject: [PATCH 0667/1247] Update pointer info to include IPC memory. IPC memory was previously returned as HSA_POINTER_ALLOCATED and had garbage in the node_id field. Due to ROCR_VISIBLE_DEVICES we need to be able to distinguish between imported memory and regular memory because imported memory may not be owned by an agent that is visible in the process. Differentiating these flags allows the users to expect null agent for the owning agent. Fixes Change-Id: Ide3489cec1ee2072dc9697fa5cb71ddb17999d14 --- include/hsakmttypes.h | 3 ++- src/fmm.c | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index ab55047739..ae5e4fdbad 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1262,7 +1262,8 @@ typedef enum _HSA_POINTER_TYPE { HSA_POINTER_UNKNOWN = 0, HSA_POINTER_ALLOCATED = 1, // Allocated with hsaKmtAllocMemory (except scratch) HSA_POINTER_REGISTERED_USER = 2, // Registered user pointer - HSA_POINTER_REGISTERED_GRAPHICS = 3 // Registered graphics buffer + HSA_POINTER_REGISTERED_GRAPHICS = 3, // Registered graphics buffer + HSA_POINTER_REGISTERED_SHARED = 4 // Registered shared buffer (IPC) // (hsaKmtRegisterGraphicsToNodes) } HSA_POINTER_TYPE; diff --git a/src/fmm.c b/src/fmm.c index e2c45b5324..4442876e43 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3276,6 +3276,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo err = HSAKMT_STATUS_ERROR; goto err_free_obj; } + obj->node_id = gpu_mem[gpu_mem_id].node_id; map_fd = importArgs.mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_mem[gpu_mem_id].drm_render_fd; ret = mmap(reservedMem, (SizeInPages << PAGE_SHIFT), @@ -3496,7 +3497,9 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) } /* Successful vm_find_object returns with the aperture locked */ - if (vm_obj->metadata) + if (vm_obj->is_imported_kfd_bo) + info->Type = HSA_POINTER_REGISTERED_SHARED; + else if (vm_obj->metadata) info->Type = HSA_POINTER_REGISTERED_GRAPHICS; else if (vm_obj->userptr) info->Type = HSA_POINTER_REGISTERED_USER; From 6b8095184f4cee0bc2fd9b8bc09f147b3247d454 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 22 Jan 2020 14:34:46 -0500 Subject: [PATCH 0668/1247] kfdtest: List source files explicitly Tests run in the order in which they are linked. Currently that order is non-deterministic. Listing source files in the Makefile explicitly makes the order deterministic. The order chosen runs basic tests before more advanced tests before stress tests. Change-Id: I5bc032bcd589f92a51db36acb518bb4d8ef778d3 Signed-off-by: Felix Kuehling --- tests/kfdtest/CMakeLists.txt | 48 ++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index ce87fc978f..ffde898f60 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -82,8 +82,52 @@ include_directories(${SP3_DIR}) include_directories(${DRM_INCLUDE_DIRS}) -aux_source_directory(${PROJECT_SOURCE_DIR}/gtest-1.6.0 SRC_FILES) -aux_source_directory(${PROJECT_SOURCE_DIR}/src SRC_FILES) +set (SRC_FILES gtest-1.6.0/gtest-all.cpp + + src/AqlQueue.cpp + src/BasePacket.cpp + src/BaseQueue.cpp + src/Dispatch.cpp + src/GoogleTestExtension.cpp + src/IndirectBuffer.cpp + src/IsaGenerator.cpp + src/IsaGenerator_Gfx10.cpp + src/IsaGenerator_Gfx72.cpp + src/IsaGenerator_Gfx8.cpp + src/IsaGenerator_Gfx9.cpp + src/LinuxOSWrapper.cpp + src/PM4Packet.cpp + src/PM4Queue.cpp + src/RDMAUtil.cpp + src/SDMAPacket.cpp + src/SDMAQueue.cpp + src/KFDBaseComponentTest.cpp + src/KFDMultiProcessTest.cpp + src/KFDTestMain.cpp + src/KFDTestUtil.cpp + src/KFDTestUtilQueue.cpp + + src/KFDOpenCloseKFDTest.cpp + src/KFDTopologyTest.cpp + src/KFDMemoryTest.cpp + src/KFDLocalMemoryTest.cpp + src/KFDEventTest.cpp + src/KFDQMTest.cpp + src/KFDCWSRTest.cpp + src/KFDExceptionTest.cpp + src/KFDGraphicsInterop.cpp + src/KFDPerfCounters.cpp + src/KFDDBGTest.cpp + src/KFDGWSTest.cpp + src/KFDIPCTest.cpp + + src/KFDEvictTest.cpp + src/KFDHWSTest.cpp + src/KFDPerformanceTest.cpp + src/KFDPMTest.cpp + + src/KFDRASTest.cpp + src/RDMATest.cpp) message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) #message( STATUS "SRC_FILES: ") From 31530da7c6f6b2f7832963bf39297c76d5d97a5c Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 6 Feb 2020 14:40:27 -0500 Subject: [PATCH 0669/1247] libhsakmt: Child process can reacquire system props If child process explicitly calls hsaKmtReleaseSystemProperties(), it fails. Allow child process to release and acquire system properties. Change-Id: I649a4600212711b2ad4474f605f3ca39a4003d03 Signed-off-by: Harish Kasiviswanathan --- src/topology.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/topology.c b/src/topology.c index 9bcf0ad243..b4266ad176 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1962,8 +1962,6 @@ out: HSAKMT_STATUS HSAKMTAPI hsaKmtReleaseSystemProperties(void) { - CHECK_KFD_OPEN(); - HSAKMT_STATUS err; pthread_mutex_lock(&hsakmt_mutex); From a360c68b0cade21b004b102b196fb1e6cbe97d1a Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 20 Jan 2020 10:06:27 -0500 Subject: [PATCH 0670/1247] Add DEB/RPM packaging for KFDTest This will allow it to be installed with the ROCm suite, and centralize things a little bit more Also update run_kfdtest.sh to reflect the changes Lastly, remove "die" reference as compute_utils.sh may not be packaged with KFDTest Change-Id: I4c30cd29979192496419e71e3685937d7417f739 --- tests/kfdtest/CMakeLists.txt | 26 +++++++++++++ tests/kfdtest/scripts/run_kfdtest.sh | 57 +++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index ffde898f60..d92a6154ad 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -28,6 +28,16 @@ cmake_minimum_required(VERSION 2.8 FATAL_ERROR) project(KFDTest) +# For DEB/RPM generation +set ( CPACK_PACKAGE_FILE_NAME "kfdtest" ) +set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) +set ( CPACK_PACKAGE_DESCRIPTION "This package includes kfdtest, the list of excluded tests for each ASIC, and a convenience script to run the test suite" ) +set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "Test suite for ROCK/KFD" ) +set ( CPACK_PACKAGE_VERSION_MAJOR "1" ) +set ( CPACK_PACKAGE_VERSION_MINOR "0" ) +set ( CPACK_PACKAGE_VERSION_PATCH "0" ) +set ( CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) + #set ( CMAKE_VERBOSE_MAKEFILE on ) find_package(PkgConfig) @@ -47,6 +57,14 @@ else() include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) endif() +## Define default variable and variables for the optional build target hsakmt-dev +set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) +set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) +set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "Default packaging prefix." ) +set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) + +# Debian package specific variables +set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) if( DEFINED ENV{LIBHSAKMT_PATH} ) set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} ) @@ -159,3 +177,11 @@ target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LIBRARIES} ${DRM_AMDGPU_ configure_file ( scripts/kfdtest.exclude kfdtest.exclude COPYONLY ) configure_file ( scripts/run_kfdtest.sh run_kfdtest.sh COPYONLY ) + +install( FILES ${CMAKE_CURRENT_BINARY_DIR}/kfdtest ${CMAKE_CURRENT_BINARY_DIR}/run_kfdtest.sh + PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_WRITE GROUP_READ WORLD_EXECUTE WORLD_READ + DESTINATION bin ) +install( FILES ${CMAKE_CURRENT_BINARY_DIR}/kfdtest.exclude + PERMISSIONS OWNER_WRITE OWNER_READ GROUP_WRITE GROUP_READ WORLD_READ + DESTINATION share/kfdtest ) +include ( CPack ) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 52a1dd20a0..6097c99c45 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -22,9 +22,57 @@ # # +# See if we can find the SHARE/BIN dirs in their expected locations +CWD="${0%/*}" +while read candidate; do + if [ -e "$candidate/kfdtest.exclude" ]; then + source "$candidate/kfdtest.exclude" + break + fi +done < Date: Wed, 5 Feb 2020 14:13:04 -0600 Subject: [PATCH 0671/1247] Update analysis_memory_exception to recognize shared memory. Add type HSA_POINTER_REGISTERED_SHARED printing. Change-Id: Ic0400a097ebabde4f035b57fbca4cca12428fc97 --- src/events.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/events.c b/src/events.c index 3d13341688..23fb710523 100644 --- a/src/events.c +++ b/src/events.c @@ -250,6 +250,9 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * pr_err("GPU address 0x%lx, node id %d, size in byte 0x%lx\n", info.GPUAddress, info.Node, info.SizeInBytes); switch (info.Type) { + case HSA_POINTER_REGISTERED_SHARED: + pr_err("Memory is registered shared buffer (IPC)\n"); + break; case HSA_POINTER_REGISTERED_GRAPHICS: pr_err("Memory is registered graphics buffer\n"); break; From 0d4c209552b96965f57f26cc79ff6da17dffca9b Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 22 Jan 2020 14:42:29 -0500 Subject: [PATCH 0672/1247] Remove RegisterForeignDeviceMem test code Use an "if 0" to remove the code, as it is not working as expected. The test is supposed to test mapping from a foreign device (like a NIC), so it uses a separate GPU to map, but this mapping can be evicted and thus the test can fail unexpectedly. Remove the code until the test can be reworked Change-Id: Ie4a15c2a018bbd8e931b06b6700d10b3be86e410 --- tests/kfdtest/src/KFDGraphicsInterop.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp index 8d97885a30..b5a408534a 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.cpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -132,6 +132,12 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { TEST_END } +#if 0 +/* This test isn't testing things the way we wanted it to. It is flaky and + * will end up failing if the memory is evicted, which isn't possible for what + * it is intended to test. It needs a rework + */ + /* Third-party device memory can be registered for GPU access in * ROCm stack. Test this feature. Third party device is mimicked * in multi-GPU system using Graphics stack (libdrm). CPU accessible @@ -222,3 +228,4 @@ TEST_F(KFDGraphicsInterop, RegisterForeignDeviceMem) { TEST_END } +#endif From 21cda69ba9dd2ef47a9c125f5820878ee1f0f9d2 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 6 Nov 2019 14:42:50 -0500 Subject: [PATCH 0673/1247] kfdtest: Adapt the CWSR test for emulators The original test takes forever to run on emulators because emulators are much slower than Asic. So intelligently detect the emulator scenarios and reduce the run time by slashing the iteration times. Change-Id: I087f43c04c2b23b5ab2ecaad07533b767c337e94 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDCWSRTest.cpp | 16 ++++++++++++++++ tests/kfdtest/src/KFDTestUtil.cpp | 25 +++++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 1 + 3 files changed, 42 insertions(+) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index bc9b9698f9..5a4dbb4340 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -114,6 +114,14 @@ void KFDCWSRTest::TearDown() { ROUTINE_END } +bool isOnEmulator() { + uint32_t isEmuMode = 0; + + fscanf_dec("/sys/module/amdgpu/parameters/emu_mode", &isEmuMode); + + return isEmuMode; +} + /** * KFDCWSRTest.BasicTest * @@ -135,6 +143,14 @@ TEST_F(KFDCWSRTest, BasicTest) { int count1 = 40000000; int count2 = 20000000; + if (isOnEmulator()) { + // Divide the iterator times by 1000 so that the test can + // finish in a reasonable time. + count1 /= 1000; + count2 /= 1000; + LOG() << "On Emulators" << std::endl; + } + unsigned int* result1 = resultBuf1.As(); unsigned int* result2 = resultBuf2.As(); const char *pIterateIsa; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index f8743cb020..b55cd80247 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -39,6 +39,31 @@ void WaitUntilInput() { } while (dummy != 10); // enter key's ascii value is 10 } +/* fscanf_dec - read a file whose content is a decimal number + * @file [IN ] file to read + * @num [OUT] number in the file + * + * It is copied from the same function in libhsakmt + */ +HSAKMT_STATUS fscanf_dec(const char *file, uint32_t *num) +{ + FILE *fd; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + + fd = fopen(file, "r"); + if (!fd) { + LOG() << "Failed to open " << file << std::endl; + return HSAKMT_STATUS_INVALID_PARAMETER; + } + if (fscanf(fd, "%u", num) != 1) { + LOG() << "Failed to parse as a decimal: " << file << std::endl;; + ret = HSAKMT_STATUS_ERROR; + } + + fclose(fd); + return ret; +} + uint64_t RoundToPowerOf2(uint64_t val) { int bytes = sizeof(uint64_t); diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 139fbd1513..28847f370e 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -36,6 +36,7 @@ class BaseQueue; #define CounterToNanoSec(x) ((x) * 1000 / (is_dgpu() ? 27 : 100)) void WaitUntilInput(); +HSAKMT_STATUS fscanf_dec(const char *file, uint32_t *num); uint64_t RoundToPowerOf2(uint64_t val); // @brief: waits until the value is written to the buffer or until time out if received through args From 7a852be42e738adca2b7ad3f934e80dbcba87e6b Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 22 Jan 2020 15:55:26 -0500 Subject: [PATCH 0674/1247] kfdtest: Clean up KFDEvictTest Move the shader code before the test case code so that all test case code is consecutive. Rectify the print messages and avoid calling GetSysMemSize() repeatedly. Change-Id: I1c4aa5552de4d74163717fe66ad9759fb09e1316 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDEvictTest.cpp | 187 +++++++++++++++-------------- 1 file changed, 100 insertions(+), 87 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index b74aedba35..adeb847f73 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -277,79 +277,6 @@ void KFDEvictTest::AmdgpuCommandSubmissionSdmaNop(int rn, amdgpu_bo_handle handl EXPECT_EQ(0, amdgpu_cs_ctx_free(contextHandle)); } -/* Evict and restore procedure basic test - * - * Use N_PROCESSES processes to allocate vram buf size larger than total vram size - * - * ALLOCATE_BUF_SIZE_MB buf allocation size - * - * buf is equal to (vramSizeMB / (vramBufSizeMB * N_PROCESSES) ) + 8 - * Total vram all processes allocated: 8GB for 4GB Fiji, and 20GB for 16GB Vega10 - * - * Eviction and restore will happen many times: - * ttm will evict buffers of another process if there is not enough free vram - * process restore will evict buffers of another process - * - * Sometimes the allocation may fail (maybe that is normal) - * ALLOCATE_RETRY_TIMES max retry times to allocate - * - * This is basic test with no queue, so vram is not used by the GPU during test - * - * TODO: - * - Synchronization between the processes, so they know for sure when - * they are done allocating memory - */ -TEST_F(KFDEvictTest, BasicTest) { - TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); - TEST_START(TESTPROFILE_RUNALL); - - HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; - - HSAuint64 vramSize = GetVramSize(defaultGPUNode) * 7 / 8; - - if (!vramSize) { - LOG() << "Skipping test: No VRAM found." << std::endl; - return; - } else { - LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; - } - - // Use 7/8 of VRAM between all processes - HSAuint32 count = vramSize / (vramBufSize * N_PROCESSES); - - LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; - - /* Fork the child processes */ - ForkChildProcesses(N_PROCESSES); - - int rn = FindDRMRenderNode(defaultGPUNode); - if (rn < 0) { - LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; - WaitChildProcesses(); - return; - } - - std::vector pBuffers; - AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); - - /* Allocate gfx vram size of at most one third system memory */ - HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; - amdgpu_bo_handle handle; - AllocAmdgpuBo(rn, size, handle); - - AmdgpuCommandSubmissionSdmaNop(rn, handle); - - FreeAmdgpuBo(handle); - LOG() << m_psName << "free buffer" << std::endl; - FreeBuffers(pBuffers, vramBufSize); - - WaitChildProcesses(); - - TEST_END -} - /* Shader to read local buffers using multiple wavefronts in parallel * until address buffer is filled with specific value 0x5678 by host program, * then each wavefront fills value 0x5678 at corresponding result buffer and quit @@ -480,6 +407,85 @@ std::string KFDEvictTest::CreateShader() { return gfx9_ReadMemory; } +/* Evict and restore procedure basic test + * + * Use N_PROCESSES processes to allocate vram buf size larger than total vram size + * + * ALLOCATE_BUF_SIZE_MB buf allocation size + * + * buf is equal to (vramSizeMB / (vramBufSizeMB * N_PROCESSES) ) + 8 + * Total vram all processes allocated: 8GB for 4GB Fiji, and 20GB for 16GB Vega10 + * + * Eviction and restore will happen many times: + * ttm will evict buffers of another process if there is not enough free vram + * process restore will evict buffers of another process + * + * Sometimes the allocation may fail (maybe that is normal) + * ALLOCATE_RETRY_TIMES max retry times to allocate + * + * This is basic test with no queue, so vram is not used by the GPU during test + * + * TODO: + * - Synchronization between the processes, so they know for sure when + * they are done allocating memory + */ +TEST_F(KFDEvictTest, BasicTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + HSAuint64 sysMemSize = GetSysMemSize(); + + if (!vramSize) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; + LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; + + // Use 7/8 of VRAM between all processes + HSAuint64 testSize = vramSize * 7 / 8; + HSAuint32 count = testSize / (vramBufSize * N_PROCESSES); + + if (count == 0) { + LOG() << "Skipping test: Not enough system memory available." << std::endl; + return; + } + + /* Fork the child processes */ + ForkChildProcesses(N_PROCESSES); + + int rn = FindDRMRenderNode(defaultGPUNode); + if (rn < 0) { + LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; + WaitChildProcesses(); + return; + } + + std::vector pBuffers; + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + + /* Allocate gfx vram size of at most one third system memory */ + HSAuint64 size = sysMemSize / 3 < testSize ? sysMemSize / 3 : testSize; + amdgpu_bo_handle handle; + AllocAmdgpuBo(rn, size, handle); + + AmdgpuCommandSubmissionSdmaNop(rn, handle); + + FreeAmdgpuBo(handle); + LOG() << m_psName << "free buffer" << std::endl; + FreeBuffers(pBuffers, vramBufSize); + + WaitChildProcesses(); + + TEST_END +} + /* Evict and restore queue test * * N_PROCESSES processes read all local buffers in parallel while buffers are evicted and restored @@ -514,19 +520,20 @@ TEST_F(KFDEvictTest, QueueTest) { } HSAuint32 i; - HSAuint64 vramSize = GetVramSize(defaultGPUNode) * 7 / 8; + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + HSAuint64 sysMemSize = GetSysMemSize(); if (!vramSize) { LOG() << "Skipping test: No VRAM found." << std::endl; return; - } else { - LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; } - // Use 7/8 of VRAM between all processes - HSAuint32 count = vramSize / (vramBufSize * N_PROCESSES); + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; + LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; - LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; + // Use 7/8 of VRAM between all processes + HSAuint64 testSize = vramSize * 7 / 8; + HSAuint32 count = testSize / (vramBufSize * N_PROCESSES); if (count == 0) { LOG() << "Skipping test: Not enough system memory available." << std::endl; @@ -562,7 +569,7 @@ TEST_F(KFDEvictTest, QueueTest) { AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); /* Allocate gfx vram size of at most one third system memory */ - HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; + HSAuint64 size = sysMemSize / 3 < testSize ? sysMemSize / 3 : testSize; amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); @@ -624,19 +631,25 @@ TEST_F(KFDEvictTest, BurstyTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; - HSAuint64 vramSize = GetVramSize(defaultGPUNode) * 7 / 8; + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + HSAuint64 sysMemSize = GetSysMemSize(); if (!vramSize) { LOG() << "Skipping test: No VRAM found." << std::endl; return; - } else { - LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; } - // Use 7/8 of VRAM between all processes - HSAuint32 count = vramSize / (vramBufSize * N_PROCESSES); + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; + LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; - LOG() << "Found System RAM of " << std::dec << (GetSysMemSize() >> 20) << "MB" << std::endl; + // Use 7/8 of VRAM between all processes + HSAuint64 testSize = vramSize * 7 / 8; + HSAuint32 count = testSize / (vramBufSize * N_PROCESSES); + + if (count == 0) { + LOG() << "Skipping test: Not enough system memory available." << std::endl; + return; + } /* Fork the child processes */ ForkChildProcesses(N_PROCESSES); @@ -655,7 +668,7 @@ TEST_F(KFDEvictTest, BurstyTest) { AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); /* Allocate gfx vram size of at most one third system memory */ - HSAuint64 size = GetSysMemSize() / 3 < vramSize ? GetSysMemSize() / 3 : vramSize; + HSAuint64 size = sysMemSize / 3 < testSize ? sysMemSize / 3 : testSize; amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); From 0e5c4d83e6988d25918c9ab328bf169a78dff1a0 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 12 Feb 2020 18:00:15 -0500 Subject: [PATCH 0675/1247] kfdtest: Delete MULTI_GPU usage in run_kfdtest.sh It is obsolete. Change-Id: Ifd137ce1ce8d9133cfa5c8bfd46aaeea461b5aa7 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/run_kfdtest.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 6097c99c45..8ad27ff118 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -78,7 +78,6 @@ fi PLATFORM="" GDB="" NODE="" -MULTI_GPU="" FORCE_HIGH="" RUN_IN_DOCKER="" @@ -106,8 +105,6 @@ printUsage() { return 0 } # Print gtest_filter for the given Platform -# If MULTI_GPU flag is set, then Multi-GPU Tests are selected. Once all tests -# pass in Multi GPU environment, this flag can be removed # param - Platform. getFilter() { # For regular platforms such as vega10, this will automatically generate @@ -219,16 +216,6 @@ getGPUCount() { echo "$gpuCount" } -# set $MULTI_GPU flag if the system has more than 1 GPU -setMultiGPUFlag() { - gpuCount=$(getGPUCount) - if [ $gpuCount -gt 1 ]; then - MULTI_GPU=1 - else - MULTI_GPU=0 - fi -} - while [ "$1" != "" ]; do case "$1" in -p | --platform ) @@ -257,7 +244,6 @@ if [ "$FORCE_HIGH" == "true" ]; then pushTrap "popGpuDpmState" EXIT fi -setMultiGPUFlag # Set HSA_DEBUG env to run KFDMemoryTest.PtraceAccessInvisibleVram export HSA_DEBUG=1 runKfdTest From 4e7b2f2e27e404f428a8db82887e589ec7433728 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Wed, 12 Feb 2020 18:03:54 -0500 Subject: [PATCH 0676/1247] kfdtest: Print a message when there is no GPU This helps the user to troubleshoot the problem. Change-Id: If6cf42c488097011285252a6c722d3d74c0f7ce7 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/run_kfdtest.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 8ad27ff118..fbeee08ece 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -163,6 +163,11 @@ runKfdTest() { if [ "$NODE" == "" ]; then hsaNodes=$(getHsaNodes) + + if [ "$hsaNodes" == "" ]; then + echo "No GPU found in the system." + exit 1 + fi else hsaNodes=$NODE fi From ebe7de1f99e2f5e6b18c6572f3084decdb35cee5 Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Thu, 5 Mar 2020 14:49:49 -0500 Subject: [PATCH 0677/1247] libhsakmt: Expose device Unique Id Read device unique id from sysfs and expose it in HsaNodeProperties. For devices not supported the value will be 0 Signed-off-by: Divya Shikre Change-Id: I97b8689dfa090971c6876de6feaa97652e28c03d --- include/hsakmttypes.h | 3 ++- src/topology.c | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index ae5e4fdbad..3aeba807ea 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -308,7 +308,8 @@ typedef struct _HsaNodeProperties HSAuint8 Reserved2; HSAuint32 Domain; // PCI domain of the GPU - HSAuint8 Reserved[28]; + HSAuint64 UniqueID; // Globally unique immutable id + HSAuint8 Reserved[20]; } HsaNodeProperties; diff --git a/src/topology.c b/src/topology.c index b4266ad176..f718ddaa05 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1020,6 +1020,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->uCodeEngineVersions.Value = (uint32_t)prop_val & 0x3ff; else if (strcmp(prop_name, "hive_id") == 0) props->HiveID = prop_val; + else if (strcmp(prop_name, "unique_id") == 0) + props->UniqueID = prop_val; else if (strcmp(prop_name, "num_sdma_engines") == 0) props->NumSdmaEngines = prop_val; else if (strcmp(prop_name, "num_sdma_xgmi_engines") == 0) From 9efefe6d52af096c5cc2735aaf03786b829231dc Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Wed, 11 Mar 2020 18:25:55 -0500 Subject: [PATCH 0678/1247] Handle EBADF when KFD file handle is still open. Signed-off-by: Sean Keely Change-Id: I23d6c87d5729f57c261030c6baeff4c977eef934 --- src/libhsakmt.c | 2 +- src/libhsakmt.h | 2 ++ src/openclose.c | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libhsakmt.c b/src/libhsakmt.c index 9236b8e280..a9e250d087 100644 --- a/src/libhsakmt.c +++ b/src/libhsakmt.c @@ -18,7 +18,7 @@ int kmtIoctl(int fd, unsigned long request, void *arg) * make any subsequent hsaKmt calls fail in CHECK_KFD_OPEN. */ pr_err("KFD file descriptor not valid in this process\n"); - hsakmt_forked = true; + is_forked_child(); } return ret; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index f929833d28..411cdec512 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -196,4 +196,6 @@ void clear_events_page(void); void fmm_clear_all_mem(void); void clear_process_doorbells(void); uint32_t get_num_sysfs_nodes(void); + +bool is_forked_child(void); #endif diff --git a/src/openclose.c b/src/openclose.c index 11f930f486..eb9b6080a0 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -48,7 +48,7 @@ int zfb_support; * because the process can fork without calling the fork function in * libc (using clone or calling the system call directly). */ -static bool is_forked_child(void) +bool is_forked_child(void) { pid_t cur_pid; From 96259b5830c59a6fd2b1307def31039af05ffe2d Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Fri, 6 Mar 2020 11:10:31 -0500 Subject: [PATCH 0679/1247] kfdtest: Provide Unique ID information. Signed-off-by: Divya Shikre Change-Id: I9a837a3d1ab38f5ad05673406874d862c9e97541 --- tests/kfdtest/src/KFDTopologyTest.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index ac9960bffb..334317e943 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -37,6 +37,13 @@ TEST_F(KFDTopologyTest , BasicTest) { for (unsigned node = 0; node < m_SystemProperties.NumNodes; node++) { pNodeProperties = m_NodeInfo.GetNodeProperties(node); if (pNodeProperties != NULL) { + HSAuint64 uniqueid; + if (!pNodeProperties->UniqueID) + uniqueid = 0; + else + uniqueid = pNodeProperties->UniqueID; + LOG() << "UniqueID : " << std::dec << uniqueid << + " Node index: " << node << std::endl; // Checking for cpu core only if it's a cpu only node or if its KAVERI apu. if (pNodeProperties->DeviceId == 0 || FamilyIdFromNode(pNodeProperties) == FAMILY_KV) { EXPECT_GT(pNodeProperties->NumCPUCores, HSAuint32(0)) << "Node index: " << node From 5dcd49f72631f43a8fc3d799ef65f33eb6b34c08 Mon Sep 17 00:00:00 2001 From: Jeffrey Poznanovic Date: Mon, 2 Mar 2020 20:15:01 +0000 Subject: [PATCH 0680/1247] Added CentOS-6 mods to support manylinux2010 Change-Id: I8c303ccfdc7d314d1b4609ed6181d46795ada621 --- CMakeLists.txt | 9 +++++++++ RPM/libhsakmt.spec | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a3e969764..cc186fd7d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -181,6 +181,13 @@ if( EXISTS "/etc/os-release" ) string( REPLACE "ID=" "" DISTRO_ID "${DISTRO_ID}" ) string( REPLACE "VERSION_ID=" "" DISTRO_RELEASE "${DISTRO_RELEASE}" ) message( STATUS "Detected distribution: ${DISTRO_ID}:${DISTRO_RELEASE}" ) +elseif( EXISTS "/etc/centos-release" ) + # Example: CentOS release 6.10 (Final) + file( STRINGS "/etc/centos-release" DISTRO_FULL_STR REGEX "release" ) + string( REGEX MATCH "^[a-zA-Z]+" DISTRO_ID "${DISTRO_FULL_STR}" ) + string( TOLOWER "${DISTRO_ID}" DISTRO_ID ) + string( REGEX MATCH "[0-9]+" DISTRO_RELEASE "${DISTRO_FULL_STR}" ) + message( STATUS "Detected distribution: ${DISTRO_ID}:${DISTRO_RELEASE}" ) else() message( STATUS "Not able to detect OS" ) endif() @@ -193,6 +200,8 @@ set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libnuma1, libpci3, zlib1g, libudev1" # RPM package specific variables if( DISTRO_ID MATCHES "sles" ) set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, libnuma-devel, pciutils, libgcc_s1") +elseif( DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6" ) + set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl, pciutils-libs, libgcc" ) else() set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl-libs, pciutils-libs, libgcc" ) endif() diff --git a/RPM/libhsakmt.spec b/RPM/libhsakmt.spec index 631d65ba40..c219ef7581 100644 --- a/RPM/libhsakmt.spec +++ b/RPM/libhsakmt.spec @@ -9,7 +9,12 @@ Summary: Thunk libraries for AMD KFD Group: System Environment/Libraries License: Advanced Micro Devices Inc. + +%if 0%{?centos} == 6 +Requires: numactl +%else Requires: numactl-libs +%endif %description From 0a1718b7533e2ce482100b9bb8154470e2bcd044 Mon Sep 17 00:00:00 2001 From: Jon Chesterfield Date: Thu, 19 Mar 2020 01:04:28 +0000 Subject: [PATCH 0681/1247] Replace libpci with new parser. libpci was only used to find a marketing string for a device. This patch looks for a pci.ids on disk and parses it to extract the same string, using 'Device xxxx' as the fallback on file i/o error or missing data from the text file. Tested by checking every vendor/ device pair against the values returned from libpci. Change-Id: I21af3157472c1824d57fcee31393c6ee8ce07330 Signed-off-by: Jon Chesterfield --- CMakeLists.txt | 13 +-- src/fmm.c | 9 +- src/libhsakmt.h | 4 +- src/pci_ids.c | 304 ++++++++++++++++++++++++++++++++++++++++++++++++ src/pci_ids.h | 46 ++++++++ src/perfctr.c | 8 +- src/topology.c | 15 ++- 7 files changed, 373 insertions(+), 26 deletions(-) create mode 100644 src/pci_ids.c create mode 100644 src/pci_ids.h diff --git a/CMakeLists.txt b/CMakeLists.txt index cc186fd7d0..aa2c4c609c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,6 +105,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/libhsakmt.c" "src/memory.c" "src/openclose.c" + "src/pci_ids.c" "src/perfctr.c" "src/pmc_table.c" "src/queues.c" @@ -127,10 +128,8 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" ) find_package(PkgConfig) -pkg_check_modules(PC_LIBPCI REQUIRED libpci) -include_directories ( ${PC_LIBPCI_INCLUDEDIR} ) target_link_libraries ( ${HSAKMT_TARGET} - pthread rt numa ${PC_LIBPCI_LIBRARIES} + pthread rt numa ) ## Define default variable and variables for the optional build target hsakmt-dev @@ -195,15 +194,15 @@ endif() # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm" ) -set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libnuma1, libpci3, zlib1g, libudev1" ) +set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1" ) # RPM package specific variables if( DISTRO_ID MATCHES "sles" ) - set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, libnuma-devel, pciutils, libgcc_s1") + set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1") elseif( DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6" ) - set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl, pciutils-libs, libgcc" ) + set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl, libgcc" ) else() - set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl-libs, pciutils-libs, libgcc" ) + set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc" ) endif() set ( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) diff --git a/src/fmm.c b/src/fmm.c index 4442876e43..fab57af6b5 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -36,7 +36,7 @@ #include #include #include -#include + #include #include #include "rbtree.h" @@ -2119,7 +2119,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; unsigned int guardPages = 1; - struct pci_access *pacc; + struct pci_ids pacc; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2166,8 +2166,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * gets called before hsaKmtAcquireSystemProperties() is called. */ - pacc = pci_alloc(); - pci_init(pacc); + pacc = pci_ids_create(); for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc); @@ -2204,7 +2203,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem_count++; } } - pci_cleanup(pacc); + pci_ids_destroy(pacc); /* The ioctl will also return Number of Nodes if * args.kfd_process_device_apertures_ptr is set to NULL. This is not diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 411cdec512..8d38eef80c 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -27,10 +27,10 @@ #define LIBHSAKMT_H_INCLUDED #include "hsakmt.h" +#include "pci_ids.h" #include #include #include -#include extern int kfd_fd; extern unsigned long kfd_open_count; @@ -155,7 +155,7 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - uint32_t *gpu_id, struct pci_access* pacc); + uint32_t *gpu_id, struct pci_ids pacc); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); bool topology_is_dgpu(uint16_t device_id); bool topology_is_svm_needed(uint16_t device_id); diff --git a/src/pci_ids.c b/src/pci_ids.c new file mode 100644 index 0000000000..1353cfa23d --- /dev/null +++ b/src/pci_ids.c @@ -0,0 +1,304 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * Parse a pci.ids text file to extract 'device_name' + * # Vendors, devices and subsystems. Please keep sorted. + * # Syntax: + * # vendor vendor_name + * # device device_name <-- single tab + * # subvendor subdevice subsystem_name <-- two tabs + */ + +/* + * Example section of file. Searching for 1002/130c + * + * 1002 Advanced Micro Devices, Inc. [AMD/ATI] + * # fields elided + * 130a Kaveri [Radeon R6 Graphics] + * 130b Kaveri [Radeon R4 Graphics] + * + * 130c Kaveri [Radeon R7 Graphics] <- result + * # ^-------------------------^ + * + * 130d Kaveri [Radeon R6 Graphics] + * 130e Kaveri [Radeon R5 Graphics] + * # fields elided + * # next vendor region starts + * 1003 ULSI Systems + * 0201 US201 + */ + +#define _GNU_SOURCE +#include + +#include "pci_ids.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static const char *const pci_ids_paths[] = { + "/usr/share/hwdata/pci.ids", // update-pciids + "/usr/share/misc/pci.ids", // debian + "/usr/share/pci.ids", // redhat + "/var/lib/pciutils/pci.ids", // also debian + "pci.ids", +}; + +static struct pci_ids pci_ids_create_from_file(const char *path) +{ + struct pci_ids failure = { + .fd = -1, + }; + + int fd = open(path, O_RDONLY, 0); + + if (fd == -1) + return failure; + + struct stat sb; + + fstat(fd, &sb); + size_t sz = sb.st_size; + + if (sz == 0) { + close(fd); + return failure; + } + + sz = (sz < UINT32_MAX) ? sz : UINT32_MAX; + void *addr = mmap(0, sz, PROT_READ, MAP_PRIVATE, fd, 0); + + if (addr == MAP_FAILED) { + close(fd); + return failure; + } + + return (struct pci_ids){ + .fd = fd, + .addr = addr, + .size = sz, + }; +} + +struct pci_ids pci_ids_create(void) +{ + size_t sz = sizeof(pci_ids_paths) / sizeof(pci_ids_paths[0]); + + for (size_t i = 0; i < sz; i++) { + struct pci_ids res = pci_ids_create_from_file(pci_ids_paths[i]); + + if (res.fd != -1) + return res; + } + + return (struct pci_ids){ .fd = -1 }; +} + +void pci_ids_destroy(struct pci_ids f) +{ + if (f.fd != -1) { + munmap(f.addr, f.size); + close(f.fd); + } +} + +struct range { + // Iterator pair, start <= end. Can dereference [start end) + unsigned char *start; + unsigned char *end; +}; +static bool empty_range(struct range r) +{ + return r.start == r.end; +} + +static void write_as_hex(uint16_t x, char *b) +{ + static const char digits[] = "0123456789abcdef"; + + for (unsigned int i = 0; i < 4; i++) { + unsigned int index = 0xf & (x >> 4 * (3 - i)); + + b[i] = digits[index]; + } +} + +static struct range find_vendor(struct range r, uint16_t VendorId) + +{ + if (empty_range(r)) + return r; + + char needle[5] = { '\n' }; + + write_as_hex(VendorId, &needle[1]); + unsigned char *s = + memmem(r.start, r.end - r.start, needle, sizeof(needle)); + + if (s) { + r.start = s; + } else { + r.start = r.end; + assert(empty_range(r)); + } + return r; +} + +static struct range trim_whitespace(struct range r) +{ + while (!empty_range(r) && isspace(r.start[0])) + r.start++; + + while (!empty_range(r) && isspace(r.end[-1])) + r.end--; + + return r; +} + +static struct range skip_vendor_id(struct range r) +{ + const struct range failure = { 0, 0 }; + + assert(!empty_range(r)); + // Expect an initial newline to skip over + if (r.start[0] != '\n') + return failure; + + r.start++; + if (empty_range(r)) + return failure; + + // Skip rest of line + r.start = memchr(r.start, '\n', r.end - r.start); + if (!r.start) + return failure; + + return r; +} + +static struct range find_device(struct range r, uint16_t DeviceId) +{ + struct range failure = { 0, 0 }; + + if (empty_range(r)) + return failure; + + r = skip_vendor_id(r); + if (empty_range(r)) + return failure; + + assert(r.start[0] == '\n'); + + char needle[6] = { '\n', '\t' }; + + write_as_hex(DeviceId, &needle[2]); + + for (;;) { + size_t width = r.end - r.start; + + if (width < sizeof(needle)) + return failure; + + unsigned char *line_end = memchr(r.start + 1, '\n', width - 1); + + if (!line_end) { + // File may not end with a newline + line_end = r.end; + } + + if (memcmp(r.start, needle, sizeof(needle)) == 0) { + // Success + r.start += sizeof(needle); + r.end = line_end; + return trim_whitespace(r); + } + + if (isxdigit(r.start[1])) { + // Reached the end of this region + return failure; + } + + // Otherwise ignore whatever is on the line, e.g. '#' + r.start = line_end; + } +} + +static void copy_range_to_buffer(struct range r, char *buf, size_t size) +{ + assert(!empty_range(r)); + size_t to_copy = (r.end - r.start); + + to_copy = to_copy < (size - 1) ? to_copy : size - 1; + + memcpy(buf, r.start, to_copy); + buf[to_copy] = '\0'; +} + +static void write_fallback_to_buffer(char *buf, size_t size, uint16_t DeviceId) +{ + char tmp[] = "Device xxxx"; + + static_assert(sizeof(tmp) == 12, ""); + write_as_hex(DeviceId, &tmp[7]); + + size_t to_copy = (sizeof(tmp) <= size) ? sizeof(tmp) : size; + + memcpy(buf, tmp, to_copy); + buf[size - 1] = '\0'; +} + +char *pci_ids_lookup(struct pci_ids f, char *buf, size_t size, + uint16_t VendorId, uint16_t DeviceId) +{ + if (f.fd == -1) { + write_fallback_to_buffer(buf, size, DeviceId); + return buf; + } + + struct range whole_file = { + .start = f.addr, + .end = (unsigned char *)f.addr + f.size, + }; + + struct range vendor = find_vendor(whole_file, VendorId); + + struct range device = find_device(vendor, DeviceId); + + if (!empty_range(device)) + copy_range_to_buffer(device, buf, size); + else + write_fallback_to_buffer(buf, size, DeviceId); + + return buf; +} diff --git a/src/pci_ids.h b/src/pci_ids.h new file mode 100644 index 0000000000..4c0d44c57e --- /dev/null +++ b/src/pci_ids.h @@ -0,0 +1,46 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef SRC_PCI_IDS_H_ +#define SRC_PCI_IDS_H_ + +#include +#include + +struct pci_ids { + int32_t fd; // -1 if file access failed + uint32_t size; + void *addr; +}; + +// Sixteen byte struct is passed in registers. Avoids calling malloc/free. +struct pci_ids pci_ids_create(void); +void pci_ids_destroy(struct pci_ids pacc); + +// Writes to buf. Returns buf. Does not fail. +char *pci_ids_lookup(struct pci_ids pacc, char *buf, size_t buf_size, + uint16_t VendorId, uint16_t DeviceId); + +#endif // SRC_PCI_IDS_H_ diff --git a/src/perfctr.c b/src/perfctr.c index 73bab19f45..546dd2358b 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -77,11 +77,11 @@ struct perf_shared_table { struct perf_counts_values { union { struct { - u64 val; - u64 ena; - u64 run; + uint64_t val; + uint64_t ena; + uint64_t run; }; - u64 values[3]; + uint64_t values[3]; }; }; diff --git a/src/topology.c b/src/topology.c index f718ddaa05..2817b035ff 100644 --- a/src/topology.c +++ b/src/topology.c @@ -31,7 +31,7 @@ #include #include #include -#include + #include #include @@ -910,7 +910,7 @@ exit: HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, - struct pci_access *pacc) + struct pci_ids pacc) { FILE *fd; char *read_buf, *p, *envvar, dummy; @@ -1068,7 +1068,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, * Retrieve the marketing name of the node using pcilib, * convert UTF8 to UTF16 */ - name = pci_lookup_name(pacc, namebuf, sizeof(namebuf), PCI_LOOKUP_DEVICE, + name = pci_ids_lookup(pacc, namebuf, sizeof(namebuf), props->VendorId, props->DeviceId); for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) props->MarketingName[i] = name[i]; @@ -1731,7 +1731,7 @@ HSAKMT_STATUS topology_take_snapshot(void) HsaSystemProperties sys_props; node_props_t *temp_props = 0; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - struct pci_access *pacc; + struct pci_ids pacc; struct proc_cpuinfo *cpuinfo; const uint32_t num_procs = get_nprocs(); @@ -1755,8 +1755,7 @@ retry: ret = HSAKMT_STATUS_NO_MEMORY; goto err; } - pacc = pci_alloc(); - pci_init(pacc); + pacc = pci_ids_create(); for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_props[i].node, @@ -1826,7 +1825,7 @@ retry: * remote node (node_to) is not accessible */ while (sys_link_id < temp_props[i].node.NumIOLinks && - link_id < sys_props.NumNodes - 1) { + link_id < sys_props.NumNodes - 1) { ret = topology_sysfs_get_iolink_props(i, sys_link_id++, &temp_props[i].link[link_id]); if (ret == HSAKMT_STATUS_NOT_SUPPORTED) { @@ -1842,7 +1841,7 @@ retry: temp_props[i].node.NumIOLinks = link_id; } } - pci_cleanup(pacc); + pci_ids_destroy(pacc); } /* All direct IO links are created in the kernel. Here we need to From 884fed4f045d6dcc5f67cb1245fbe4bc818f9f1e Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Thu, 26 Mar 2020 16:22:25 -0500 Subject: [PATCH 0682/1247] Correct initial kfd_open_count increment. Don't set kfd_open_count=1 unless hsaKmtOpenKFD actually succeeds. This prevents returning HSAKMT_STATUS_KERNEL_ALREADY_OPENED in subsequent calls when KFD is actually closed. Signed-off-by: Sean Keely Change-Id: Ia870b5faa8626826a6c8795aa10784d376cf2e80 --- src/openclose.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/openclose.c b/src/openclose.c index eb9b6080a0..72dfd66a1f 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -194,7 +194,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) } kfd_fd = fd; - kfd_open_count = 1; init_page_size(); @@ -210,6 +209,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (result != HSAKMT_STATUS_SUCCESS) goto init_doorbell_failed; + kfd_open_count = 1; + if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) pr_warn("Insufficient Memory. Debugging unavailable\n"); From 5e353648382fb2d7fc72db37ddf2808840009db9 Mon Sep 17 00:00:00 2001 From: Srinivasan Subramanian Date: Wed, 1 Apr 2020 20:11:42 -0700 Subject: [PATCH 0683/1247] libhsakmt: check ret and errno for EBADF Change-Id: I9fcbf955d8b7b01ff1025534a8c2eaa8e6790565 Signed-off-by: Srinivasan Subramanian --- src/libhsakmt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libhsakmt.c b/src/libhsakmt.c index a9e250d087..b742e799c6 100644 --- a/src/libhsakmt.c +++ b/src/libhsakmt.c @@ -13,7 +13,7 @@ int kmtIoctl(int fd, unsigned long request, void *arg) ret = ioctl(fd, request, arg); } while (ret == -1 && (errno == EINTR || errno == EAGAIN)); - if (errno == EBADF) { + if (ret == -1 && errno == EBADF) { /* In case pthread_atfork didn't catch it, this will * make any subsequent hsaKmt calls fail in CHECK_KFD_OPEN. */ From 7ee9e01587016ad2f5a6e6cd5842149b5a5c3d39 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 14 Apr 2020 11:29:45 -0400 Subject: [PATCH 0684/1247] KFDTest: Use SMI for HIGH clocks, if possible Some systems don't support coarse-grained DPM, so performance level will fail. Remove the compute_utils.sh references, and just use the SMI if we request clocks be high, without throwing errors if it fails. Change-Id: Ic5beda9921128be36ac2d58cae3f0608618a8e21 --- tests/kfdtest/scripts/run_kfdtest.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index fbeee08ece..de4c6783ec 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -243,12 +243,25 @@ while [ "$1" != "" ]; do shift 1 done -# If compute_utils.sh doesn't exist, this will just silently do nothing +# If the SMI is missing, just report and continue if [ "$FORCE_HIGH" == "true" ]; then - pushGpuDpmState high - pushTrap "popGpuDpmState" EXIT + if [ -e "/opt/rocm/bin/rocm-smi" ]; then + OLDPERF=$(/opt/rocm/bin/rocm-smi -p | awk '/Performance Level/ {print $NF; exit}') + $(/opt/rocm/bin/rocm-smi --setperflevel high &> /dev/null) + if [ $? != 0 ]; then + echo "SMI failed to set perf level" + OLDPERF="" + fi + else + echo "Unable to set clocks to high" + fi fi # Set HSA_DEBUG env to run KFDMemoryTest.PtraceAccessInvisibleVram export HSA_DEBUG=1 runKfdTest + +# OLDPERF is only set if FORCE_HIGH and SMI both exist +if [ -n "$OLDPERF" ]; then + /opt/rocm/bin/rocm-smi --setperflevel $OLDPERF &> /dev/null +fi From 8ee763d94a74a5abdd7db4fad8b28076a77d9534 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 5 Mar 2020 11:27:27 -0500 Subject: [PATCH 0685/1247] kfdtest: Fix problems finding kfdtest.exclude When running run_kfdtest.sh through a wrapper script that sources run_kfdtest.sh, kfdtest.exclude isn't found because $0 points to the location of the wrapper script. User $BASH_SOURCE instead of $0 to find the location of the correct run_kfdtest.sh script. Change-Id: I0ae7899e527e6d98bb8651197484e5ee03a5fd7b Signed-off-by: Felix Kuehling --- tests/kfdtest/scripts/run_kfdtest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index de4c6783ec..12ef8af51d 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -23,7 +23,7 @@ # # See if we can find the SHARE/BIN dirs in their expected locations -CWD="${0%/*}" +CWD="${BASH_SOURCE%/*}" while read candidate; do if [ -e "$candidate/kfdtest.exclude" ]; then source "$candidate/kfdtest.exclude" From af249159ee44c1d95dee5d4c9dd983f51511fc8b Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 23 Jan 2020 15:45:13 -0500 Subject: [PATCH 0686/1247] kfdtest: do not request host accessible memory for P2P tests Do not request host accessible memory otherwise small-bar XGMI fails. Change-Id: I6b1e750839ae66a34c85405fa8d0a4aa455399ef Signed-off-by: Jonathan Kim --- tests/kfdtest/src/KFDPerformanceTest.cpp | 2 +- tests/kfdtest/src/KFDQMTest.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index 50cff1544d..e80e4e71d6 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -73,7 +73,7 @@ testNodeToNodes(HSAuint32 n1, const HSAuint32 *const n2Array, int n, P2PDirectio void *n1Mem, *n2Mem[n]; HsaMemFlags memFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - memFlags.ui32.HostAccess = 1; + memFlags.ui32.HostAccess = 0; memFlags.ui32.NonPaged = 1; SDMACopyParams array[n * 4]; int array_count = 0; diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 6a0d60db46..10f74ae808 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1671,7 +1671,7 @@ TEST_F(KFDQMTest, P2PTest) { HsaMemFlags memFlags = {0}; HsaMemMapFlags mapFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - memFlags.ui32.HostAccess = 1; + memFlags.ui32.HostAccess = 0; memFlags.ui32.NonPaged = 1; memFlags.ui32.NoNUMABind = 1; unsigned int end = size / sizeof(HSAuint32) - 1; From b72bbeac3e679a3b6507e36df913f846122c2109 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 7 Feb 2020 08:44:30 -0500 Subject: [PATCH 0687/1247] Fix naming conventions again for -dev package Using the building OS isn't guaranteed, as we can theoretically build RPMs in Ubuntu or DEBs in CentOS. Use CPack's DEB/RPM-specific variables to get around this issue Change-Id: I404246c070eac2c74b45ae4b763c612891d66de1 Signed-off-by: Kent Russell --- hsakmt-dev.txt | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt index 5286fc0f72..e22db6d836 100644 --- a/hsakmt-dev.txt +++ b/hsakmt-dev.txt @@ -25,17 +25,7 @@ cmake_minimum_required ( VERSION 3.5.0 ) -#checking for OS-version -if( EXISTS "/etc/os-release" ) - file( STRINGS "/etc/os-release" DISTRO_ID REGEX "^ID=" ) - string( REPLACE "ID=" "" DISTRO_ID "${DISTRO_ID}" ) -endif() - -if (DISTRO_ID STREQUAL "ubuntu" ) - set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-dev" ) -else() - set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-devel" ) -endif() +set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-dev" ) project ( ${HSAKMT_DEV_PACKAGE} ) @@ -61,7 +51,8 @@ install ( DIRECTORY ${SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDI set ( CPACK_GENERATOR "DEB;RPM;TGZ" CACHE STRING "Default packaging generators." ) ## Packaging directives -set ( CPACK_PACKAGE_NAME ${HSAKMT_DEV_PACKAGE} ) +set ( CPACK_DEBIAN_PACKAGE_NAME "hsakmt-roct-dev" ) +set ( CPACK_RPM_PACKAGE_NAME "hsakmt-roct-devel" ) set ( CPACK_PACKAGE_VENDOR "AMD" ) set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) @@ -69,7 +60,8 @@ set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT development package." ) set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) -set ( CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64" ) +set ( CPACK_DEBIAN_FILE_NAME "${CPACK_DEBIAN_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.deb" ) +set ( CPACK_RPM_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) From ddd38deab7e4eb4a665c16182155ff4708f009b7 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 21 Apr 2020 11:09:39 -0400 Subject: [PATCH 0688/1247] Make thunk lib type be defined by cmake Make the hsakmt library take the value from CMake regarding static/shared KFDTest automatically grabs the right one due to it checking the normal shared folders. Tested locally (and via automation by the time that this is merged) Also set the default to building SOs if BUILD_SHARED_LIBS is not defined Change-Id: I7f8b76a7e60f3b41e5981f472b388301ae09e2c6 Signed-off-by: Kent Russell --- CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa2c4c609c..1b1b66dd81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,6 +32,11 @@ set ( HSAKMT_TARGET "${HSAKMT}" ) project ( ${HSAKMT_TARGET} VERSION 1.9.0) +if ( NOT DEFINED BUILD_SHARED_LIBS ) + set ( BUILD_SHARED_LIBS "on" ) +endif() +set ( BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS} CACHE BOOL "Build shared library (.so) or not.") + # Optionally, build HSAKMT with ccache. set(ROCM_CCACHE_BUILD OFF CACHE BOOL "Set to ON for a ccache enabled build") if (ROCM_CCACHE_BUILD) @@ -119,7 +124,7 @@ include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/include ) include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/src ) ## Add the library -add_library ( ${HSAKMT_TARGET} SHARED ${HSAKMT_SRC} ) +add_library ( ${HSAKMT_TARGET} ${HSAKMT_SRC} ) target_compile_options(${HSAKMT_TARGET} PRIVATE ${HSAKMT_C_FLAGS}) set_property(TARGET ${HSAKMT_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_LINK_FLAGS}) From e5062c438315ba1bead2e2eb48f5e881ed946e4f Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 2 May 2020 15:46:58 -0400 Subject: [PATCH 0689/1247] libhsakmt: Add PROT_NONE CPU mapping for scratch mappings This is needed to allow gdb to access the memory. Change-Id: I96c084b714e952d7b7000f0dd41e1c530fdd092f Signed-off-by: Felix Kuehling --- src/fmm.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index fab57af6b5..6867c69599 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2603,6 +2603,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert bool is_debugger = 0; void *mmap_ret = NULL; uint64_t mmap_offset = 0; + int map_fd; vm_object_t *obj; /* Retrieve gpu_mem id according to gpu_id */ @@ -2622,18 +2623,27 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* allocate object within the scratch backing aperture */ if (!ret && !is_debugger) { obj = fmm_allocate_memory_object( - gpu_id, address, size, aperture, NULL, + gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); if (!obj) return -1; - } else { - int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : + /* Create a CPU mapping for the debugger */ + map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_mem[gpu_mem_id].drm_render_fd; + mmap_ret = mmap(address, size, PROT_NONE, + MAP_PRIVATE | MAP_FIXED, map_fd, mmap_offset); + if (mmap_ret == MAP_FAILED) { + __fmm_release(obj, aperture); + return -1; + } + } else { obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_GTT | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); + map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : + gpu_mem[gpu_mem_id].drm_render_fd; mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); @@ -2841,6 +2851,11 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, args.n_success = 0; ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + /* unmap from CPU while keeping the address space reserved */ + mmap(address, object->size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, args.n_success * sizeof(uint32_t)); From 93c333711a6503a739555097fa3c35a63277dc90 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 4 May 2020 10:31:20 -0400 Subject: [PATCH 0690/1247] libhsakmt: queue suspend/resume can return non-zero positive values New queue suspend/resume update can now return the number of successful queue requests so return success if IOCTL return is non-negative. This should be backwards compatible since old queue suspend/resume returns 0 on success. Signed-off-by: Jonathan Kim Change-Id: I06b70d95d203b2bfc19a0cc1b88c5719c695159a --- src/debug.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index 92cbaeb186..f7d9cbd22d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -325,7 +325,11 @@ static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, if (argout) *argout = args; - if (err == 0) + if ((op == KFD_IOC_DBG_TRAP_NODE_SUSPEND || + op == KFD_IOC_DBG_TRAP_NODE_RESUME) && err >= 0 && + err <= args.data2) + result = HSAKMT_STATUS_SUCCESS; + else if (err == 0) result = HSAKMT_STATUS_SUCCESS; else result = HSAKMT_STATUS_ERROR; From dc784ba482aef13fbce998d076443ced1c91f3c9 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 6 May 2020 14:01:59 -0400 Subject: [PATCH 0691/1247] Fix HostAccess flags for XGMI configs in P2PTest We need HostAccess for large-bar XGMI configs for this test, but we can't use it on small-bar XGMI configs, so deal with it appropriately Change-Id: Ic16b810de03adfc63de21a00c55e5f3ea1ce66d1 --- tests/kfdtest/src/KFDQMTest.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 10f74ae808..f2b68ce233 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1671,11 +1671,16 @@ TEST_F(KFDQMTest, P2PTest) { HsaMemFlags memFlags = {0}; HsaMemMapFlags mapFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - memFlags.ui32.HostAccess = 0; + memFlags.ui32.HostAccess = 1; memFlags.ui32.NonPaged = 1; memFlags.ui32.NoNUMABind = 1; unsigned int end = size / sizeof(HSAuint32) - 1; + if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId) && + m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + memFlags.ui32.HostAccess = 0; + } + /* 1. Allocate a system buffer and allow the access to GPUs */ EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, reinterpret_cast(&sysBuf))); From 46f5e830660ea88a54e29f75e7e4ece06f5ae681 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Mon, 18 Nov 2019 10:40:47 -0500 Subject: [PATCH 0692/1247] Create KFDTest for SDMA Fault The purpose of this KFDTest is to investigate the behaviour of an SDMA queue when an invalid memory address is used. v2: Don't wait for SDMA queue to finish - it won't finish because of the gpuvm fault. v3: Create kfd event before SDMA queue submission. This fix the issue that gpuvm fault happens earlier than kfd event is created then KFD exception handler can't find the kfd event (to wake up kfd test) v4: Instead of using 0x12345678 as the invalid VA, map one page of FB to gpu and unmap it. Use the mapped GPUVA as the invalid address Change-Id: I58af1511f75d869adddede302b238c2725f3fe5a Signed-off-by: Ori Messinger Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDExceptionTest.cpp | 104 +++++++++++++++++++++++++ tests/kfdtest/src/KFDExceptionTest.hpp | 1 + 2 files changed, 105 insertions(+) diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 6656317304..4976d0e319 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -113,6 +113,59 @@ queuefail: queue.Destroy(); } +void KFDExceptionTest::TestSdmaException(int defaultGPUNode, void *pDst) { + SDMAQueue queue; + HsaEvent *vmFaultEvent; + HSAuint64 faultAddress, page_mask = ~((HSAuint64)PAGE_SIZE - 1); + + + HsaEventDescriptor eventDesc; + eventDesc.EventType = HSA_EVENTTYPE_MEMORY; + eventDesc.NodeId = defaultGPUNode; + eventDesc.SyncVar.SyncVar.UserData = NULL; + eventDesc.SyncVar.SyncVarSize = 0; + + m_ChildStatus = queue.Create(defaultGPUNode); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "Queue create failed" << std::endl; + return; + } + + m_ChildStatus = hsaKmtCreateEvent(&eventDesc, true, false, &vmFaultEvent); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "Event create failed" << std::endl; + goto queuefail; + } + + queue.PlaceAndSubmitPacket(SDMAWriteDataPacket(queue.GetFamilyId(), + reinterpret_cast(pDst), + 0x02020202)); + + m_ChildStatus = hsaKmtWaitOnEvent(vmFaultEvent, g_TestTimeOut); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "Wait failed. No Exception triggered" << std::endl; + goto eventfail; + } + + if (vmFaultEvent->EventData.EventType != HSA_EVENTTYPE_MEMORY) { + WARN() << "Unexpected Event Received " << vmFaultEvent->EventData.EventType + << std::endl; + m_ChildStatus = HSAKMT_STATUS_ERROR; + goto eventfail; + } + faultAddress = vmFaultEvent->EventData.EventData.MemoryAccessFault.VirtualAddress; + if (faultAddress != ((HSAuint64)pDst & page_mask) ) { + WARN() << "Unexpected Fault Address " << faultAddress + << " expected " << ((HSAuint64)pDst & page_mask) << std::endl; + m_ChildStatus = HSAKMT_STATUS_ERROR; + } + +eventfail: + hsaKmtDestroyEvent(vmFaultEvent); +queuefail: + queue.Destroy(); +} + /* Test Bad Address access in a child process */ TEST_F(KFDExceptionTest, AddressFault) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); @@ -244,3 +297,54 @@ TEST_F(KFDExceptionTest, FaultStorm) { TEST_END } + +/* + */ +TEST_F(KFDExceptionTest, SdmaQueueException) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; + return; + } + + HSAKMT_STATUS status; + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + unsigned int* pDb = NULL; + unsigned int *nullPtr = NULL; + m_ChildStatus = hsaKmtOpenKFD(); + if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in child process" << std::endl; + return; + } + m_MemoryFlags.ui32.NonPaged = 1; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, m_MemoryFlags, + reinterpret_cast(&pDb))); + // verify that pDb is not null before it's being used + ASSERT_NE(nullPtr, pDb) << "hsaKmtAllocMemory returned a null pointer"; + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(pDb, PAGE_SIZE, NULL)); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); + + TestSdmaException(defaultGPUNode, pDb); + EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, PAGE_SIZE)); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + if (is_dgpu()) { + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } else { + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + } + + TEST_END +} diff --git a/tests/kfdtest/src/KFDExceptionTest.hpp b/tests/kfdtest/src/KFDExceptionTest.hpp index bf6bd7812b..00b45fe5db 100644 --- a/tests/kfdtest/src/KFDExceptionTest.hpp +++ b/tests/kfdtest/src/KFDExceptionTest.hpp @@ -54,6 +54,7 @@ class KFDExceptionTest : public KFDBaseComponentTest { void TestMemoryException(int defaultGPUNode, HSAuint64 pSrc, HSAuint64 pDst, unsigned int dimX = 1, unsigned int dimY = 1, unsigned int dimZ = 1); + void TestSdmaException(int defaultGPUNode, void *pDst); protected: // Members pid_t m_ChildPid; From a37a88ddcbb2a2d9f67dd0c8d6b2ed02d9f35662 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Mon, 27 Apr 2020 09:14:37 -0400 Subject: [PATCH 0693/1247] libhsakmt: Sync up the ioctl number Update the non-upstream ioctl numbers to align with the change in the kernel. Signed-off-by: Amber Lin Change-Id: Ie0ddccb343a023b55eb18477c59341acaa666e99 --- include/linux/kfd_ioctl.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index fa14bd44a3..aa4292e629 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -692,19 +692,23 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) +#define AMDKFD_COMMAND_START 0x01 +#define AMDKFD_COMMAND_END 0x1F + +/* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ - AMDKFD_IOWR(0x1F, struct kfd_ioctl_ipc_import_handle_args) + AMDKFD_IOWR(0x80, struct kfd_ioctl_ipc_import_handle_args) #define AMDKFD_IOC_IPC_EXPORT_HANDLE \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_ipc_export_handle_args) + AMDKFD_IOWR(0x81, struct kfd_ioctl_ipc_export_handle_args) #define AMDKFD_IOC_DBG_TRAP \ - AMDKFD_IOWR(0x21, struct kfd_ioctl_dbg_trap_args) + AMDKFD_IOWR(0x82, struct kfd_ioctl_dbg_trap_args) #define AMDKFD_IOC_CROSS_MEMORY_COPY \ - AMDKFD_IOWR(0x22, struct kfd_ioctl_cross_memory_copy_args) + AMDKFD_IOWR(0x83, struct kfd_ioctl_cross_memory_copy_args) -#define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x22 +#define AMDKFD_COMMAND_START_2 0x80 +#define AMDKFD_COMMAND_END_2 0x84 #endif From aa185380f96e2f6aa1071de9370c022cdeaef6fa Mon Sep 17 00:00:00 2001 From: Ranieri Althoff <1993083+ranisalt@users.noreply.github.com> Date: Fri, 8 May 2020 16:53:44 +0000 Subject: [PATCH 0694/1247] Avoid calculating strlen multiple times Change-Id: Iec66c7d35e5d6cd2deb02c94ee070d0fa1335147 Signed-off-by: Ranieri Althoff Signed-off-by: Felix Kuehling Reviewed-by: Felix Kuehling --- src/topology.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/topology.c b/src/topology.c index 2817b035ff..23d2e1bcaa 100644 --- a/src/topology.c +++ b/src/topology.c @@ -831,6 +831,7 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, char read_buf[256]; char *p; uint32_t proc = 0; + size_t p_len; const char *proc_cpuinfo_path = "/proc/cpuinfo"; if (!cpuinfo) { @@ -879,10 +880,11 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { p = strchr(read_buf, ':'); p += 2; /* remove ": " */ - if (strlen(p) < HSA_PUBLIC_NAME_SIZE) { + p_len = strlen(p); + if (p_len < HSA_PUBLIC_NAME_SIZE) { /* -1 to remove \n from p */ - strncpy(cpuinfo[proc].model_name, p, strlen(p) - 1); - cpuinfo[proc].model_name[strlen(p) - 1] = '\0'; + strncpy(cpuinfo[proc].model_name, p, p_len - 1); + cpuinfo[proc].model_name[p_len - 1] = '\0'; } else strncpy(cpuinfo[proc].model_name, p, HSA_PUBLIC_NAME_SIZE); continue; From df16950a0c164890d3dc8a09c26d2ef85338e10d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 15 May 2020 01:34:10 -0400 Subject: [PATCH 0695/1247] libhsakmt: use _Static_assert instead of static_assert Using static_assert breaks in "Many Linux" build environment. It is not supported by that libc version. _Static_assert is a compiler built-in and does not depend on the libc version. Change-Id: I37cf0ad10de94d8f6fc8cefc4fdda55c9520d599 Signed-off-by: Felix Kuehling --- src/pci_ids.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pci_ids.c b/src/pci_ids.c index 1353cfa23d..0c8df69d28 100644 --- a/src/pci_ids.c +++ b/src/pci_ids.c @@ -269,7 +269,7 @@ static void write_fallback_to_buffer(char *buf, size_t size, uint16_t DeviceId) { char tmp[] = "Device xxxx"; - static_assert(sizeof(tmp) == 12, ""); + _Static_assert(sizeof(tmp) == 12, ""); write_as_hex(DeviceId, &tmp[7]); size_t to_copy = (sizeof(tmp) <= size) ? sizeof(tmp) : size; From 0a55f31463f457030d2a269c4be9f014d46b5cae Mon Sep 17 00:00:00 2001 From: "Philip.Cox@amd.com" Date: Wed, 1 Apr 2020 13:39:26 -0400 Subject: [PATCH 0696/1247] Initial kfd debugger address watch support Code for new kfd debugger address watch code. -- Adding support for: -- add address watch -- clear address watch Change-Id: I9b014e7cee03897157b997b9e5b39b6ed403b8e1 Signed-off-by: Philip.Cox@amd.com --- include/hsakmt.h | 2 ++ include/hsakmttypes.h | 4 +-- include/linux/kfd_ioctl.h | 39 ++++++++++++++++++++++-- src/debug.c | 62 +++++++++++++++++++++++++++++++++++++++ src/libhsakmt.ver | 2 ++ 5 files changed, 104 insertions(+), 5 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index cc96e492b6..237f80bbc7 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -990,6 +990,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetAddressWatch( HSAuint32 NodeId, //IN + HSAuint32 Pid, //IN HSA_DBG_WATCH_MODE WatchMode, //IN void* WatchAddress, //IN HSAuint64 WatchAddrMask, //IN @@ -1013,6 +1014,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtClearAddressWatch( HSAuint32 NodeId, //IN + HSAuint32 Pid, //IN HSAuint32 WatchId //IN ); diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 3aeba807ea..9abf4963c5 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -223,10 +223,10 @@ typedef union HSAuint64 Value; struct { - HSAuint64 WatchAddrMaskLoBit: 6; // Only bits + HSAuint64 WatchAddrMaskLoBit: 4; // Only bits // WatchAddrMaskLoBit..WatchAddrMaskHiBit // of the - HSAuint64 WatchAddrMaskHiBit: 4; // watch address mask are used. + HSAuint64 WatchAddrMaskHiBit: 6; // watch address mask are used. // 0 is the least significant bit. HSAuint64 TrapDataCount: 4; // Number of 32 bit TrapData // registers supported. diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index aa4292e629..6a7dd1f44a 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -28,8 +28,25 @@ #define KFD_IOCTL_MAJOR_VERSION 1 #define KFD_IOCTL_MINOR_VERSION 2 -#define KFD_IOCTL_DBG_MAJOR_VERSION 1 -#define KFD_IOCTL_DBG_MINOR_VERSION 0 + +/* + * Debug revision change log + * + * 0.1 - Initial revision + * 0.2 - Fix to include querying pending event that is both trap and vmfault + * 1.0 - Removed function to set debug data (renumbering functions broke ABI) + * 1.1 - Allow attaching to processes that have not opened /dev/kfd yet + * 1.2 - Allow flag option to clear queue status on queue suspend + * 1.3 - Fix race condition between clear on suspend and trap event handling + * 1.3 - Fix race condition between clear on suspend and trap event handling + * 1.4 - Fix bad kfifo free + * 1.5 - Fix ABA issue between queue snapshot and suspend + * 2.0 - Return number of queues suspended/resumed and mask invalid/error + * array slots + * 2.1 - Add Set Address Watch, and Clear Address Watch support. + */ +#define KFD_IOCTL_DBG_MAJOR_VERSION 2 +#define KFD_IOCTL_DBG_MINOR_VERSION 1 struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ @@ -257,6 +274,7 @@ struct kfd_ioctl_dbg_wave_control_args { * data2: flags (IN) * data3: suspend[2:2], event type [1:0] (OUT) */ + #define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 5 /* KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT: @@ -268,13 +286,28 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT 6 /* KFD_IOC_DBG_TRAP_GET_VERSION: - * prt: unsused + * ptr: unsused * data1: major version (OUT) * data2: minor version (OUT) * data3: unused */ #define KFD_IOC_DBG_TRAP_GET_VERSION 7 +/* KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH: + * ptr: unused + * data1: watch ID + * data2: unused + * data3: unused + */ +#define KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH 8 + +/* KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH: + * ptr: Watch address + * data1: Watch ID (OUT) + * data2: watch_mode: 0=read, 1=nonread, 2=atomic, 3=all + * data3: watch address mask + */ +#define KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH 9 struct kfd_ioctl_dbg_trap_args { __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ diff --git a/src/debug.c b/src/debug.c index f7d9cbd22d..caddc79a8e 100644 --- a/src/debug.c +++ b/src/debug.c @@ -30,6 +30,8 @@ #include static bool *is_device_debugged; + + int debug_get_reg_status(uint32_t node_id, bool *is_debugged); HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes) @@ -669,3 +671,63 @@ hsaKmtGetQueueSnapshot( return 0; } + +HSAKMT_STATUS HSAKMTAPI hsaKmtSetAddressWatch( + HSAuint32 NodeId, + HSAuint32 Pid, + HSA_DBG_WATCH_MODE WatchMode, + void *WatchAddress, + HSAuint64 WatchAddrMask, + HSAuint32 *WatchId + ) +{ + + HSAKMT_STATUS result; + HSAuint32 TruncatedWatchAddressMask; + struct kfd_ioctl_dbg_trap_args argout = {0}; + + /* Right now we only support 32 bit watch address masks, so we need + * to check that we aren't losing data when we truncate the mask + * to be passed to the kernel. + */ + if (WatchAddrMask > (HSAuint64) UINT_MAX) + { + return HSAKMT_STATUS_INVALID_PARAMETER; + } + TruncatedWatchAddressMask = (HSAuint32) WatchAddrMask; + + if (WatchId == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + result = debug_trap(NodeId, + KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH, // op + *WatchId, + WatchMode, + TruncatedWatchAddressMask, + Pid, + (HSAuint64) WatchAddress, + &argout); + *WatchId = argout.data1; + + return result; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtClearAddressWatch( + HSAuint32 NodeId, + HSAuint32 Pid, + HSAuint32 WatchId + ) +{ + + HSAKMT_STATUS result; + + result = debug_trap(NodeId, + KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH, // op + WatchId, + 0, + 0, + Pid, + 0, + NULL); + return result; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index fc9564f3ec..d47f68d2d0 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -69,6 +69,8 @@ hsaKmtQueueResume; hsaKmtAllocQueueGWS; hsaKmtGetKernelDebugTrapVersionInfo; hsaKmtGetThunkDebugTrapVersionInfo; +hsaKmtSetAddressWatch; +hsaKmtClearAddressWatch; local: *; }; From 5ddd8fb68bdade59e5b49602a224a540f981e220 Mon Sep 17 00:00:00 2001 From: Joseph Greathouse Date: Fri, 22 May 2020 10:39:13 -0500 Subject: [PATCH 0697/1247] Pull out ASIC revision from HSA capability KFD now passes the ASIC revision to user level through some bits in the HSA topology's capability field. Some user-level software wants this because different ASIC revisions may require user-level software to do different things (e.g. patch code for things that are changed in later hardware revisions). Change-Id: I16f2a15ae0875edd01ebdb1f1685ec7865f7049e --- include/hsakmttypes.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 9abf4963c5..7a7dc2ad10 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -212,7 +212,8 @@ typedef union unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active unsigned int Mem_EDCSupoort: 1; // Indicates if GFX internal DRAM/HBM EDC/ECC functionality is active unsigned int RASEventNotify: 1; // Indicates if GFX extended RASFeatures and RAS EventNotify status is available - unsigned int Reserved : 10; + unsigned int ASICRevision: 4; // Indicates the ASIC revision of the chip on this node. + unsigned int Reserved : 6; } ui32; } HSA_CAPABILITY; From d675d1cce1a2349ec8ea2ac63c3fdb62f1a00654 Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Mon, 27 Apr 2020 17:58:56 -0400 Subject: [PATCH 0698/1247] libhsakmt: add Streaming Performance Monitors APIs Signed-off-by: Gang Ba Change-Id: I5c23a8dacf9bc50c740908aabe391432f2c7112e Signed-off-by: Gang Ba --- CMakeLists.txt | 1 + include/hsakmt.h | 52 ++++++++++++++++ include/linux/kfd_ioctl.h | 53 ++++++++++++++++- src/libhsakmt.ver | 3 + src/spm.c | 121 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 src/spm.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b1b66dd81..f24a2bb3e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/time.c" "src/topology.c" "src/rbtree.c" + "src/spm.c" "src/version.c") ## Include paths diff --git a/include/hsakmt.h b/include/hsakmt.h index 237f80bbc7..4938b8a96e 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -1209,6 +1209,58 @@ hsaKmtSetMemoryUserData( void * UserData //IN ); +/** + Acquire request exclusive use of SPM +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSPMAcquire( + HSAuint32 PreferredNode //IN + ); + + +/** + Disable SPM Streamimg +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSPMRelease( + HSAuint32 PreferredNode //IN + ); + +/** + Set up the destination user mode buffer for stream performance + counter data. + + PreferredNode -- note Id + + SizeInBytes -- number of byte of Destination buffer for performance + counter data + + timeout -- wait to collect performance counter data + + DestMemoryAddress -- point to user mode buffer for stream performance + counter data + + SPMMemoryAddress -- user mode buffer that contains stream performance + counter data, it ponit to current or one of DestMemoryAddress that set + before. + + SizeCopied -- number of bytes of data copied from Stream Ring buffer. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSPMSetDestBuffer( + HSAuint32 PreferredNode, //IN + HSAuint64 SizeInBytes, //IN + HSAuint32 timeout, //IN + void* DestMemoryAddress, //IN + void ** SPMMemoryAddress, //OUT + HSAuint32 *SizeCopied //OUT + ); + + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 6a7dd1f44a..96df15f3fc 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -628,6 +628,53 @@ struct kfd_ioctl_cross_memory_copy_args { __u64 bytes_copied; }; +/** + * kfd_ioctl_spm_op - SPM ioctl operations + * + * @KFD_IOCTL_SPM_OP_ACQUIRE: acquire exclusive access to SPM + * @KFD_IOCTL_SPM_OP_RELEASE: release exclusive access to SPM + * @KFD_IOCTL_SPM_OP_SET_DEST_BUF: set or unset destination buffer for SPM streaming + */ +enum kfd_ioctl_spm_op { + KFD_IOCTL_SPM_OP_ACQUIRE, + KFD_IOCTL_SPM_OP_RELEASE, + KFD_IOCTL_SPM_OP_SET_DEST_BUF +}; + + +/** + * kfd_ioctl_spm_args - Arguments for SPM ioctl + * + * @op: specifies the operation to perform + * @destptr:used for the address of the destination buffer in @KFD_IOCTL_SPM_SET_DEST_BUFFER + * @buf_size:size of the destination buffer in @KFD_IOCTL_SPM_SET_DEST_BUFFER + * @timeout: timeout to wait for the buffer to get filled + * @gpu_id: gpu ID + * @bytes_copied: bytes copied from streaming performance ring buffer + * + * If @ptr is NULL, the destination buffer address is unset and copying of counters + * is stopped. + * + * Returns negative error code on failure. On success, @KFD_IOCTL_SPM_OP_ACQUIRE and + * @KFD_IOCTL_SPM_OP_RELEASE return 0, @KFD_IOCTL_SPM_OP_SET_DEST_BUF returns the fill + * level of the previous buffer. + */ +struct kfd_ioctl_spm_args { + __u64 destptr; + __u64 spmtptr; + __u32 buf_size; + __u32 op; + __u32 timeout; + __u32 gpu_id; /* to KFD */ + /* from KFD: Total amount of bytes copied */ + __u64 bytes_copied; +}; + +struct kfd_spm_set_reg { + __u64 reg; + __u32 value; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -725,8 +772,12 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) +#define AMDKFD_IOC_RLC_SPM \ + AMDKFD_IOWR(0x20, struct kfd_ioctl_spm_args) + + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1F +#define AMDKFD_COMMAND_END 0x21 /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index d47f68d2d0..97e2cf6beb 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -71,6 +71,9 @@ hsaKmtGetKernelDebugTrapVersionInfo; hsaKmtGetThunkDebugTrapVersionInfo; hsaKmtSetAddressWatch; hsaKmtClearAddressWatch; +hsaKmtSPMAcquire; +hsaKmtSPMRelease; +hsaKmtSPMSetDestBuffer; local: *; }; diff --git a/src/spm.c b/src/spm.c new file mode 100644 index 0000000000..33b2b0a843 --- /dev/null +++ b/src/spm.c @@ -0,0 +1,121 @@ +/* + * Copyright © 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include "libhsakmt.h" +#include "linux/kfd_ioctl.h" +#include +#include + + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMAcquire(HSAuint32 PreferredNode) +{ + int ret; + struct kfd_ioctl_spm_args args = {0}; + uint32_t gpu_id; + + pr_debug("[%s]\n", __func__); + + ret = validate_nodeid(PreferredNode, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode); + return ret; + } + + ret = HSAKMT_STATUS_SUCCESS; + args.destptr = 0; + args.buf_size = 0; + args.op = KFD_IOCTL_SPM_OP_ACQUIRE; + args.gpu_id = gpu_id; + args.bytes_copied = 0; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); + + return ret; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMSetDestBuffer(HSAuint32 PreferredNode, + HSAuint64 SizeInBytes, + HSAuint32 timeout, + void* DestMemoryAddress, + void **SPMMemoryAddress, + HSAuint32 *SizeCopied) +{ + int ret; + struct kfd_ioctl_spm_args args = {0}; + uint32_t gpu_id; + + ret = HSAKMT_STATUS_SUCCESS; + + pr_debug("[%s]\n", __func__); + ret = validate_nodeid(PreferredNode, &gpu_id); + + args.timeout = timeout; + args.destptr = (uint64_t)DestMemoryAddress; + + args.buf_size = SizeInBytes; + args.spmtptr = 0; + args.op = KFD_IOCTL_SPM_OP_SET_DEST_BUF; + args.gpu_id = gpu_id; + args.bytes_copied = 0; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); + + if (ret != HSAKMT_STATUS_SUCCESS) { + *SPMMemoryAddress = NULL; + *SizeCopied = 0; + return ret; + } + + *SPMMemoryAddress = (void *)args.spmtptr; + *SizeCopied = args.bytes_copied; + + return ret; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMRelease(HSAuint32 PreferredNode) +{ + int ret = HSAKMT_STATUS_SUCCESS; + struct kfd_ioctl_spm_args args = {0}; + uint32_t gpu_id; + + pr_debug("[%s]\n", __func__); + + ret = validate_nodeid(PreferredNode, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode); + return ret; + } + + args.destptr = 0; + args.buf_size = 0; + args.op = KFD_IOCTL_SPM_OP_RELEASE; + args.gpu_id = gpu_id; + args.bytes_copied = 0; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); + + return ret; +} + + From 06add334b9aa2e200dc1478c5835200798dcb2c1 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Fri, 22 May 2020 16:50:45 -0500 Subject: [PATCH 0699/1247] Update to conform with modern cmake best practices. Immediate benefit is that this enables cmake native find_package support including transitive passthrough of link dependencies, options, and include paths. It does require a bump to cmake 3.6 to enable proper package file names. Change-Id: I3f4e2f2db745c04a7f686111b080098a0d92fbf5 Signed-off-by: Sean Keely --- CMakeLists.txt | 152 ++++++++++++++++++++++++++++------------- hsakmt-config.cmake.in | 14 ++-- hsakmt-dev.txt | 102 --------------------------- 3 files changed, 114 insertions(+), 154 deletions(-) delete mode 100644 hsakmt-dev.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index f24a2bb3e6..cced60f616 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,9 +79,6 @@ if ( DEFINED VERSION_BUILD AND NOT ${VERSION_BUILD} STREQUAL "" ) endif () set ( BUILD_VERSION_STRING "${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) -## Verbose output. -#set ( CMAKE_VERBOSE_MAKEFILE on ) - ## Compiler flags set (HSAKMT_C_FLAGS -fPIC -W -Wall -Wextra -Wno-unused-parameter -Wformat-security -Wswitch-default -Wundef -Wshadow -Wpointer-arith -Wbad-function-cast -Wcast-qual -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls -Wunreachable-code -std=gnu99 -fvisibility=hidden) if ( CMAKE_COMPILER_IS_GNUCC ) @@ -120,12 +117,21 @@ set ( HSAKMT_SRC "src/debug.c" "src/spm.c" "src/version.c") -## Include paths -include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/include ) -include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/src ) +## Declare the library target name +add_library ( ${HSAKMT_TARGET} "") + +## Add sources +target_sources ( ${HSAKMT_TARGET} PRIVATE ${HSAKMT_SRC} ) + +## Add headers. The public headers need to point at their location in both build and install +## directory layouts. This declaration allows publishing library use data to downstream clients. +target_include_directories( ${HSAKMT_TARGET} + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src ) -## Add the library -add_library ( ${HSAKMT_TARGET} ${HSAKMT_SRC} ) target_compile_options(${HSAKMT_TARGET} PRIVATE ${HSAKMT_C_FLAGS}) set_property(TARGET ${HSAKMT_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_LINK_FLAGS}) @@ -133,51 +139,91 @@ set_property(TARGET ${HSAKMT_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_LINK_FLAGS}) set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" ) set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" ) -find_package(PkgConfig) +#find_package(PkgConfig) target_link_libraries ( ${HSAKMT_TARGET} - pthread rt numa + PRIVATE pthread rt numa ) -## Define default variable and variables for the optional build target hsakmt-dev -set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) -set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) +## Define default paths and packages. +if( CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT ) + set ( CMAKE_INSTALL_PREFIX "/opt/rocm" ) +endif() +set ( CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX} CACHE STRING "Default installation directory." FORCE ) set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "Default packaging prefix." ) set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) -## Specify build, install and package targets hsakmt-dev -set ( DEV_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/hsakmt-roct-dev ) -configure_file ( hsakmt-dev.txt ${DEV_BUILD_DIR}/CMakeLists.txt @ONLY ) -add_custom_target ( build-dev - COMMAND ${CMAKE_COMMAND} - -DSOURCE_DIR="${SOURCE_DIR}" - -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" - -DCPACK_RPM_PACKAGE_REQUIRES="hsakmt-roct" - -DCPACK_PACKAGING_INSTALL_PREFIX="${CPACK_PACKAGING_INSTALL_PREFIX}" - -DVERSION="${BUILD_VERSION_STRING}" - -DHSAKMT_BIN_NAME="${HSAKMT}" - WORKING_DIRECTORY ${DEV_BUILD_DIR} ) +# Installs binaries and exports the library usage data to ${HSAKMT_TARGET}Targets +install ( TARGETS ${HSAKMT_TARGET} EXPORT ${HSAKMT_TARGET}Targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT binary + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT binary ) -## Custom targets for the devel package -add_custom_target ( install-dev DEPENDS build-dev COMMAND make install WORKING_DIRECTORY ${DEV_BUILD_DIR} ) -add_custom_target ( package-dev DEPENDS build-dev - COMMAND rm -rf *.deb *.rpm *.tar.gz - COMMAND make package - WORKING_DIRECTORY ${DEV_BUILD_DIR} ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT binary ) -## Add the install directives for the runtime library. -install ( TARGETS ${HSAKMT_TARGET} DESTINATION ${CMAKE_INSTALL_LIBDIR} ) -install ( FILES ${SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} ) +# Install public headers +install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + COMPONENT devel PATTERN "linux" EXCLUDE ) + +# Record our usage data for clients find_package calls. +install ( EXPORT ${HSAKMT_TARGET}Targets + FILE ${HSAKMT_TARGET}Targets.cmake + NAMESPACE ${HSAKMT_TARGET}:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} + COMPONENT devel) + +# Adds the target alias hsakmt::hsakmt to the local cmake cache. +# This isn't necessary today. It's harmless preparation for some +# hypothetical future in which the we might be included by add_subdirectory() +# in some other project's cmake file. It allows uniform use of find_package +# and target_link_library() without regard to whether a target is external or +# a subdirectory of the current build. +add_library( ${HSAKMT_TARGET}::${HSAKMT_TARGET} ALIAS ${HSAKMT_TARGET} ) + +# Create cmake configuration files +include(CMakePackageConfigHelpers) + +configure_package_config_file(${HSAKMT_TARGET}-config.cmake.in + ${HSAKMT_TARGET}-config.cmake + INSTALL_DESTINATION + ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} ) + +write_basic_package_version_file(${HSAKMT_TARGET}-config-version.cmake + VERSION ${BUILD_VERSION_STRING} + COMPATIBILITY + AnyNewerVersion) + +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_TARGET}-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_TARGET}-config-version.cmake + DESTINATION + ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} + COMPONENT devel) + +# CPACK_PACKAGING_INSTALL_PREFIX is needed in libhsakmt.pc.in +# TODO: Add support for relocatable packages. +configure_file ( libhsakmt.pc.in libhsakmt.pc @ONLY ) + +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.pc DESTINATION ${CMAKE_INSTALL_DATADIR}/pkgconfig COMPONENT devel) + +# Use component packaging +set ( CPACK_COMPONENTS_GROUPING IGNORE ) +set ( CPACK_COMPONENTS_ALL binary devel ) +set ( CPACK_DEB_COMPONENT_INSTALL ON ) +set ( CPACK_RPM_COMPONENT_INSTALL ON ) ## Add the packaging directives for the runtime library. -set ( CPACK_PACKAGE_NAME ${HSAKMT_PACKAGE} ) set ( CPACK_PACKAGE_VENDOR "AMD" ) set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) -set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT library for AMD KFD support" ) -set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) -set ( CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64" ) +set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md" ) + +## Component packaging, BINARY +set ( CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support" ) +set ( CPACK_DEBIAN_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) +set ( CPACK_DEBIAN_BINARY_FILE_NAME "${CPACK_DEBIAN_BINARY_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.deb" ) +set ( CPACK_RPM_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) +set ( CPACK_RPM_BINARY_FILE_NAME "${CPACK_RPM_BINARY_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) #checking for OS-version if( EXISTS "/etc/os-release" ) @@ -199,21 +245,33 @@ endif() # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) -set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm" ) -set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1" ) +set ( CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm" ) +set ( CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1" ) # RPM package specific variables if( DISTRO_ID MATCHES "sles" ) - set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1") + set ( CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1") elseif( DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6" ) - set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl, libgcc" ) + set ( CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl, libgcc" ) else() - set ( CPACK_RPM_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc" ) + set ( CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc" ) endif() -set ( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) -set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) +set ( CPACK_RPM_BINARY_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) +set ( CPACK_RPM_BINARY_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) -# CPACK_PACKAGING_INSTALL_PREFIX is needed in libhsakmt.pc.in -configure_file ( libhsakmt.pc.in ${DEV_BUILD_DIR}/libhsakmt.pc @ONLY ) +## Component packaging, DEVEL +set ( CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package." ) + +# Debian package specific variables +set ( CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev" ) +set ( CPACK_DEBIAN_DEVEL_FILE_NAME "${CPACK_DEBIAN_DEVEL_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.deb" ) +set ( CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) + +# RPM package specific variables +set ( CPACK_RPM_DEVEL_PACKAGE_NAME "hsakmt-roct-devel" ) +set ( CPACK_RPM_DEVEL_FILE_NAME "${CPACK_RPM_DEVEL_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) +set ( CPACK_RPM_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} = ${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) +# Since we changed the package name to match RPM specs, take care of older builds that had -dev installed +set ( CPACK_RPM_DEVEL_PACKAGE_OBSOLETES "hsakmt-roct-dev" ) include ( CPack ) diff --git a/hsakmt-config.cmake.in b/hsakmt-config.cmake.in index 91f73898b5..9b162dae71 100644 --- a/hsakmt-config.cmake.in +++ b/hsakmt-config.cmake.in @@ -1,9 +1,13 @@ @PACKAGE_INIT@ -set_and_check(HSAKMT_INCLUDE_DIR @PACKAGE_INCLUDE_INSTALL_DIR@) -set_and_check(HSAKMT_LIB_DIR @PACKAGE_LIB_INSTALL_DIR@) +include( CMakeFindDependencyMacro ) -set(HSAKMT_LIBRARY @PACKAGE_LIB_INSTALL_DIR@/@HSAKMT_LIBRARY_NAME@) -set(HSAKMT_LIBRARIES @PACKAGE_LIB_INSTALL_DIR@/@HSAKMT_LIBRARY_NAME@) +# Locate dependent packages here. Finding them propagates usage requirements, +# if any, to our clients and ensures that their target names are in scope for +# the build. hsakmt has no cmake project dependencies so there is nothing to +# find. If we switch to use find_package with external (to ROCm) library +# dependencies (ie libnuma) then those packages should be located here using +# find_dependencies as shown below. +#find_dependency(Bar, 2.0) -mark_as_advanced(HSAKMT_LIBRARY HSAKMT_INCLUDE_DIR) +include( "${CMAKE_CURRENT_LIST_DIR}/@HSAKMT_TARGET@Targets.cmake" ) diff --git a/hsakmt-dev.txt b/hsakmt-dev.txt deleted file mode 100644 index e22db6d836..0000000000 --- a/hsakmt-dev.txt +++ /dev/null @@ -1,102 +0,0 @@ -################################################################################ -## -## Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved. -## -## MIT LICENSE: -## Permission is hereby granted, free of charge, to any person obtaining a copy of -## this software and associated documentation files (the "Software"), to deal in -## the Software without restriction, including without limitation the rights to -## use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -## of the Software, and to permit persons to whom the Software is furnished to do -## so, subject to the following conditions: -## -## The above copyright notice and this permission notice shall be included in all -## copies or substantial portions of the Software. -## -## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -## OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -## SOFTWARE. -## -################################################################################ - -cmake_minimum_required ( VERSION 3.5.0 ) - -set ( HSAKMT_DEV_PACKAGE "hsakmt-roct-dev" ) - -project ( ${HSAKMT_DEV_PACKAGE} ) - -include ( GNUInstallDirs ) - -## Set the runtime package name. -set ( HSAKMT_PACKAGE @HSAKMT_PACKAGE@ ) - -## Setup the package version. -set ( BUILD_VERSION_MAJOR @BUILD_VERSION_MAJOR@ ) -set ( BUILD_VERSION_MINOR @BUILD_VERSION_MINOR@ ) -set ( BUILD_VERSION_PATCH @BUILD_VERSION_PATCH@ ) - -## Verbose output. -set ( CMAKE_VERBOSE_MAKEFILE on ) - -## Set the install targets -install ( FILES libhsakmt.pc DESTINATION ${CMAKE_INSTALL_DATADIR}/pkgconfig ) -install ( DIRECTORY ${SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_COMPONENT} PATTERN "linux" EXCLUDE ) - - -## Set the default generator types for the devel package. -set ( CPACK_GENERATOR "DEB;RPM;TGZ" CACHE STRING "Default packaging generators." ) - -## Packaging directives -set ( CPACK_DEBIAN_PACKAGE_NAME "hsakmt-roct-dev" ) -set ( CPACK_RPM_PACKAGE_NAME "hsakmt-roct-devel" ) -set ( CPACK_PACKAGE_VENDOR "AMD" ) -set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) -set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) -set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) -set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) -set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "HSAKMT development package." ) -set ( CPACK_RESOURCE_FILE_LICENSE "${SOURCE_DIR}/LICENSE.md" ) -set ( CPACK_DEBIAN_FILE_NAME "${CPACK_DEBIAN_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.deb" ) -set ( CPACK_RPM_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) -# Debian package specific variables -set ( CPACK_DEBIAN_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) -set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) - -# RPM package specific variables -set ( CPACK_RPM_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} = ${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) -# Since we changed the package name to match RPM specs, take care of older builds that had -dev installed -set ( CPACK_RPM_PACKAGE_OBSOLETES "hsakmt-roct-dev" ) - -# Create cmake configuration files -include(CMakePackageConfigHelpers) - -set(HSAKMT_LIBRARY_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${PROJECT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - -set(INCLUDE_INSTALL_DIR ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME} CACHE INTERNAL "") -set(LIB_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR} CACHE INTERNAL "") - -configure_package_config_file(${SOURCE_DIR}/${HSAKMT_BIN_NAME}-config.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_BIN_NAME}-config.cmake - PATH_VARS - INCLUDE_INSTALL_DIR LIB_INSTALL_DIR - INSTALL_DESTINATION - ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_BIN_NAME}) - -write_basic_package_version_file(${HSAKMT_BIN_NAME}-config-version.cmake - VERSION ${VERSION} - COMPATIBILITY - AnyNewerVersion) - -install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_BIN_NAME}-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_BIN_NAME}-config-version.cmake - DESTINATION - ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_BIN_NAME} - COMPONENT - devel) - -include ( CPack ) From f0b05559961b1a857d4cbf0479b9f3b8a2b6904a Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Thu, 4 Jun 2020 20:57:56 -0500 Subject: [PATCH 0700/1247] Add optional support for the cmake per user package registry. Default is OFF to conform to latest cmake standard (3.15) and because this feature can cause some confusion for unaware developers. Change-Id: I6fdbd5ab76c90b73ac6451a383334a83cabef438 Signed-off-by: Sean Keely --- CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index cced60f616..56292ed135 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -198,6 +198,20 @@ install(FILES ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} COMPONENT devel) +# Optionally record the package's find module in the user's package cache. +if ( NOT DEFINED EXPORT_TO_USER_PACKAGE_REGISTRY ) + set ( EXPORT_TO_USER_PACKAGE_REGISTRY "off" ) +endif() +set ( EXPORT_TO_USER_PACKAGE_REGISTRY ${EXPORT_TO_USER_PACKAGE_REGISTRY} CACHE BOOL "Add cmake package config location to the user's cmake package registry.") +if(${EXPORT_TO_USER_PACKAGE_REGISTRY}) + # Enable writing to the registry + set(CMAKE_EXPORT_PACKAGE_REGISTRY ON) + # Generate a target file for the build + export(TARGETS ${HSAKMT_TARGET} NAMESPACE ${HSAKMT_TARGET}:: FILE ${HSAKMT_TARGET}Targets.cmake) + # Record the package in the user's cache. + export(PACKAGE ${HSAKMT_TARGET}) +endif() + # CPACK_PACKAGING_INSTALL_PREFIX is needed in libhsakmt.pc.in # TODO: Add support for relocatable packages. configure_file ( libhsakmt.pc.in libhsakmt.pc @ONLY ) From 6a762ec717bc91527369abce3eee41bab538958a Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Fri, 22 Nov 2019 21:04:31 -0500 Subject: [PATCH 0701/1247] libhsakmt: Improve the comment regarding queue doorbells The comment failed to convey the fact. Change-Id: Ia9b1d1c2583e288a6308d2bc81d42055064a5f4f Signed-off-by: Yong Zhao --- src/queues.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/queues.c b/src/queues.c index 85d1f549ba..ac633479fe 100644 --- a/src/queues.c +++ b/src/queues.c @@ -676,9 +676,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, if (IS_SOC15(dev_info->asic_family)) { /* On SOC15 chips, the doorbell offset within the * doorbell page is included in the doorbell offset - * returned by KFD. This allows doorbells to be - * allocated per-device, independent of the - * per-process queue ID. + * returned by KFD. This allows CP queue doorbells to be + * allocated dynamically (while SDMA queue doorbells fixed) + * rather than based on the its process queue ID. */ doorbell_mmap_offset = args.doorbell_offset & ~(HSAuint64)(doorbells[NodeId].size - 1); From 0003b6d14711f2fbe0eefd75d6ec62f3b849f1c1 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Tue, 16 Jun 2020 10:33:37 -0400 Subject: [PATCH 0702/1247] kfdtest: change shader command to avoid compile error for gfx90a Signed-off-by: Eric Huang Change-Id: I531ebac162403cc957802bbcdb34cdc022be3a50 --- tests/kfdtest/src/KFDQMTest.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index f2b68ce233..eab1b54885 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -712,28 +712,28 @@ s_waitcnt lgkmcnt(0)\n\ v_add_f32 v24, s14, v17\n\ v_add_f32 v17, s15, v17\n\ v_log_f32 v25, v18\n\ - v_mul_legacy_f32 v25, v22, v25\n\ + v_mul_f32 v25, v22, v25\n\ v_exp_f32 v25, v25\n\ v_log_f32 v26, v19\n\ - v_mul_legacy_f32 v26, v23, v26\n\ + v_mul_f32 v26, v23, v26\n\ v_exp_f32 v26, v26\n\ v_log_f32 v27, v20\n\ - v_mul_legacy_f32 v27, v24, v27\n\ + v_mul_f32 v27, v24, v27\n\ v_exp_f32 v27, v27\n\ v_log_f32 v28, v21\n\ - v_mul_legacy_f32 v28, v17, v28\n\ + v_mul_f32 v28, v17, v28\n\ v_exp_f32 v28, v28\n\ v_add_f32 v5, v5, v25\n\ v_add_f32 v6, v6, v26\n\ v_add_f32 v7, v7, v27\n\ v_add_f32 v8, v8, v28\n\ - v_mul_legacy_f32 v18, 0x3fb8aa3b, v18\n\ + v_mul_f32 v18, 0x3fb8aa3b, v18\n\ v_exp_f32 v18, v18\n\ - v_mul_legacy_f32 v19, 0x3fb8aa3b, v19\n\ + v_mul_f32 v19, 0x3fb8aa3b, v19\n\ v_exp_f32 v19, v19\n\ - v_mul_legacy_f32 v20, 0x3fb8aa3b, v20\n\ + v_mul_f32 v20, 0x3fb8aa3b, v20\n\ v_exp_f32 v20, v20\n\ - v_mul_legacy_f32 v21, 0x3fb8aa3b, v21\n\ + v_mul_f32 v21, 0x3fb8aa3b, v21\n\ v_exp_f32 v21, v21\n\ v_add_f32 v9, v9, v18\n\ v_add_f32 v10, v10, v19\n\ From f101875de8bfcbe96964961442c94e8d73d594c0 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Fri, 19 Jun 2020 20:11:34 -0500 Subject: [PATCH 0703/1247] Add warning about incorrect package names when using CMake 3.5 Signed-off-by: Sean Keely Change-Id: Ie35b287224ec6b963bc317baf60853fbe223fc17 --- CMakeLists.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 56292ed135..9d99b40e57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,18 @@ cmake_minimum_required ( VERSION 3.5.0 ) +## Cosmetic Cmake version warnings. +if(NOT EXISTS VERSION_WARNED) + set (VERSION_WARNED FALSE CACHE BOOL "") +endif() +if(${CMAKE_VERSION} VERSION_LESS "3.6.0" AND NOT ${VERSION_WARNED}) + message("Your CMake version is too old for full functionality. +Generated package file names may be incorrect. +Please update to CMake 3.6 or newer to generate correct package file names") + set( VERSION_WARNED TRUE CACHE BOOL "Suppress cosmetic build errors due to CMake version after first warning." FORCE ) + mark_as_advanced( FORCE VERSION_WARNED ) +endif() + set ( HSAKMT "hsakmt" ) set ( HSAKMT_PACKAGE "hsakmt-roct" ) set ( HSAKMT_COMPONENT "lib${HSAKMT}" ) From 5fb771a195300e2f959ce74f3e9756e4aa7e60bc Mon Sep 17 00:00:00 2001 From: Vlad Sytchenko Date: Wed, 24 Jun 2020 12:48:16 -0400 Subject: [PATCH 0704/1247] Correctly set CMAKE_MODULE_PATH Since CMAKE_MODULE_PATH can already be set by another project, we should just append the libhsamkt cmake module directory to it. Change-Id: I999dc52a2862e4bbff02e0a8e8b39530f4dae2cd Signed-off-by: Vlad Sytchenko --- CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d99b40e57..362f3a2e06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,10 +60,7 @@ if (ROCM_CCACHE_BUILD) endif() # if (CCACHE_PROGRAM) endif() # if (ROCM_CCACHE_BUILD) -## Set default module path if not already set -if ( NOT DEFINED CMAKE_MODULE_PATH ) - set ( CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/" ) -endif () +list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules" ) ## Include common cmake modules include ( utils ) From fec3780c1abe50b073c47a9677c37309c0464e68 Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Sun, 31 May 2020 11:14:29 -0400 Subject: [PATCH 0705/1247] Revert "libhsakmt: add Streaming Performance Monitors APIs" This reverts commit d675d1cce1a2349ec8ea2ac63c3fdb62f1a00654. Reason for revert: Change was submitted by accident Change-Id: If05c705e22296fd3ca789f269737d379a933361d Signed-off-by: Felix Kuehling --- CMakeLists.txt | 1 - include/hsakmt.h | 52 ---------------- include/linux/kfd_ioctl.h | 53 +---------------- src/libhsakmt.ver | 3 - src/spm.c | 121 -------------------------------------- 5 files changed, 1 insertion(+), 229 deletions(-) delete mode 100644 src/spm.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 362f3a2e06..d7a63d57fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,7 +123,6 @@ set ( HSAKMT_SRC "src/debug.c" "src/time.c" "src/topology.c" "src/rbtree.c" - "src/spm.c" "src/version.c") ## Declare the library target name diff --git a/include/hsakmt.h b/include/hsakmt.h index 4938b8a96e..237f80bbc7 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -1209,58 +1209,6 @@ hsaKmtSetMemoryUserData( void * UserData //IN ); -/** - Acquire request exclusive use of SPM -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSPMAcquire( - HSAuint32 PreferredNode //IN - ); - - -/** - Disable SPM Streamimg -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSPMRelease( - HSAuint32 PreferredNode //IN - ); - -/** - Set up the destination user mode buffer for stream performance - counter data. - - PreferredNode -- note Id - - SizeInBytes -- number of byte of Destination buffer for performance - counter data - - timeout -- wait to collect performance counter data - - DestMemoryAddress -- point to user mode buffer for stream performance - counter data - - SPMMemoryAddress -- user mode buffer that contains stream performance - counter data, it ponit to current or one of DestMemoryAddress that set - before. - - SizeCopied -- number of bytes of data copied from Stream Ring buffer. -*/ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSPMSetDestBuffer( - HSAuint32 PreferredNode, //IN - HSAuint64 SizeInBytes, //IN - HSAuint32 timeout, //IN - void* DestMemoryAddress, //IN - void ** SPMMemoryAddress, //OUT - HSAuint32 *SizeCopied //OUT - ); - - #ifdef __cplusplus } //extern "C" #endif diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 96df15f3fc..6a7dd1f44a 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -628,53 +628,6 @@ struct kfd_ioctl_cross_memory_copy_args { __u64 bytes_copied; }; -/** - * kfd_ioctl_spm_op - SPM ioctl operations - * - * @KFD_IOCTL_SPM_OP_ACQUIRE: acquire exclusive access to SPM - * @KFD_IOCTL_SPM_OP_RELEASE: release exclusive access to SPM - * @KFD_IOCTL_SPM_OP_SET_DEST_BUF: set or unset destination buffer for SPM streaming - */ -enum kfd_ioctl_spm_op { - KFD_IOCTL_SPM_OP_ACQUIRE, - KFD_IOCTL_SPM_OP_RELEASE, - KFD_IOCTL_SPM_OP_SET_DEST_BUF -}; - - -/** - * kfd_ioctl_spm_args - Arguments for SPM ioctl - * - * @op: specifies the operation to perform - * @destptr:used for the address of the destination buffer in @KFD_IOCTL_SPM_SET_DEST_BUFFER - * @buf_size:size of the destination buffer in @KFD_IOCTL_SPM_SET_DEST_BUFFER - * @timeout: timeout to wait for the buffer to get filled - * @gpu_id: gpu ID - * @bytes_copied: bytes copied from streaming performance ring buffer - * - * If @ptr is NULL, the destination buffer address is unset and copying of counters - * is stopped. - * - * Returns negative error code on failure. On success, @KFD_IOCTL_SPM_OP_ACQUIRE and - * @KFD_IOCTL_SPM_OP_RELEASE return 0, @KFD_IOCTL_SPM_OP_SET_DEST_BUF returns the fill - * level of the previous buffer. - */ -struct kfd_ioctl_spm_args { - __u64 destptr; - __u64 spmtptr; - __u32 buf_size; - __u32 op; - __u32 timeout; - __u32 gpu_id; /* to KFD */ - /* from KFD: Total amount of bytes copied */ - __u64 bytes_copied; -}; - -struct kfd_spm_set_reg { - __u64 reg; - __u32 value; -}; - #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -772,12 +725,8 @@ struct kfd_spm_set_reg { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) -#define AMDKFD_IOC_RLC_SPM \ - AMDKFD_IOWR(0x20, struct kfd_ioctl_spm_args) - - #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x21 +#define AMDKFD_COMMAND_END 0x1F /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 97e2cf6beb..d47f68d2d0 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -71,9 +71,6 @@ hsaKmtGetKernelDebugTrapVersionInfo; hsaKmtGetThunkDebugTrapVersionInfo; hsaKmtSetAddressWatch; hsaKmtClearAddressWatch; -hsaKmtSPMAcquire; -hsaKmtSPMRelease; -hsaKmtSPMSetDestBuffer; local: *; }; diff --git a/src/spm.c b/src/spm.c deleted file mode 100644 index 33b2b0a843..0000000000 --- a/src/spm.c +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright © 2014 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including - * the next paragraph) shall be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#include "libhsakmt.h" -#include "linux/kfd_ioctl.h" -#include -#include - - -HSAKMT_STATUS HSAKMTAPI hsaKmtSPMAcquire(HSAuint32 PreferredNode) -{ - int ret; - struct kfd_ioctl_spm_args args = {0}; - uint32_t gpu_id; - - pr_debug("[%s]\n", __func__); - - ret = validate_nodeid(PreferredNode, &gpu_id); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode); - return ret; - } - - ret = HSAKMT_STATUS_SUCCESS; - args.destptr = 0; - args.buf_size = 0; - args.op = KFD_IOCTL_SPM_OP_ACQUIRE; - args.gpu_id = gpu_id; - args.bytes_copied = 0; - - ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); - - return ret; -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtSPMSetDestBuffer(HSAuint32 PreferredNode, - HSAuint64 SizeInBytes, - HSAuint32 timeout, - void* DestMemoryAddress, - void **SPMMemoryAddress, - HSAuint32 *SizeCopied) -{ - int ret; - struct kfd_ioctl_spm_args args = {0}; - uint32_t gpu_id; - - ret = HSAKMT_STATUS_SUCCESS; - - pr_debug("[%s]\n", __func__); - ret = validate_nodeid(PreferredNode, &gpu_id); - - args.timeout = timeout; - args.destptr = (uint64_t)DestMemoryAddress; - - args.buf_size = SizeInBytes; - args.spmtptr = 0; - args.op = KFD_IOCTL_SPM_OP_SET_DEST_BUF; - args.gpu_id = gpu_id; - args.bytes_copied = 0; - - ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); - - if (ret != HSAKMT_STATUS_SUCCESS) { - *SPMMemoryAddress = NULL; - *SizeCopied = 0; - return ret; - } - - *SPMMemoryAddress = (void *)args.spmtptr; - *SizeCopied = args.bytes_copied; - - return ret; -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtSPMRelease(HSAuint32 PreferredNode) -{ - int ret = HSAKMT_STATUS_SUCCESS; - struct kfd_ioctl_spm_args args = {0}; - uint32_t gpu_id; - - pr_debug("[%s]\n", __func__); - - ret = validate_nodeid(PreferredNode, &gpu_id); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode); - return ret; - } - - args.destptr = 0; - args.buf_size = 0; - args.op = KFD_IOCTL_SPM_OP_RELEASE; - args.gpu_id = gpu_id; - args.bytes_copied = 0; - - ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); - - return ret; -} - - From 141246eeeb766119adadfb48e4adbec18e630564 Mon Sep 17 00:00:00 2001 From: Cole Nelson Date: Fri, 26 Jun 2020 11:21:01 -0700 Subject: [PATCH 0706/1247] libhsakmt: set packaging arch, field separators, vendor info Change-Id: Ia1392285f5ee0beeaae136ad7d68e1723bc590ba Signed-off-by: Cole Nelson --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d7a63d57fd..0bfe9e1d73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -233,17 +233,17 @@ set ( CPACK_DEB_COMPONENT_INSTALL ON ) set ( CPACK_RPM_COMPONENT_INSTALL ON ) ## Add the packaging directives for the runtime library. -set ( CPACK_PACKAGE_VENDOR "AMD" ) +set ( CPACK_PACKAGE_VENDOR "Advanced Micro Devices, Inc." ) set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) -set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) +set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices, Inc." ) set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md" ) ## Component packaging, BINARY set ( CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support" ) set ( CPACK_DEBIAN_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) -set ( CPACK_DEBIAN_BINARY_FILE_NAME "${CPACK_DEBIAN_BINARY_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.deb" ) +set ( CPACK_DEBIAN_BINARY_FILE_NAME "${CPACK_DEBIAN_BINARY_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}_amd64.deb" ) set ( CPACK_RPM_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) set ( CPACK_RPM_BINARY_FILE_NAME "${CPACK_RPM_BINARY_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) @@ -286,7 +286,7 @@ set ( CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package." ) # Debian package specific variables set ( CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev" ) -set ( CPACK_DEBIAN_DEVEL_FILE_NAME "${CPACK_DEBIAN_DEVEL_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.deb" ) +set ( CPACK_DEBIAN_DEVEL_FILE_NAME "${CPACK_DEBIAN_DEVEL_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}_amd64.deb" ) set ( CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) # RPM package specific variables From 7c74069d6aa5484ac163a43ab62c630df69e7530 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 17 Sep 2019 19:04:07 -0400 Subject: [PATCH 0707/1247] libhsakmt: Prepare for gfx1030 support PCI IDs have yet to be added later. Change-Id: Iac303fc1346f4ed5c4da5300b1e311c1c6938ee2 Signed-off-by: Yong Zhao --- src/libhsakmt.h | 1 + src/queues.c | 7 +++++++ src/topology.c | 1 - 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 8d38eef80c..cb20026069 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -128,6 +128,7 @@ enum asic_family_type { CHIP_NAVI10, /* 15 */ CHIP_NAVI12, /* 16 */ CHIP_NAVI14, /* 17 */ + CHIP_SIENNA_CICHLID, /* 18 */ CHIP_LAST }; diff --git a/src/queues.c b/src/queues.c index ac633479fe..206e66e379 100644 --- a/src/queues.c +++ b/src/queues.c @@ -164,6 +164,12 @@ const struct device_info navi14_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info sienna_cichlid_device_info = { + .asic_family = CHIP_SIENNA_CICHLID, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -183,6 +189,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_NAVI10] = &navi10_device_info, [CHIP_NAVI12] = &navi12_device_info, [CHIP_NAVI14] = &navi14_device_info, + [CHIP_SIENNA_CICHLID] = &sienna_cichlid_device_info, }; struct queue { diff --git a/src/topology.c b/src/topology.c index 23d2e1bcaa..502337dff1 100644 --- a/src/topology.c +++ b/src/topology.c @@ -235,7 +235,6 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Navi12 */ { 0x7360, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, { 0x7362, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, - }; /* information from /proc/cpuinfo */ From 58ca2b745c88d2351f71a8c24d30bb496d37ae62 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 12 Dec 2019 19:41:34 -0500 Subject: [PATCH 0708/1247] libhsakmt: Support gfx90a Change-Id: I1ad594eab093f5aa30143ade4e72f2379c9e3616 Signed-off-by: Yong Zhao --- src/libhsakmt.h | 1 + src/pmc_table.c | 1 + src/queues.c | 10 +++++++++- src/topology.c | 6 ++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index cb20026069..4fa6f9c6d3 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -129,6 +129,7 @@ enum asic_family_type { CHIP_NAVI12, /* 16 */ CHIP_NAVI14, /* 17 */ CHIP_SIENNA_CICHLID, /* 18 */ + CHIP_ALDEBARAN, /* 19 */ CHIP_LAST }; diff --git a/src/pmc_table.c b/src/pmc_table.c index 5d30391497..c4a2dec6a4 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -2128,6 +2128,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, case CHIP_RAVEN: case CHIP_RENOIR: case CHIP_ARCTURUS: + case CHIP_ALDEBARAN: *block = vega_blocks[block_id]; break; case CHIP_NAVI10: diff --git a/src/queues.c b/src/queues.c index 206e66e379..aaf667c296 100644 --- a/src/queues.c +++ b/src/queues.c @@ -42,7 +42,8 @@ #define DOORBELL_SIZE_GFX9 8 #define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) -#define VGPR_SIZE_PER_CU(asic_family) (asic_family == CHIP_ARCTURUS ? 0x80000 : 0x40000) +#define VGPR_SIZE_PER_CU(asic_family) ((asic_family == CHIP_ARCTURUS || \ + asic_family == CHIP_ALDEBARAN) ? 0x80000 : 0x40000) #define SGPR_SIZE_PER_CU 0x4000 #define LDS_SIZE_PER_CU 0x10000 #define HWREG_SIZE_PER_CU 0x1000 @@ -146,6 +147,12 @@ const struct device_info arcturus_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info aldebaran_device_info = { + .asic_family = CHIP_ALDEBARAN, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + const struct device_info navi10_device_info = { .asic_family = CHIP_NAVI10, .eop_buffer_size = 4096, @@ -186,6 +193,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_RAVEN] = &raven_device_info, [CHIP_RENOIR] = &renoir_device_info, [CHIP_ARCTURUS] = &arcturus_device_info, + [CHIP_ALDEBARAN] = &aldebaran_device_info, [CHIP_NAVI10] = &navi10_device_info, [CHIP_NAVI12] = &navi12_device_info, [CHIP_NAVI14] = &navi14_device_info, diff --git a/src/topology.c b/src/topology.c index 502337dff1..368be3aaa8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -222,6 +222,12 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x738C, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, { 0x738E, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, { 0x7390, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, + /* Aldebaran */ + { 0x50, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x51, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x52, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x60, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x62, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, /* Navi10 */ { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x7312, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, From 76a6781c2d126e094f756dbd0dc3f5478644f74c Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Tue, 17 Sep 2019 19:05:53 -0400 Subject: [PATCH 0709/1247] kfdtest: Add gfx1030 support Change-Id: I5fd0d9af09cbefc0acbe24397a7e2f3aa14a7c97 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 9ecbcc1b55..7d71f69d68 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -198,3 +198,9 @@ FILTER[navi10]=\ FILTER[navi14]=\ "$BLACKLIST_ALL_ASICS" + +FILTER[sienna_cichlid]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDDBGTest.*:"\ +"KFDPerfCountersTest.*:"\ From b0b0a1b479723beed82d2bccca06a61af746b63a Mon Sep 17 00:00:00 2001 From: Ramesh Errabolu Date: Tue, 30 Jun 2020 15:00:26 -0500 Subject: [PATCH 0710/1247] Fix typo in the field name Mem_EDCSupport of HSA_CAPABILITY entity Change-Id: Ic1aba0b2c6e7bc8324fe756df84998f32a8c09da --- include/hsakmttypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 7a7dc2ad10..b4fd9bdcb3 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -210,7 +210,7 @@ typedef union unsigned int WaveLaunchModeSupported: 1; // Indicates if Wave Launch Mode is supported on the node. unsigned int PreciseMemoryOperationsSupported: 1; // Indicates if Precise Memory Operations are supported on the node. unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active - unsigned int Mem_EDCSupoort: 1; // Indicates if GFX internal DRAM/HBM EDC/ECC functionality is active + unsigned int Mem_EDCSupport: 1; // Indicates if GFX internal DRAM/HBM EDC/ECC functionality is active unsigned int RASEventNotify: 1; // Indicates if GFX extended RASFeatures and RAS EventNotify status is available unsigned int ASICRevision: 4; // Indicates the ASIC revision of the chip on this node. unsigned int Reserved : 6; From bf1a7acea3a6bd86ee2f3c9b8cb9a9022abc3c1a Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Mon, 8 Jun 2020 11:00:29 +0800 Subject: [PATCH 0711/1247] libhsakmt: Add gfx1031 support Signed-off-by: Chengming Gui Change-Id: Ic1e78e5c3a453eb01f725612cf9ecc702ce2e132 --- src/libhsakmt.h | 1 + src/queues.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index cb20026069..757ebeb0c6 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -129,6 +129,7 @@ enum asic_family_type { CHIP_NAVI12, /* 16 */ CHIP_NAVI14, /* 17 */ CHIP_SIENNA_CICHLID, /* 18 */ + CHIP_NAVY_FLOUNDER, /* 19 */ CHIP_LAST }; diff --git a/src/queues.c b/src/queues.c index 206e66e379..a2e59900a9 100644 --- a/src/queues.c +++ b/src/queues.c @@ -170,6 +170,12 @@ const struct device_info sienna_cichlid_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info navy_flounder_device_info = { + .asic_family = CHIP_NAVY_FLOUNDER, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -190,6 +196,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_NAVI12] = &navi12_device_info, [CHIP_NAVI14] = &navi14_device_info, [CHIP_SIENNA_CICHLID] = &sienna_cichlid_device_info, + [CHIP_NAVY_FLOUNDER] = &navy_flounder_device_info, }; struct queue { From 72294266c228ad3324aa8d21253cea6d54057af3 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Mon, 8 Jun 2020 11:08:55 +0800 Subject: [PATCH 0712/1247] kfdtest: Add gfx1031 support Signed-off-by: Chengming Gui Change-Id: I3806ec33eefd84ff1dd690dd142889e076b7ef43 --- tests/kfdtest/scripts/kfdtest.exclude | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 7d71f69d68..244f3ca36f 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -204,3 +204,9 @@ FILTER[sienna_cichlid]=\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ + +FILTER[navy_flounder]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDDBGTest.*:"\ +"KFDPerfCountersTest.*:"\ From 9e9771a7d9af826ef90575075079953122a490a1 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 28 Jul 2020 13:33:14 -0400 Subject: [PATCH 0713/1247] libhsakmt: always use render fd to create CPU mapping The option to use kfd_fd for cpu mapping is for very old broken KFD version, it is not used in upstreaming process. This causes issue when multiple process uses shared system memory because the GTT address is over 40 bits. Change to always use render node fd to create CPU mapping. Change-Id: Id7e7b2a2e2f13c6e62c5de170589abfff4d456b0 Signed-off-by: Philip Yang --- src/fmm.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 6867c69599..8bcfa033c7 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1349,8 +1349,7 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB } if (mem) { - int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : - gpu_mem[gpu_mem_id].drm_render_fd; + int map_fd = gpu_mem[gpu_mem_id].drm_render_fd; int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; int flag = flags.ui32.HostAccess ? MAP_SHARED | MAP_FIXED : @@ -1588,7 +1587,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, &mmap_offset, ioc_flags, &vm_obj); if (mem && flags.ui32.HostAccess) { - int map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : gpu_drm_fd; + int map_fd = gpu_drm_fd; void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); @@ -2629,8 +2628,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert if (!obj) return -1; /* Create a CPU mapping for the debugger */ - map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : - gpu_mem[gpu_mem_id].drm_render_fd; + map_fd = gpu_mem[gpu_mem_id].drm_render_fd; mmap_ret = mmap(address, size, PROT_NONE, MAP_PRIVATE | MAP_FIXED, map_fd, mmap_offset); if (mmap_ret == MAP_FAILED) { @@ -2642,8 +2640,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_GTT | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); - map_fd = mmap_offset >= (1ULL<<40) ? kfd_fd : - gpu_mem[gpu_mem_id].drm_render_fd; + map_fd = gpu_mem[gpu_mem_id].drm_render_fd; mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); @@ -3291,8 +3288,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo goto err_free_obj; } obj->node_id = gpu_mem[gpu_mem_id].node_id; - map_fd = importArgs.mmap_offset >= (1ULL<<40) ? kfd_fd : - gpu_mem[gpu_mem_id].drm_render_fd; + map_fd = gpu_mem[gpu_mem_id].drm_render_fd; ret = mmap(reservedMem, (SizeInPages << PAGE_SHIFT), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, importArgs.mmap_offset); From 6085baa2dc25e0b315a51afba1adfd0d1c363bf6 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 13 Aug 2020 06:33:03 -0400 Subject: [PATCH 0714/1247] Fix typo lager->larger Signed-off-by: Kent Russell Change-Id: I188d629b6441e5ebb14f104869e871d003c78c9d --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 502337dff1..04f3f9ec50 100644 --- a/src/topology.c +++ b/src/topology.c @@ -858,7 +858,7 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, p += 2; /* remove ": " */ proc = atoi(p); if (proc >= num_procs) { - pr_warn("cpuinfo contains processor %d lager than %u\n", + pr_warn("cpuinfo contains processor %d larger than %u\n", proc, num_procs); ret = HSAKMT_STATUS_NO_MEMORY; goto exit; From 04f6b9e16bbda987a0afeee396b6892347362a12 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 13 Aug 2020 11:01:21 -0400 Subject: [PATCH 0715/1247] Fix GCC warning regarding strncpy in CPU info strlen(src) should not be used as the length in strncpy. Use memcpy since we know the length of the string, and ensure that we NULL-terminate regardless of length Signed-off-by: Kent Russell Change-Id: I21cc6d106510c69464e7ac9d3fc7da3a1e6d1a68 --- src/topology.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/topology.c b/src/topology.c index 04f3f9ec50..b92342dd6e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -879,13 +879,10 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { p = strchr(read_buf, ':'); p += 2; /* remove ": " */ - p_len = strlen(p); - if (p_len < HSA_PUBLIC_NAME_SIZE) { - /* -1 to remove \n from p */ - strncpy(cpuinfo[proc].model_name, p, p_len - 1); - cpuinfo[proc].model_name[p_len - 1] = '\0'; - } else - strncpy(cpuinfo[proc].model_name, p, HSA_PUBLIC_NAME_SIZE); + p_len = (strlen(p) > HSA_PUBLIC_NAME_SIZE ? + HSA_PUBLIC_NAME_SIZE : strlen(p)); + memcpy(cpuinfo[proc].model_name, p, p_len); + cpuinfo[proc].model_name[p_len - 1] = '\0'; continue; } From a69830917f81666cedf4cba4c281cbd7174f6b75 Mon Sep 17 00:00:00 2001 From: "Jinzhou.Su" Date: Mon, 17 Aug 2020 17:31:03 +0800 Subject: [PATCH 0716/1247] kfdtest: remove test cases on gfx90c which block IQE test SDMA_BLACKLIST, KFDMemoryTest.LargestSysBufferTest, KFDQMTest.BasicCuMaskingEven Change-Id: Ic603452ede5ef14f5fcb49925d57d3a656d48a08 Signed-off-by: Jinzhou.Su --- tests/kfdtest/scripts/kfdtest.exclude | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 244f3ca36f..86e65fade8 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -180,8 +180,11 @@ FILTER[raven]=\ FILTER[renoir]=\ "$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ "KFDQMTest.QueueLatency:"\ -"KFDQMTest.SdmaEventInterrupt" +"KFDQMTest.SdmaEventInterrupt:"\ +"KFDMemoryTest.LargestSysBufferTest:"\ +"KFDQMTest.BasicCuMaskingEven" # KFDExceptionTest.* (KFD-435) # KFDEvictTest.BurstyTest (KFD-425) From f7a3427c99763624ef5a3b236031e657a3e4e90c Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Wed, 19 Aug 2020 11:54:33 -0400 Subject: [PATCH 0717/1247] libhsakmt: call madvise() from fmm_allocate_device This is needed to avoid additional references to mapped BOs in child processes that can prevent freeing memory in the parent process and lead to out-of-memory conditions. Change-Id: I25c90510a14dde515cc23ea5dc1f68e8d7e37a66 Signed-off-by: Philip Cox --- src/fmm.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index 8bcfa033c7..411fc0ee32 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1361,6 +1361,13 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB __fmm_release(vm_obj, aperture); return NULL; } + /* + * This madvise() call is needed to avoid additional references + * to mapped BOs in child processes that can prevent freeing + * memory in the parent process and lead to out-of-memory + * conditions. + */ + madvise(mem, MemorySizeInBytes, MADV_DONTFORK); } return mem; From 3d3b28b670bd4ee7713ff561cc8a1413780179f1 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 1 Sep 2020 21:29:22 -0700 Subject: [PATCH 0718/1247] CWSR control stack size calculation for gfx10 Gfx10 need 12bytes/wave control stack Change-Id: I6c6f2819572e6b43aa3140d4dbe79d930e4c1c9c Signed-off-by: Oak Zeng Signed-off-by: Jay Cornwall --- src/queues.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/queues.c b/src/queues.c index a2e59900a9..539b06b139 100644 --- a/src/queues.c +++ b/src/queues.c @@ -48,7 +48,7 @@ #define HWREG_SIZE_PER_CU 0x1000 #define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) #define WAVES_PER_CU 32 -#define CNTL_STACK_BYTES_PER_WAVE 8 +#define CNTL_STACK_BYTES_PER_WAVE(asic_family) (asic_family >= CHIP_NAVI10 ? 12 : 8) struct device_info { enum asic_family_type asic_family; @@ -425,7 +425,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) uint32_t ctl_stack_size, wg_data_size; uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; - ctl_stack_size = cu_num * WAVES_PER_CU * CNTL_STACK_BYTES_PER_WAVE + 8; + ctl_stack_size = cu_num * WAVES_PER_CU * CNTL_STACK_BYTES_PER_WAVE(q->dev_info->asic_family) + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->dev_info->asic_family); q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size + sizeof(HsaUserContextSaveAreaHeader)); From 12813691a225c2973d4abc2411e25b5ca698ef8d Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Thu, 30 Jul 2020 10:22:23 +0800 Subject: [PATCH 0719/1247] libhsakmt: implement the method that using flag which exposed by kfd to configure is_dgpu KFD already implemented the fallback path for APU. Thunk will use flag which exposed by kfd to configure is_dgpu instead of hardcode before. Signed-off-by: Huang Rui Change-Id: I445f6cf668f9484dd06cd9ae1bb3cfe7428ec7eb --- src/fmm.c | 16 +++++++++++----- src/libhsakmt.h | 2 +- src/queues.c | 2 +- src/topology.c | 17 ++++++----------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 411fc0ee32..d3f289761d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -931,7 +931,7 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, args.flags = flags | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; args.va_addr = (uint64_t)mem; - if (!topology_is_dgpu(get_device_id_by_gpu_id(gpu_id)) && + if (!is_dgpu && (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) args.va_addr = VOID_PTRS_SUB(mem, aperture->base); if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) @@ -1150,7 +1150,7 @@ static void fmm_release_scratch(uint32_t gpu_id) size = VOID_PTRS_SUB(aperture->limit, aperture->base) + 1; - if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + if (is_dgpu) { /* unmap and remove all remaining objects */ pthread_mutex_lock(&aperture->fmm_mutex); while ((n = rbtree_node_any(&aperture->tree, MID))) { @@ -1217,7 +1217,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeIn return NULL; /* Allocate address space for scratch backing, 64KB aligned */ - if (topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) { + if (is_dgpu) { pthread_mutex_lock(&svm.dgpu_aperture->fmm_mutex); mem = aperture_allocate_area_aligned( svm.dgpu_aperture, address, @@ -2173,12 +2173,17 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) */ pacc = pci_ids_create(); + + is_dgpu = false; + for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; + topology_setup_is_dgpu_param(&props); + /* Skip non-GPU nodes */ if (gpu_id != 0) { int fd = open_drm_render_device(props.DrmRenderMinor); @@ -2209,6 +2214,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem_count++; } } + pci_ids_destroy(pacc); /* The ioctl will also return Number of Nodes if @@ -2617,7 +2623,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert if (gpu_mem_id < 0) return -1; - if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) + if (!is_dgpu) return 0; /* Nothing to do on APU */ /* sanity check the address */ @@ -2830,7 +2836,7 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, if (gpu_mem_id < 0) return -1; - if (!topology_is_dgpu(gpu_mem[gpu_mem_id].device_id)) + if (!is_dgpu) return 0; /* Nothing to do on APU */ pthread_mutex_lock(&aperture->fmm_mutex); diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 757ebeb0c6..e5e650bfe6 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -159,7 +159,7 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, struct pci_ids pacc); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); -bool topology_is_dgpu(uint16_t device_id); +void topology_setup_is_dgpu_param(HsaNodeProperties *props); bool topology_is_svm_needed(uint16_t device_id); HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, enum asic_family_type *asic); diff --git a/src/queues.c b/src/queues.c index 539b06b139..1ddf6805ee 100644 --- a/src/queues.c +++ b/src/queues.c @@ -272,7 +272,7 @@ static void get_doorbell_map_info(uint16_t dev_id, * GPUVM doorbell on Tonga requires a workaround for VM TLB ACTIVE bit * lookup bug. Remove ASIC check when this is implemented in amdgpu. */ - doorbell->use_gpuvm = (topology_is_dgpu(dev_id) && + doorbell->use_gpuvm = (is_dgpu && dev_info->asic_family != CHIP_TONGA); doorbell->size = DOORBELLS_PAGE_SIZE(dev_info->doorbell_size); } diff --git a/src/topology.c b/src/topology.c index b92342dd6e..3e6ca1e2d4 100644 --- a/src/topology.c +++ b/src/topology.c @@ -753,24 +753,19 @@ HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, return HSAKMT_STATUS_SUCCESS; } -bool topology_is_dgpu(uint16_t device_id) -{ - const struct hsa_gfxip_table *hsa_gfxip = - find_hsa_gfxip_device(device_id); - if (hsa_gfxip && hsa_gfxip->is_dgpu) { +void topology_setup_is_dgpu_param(HsaNodeProperties *props) +{ + /* if we found a dGPU node, then treat the whole system as dGPU */ + if (!props->NumCPUCores && props->NumFComputeCores) is_dgpu = true; - return true; - } - is_dgpu = false; - return false; } bool topology_is_svm_needed(uint16_t device_id) { const struct hsa_gfxip_table *hsa_gfxip; - if (topology_is_dgpu(device_id)) + if (is_dgpu) return true; hsa_gfxip = find_hsa_gfxip_device(device_id); @@ -2004,7 +1999,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, /* For CPU only node don't add any additional GPU memory banks. */ if (gpu_id) { uint64_t base, limit; - if (topology_is_dgpu(get_device_id_by_gpu_id(gpu_id))) + if (is_dgpu) NodeProperties->NumMemoryBanks += NUM_OF_DGPU_HEAPS; else NodeProperties->NumMemoryBanks += NUM_OF_IGPU_HEAPS; From ad87f38dad621c4306d15175c0d16cef16938f6b Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Tue, 18 Aug 2020 18:54:05 +0800 Subject: [PATCH 0720/1247] libhsakmt: remove is_dgpu flag in the hsa_gfxip_table Whether use dgpu path will check the props which exposed from kernel. We won't need hard code in the ASIC table. Signed-off-by: Huang Rui Change-Id: I0c018a26b219914a41197ff36dbec7a75945d452 --- src/libhsakmt.h | 1 - src/openclose.c | 9 +- src/topology.c | 252 ++++++++++++++++++++++++------------------------ 3 files changed, 130 insertions(+), 132 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index e5e650bfe6..2bbec6f211 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -138,7 +138,6 @@ struct hsa_gfxip_table { unsigned char major; // GFXIP Major engine version unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info - unsigned char is_dgpu; // Predicate for dGPU devices const char *amd_name; // CALName of the device enum asic_family_type asic_family; // Device family id }; diff --git a/src/openclose.c b/src/openclose.c index 72dfd66a1f..2ffe9a8268 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -141,12 +141,12 @@ static HSAKMT_STATUS init_vars_from_env(void) */ envvar = getenv("HSA_FORCE_ASIC_TYPE"); if (envvar) { - uint32_t major, minor, step, dgpu, asic_family; + uint32_t major, minor, step, asic_family; - if ((sscanf(envvar, "%u.%u.%u %u %63s %u", &major, &minor, &step, - &dgpu, force_asic_name, &asic_family) != 6) + if ((sscanf(envvar, "%u.%u.%u %63s %u", &major, &minor, &step, + force_asic_name, &asic_family) != 6) || (major > 63 || minor > 255 || step > 255) - || dgpu > 1 || asic_family >= CHIP_LAST) { + || asic_family >= CHIP_LAST) { pr_err("HSA_FORCE_ASIC_TYPE %s is invalid\n", envvar); return HSAKMT_STATUS_ERROR; } @@ -154,7 +154,6 @@ static HSAKMT_STATUS init_vars_from_env(void) force_asic_entry.major = major; force_asic_entry.minor = minor; force_asic_entry.stepping = step; - force_asic_entry.is_dgpu = dgpu; force_asic_entry.asic_family = asic_family; diff --git a/src/topology.c b/src/topology.c index 3e6ca1e2d4..eb05be8c57 100644 --- a/src/topology.c +++ b/src/topology.c @@ -92,149 +92,149 @@ struct hsa_gfxip_table force_asic_entry = { static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Kaveri Family */ - { 0x1304, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1305, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1306, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1307, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1309, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130A, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130B, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130C, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130D, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130E, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130F, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1310, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1311, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1312, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, - { 0x1313, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1315, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1316, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, - { 0x1317, 7, 0, 0, 0, "Spooky", CHIP_KAVERI }, - { 0x1318, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x131B, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x131C, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x131D, 7, 0, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1304, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1305, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1306, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1307, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1309, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130A, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130B, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130C, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130D, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130E, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x130F, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1310, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1311, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1312, 7, 0, 0, "Spooky", CHIP_KAVERI }, + { 0x1313, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1315, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1316, 7, 0, 0, "Spooky", CHIP_KAVERI }, + { 0x1317, 7, 0, 0, "Spooky", CHIP_KAVERI }, + { 0x1318, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x131B, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x131C, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x131D, 7, 0, 0, "Spectre", CHIP_KAVERI }, /* Hawaii Family */ - { 0x67A0, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A1, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A2, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A8, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A9, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67AA, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B0, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B1, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B8, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B9, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67BA, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, - { 0x67BE, 7, 0, 1, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A0, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A1, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A2, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A8, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A9, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67AA, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B0, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B1, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B8, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67B9, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67BA, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67BE, 7, 0, 1, "Hawaii", CHIP_HAWAII }, /* Carrizo Family */ - { 0x9870, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, - { 0x9874, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, - { 0x9875, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, - { 0x9876, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, - { 0x9877, 8, 0, 1, 0, "Carrizo", CHIP_CARRIZO }, + { 0x9870, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, + { 0x9874, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, + { 0x9875, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, + { 0x9876, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, + { 0x9877, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, /* Tonga Family */ - { 0x6920, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x6921, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x6928, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x6929, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x692B, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x692F, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x6930, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x6938, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, - { 0x6939, 8, 0, 2, 1, "Tonga", CHIP_TONGA }, + { 0x6920, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x6921, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x6928, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x6929, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x692B, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x692F, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x6930, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x6938, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x6939, 8, 0, 2, "Tonga", CHIP_TONGA }, /* Fiji */ - { 0x7300, 8, 0, 3, 1, "Fiji", CHIP_FIJI }, - { 0x730F, 8, 0, 3, 1, "Fiji", CHIP_FIJI }, + { 0x7300, 8, 0, 3, "Fiji", CHIP_FIJI }, + { 0x730F, 8, 0, 3, "Fiji", CHIP_FIJI }, /* Polaris10 */ - { 0x67C0, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67C1, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67C2, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67C4, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67C7, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67C8, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67C9, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67CA, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67CC, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67CF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67D0, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x67DF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, - { 0x6FDF, 8, 0, 3, 1, "Polaris10", CHIP_POLARIS10 }, + { 0x67C0, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67C1, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67C2, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67C4, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67C7, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67C8, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67C9, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67CA, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67CC, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67CF, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67D0, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67DF, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x6FDF, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, /* Polaris11 */ - { 0x67E0, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67E1, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67E3, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67E7, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67E8, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67E9, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67EB, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67EF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, - { 0x67FF, 8, 0, 3, 1, "Polaris11", CHIP_POLARIS11 }, + { 0x67E0, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67E1, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67E3, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67E7, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67E8, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67E9, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67EB, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67EF, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67FF, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, /* Polaris12 */ - { 0x6980, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, - { 0x6981, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, - { 0x6985, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, - { 0x6986, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, - { 0x6987, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, - { 0x6995, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, - { 0x6997, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, - { 0x699F, 8, 0, 3, 1, "Polaris12", CHIP_POLARIS12 }, + { 0x6980, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x6981, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x6985, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x6986, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x6987, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x6995, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x6997, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x699F, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, /* VegaM */ - { 0x694C, 8, 0, 3, 1, "VegaM", CHIP_VEGAM }, - { 0x694E, 8, 0, 3, 1, "VegaM", CHIP_VEGAM }, - { 0x694F, 8, 0, 3, 1, "VegaM", CHIP_VEGAM }, + { 0x694C, 8, 0, 3, "VegaM", CHIP_VEGAM }, + { 0x694E, 8, 0, 3, "VegaM", CHIP_VEGAM }, + { 0x694F, 8, 0, 3, "VegaM", CHIP_VEGAM }, /* Vega10 */ - { 0x6860, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6861, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6862, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6863, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6864, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6867, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6868, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x6869, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x686A, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x686B, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x686C, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x686D, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x686E, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, - { 0x687F, 9, 0, 0, 1, "Vega10", CHIP_VEGA10 }, + { 0x6860, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6861, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6862, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6863, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6864, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6867, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6868, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6869, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x686A, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x686B, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x686C, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x686D, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x686E, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x687F, 9, 0, 0, "Vega10", CHIP_VEGA10 }, /* Vega12 */ - { 0x69A0, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, - { 0x69A1, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, - { 0x69A2, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, - { 0x69A3, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, - { 0x69Af, 9, 0, 4, 1, "Vega12", CHIP_VEGA12 }, + { 0x69A0, 9, 0, 4, "Vega12", CHIP_VEGA12 }, + { 0x69A1, 9, 0, 4, "Vega12", CHIP_VEGA12 }, + { 0x69A2, 9, 0, 4, "Vega12", CHIP_VEGA12 }, + { 0x69A3, 9, 0, 4, "Vega12", CHIP_VEGA12 }, + { 0x69Af, 9, 0, 4, "Vega12", CHIP_VEGA12 }, /* Raven */ - { 0x15DD, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, - { 0x15D8, 9, 0, 2, 0, "Raven", CHIP_RAVEN }, + { 0x15DD, 9, 0, 2, "Raven", CHIP_RAVEN }, + { 0x15D8, 9, 0, 2, "Raven", CHIP_RAVEN }, /* Renoir */ - { 0x1636, 9, 0, 0, 1, "Renoir", CHIP_RENOIR }, + { 0x1636, 9, 0, 0, "Renoir", CHIP_RENOIR }, /* Vega20 */ - { 0x66A0, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, - { 0x66A1, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, - { 0x66A2, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, - { 0x66A3, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, - { 0x66A4, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, - { 0x66A7, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, - { 0x66AF, 9, 0, 6, 1, "Vega20", CHIP_VEGA20 }, + { 0x66A0, 9, 0, 6, "Vega20", CHIP_VEGA20 }, + { 0x66A1, 9, 0, 6, "Vega20", CHIP_VEGA20 }, + { 0x66A2, 9, 0, 6, "Vega20", CHIP_VEGA20 }, + { 0x66A3, 9, 0, 6, "Vega20", CHIP_VEGA20 }, + { 0x66A4, 9, 0, 6, "Vega20", CHIP_VEGA20 }, + { 0x66A7, 9, 0, 6, "Vega20", CHIP_VEGA20 }, + { 0x66AF, 9, 0, 6, "Vega20", CHIP_VEGA20 }, /* Arcturus */ - { 0x7388, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, - { 0x738C, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, - { 0x738E, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, - { 0x7390, 9, 0, 8, 1, "Arcturus", CHIP_ARCTURUS }, + { 0x7388, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, + { 0x738C, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, + { 0x738E, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, + { 0x7390, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, /* Navi10 */ - { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, - { 0x7312, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, - { 0x7318, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, - { 0x731A, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, - { 0x731F, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, + { 0x7310, 10, 1, 0, "Navi10", CHIP_NAVI10 }, + { 0x7312, 10, 1, 0, "Navi10", CHIP_NAVI10 }, + { 0x7318, 10, 1, 0, "Navi10", CHIP_NAVI10 }, + { 0x731A, 10, 1, 0, "Navi10", CHIP_NAVI10 }, + { 0x731F, 10, 1, 0, "Navi10", CHIP_NAVI10 }, /* Navi14 */ - { 0x7340, 10, 1, 2, 1, "Navi14", CHIP_NAVI14 }, - { 0x7341, 10, 1, 2, 1, "Navi14", CHIP_NAVI14 }, - { 0x7347, 10, 1, 2, 1, "Navi14", CHIP_NAVI14 }, + { 0x7340, 10, 1, 2, "Navi14", CHIP_NAVI14 }, + { 0x7341, 10, 1, 2, "Navi14", CHIP_NAVI14 }, + { 0x7347, 10, 1, 2, "Navi14", CHIP_NAVI14 }, /* Navi12 */ - { 0x7360, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, - { 0x7362, 10, 1, 1, 1, "Navi12", CHIP_NAVI12 }, + { 0x7360, 10, 1, 1, "Navi12", CHIP_NAVI12 }, + { 0x7362, 10, 1, 1, "Navi12", CHIP_NAVI12 }, }; /* information from /proc/cpuinfo */ From ae4e980ea8a067a5e079c0db957d1d15c0e38878 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 6 Aug 2020 12:10:25 -0400 Subject: [PATCH 0721/1247] README: Update README info and installation instructions Added caveat regarding Ubuntu 20.04 /dev/kfd permissions Edit copyright date, remove outdated HW support list Change-Id: I3138058a01db6c46cf624bcfb8cfb04c164d9133 Signed-off-by: Kent Russell --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2d1d4ea065..a3dc989f26 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ ### ROCt Library -This repository includes the user-mode API interfaces used to interact with the ROCk driver. Currently supported agents include only the AMD/ATI Fiji family of discrete GPUs. +This repository includes the user-mode API interfaces used to interact with the ROCk driver. Starting at 1.7 release, ROCt uses drm render device. This requires the user to belong to video group. Add the user account to video group with "sudo usermod -a -G video _username_" command if the user if not part of video group yet. +NOTE: Users of Ubuntu 20.04 will need to add the user to the new "render" group, as Ubuntu has changed the owner:group of /dev/kfd to render:render as of that release #### ROCk Driver @@ -30,6 +31,11 @@ If the hsakmt-roct and hsakmt-roct-dev packages are desired: make package make package-dev ``` +If you choose not to build and install packages, manual installation of the binaries and header files can be done via: +```bash + make install + make install-dev +``` #### Disclaimer @@ -37,4 +43,4 @@ The information contained herein is for informational purposes only, and is subj AMD, the AMD Arrow logo, and combinations thereof are trademarks of Advanced Micro Devices, Inc. Other product names used in this publication are for identification purposes only and may be trademarks of their respective companies. -Copyright (c) 2014-2017 Advanced Micro Devices, Inc. All rights reserved. +Copyright (c) 2014-2020 Advanced Micro Devices, Inc. All rights reserved. From 0a4b23d625fe65ec2adc2c7176c5cf85700df345 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 21 Aug 2020 08:35:55 -0400 Subject: [PATCH 0722/1247] Fix lintian errors in deb packages Add extended descriptions and e-mail address to CMakeLists A lintian error will remain regarding stripping the .so, as we will not be doing this for Release versions of the hsakmt .so Signed-off-by: Kent Russell Change-Id: I41c768dee28c0564d92b9c103a6e2d97590e4589 --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bfe9e1d73..7bd07ad6bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,11 +237,11 @@ set ( CPACK_PACKAGE_VENDOR "Advanced Micro Devices, Inc." ) set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) -set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices, Inc." ) +set ( CPACK_PACKAGE_CONTACT "AMD GFX mailing list " ) set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md" ) ## Component packaging, BINARY -set ( CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support" ) +set ( CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support\n This repository includes the user-mode API interfaces\n used to interact with the ROCk driver." ) set ( CPACK_DEBIAN_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) set ( CPACK_DEBIAN_BINARY_FILE_NAME "${CPACK_DEBIAN_BINARY_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}_amd64.deb" ) set ( CPACK_RPM_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) @@ -282,7 +282,7 @@ set ( CPACK_RPM_BINARY_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/ set ( CPACK_RPM_BINARY_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) ## Component packaging, DEVEL -set ( CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package." ) +set ( CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package.\n This package contains the headers, pkgonfig and\n cmake files for the ROCT package" ) # Debian package specific variables set ( CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev" ) From 654ee83ac8ca8630d872e62559e73b61ca58c58f Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 9 Sep 2020 11:26:44 -0400 Subject: [PATCH 0723/1247] Remove -dev make instructions These are removed now that we've consolidated the dev package information into CMakeLists.txt from hsakmt-dev.txt. Change-Id: I49496ec5def85b0af7fa6b15110910528a8e0be0 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a3dc989f26..b274bb7202 100644 --- a/README.md +++ b/README.md @@ -29,14 +29,14 @@ If the hsakmt-roct and hsakmt-roct-dev packages are desired: cd build cmake .. make package - make package-dev ``` If you choose not to build and install packages, manual installation of the binaries and header files can be done via: ```bash make install - make install-dev ``` +NOTE: For older versions of the thunk where hsakmt-dev.txt is present, "make package-dev" and "make install-dev" are required to generate/install the developer packages. Currently, these are created via the "make package" and "make install" commands + #### Disclaimer The information contained herein is for informational purposes only, and is subject to change without notice. While every precaution has been taken in the preparation of this document, it may contain technical inaccuracies, omissions and typographical errors, and AMD is under no obligation to update or otherwise correct this information. Advanced Micro Devices, Inc. makes no representations or warranties with respect to the accuracy or completeness of the contents of this document, and assumes no liability of any kind, including the implied warranties of noninfringement, merchantability or fitness for particular purposes, with respect to the operation or use of AMD hardware, software or other products described herein. No license, including implied or arising by estoppel, to any intellectual property rights is granted by this document. Terms and limitations applicable to the purchase or use of AMD's products are as set forth in a signed agreement between the parties or in AMD's Standard Terms and Conditions of Sale. From 8ea0d4933786e481910c01d781a6170b1501babb Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Wed, 19 Aug 2020 13:22:35 +0800 Subject: [PATCH 0724/1247] libhsakmt: update gfx90c isa version gfx90c should use GFX902 which is the same with gfx902. Signed-off-by: Huang Rui Change-Id: Id24dc2c85c9f49f36b00889c3b8b1b19cce34e09 --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index eb05be8c57..5eed99d079 100644 --- a/src/topology.c +++ b/src/topology.c @@ -208,7 +208,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x15DD, 9, 0, 2, "Raven", CHIP_RAVEN }, { 0x15D8, 9, 0, 2, "Raven", CHIP_RAVEN }, /* Renoir */ - { 0x1636, 9, 0, 0, "Renoir", CHIP_RENOIR }, + { 0x1636, 9, 0, 2, "Renoir", CHIP_RENOIR }, /* Vega20 */ { 0x66A0, 9, 0, 6, "Vega20", CHIP_VEGA20 }, { 0x66A1, 9, 0, 6, "Vega20", CHIP_VEGA20 }, From 8fc816affeaa47848591d60af5eabc24ebcaba08 Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Fri, 11 Sep 2020 11:27:31 +0800 Subject: [PATCH 0725/1247] libhsakmt: fix to update the param number after remove to dgpu input This patch is the hot fix to fix the param number checking after remove dgpu input. Signed-off-by: Huang Rui Change-Id: Ic980588f78616f99076de742af580afb4273fb2f --- src/openclose.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openclose.c b/src/openclose.c index 2ffe9a8268..784ee51f2a 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -136,15 +136,15 @@ static HSAKMT_STATUS init_vars_from_env(void) zfb_support = atoi(envvar); /* Force all the GPUs to a certain type, use the below command: - * export HSA_FORCE_ASIC_TYPE="10.1.0 1 Navi10 14" - * meaning major.minor.step dgpu asic_name asic_id + * export HSA_FORCE_ASIC_TYPE="10.1.0 Navi10 14" + * meaning major.minor.step asic_name asic_id */ envvar = getenv("HSA_FORCE_ASIC_TYPE"); if (envvar) { uint32_t major, minor, step, asic_family; if ((sscanf(envvar, "%u.%u.%u %63s %u", &major, &minor, &step, - force_asic_name, &asic_family) != 6) + force_asic_name, &asic_family) != 5) || (major > 63 || minor > 255 || step > 255) || asic_family >= CHIP_LAST) { pr_err("HSA_FORCE_ASIC_TYPE %s is invalid\n", envvar); From 36249ddc0e31a637a8602e4e6e81d13622755d54 Mon Sep 17 00:00:00 2001 From: "Jinzhou.Su" Date: Wed, 9 Sep 2020 08:47:41 +0800 Subject: [PATCH 0726/1247] kfdtest:Update gfx90c blacklist 1. Add KFDEvictTest.* for gfx90c based on CI test results 2. Remove SDMA blacklist based on SDMA issue fixed: Change-Id: I86910fc98a5141f29959b35248a900f0c098a6e8 --- tests/kfdtest/scripts/kfdtest.exclude | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 86e65fade8..a2c4f1cb70 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -180,11 +180,12 @@ FILTER[raven]=\ FILTER[renoir]=\ "$BLACKLIST_ALL_ASICS:"\ -"$SDMA_BLACKLIST:"\ -"KFDQMTest.QueueLatency:"\ +"KFDEvictTest.*:"\ "KFDQMTest.SdmaEventInterrupt:"\ "KFDMemoryTest.LargestSysBufferTest:"\ -"KFDQMTest.BasicCuMaskingEven" +"KFDQMTest.BasicCuMaskingEven"\ +"KFDMemoryTest.MMBench:"\ +"KFDMemoryTest.SignalHandling:" # KFDExceptionTest.* (KFD-435) # KFDEvictTest.BurstyTest (KFD-425) From 7efa823ec8c6fa7067bec089001d830b4fe3dc51 Mon Sep 17 00:00:00 2001 From: "Jinzhou.Su" Date: Wed, 16 Sep 2020 09:36:59 +0800 Subject: [PATCH 0727/1247] kfdtest: fix gfx90c blacklist issue This patch is the hot fix to update gfx90c blacklist Change-Id: I41f48154ad5ec3035fcb7891a224fc940dca991f --- tests/kfdtest/scripts/kfdtest.exclude | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index a2c4f1cb70..dc8e57977d 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -183,9 +183,9 @@ FILTER[renoir]=\ "KFDEvictTest.*:"\ "KFDQMTest.SdmaEventInterrupt:"\ "KFDMemoryTest.LargestSysBufferTest:"\ -"KFDQMTest.BasicCuMaskingEven"\ +"KFDQMTest.BasicCuMaskingEven:"\ "KFDMemoryTest.MMBench:"\ -"KFDMemoryTest.SignalHandling:" +"KFDMemoryTest.SignalHandling" # KFDExceptionTest.* (KFD-435) # KFDEvictTest.BurstyTest (KFD-425) From befc7edaea7ae81069c0b1ba6d19843636d7a10e Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Thu, 10 Sep 2020 18:31:06 +0800 Subject: [PATCH 0728/1247] kfdtest: fix MTYPE error when init sdma fence packet Signed-off-by: Chengming Gui Change-Id: I14a1ef204dbd5ab1e9f1840b9555f88b0df361c0 --- tests/kfdtest/src/SDMAPacket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/tests/kfdtest/src/SDMAPacket.cpp index ee12e358b2..f5a8a5787d 100644 --- a/tests/kfdtest/src/SDMAPacket.cpp +++ b/tests/kfdtest/src/SDMAPacket.cpp @@ -189,7 +189,7 @@ void SDMAFencePacket::InitPacketNV(void * destAddr,unsigned int data) { * System = 1 because the memory is system memory * mtype = uncached, for the purpose of CPU coherent, L2 policy doesn't matter in this case */ - packetData.HEADER_UNION.DW_0_DATA = (0 << 23) | (1 << 22) | (1 << 20) | (3 << 15) | SDMA_OP_FENCE; + packetData.HEADER_UNION.DW_0_DATA = (0 << 23) | (1 << 22) | (1 << 20) | (3 << 16) | SDMA_OP_FENCE; SplitU64(reinterpret_cast(destAddr), packetData.ADDR_LO_UNION.DW_1_DATA, /*dst_addr_31_0*/ From 3311785c7b3560943e427ef61496cafaef060f85 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 21 Sep 2020 14:13:11 -0400 Subject: [PATCH 0729/1247] libhsakmt: add device ID for gfx1030 Signed-off-by: Harish Kasiviswanathan Change-Id: I3024bf706f88c0e612391f6d8045020101007bdc --- src/topology.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/topology.c b/src/topology.c index 5eed99d079..36ef98742d 100644 --- a/src/topology.c +++ b/src/topology.c @@ -235,6 +235,13 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Navi12 */ { 0x7360, 10, 1, 1, "Navi12", CHIP_NAVI12 }, { 0x7362, 10, 1, 1, "Navi12", CHIP_NAVI12 }, + /* SIENNA_CICHLID */ + { 0x73A0, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + { 0x73A2, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + { 0x73A3, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + { 0x73AB, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + { 0x73AE, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + { 0x73BF, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, }; /* information from /proc/cpuinfo */ From 44f80d170df3e3f098bb014a36bdd1881294b5c5 Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Mon, 28 Sep 2020 15:42:13 -0500 Subject: [PATCH 0730/1247] libhsakmt: Limit control stack size on gfx10 The queue control stack size cannot exceed 0x7000 on ASICs gfx1010 through gfx1031. The lower limit is not achievable with AQL so this should have no practical effect. Fixes control stack size overflow on large ASICs. Change-Id: Ib78cf6e4c5f096044bf8de24debe211689891caa Signed-off-by: Jay Cornwall --- src/queues.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/queues.c b/src/queues.c index 1ddf6805ee..1bfcee47b7 100644 --- a/src/queues.c +++ b/src/queues.c @@ -430,6 +430,15 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size + sizeof(HsaUserContextSaveAreaHeader)); + if (q->dev_info->asic_family >= CHIP_NAVI10 && + q->dev_info->asic_family <= CHIP_NAVY_FLOUNDER) { + /* HW design limits control stack size to 0x7000. + * This is insufficient for theoretical PM4 cases + * but sufficient for AQL, limited by SPI events. + */ + q->ctl_stack_size = MIN(q->ctl_stack_size, 0x7000); + } + q->ctx_save_restore_size = q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size); return true; From 2ed2e46b9b65e83d52ffdf2e19d941552205fed4 Mon Sep 17 00:00:00 2001 From: Laurent Morichetti Date: Thu, 10 Sep 2020 11:39:15 -0700 Subject: [PATCH 0731/1247] Update the context save area size Reserve some space in the context save area for the debugger's use. There should be 32 bytes per wave for a given queue. Change-Id: I65ddb6123d0f6afd3149844617ad19023009101d --- include/hsakmttypes.h | 24 +++++++++++++++--------- src/queues.c | 23 +++++++++++++++++------ 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index b4fd9bdcb3..ef97539ea1 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -652,15 +652,16 @@ typedef enum _HSA_QUEUE_TYPE } HSA_QUEUE_TYPE; /** - The user context save area starts at offset 0 with the - HsaUserContextSaveAreaHeader header followed by the space for a - user space copy of the control stack and the user space wave save - state. The area must be dword aligned. The context save area is - valid for the duration that the associated queue exists. When a - context save occurs, the HsaUserContextSaveAreaHeader header will - be updated with information about the context save. The context save - area is not modified by any other operation, including a context - resume. + The user context save area is page aligned. The HsaUserContextSaveAreaHeader + header starts at offset 0. Space for a user space copy of the control stack + comes next and is immediately followed by the user space wave save state. The + start of the user space wave save state is page aligned. The debugger reserved + area comes next and is 64 byte aligned. + + The user context save area is valid for the duration that the associated + queue exists. When a context save occurs, the HsaUserContextSaveAreaHeader + header will be updated with information about the context save. The context + save area is not modified by any other operation, including a context resume. */ typedef struct @@ -676,6 +677,11 @@ typedef struct // of wave state data. Must be 4 byte aligned. HSAuint32 WaveStateSize; // Byte size of the last saved wave state data. // Must be 4 byte aligned. + HSAuint32 DebugOffset; // Byte offset from start of the user context + // save area to the memory reserved for the + // debugger. Must be 64 byte aligned. + HSAuint32 DebugSize; // Byte size of the memory reserved for the + // debugger. Must be 64 byte aligned. } HsaUserContextSaveAreaHeader; diff --git a/src/queues.c b/src/queues.c index 1bfcee47b7..d46bc7e6c6 100644 --- a/src/queues.c +++ b/src/queues.c @@ -48,7 +48,9 @@ #define HWREG_SIZE_PER_CU 0x1000 #define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) #define WAVES_PER_CU 32 -#define CNTL_STACK_BYTES_PER_WAVE(asic_family) (asic_family >= CHIP_NAVI10 ? 12 : 8) +#define CNTL_STACK_BYTES_PER_CU(asic_family) (WAVES_PER_CU * (asic_family >= CHIP_NAVI10 ? 12 : 8)) +#define DEBUGGER_BYTES_ALIGN 64 +#define DEBUGGER_BYTES_PER_CU(asic_family) (WAVES_PER_CU * 32) struct device_info { enum asic_family_type asic_family; @@ -207,6 +209,7 @@ struct queue { void *ctx_save_restore; uint32_t ctx_save_restore_size; uint32_t ctl_stack_size; + uint32_t debug_memory_size; const struct device_info *dev_info; bool use_ats; /* This queue structure is allocated from GPU with page aligned size @@ -425,11 +428,10 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) uint32_t ctl_stack_size, wg_data_size; uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; - ctl_stack_size = cu_num * WAVES_PER_CU * CNTL_STACK_BYTES_PER_WAVE(q->dev_info->asic_family) + 8; + ctl_stack_size = cu_num * CNTL_STACK_BYTES_PER_CU(q->dev_info->asic_family) + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->dev_info->asic_family); - q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size - + sizeof(HsaUserContextSaveAreaHeader)); - + q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + + ctl_stack_size); if (q->dev_info->asic_family >= CHIP_NAVI10 && q->dev_info->asic_family <= CHIP_NAVY_FLOUNDER) { /* HW design limits control stack size to 0x7000. @@ -439,8 +441,11 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) q->ctl_stack_size = MIN(q->ctl_stack_size, 0x7000); } + q->debug_memory_size = + ALIGN_UP(cu_num * DEBUGGER_BYTES_PER_CU(q->dev_info->asic_family), DEBUGGER_BYTES_ALIGN); + q->ctx_save_restore_size = q->ctl_stack_size - + PAGE_ALIGN_UP(wg_data_size); + + PAGE_ALIGN_UP(wg_data_size + q->debug_memory_size); return true; } return false; @@ -568,6 +573,8 @@ static int handle_concrete_asic(struct queue *q, ret = update_ctx_save_restore_size(NodeId, q); if (ret) { + HsaUserContextSaveAreaHeader *header; + args->ctx_save_restore_size = q->ctx_save_restore_size; args->ctl_stack_size = q->ctl_stack_size; q->ctx_save_restore = @@ -578,6 +585,10 @@ static int handle_concrete_asic(struct queue *q, return HSAKMT_STATUS_NO_MEMORY; args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; + + header = (HsaUserContextSaveAreaHeader *)q->ctx_save_restore; + header->DebugOffset = q->ctx_save_restore_size - q->debug_memory_size; + header->DebugSize = q->debug_memory_size; } return HSAKMT_STATUS_SUCCESS; From b543f4f77cd1a902027b873d4de6c049d4bf1551 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Fri, 24 Jul 2020 14:11:13 +0800 Subject: [PATCH 0732/1247] libhsakmt: Prepare gfx1032 support PCI IDs have yet to be added later. Signed-off-by: Chengming Gui Change-Id: I28f657201868423012e856df4310a493b7cd5752 --- src/libhsakmt.h | 1 + src/queues.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 2bbec6f211..1cae43a12a 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -130,6 +130,7 @@ enum asic_family_type { CHIP_NAVI14, /* 17 */ CHIP_SIENNA_CICHLID, /* 18 */ CHIP_NAVY_FLOUNDER, /* 19 */ + CHIP_DIMGREY_CAVEFISH, /* 20 */ CHIP_LAST }; diff --git a/src/queues.c b/src/queues.c index d46bc7e6c6..33b81fc09c 100644 --- a/src/queues.c +++ b/src/queues.c @@ -178,6 +178,12 @@ const struct device_info navy_flounder_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info dimgrey_cavefish_device_info = { + .asic_family = CHIP_DIMGREY_CAVEFISH, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -199,6 +205,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_NAVI14] = &navi14_device_info, [CHIP_SIENNA_CICHLID] = &sienna_cichlid_device_info, [CHIP_NAVY_FLOUNDER] = &navy_flounder_device_info, + [CHIP_DIMGREY_CAVEFISH] = &dimgrey_cavefish_device_info, }; struct queue { From 4a9d55c414c4264a7219b07906afbff122b77993 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Fri, 24 Jul 2020 14:14:13 +0800 Subject: [PATCH 0733/1247] kfdtest: Add gfx1032 support Signed-off-by: Chengming Gui Change-Id: I04cd412a5e243dfe7aa7596287341e1671c1521a --- tests/kfdtest/scripts/kfdtest.exclude | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index dc8e57977d..40183489ee 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -214,3 +214,9 @@ FILTER[navy_flounder]=\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ + +FILTER[dimgrey_cavefish]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDDBGTest.*:"\ +"KFDPerfCountersTest.*:"\ From 783e3467779b86f228222c86f82b4bb6e6c4bcbb Mon Sep 17 00:00:00 2001 From: Laurent Morichetti Date: Fri, 16 Oct 2020 00:12:38 -0700 Subject: [PATCH 0734/1247] libhsakmt: Fix the ctrl stack size calculation On gfx9, the maximum number of wavefronts per queue is the minimum of 40 waves per compute units, or 512 waves per shader engine. On gfx10, there can only be 32 waves per compute units. Signed-off-by: Laurent Morichetti Change-Id: I148d1a4fe6c07cdbfaa1f77939eb29311c81c008 --- src/queues.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/queues.c b/src/queues.c index 33b81fc09c..29949c9092 100644 --- a/src/queues.c +++ b/src/queues.c @@ -47,10 +47,9 @@ #define LDS_SIZE_PER_CU 0x10000 #define HWREG_SIZE_PER_CU 0x1000 #define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) -#define WAVES_PER_CU 32 -#define CNTL_STACK_BYTES_PER_CU(asic_family) (WAVES_PER_CU * (asic_family >= CHIP_NAVI10 ? 12 : 8)) +#define CNTL_STACK_BYTES_PER_WAVE(asic_family) (asic_family >= CHIP_NAVI10 ? 12 : 8) #define DEBUGGER_BYTES_ALIGN 64 -#define DEBUGGER_BYTES_PER_CU(asic_family) (WAVES_PER_CU * 32) +#define DEBUGGER_BYTES_PER_WAVE(asic_family) 32 struct device_info { enum asic_family_type asic_family; @@ -434,8 +433,11 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) if (node.NumFComputeCores && node.NumSIMDPerCU) { uint32_t ctl_stack_size, wg_data_size; uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; + uint32_t wave_num = (q->dev_info->asic_family < CHIP_NAVI10) + ? MIN(cu_num * 40, node.NumShaderBanks / node.NumArrays * 512) + : cu_num * 32; - ctl_stack_size = cu_num * CNTL_STACK_BYTES_PER_CU(q->dev_info->asic_family) + 8; + ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(q->dev_info->asic_family) + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->dev_info->asic_family); q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + ctl_stack_size); @@ -449,7 +451,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) } q->debug_memory_size = - ALIGN_UP(cu_num * DEBUGGER_BYTES_PER_CU(q->dev_info->asic_family), DEBUGGER_BYTES_ALIGN); + ALIGN_UP(wave_num * DEBUGGER_BYTES_PER_WAVE(q->dev_info->asic_family), DEBUGGER_BYTES_ALIGN); q->ctx_save_restore_size = q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size + q->debug_memory_size); From 8f26c0c40c6079bbbf29fbab48f7f05fc121661a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 26 Oct 2020 18:44:20 -0400 Subject: [PATCH 0735/1247] libhsakmt: optimize system memory allocation Use MAP_POPULATE when allocating anonymous system memory for later GPU mapping as a userptr. This can speed up large allocations by more than factor 2. I suspect populating pages in this way is more efficient than the CPU page fault code path triggered by get_user_pages in the kernel. Change-Id: I188bbc1462ccb650d48cbfb1080dbb8eb7ada8b5 Signed-off-by: Felix Kuehling --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index d3f289761d..959e2d21dc 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1566,8 +1566,8 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) - == MAP_FAILED) + MAP_ANONYMOUS | MAP_POPULATE | MAP_PRIVATE | MAP_FIXED, + -1, 0) == MAP_FAILED) goto out_release_area; /* Bind to NUMA node */ From c7e6f5a2742abc3ce3186f4637d7d5ad02ec7382 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 21 Oct 2020 13:23:51 -0400 Subject: [PATCH 0736/1247] libhsakmt: handle GPU mapping errors Don't update the vm_object if GPU mapping failed. Print an error message to help diagnose underlying problems. Change-Id: I801ab6fe6c155bd25e6c0358007c106a4a019480 Signed-off-by: Felix Kuehling --- src/fmm.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index 959e2d21dc..285d1c7910 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2583,6 +2583,11 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, args.n_success = 0; ret = kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args); + if (ret) { + pr_err("GPU mapping failed (%d) for obj at %p, userptr %p, size %lu", + ret, object->start, object->userptr, object->size); + goto err_map_failed; + } add_device_ids_to_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, @@ -2601,6 +2606,7 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, exit_ok: err_object_not_found: +err_map_failed: if (!obj) pthread_mutex_unlock(&aperture->fmm_mutex); From 025036a662c3e133f09de314ff000fcd842fa443 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 13 Oct 2020 11:00:48 -0400 Subject: [PATCH 0737/1247] Remove extra strlen call While the ternary is nice to read, strlen in general is an expensive call, so call it once and check if the value is greater than our maximum allowable string length and adjust accordingly Signed-off-by: Kent Russell Change-Id: Id744f2ba0eb81bb2b3c52eb69f38a615398a655d --- src/topology.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/topology.c b/src/topology.c index 36ef98742d..da1d7479de 100644 --- a/src/topology.c +++ b/src/topology.c @@ -881,8 +881,9 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { p = strchr(read_buf, ':'); p += 2; /* remove ": " */ - p_len = (strlen(p) > HSA_PUBLIC_NAME_SIZE ? - HSA_PUBLIC_NAME_SIZE : strlen(p)); + p_len = strlen(p); + if (p_len > HSA_PUBLIC_NAME_SIZE) + p_len = HSA_PUBLIC_NAME_SIZE; memcpy(cpuinfo[proc].model_name, p, p_len); cpuinfo[proc].model_name[p_len - 1] = '\0'; continue; From 761d9d84d2f0ee3e2e7f6a2c833519ae5da469fb Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 29 Oct 2020 08:03:55 -0400 Subject: [PATCH 0738/1247] kfdtest: Add missing default switch label in GetQueue function There is no default case, and we were missing a few types defined from hsakmttypes.h. This was found via clang Change-Id: I26193cb111a9d8220b1eff21c7313fe060288f36 --- tests/kfdtest/src/BaseQueue.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 835ab4c71d..e66d3dd784 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -186,6 +186,8 @@ BaseQueue* QueueArray::GetQueue(unsigned int Node) { case HSA_QUEUE_COMPUTE_AQL: pQueue = new AqlQueue(); break; + default: + return NULL; } if (pQueue) { From e34dfa8ebd00822a502fdfa6ab61fcb4ac04df8c Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 29 Oct 2020 09:27:56 -0400 Subject: [PATCH 0739/1247] kfdtest: Fix bit compare logic by using | instead of || We want to compare bits, not check if a defined value is true Change-Id: Ie51ede96d18eae01aff6677d852a056ee12bd9c6 --- tests/kfdtest/src/KFDRASTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/tests/kfdtest/src/KFDRASTest.cpp index 428db0e16d..69a4719a1d 100644 --- a/tests/kfdtest/src/KFDRASTest.cpp +++ b/tests/kfdtest/src/KFDRASTest.cpp @@ -58,8 +58,8 @@ void KFDRASTest::SetUp() { AMDGPU_INFO_RAS_ENABLED_FEATURES, sizeof(uint32_t), &rasFeatures); if (!(rasFeatures & - (AMDGPU_INFO_RAS_ENABLED_SDMA || - AMDGPU_INFO_RAS_ENABLED_UMC || + (AMDGPU_INFO_RAS_ENABLED_SDMA | + AMDGPU_INFO_RAS_ENABLED_UMC | AMDGPU_INFO_RAS_ENABLED_GFX))) { LOG() << "Skipping test: GPU doesn't support RAS features!" << std::endl; throw; From 8026ba250c40df208b3fe7f7aacaa1705f4312b6 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Mon, 2 Nov 2020 16:46:45 -0500 Subject: [PATCH 0740/1247] kfdtest: Use ldflags for drm This fixes a build issue with kfdtest and the amdgpu pro driver build. This was requested as kfdtest is needed for regular testing due to the inclusion of the ROCr/KFD stack in the amdgpu pro driver (OSGSUP-199) Change-Id: I224d2e9ee3f02065596890b4d8226484f4fac04f Signed-off-by: Jeremy Newton --- tests/kfdtest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index d92a6154ad..9a0ac90960 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -173,7 +173,7 @@ link_directories(${SP3_DIR}) add_executable(kfdtest ${SRC_FILES}) -target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LIBRARIES} ${DRM_AMDGPU_LIBRARIES} pthread m stdc++ rt amdsp3 numa) +target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread m stdc++ rt amdsp3 numa) configure_file ( scripts/kfdtest.exclude kfdtest.exclude COPYONLY ) configure_file ( scripts/run_kfdtest.sh run_kfdtest.sh COPYONLY ) From e515fd818b6803a58cf9275044d2d64021d2a98d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 2 Nov 2020 20:46:01 -0500 Subject: [PATCH 0741/1247] Revert "libhsakmt: optimize system memory allocation" This reverts commit 8f26c0c40c6079bbbf29fbab48f7f05fc121661a. Reason for revert: This commit caused a regression rocrtst memory subtest: Maximum Single Allocation in Memory Pools failed. Change-Id: I15330625603f893200a08cd8b5b097f9bf95361f --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 285d1c7910..19fe48e0f9 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1566,8 +1566,8 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_POPULATE | MAP_PRIVATE | MAP_FIXED, - -1, 0) == MAP_FAILED) + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, -1, 0) + == MAP_FAILED) goto out_release_area; /* Bind to NUMA node */ From f283fe28547bc20a4d916beb5cd4ab2cc1c0cad9 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Thu, 22 Oct 2020 19:00:06 +0800 Subject: [PATCH 0742/1247] kfdtest: update shader code for gfx10.3 kfdtest s_store_* instruction set was retired from gfx10.3 Signed-off-by: Chengming Gui Change-Id: Ibe41a3fe7e053fb345b1af6ad4abc22a0885bc81 --- tests/kfdtest/src/KFDMemoryTest.cpp | 91 +++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index aa816b84e4..f5e102be55 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -131,6 +131,26 @@ type(CS)\n\ end\n\ "; +const char* gfx10_PollMemory = +"\ +shader ReadMemory\n\ +wave_size(32)\n\ +type(CS)\n\ +/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ + s_movk_i32 s18, 0x5678\n\ + v_mov_b32 v0, s2\n\ + v_mov_b32 v1, s3\n\ + v_mov_b32 v2, 0x5678\n\ + LOOP:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 LOOP\n\ + flat_store_dword v[0,1], v2 slc\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + s_endpgm\n\ + end\n\ +"; + /* Input: A buffer of at least 3 dwords. * DW0: used as a signal. 0xcafe means it is signaled * DW1: Input buffer for device to read. @@ -159,6 +179,28 @@ POLLSIGNAL:\n\ end\n\ "; +const char* gfx10_CopyOnSignal = +"\ +shader CopyOnSignal\n\ +wave_size(32)\n\ +type(CS)\n\ +/* Assume input buffer in s0, s1 */\n\ + s_mov_b32 s18, 0xcafe\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ +POLLSIGNAL:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 POLLSIGNAL\n\ + s_load_dword s17, s[0:1], 0x4 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + v_mov_b32 v2, s17\n\ + flat_store_dword v[0,1], v2 offset:0x8 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_endpgm\n\ + end\n\ +"; + /* Input0: A buffer of at least 2 dwords. * DW0: used as a signal. Write 0xcafe to signal * DW1: Write to this buffer for other device to read. @@ -180,6 +222,26 @@ type(CS)\n\ end\n\ "; +const char* gfx10_WriteAndSignal = +"\ +shader WriteAndSignal\n\ +wave_size(32)\n\ +type(CS)\n\ +/* Assume input buffer in s0, s1 */\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ + v_mov_b32 v18, 0xbeef\n\ + flat_store_dword v18, v[0:1] offset:0x4 glc\n\ + v_mov_b32 v18, 0x1\n\ + flat_store_dword v18, v[2:3] offset:0x0 glc\n\ + v_mov_b32 v18, 0xcafe\n\ + flat_store_dword v18, v[0:1] offset:0x0 glc\n\ + s_endpgm\n\ + end\n\ +"; + //These gfx9_PullMemory, gfx9_CopyOnSignal, gfx9_WriteAndSignal shaders can be used by both gfx9 and gfx10 void KFDMemoryTest::SetUp() { @@ -313,7 +375,13 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + const char *pReadMemory; + if (m_FamilyId < FAMILY_NV) + pReadMemory = gfx9_PollMemory; + else + pReadMemory = gfx10_PollMemory; + + m_pIsaGen->CompileShader(pReadMemory, "ReadMemory", isaBuffer); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); @@ -1872,7 +1940,12 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_CopyOnSignal, "CopyOnSignal", isaBuffer); + const char *pCopyOnSignal; + if (m_FamilyId < FAMILY_NV) + pCopyOnSignal = gfx9_CopyOnSignal; + else + pCopyOnSignal = gfx10_CopyOnSignal; + m_pIsaGen->CompileShader(pCopyOnSignal, "CopyOnSignal", isaBuffer); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(buffer, NULL); dispatch0.Submit(queue); @@ -1989,7 +2062,12 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(nodes[0])); HsaMemoryBuffer isaBuffer(PAGE_SIZE, nodes[0], true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_CopyOnSignal, "CopyOnSignal", isaBuffer); + const char *pCopyOnSignal; + if (m_FamilyId < FAMILY_NV) + pCopyOnSignal = gfx9_CopyOnSignal; + else + pCopyOnSignal = gfx10_CopyOnSignal; + m_pIsaGen->CompileShader(pCopyOnSignal, "CopyOnSignal", isaBuffer); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, NULL); dispatch.Submit(queue); @@ -1997,7 +2075,12 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue0; ASSERT_SUCCESS(queue0.Create(nodes[1])); HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_WriteAndSignal, "WriteAndSignal", isaBuffer0); + const char *pWriteAndSignal; + if (m_FamilyId < FAMILY_NV) + pWriteAndSignal = gfx9_WriteAndSignal; + else + pWriteAndSignal = gfx10_WriteAndSignal; + m_pIsaGen->CompileShader(pWriteAndSignal, "WriteAndSignal", isaBuffer0); Dispatch dispatch0(isaBuffer0); dispatch0.SetArgs(buffer, mmioBase); dispatch0.Submit(queue0); From 12315704418ffd360b50387d5b4100efdf0ff300 Mon Sep 17 00:00:00 2001 From: "Tianci.Yin" Date: Tue, 20 Oct 2020 14:56:51 +0800 Subject: [PATCH 0743/1247] kfdtest: add DID for gfx1010 blockchain SKU Change-Id: Icd52c5db4dd975086fcfb13deb6727919c1f5809 Signed-off-by: Tianci.Yin --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index da1d7479de..c080cb4b35 100644 --- a/src/topology.c +++ b/src/topology.c @@ -227,6 +227,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x7312, 10, 1, 0, "Navi10", CHIP_NAVI10 }, { 0x7318, 10, 1, 0, "Navi10", CHIP_NAVI10 }, { 0x731A, 10, 1, 0, "Navi10", CHIP_NAVI10 }, + { 0x731E, 10, 1, 0, "Navi10", CHIP_NAVI10 }, { 0x731F, 10, 1, 0, "Navi10", CHIP_NAVI10 }, /* Navi14 */ { 0x7340, 10, 1, 2, "Navi14", CHIP_NAVI14 }, From aefc173da4f965ea3de7e749ab966d20a6878dfc Mon Sep 17 00:00:00 2001 From: Ashutosh Mishra Date: Fri, 8 May 2020 09:49:21 -0700 Subject: [PATCH 0744/1247] Standardizing Package names : Enables standards compliant package naming for debian and rpm Signed-off-by: Ashutosh Mishra Change-Id: I177af15ec7a3f909d05135be30a0acc7b0b20745 --- CMakeLists.txt | 144 ++++++++++++++++++-------------------- cmake_modules/utils.cmake | 23 ++++++ 2 files changed, 92 insertions(+), 75 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bd07ad6bd..684dabe59d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,19 +23,7 @@ ## ################################################################################ -cmake_minimum_required ( VERSION 3.5.0 ) - -## Cosmetic Cmake version warnings. -if(NOT EXISTS VERSION_WARNED) - set (VERSION_WARNED FALSE CACHE BOOL "") -endif() -if(${CMAKE_VERSION} VERSION_LESS "3.6.0" AND NOT ${VERSION_WARNED}) - message("Your CMake version is too old for full functionality. -Generated package file names may be incorrect. -Please update to CMake 3.6 or newer to generate correct package file names") - set( VERSION_WARNED TRUE CACHE BOOL "Suppress cosmetic build errors due to CMake version after first warning." FORCE ) - mark_as_advanced( FORCE VERSION_WARNED ) -endif() +cmake_minimum_required ( VERSION 3.6.3 ) set ( HSAKMT "hsakmt" ) set ( HSAKMT_PACKAGE "hsakmt-roct" ) @@ -226,74 +214,80 @@ configure_file ( libhsakmt.pc.in libhsakmt.pc @ONLY ) install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.pc DESTINATION ${CMAKE_INSTALL_DATADIR}/pkgconfig COMPONENT devel) +########################### +# Packaging directives +########################### # Use component packaging -set ( CPACK_COMPONENTS_GROUPING IGNORE ) -set ( CPACK_COMPONENTS_ALL binary devel ) -set ( CPACK_DEB_COMPONENT_INSTALL ON ) -set ( CPACK_RPM_COMPONENT_INSTALL ON ) - -## Add the packaging directives for the runtime library. -set ( CPACK_PACKAGE_VENDOR "Advanced Micro Devices, Inc." ) -set ( CPACK_PACKAGE_VERSION_MAJOR ${BUILD_VERSION_MAJOR} ) -set ( CPACK_PACKAGE_VERSION_MINOR ${BUILD_VERSION_MINOR} ) -set ( CPACK_PACKAGE_VERSION_PATCH ${BUILD_VERSION_PATCH} ) -set ( CPACK_PACKAGE_CONTACT "AMD GFX mailing list " ) -set ( CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md" ) - -## Component packaging, BINARY -set ( CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support\n This repository includes the user-mode API interfaces\n used to interact with the ROCk driver." ) -set ( CPACK_DEBIAN_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) -set ( CPACK_DEBIAN_BINARY_FILE_NAME "${CPACK_DEBIAN_BINARY_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}_amd64.deb" ) -set ( CPACK_RPM_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE} ) -set ( CPACK_RPM_BINARY_FILE_NAME "${CPACK_RPM_BINARY_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) - -#checking for OS-version -if( EXISTS "/etc/os-release" ) - file( STRINGS "/etc/os-release" DISTRO_ID REGEX "^ID=" ) - file( STRINGS "/etc/os-release" DISTRO_RELEASE REGEX "^VERSION_ID=" ) - string( REPLACE "ID=" "" DISTRO_ID "${DISTRO_ID}" ) - string( REPLACE "VERSION_ID=" "" DISTRO_RELEASE "${DISTRO_RELEASE}" ) - message( STATUS "Detected distribution: ${DISTRO_ID}:${DISTRO_RELEASE}" ) -elseif( EXISTS "/etc/centos-release" ) - # Example: CentOS release 6.10 (Final) - file( STRINGS "/etc/centos-release" DISTRO_FULL_STR REGEX "release" ) - string( REGEX MATCH "^[a-zA-Z]+" DISTRO_ID "${DISTRO_FULL_STR}" ) - string( TOLOWER "${DISTRO_ID}" DISTRO_ID ) - string( REGEX MATCH "[0-9]+" DISTRO_RELEASE "${DISTRO_FULL_STR}" ) - message( STATUS "Detected distribution: ${DISTRO_ID}:${DISTRO_RELEASE}" ) -else() - message( STATUS "Not able to detect OS" ) -endif() - -# Debian package specific variables -set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) -set ( CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm" ) -set ( CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1" ) - -# RPM package specific variables -if( DISTRO_ID MATCHES "sles" ) - set ( CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1") -elseif( DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6" ) - set ( CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl, libgcc" ) -else() - set ( CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc" ) -endif() -set ( CPACK_RPM_BINARY_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post" ) -set ( CPACK_RPM_BINARY_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun" ) +set(CPACK_COMPONENTS_GROUPING IGNORE) +set(CPACK_DEB_COMPONENT_INSTALL ON) +set(CPACK_RPM_COMPONENT_INSTALL ON) +set(CPACK_DEBIAN_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE}) +set(CPACK_RPM_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE}) +set(CPACK_PACKAGE_VENDOR "Advanced Micro Devices, Inc.") +set(CPACK_PACKAGE_VERSION_MAJOR ${VERSION_MAJOR}) +set(CPACK_PACKAGE_VERSION_MINOR ${VERSION_MINOR}) +set(CPACK_PACKAGE_VERSION_PATCH ${VERSION_PATCH}) +set(CPACK_PACKAGE_CONTACT "AMD GFX mailing list ") +set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md") +set(CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support\n This repository includes the user-mode API interfaces\n used to interact with the ROCk driver.") ## Component packaging, DEVEL -set ( CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package.\n This package contains the headers, pkgonfig and\n cmake files for the ROCT package" ) +set(CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package.\n This package contains the headers, pkgonfig and\n cmake files for the ROCT package") + +# Make proper version for appending +# Default Value is 99999, setting it first +set(ROCM_VERSION_FOR_PACKAGE "99999") +if(DEFINED ENV{ROCM_LIBPATCH_VERSION}) + set(ROCM_VERSION_FOR_PACKAGE $ENV{ROCM_LIBPATCH_VERSION}) +endif() # Debian package specific variables -set ( CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev" ) -set ( CPACK_DEBIAN_DEVEL_FILE_NAME "${CPACK_DEBIAN_DEVEL_PACKAGE_NAME}_${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}_amd64.deb" ) -set ( CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH})" ) +set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface") +set(CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm") +set(CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1") +set(CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev") +if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) + set(CPACK_DEBIAN_PACKAGE_RELEASE $ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) +else() + set(CPACK_DEBIAN_PACKAGE_RELEASE "local") +endif() # RPM package specific variables -set ( CPACK_RPM_DEVEL_PACKAGE_NAME "hsakmt-roct-devel" ) -set ( CPACK_RPM_DEVEL_FILE_NAME "${CPACK_RPM_DEVEL_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.x86_64.rpm" ) -set ( CPACK_RPM_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} = ${BUILD_VERSION_MAJOR}.${BUILD_VERSION_MINOR}.${BUILD_VERSION_PATCH}" ) +# get OS-info +get_os_info() +if(DISTRO_ID MATCHES "sles" ) + set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1") +elseif(DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6") + set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl, libgcc") +else() + set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc") +endif() +set(CPACK_RPM_BINARY_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post") +set(CPACK_RPM_BINARY_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun") +if(DEFINED CPACK_PACKAGING_INSTALL_PREFIX) + set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX} ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}") +endif() +set(CPACK_RPM_DEVEL_PACKAGE_NAME "hsakmt-roct-devel") # Since we changed the package name to match RPM specs, take care of older builds that had -dev installed -set ( CPACK_RPM_DEVEL_PACKAGE_OBSOLETES "hsakmt-roct-dev" ) +set(CPACK_RPM_DEVEL_PACKAGE_OBSOLETES "hsakmt-roct-dev") +if(DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE}) + set(CPACK_RPM_PACKAGE_RELEASE $ENV{CPACK_RPM_PACKAGE_RELEASE}) +else() + set(CPACK_RPM_PACKAGE_RELEASE "local") +endif() +# Set rpm distro +set(CPACK_RPM_PACKAGE_RELEASE_DIST ON) -include ( CPack ) +# Prepare final version for the CPACK use +set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${ROCM_VERSION_FOR_PACKAGE}") +set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") + +# Setting devel package dependendent version +set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE})") +set(CPACK_RPM_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} =${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") + +# Set the names now using CPACK utility +set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") +set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") + +include(CPack) diff --git a/cmake_modules/utils.cmake b/cmake_modules/utils.cmake index 096b3de7b7..28dafebe74 100644 --- a/cmake_modules/utils.cmake +++ b/cmake_modules/utils.cmake @@ -114,3 +114,26 @@ function ( get_version DEFAULT_VERSION_STRING ) set( VERSION_BUILD "${VERSION_BUILD}" PARENT_SCOPE ) endfunction() + +#get the OS version +function(get_os_info) +if( EXISTS "/etc/os-release") + file(STRINGS "/etc/os-release" DISTRO_ID REGEX "^ID=") + file(STRINGS "/etc/os-release" DISTRO_RELEASE REGEX "^VERSION_ID=") + string(REPLACE "ID=" "" DISTRO_ID "${DISTRO_ID}") + string(REPLACE "VERSION_ID=" "" DISTRO_RELEASE "${DISTRO_RELEASE}") + message(STATUS "Detected distribution: ${DISTRO_ID}:${DISTRO_RELEASE}") +elseif(EXISTS "/etc/centos-release" ) + # Example: CentOS release 6.10 (Final) + file(STRINGS "/etc/centos-release" DISTRO_FULL_STR REGEX "release") + string(REGEX MATCH "^[a-zA-Z]+" DISTRO_ID "${DISTRO_FULL_STR}") + string(TOLOWER "${DISTRO_ID}" DISTRO_ID) + string(REGEX MATCH "[0-9]+" DISTRO_RELEASE "${DISTRO_FULL_STR}") + message(STATUS "Detected distribution: ${DISTRO_ID}:${DISTRO_RELEASE}") +else() + message(STATUS "Not able to detect OS") +endif() + set(DISTRO_ID "${DISTRO_ID}" PARENT_SCOPE ) + set(DISTRO_RELEASE "${DISTRO_RELEASE}" PARENT_SCOPE ) + +endfunction() From e8c0426c54eea2929de0c84032d61874d9f3edeb Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Mon, 2 Nov 2020 22:45:54 -0500 Subject: [PATCH 0745/1247] libhsakmt: add Streaming Performance Monitors APIs Signed-off-by: Gang Ba Change-Id: Iab9a98fa2079b7cae7158c524479dfc3fa672407 --- CMakeLists.txt | 1 + include/hsakmt.h | 35 +++++++++++++ include/linux/kfd_ioctl.h | 100 +++++++++++++++++++++++++++++++++++- src/libhsakmt.ver | 3 ++ src/spm.c | 103 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 src/spm.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 684dabe59d..01b02dbf8d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/time.c" "src/topology.c" "src/rbtree.c" + "src/spm.c" "src/version.c") ## Declare the library target name diff --git a/include/hsakmt.h b/include/hsakmt.h index 237f80bbc7..83f41cac90 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -1209,6 +1209,41 @@ hsaKmtSetMemoryUserData( void * UserData //IN ); +/** + Acquire request exclusive use of SPM +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSPMAcquire( + HSAuint32 PreferredNode //IN + ); + + +/** + Release exclusive use of SPM +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSPMRelease( + HSAuint32 PreferredNode //IN + ); + +/** + Set up the destination user mode buffer for stream performance + counter data. +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSPMSetDestBuffer( + HSAuint32 PreferredNode, //IN + HSAuint32 SizeInBytes, //IN + HSAuint32 * timeout, //IN/OUT + HSAuint32 * SizeCopied, //OUT + void *DestMemoryAddress, //IN + bool *isSPMDataLoss //OUT + ); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 6a7dd1f44a..62a84e4e7e 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -574,6 +574,99 @@ struct kfd_ioctl_import_dmabuf_args { __u32 dmabuf_fd; /* to KFD */ }; +/* + * KFD SMI(System Management Interface) events + */ +enum kfd_smi_event { + KFD_SMI_EVENT_NONE = 0, /* not used */ + KFD_SMI_EVENT_VMFAULT = 1, /* event start counting at 1 */ + KFD_SMI_EVENT_THERMAL_THROTTLE = 2, + KFD_SMI_EVENT_GPU_PRE_RESET = 3, + KFD_SMI_EVENT_GPU_POST_RESET = 4, +}; + +#define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1)) + +struct kfd_ioctl_smi_events_args { + __u32 gpuid; /* to KFD */ + __u32 anon_fd; /* from KFD */ +}; + +/** + * kfd_ioctl_spm_op - SPM ioctl operations + * + * @KFD_IOCTL_SPM_OP_ACQUIRE: acquire exclusive access to SPM + * @KFD_IOCTL_SPM_OP_RELEASE: release exclusive access to SPM + * @KFD_IOCTL_SPM_OP_SET_DEST_BUF: set or unset destination buffer for SPM streaming + */ +enum kfd_ioctl_spm_op { + KFD_IOCTL_SPM_OP_ACQUIRE, + KFD_IOCTL_SPM_OP_RELEASE, + KFD_IOCTL_SPM_OP_SET_DEST_BUF +}; + +/** + * kfd_ioctl_spm_args - Arguments for SPM ioctl + * + * @op[in]: specifies the operation to perform + * @gpu_id[in]: GPU ID of the GPU to profile + * @dst_buf[in]: used for the address of the destination buffer + * in @KFD_IOCTL_SPM_SET_DEST_BUFFER + * @buf_size[in]: size of the destination buffer + * @timeout[in/out]: [in]: timeout in milliseconds, [out]: amount of time left + * `in the timeout window + * @bytes_copied[out]: amount of data that was copied to the previous dest_buf + * @has_data_loss: boolean indicating whether data was lost + * (e.g. due to a ring-buffer overflow) + * + * This ioctl performs different functions depending on the @op parameter. + * + * KFD_IOCTL_SPM_OP_ACQUIRE + * ------------------------ + * + * Acquires exclusive access of SPM on the specified @gpu_id for the calling process. + * This must be called before using KFD_IOCTL_SPM_OP_SET_DEST_BUF. + * + * KFD_IOCTL_SPM_OP_RELEASE + * ------------------------ + * + * Releases exclusive access of SPM on the specified @gpu_id for the calling process, + * which allows another process to acquire it in the future. + * + * KFD_IOCTL_SPM_OP_SET_DEST_BUF + * ----------------------------- + * + * If @dst_buf is NULL, the destination buffer address is unset and copying of counters + * is stopped. + * + * If @dst_buf is not NULL, it specifies the pointer to a new destination buffer. + * @buf_size specifies the size of the buffer. + * + * If @timeout is non-0, the call will wait for up to @timeout ms for the previous + * buffer to be filled. If previous buffer to be filled before timeout, the @timeout + * will be updated value with the time remaining. If the timeout is exceeded, the function + * copies any partial data available into the previous user buffer and returns success. + * The amount of valid data in the previous user buffer is indicated by @bytes_copied. + * + * If @timeout is 0, the function immediately replaces the previous destination buffer + * without waiting for the previous buffer to be filled. That means the previous buffer + * may only be partially filled, and @bytes_copied will indicate how much data has been + * copied to it. + * + * If data was lost, e.g. due to a ring buffer overflow, @has_data_loss will be non-0. + * + * Returns negative error code on failure, 0 on success. + */ +struct kfd_ioctl_spm_args { + __u64 dest_buf; + __u32 buf_size; + __u32 op; + __u32 timeout; + __u32 gpu_id; + __u32 bytes_copied; + __u32 has_data_loss; +}; + /* Register offset inside the remapped mmio page */ enum kfd_mmio_remap { @@ -741,7 +834,12 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x83, struct kfd_ioctl_cross_memory_copy_args) +#define AMDKFD_IOC_RLC_SPM \ + AMDKFD_IOWR(0x84, struct kfd_ioctl_spm_args) + + + #define AMDKFD_COMMAND_START_2 0x80 -#define AMDKFD_COMMAND_END_2 0x84 +#define AMDKFD_COMMAND_END_2 0x85 #endif diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index d47f68d2d0..97e2cf6beb 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -71,6 +71,9 @@ hsaKmtGetKernelDebugTrapVersionInfo; hsaKmtGetThunkDebugTrapVersionInfo; hsaKmtSetAddressWatch; hsaKmtClearAddressWatch; +hsaKmtSPMAcquire; +hsaKmtSPMRelease; +hsaKmtSPMSetDestBuffer; local: *; }; diff --git a/src/spm.c b/src/spm.c new file mode 100644 index 0000000000..c4eb33e5b1 --- /dev/null +++ b/src/spm.c @@ -0,0 +1,103 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include "linux/kfd_ioctl.h" +#include +#include + + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMAcquire(HSAuint32 PreferredNode) +{ + int ret; + struct kfd_ioctl_spm_args args = {0}; + uint32_t gpu_id; + + ret = validate_nodeid(PreferredNode, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode); + return ret; + } + + ret = HSAKMT_STATUS_SUCCESS; + args.op = KFD_IOCTL_SPM_OP_ACQUIRE; + args.gpu_id = gpu_id; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); + + return ret; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMSetDestBuffer(HSAuint32 PreferredNode, + HSAuint32 SizeInBytes, + HSAuint32 * timeout, + HSAuint32 * SizeCopied, + void *DestMemoryAddress, + bool *isSPMDataLoss) +{ + int ret; + struct kfd_ioctl_spm_args args = {0}; + uint32_t gpu_id; + + ret = HSAKMT_STATUS_SUCCESS; + + ret = validate_nodeid(PreferredNode, &gpu_id); + + args.timeout = *timeout; + args.dest_buf = (uint64_t)DestMemoryAddress; + args.buf_size = SizeInBytes; + args.op = KFD_IOCTL_SPM_OP_SET_DEST_BUF; + args.gpu_id = gpu_id; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); + + *SizeCopied = args.bytes_copied; + *isSPMDataLoss = args.has_data_loss; + *timeout = args.timeout; + + return ret; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtSPMRelease(HSAuint32 PreferredNode) +{ + int ret = HSAKMT_STATUS_SUCCESS; + struct kfd_ioctl_spm_args args = {0}; + uint32_t gpu_id; + + ret = validate_nodeid(PreferredNode, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, PreferredNode); + return ret; + } + + args.op = KFD_IOCTL_SPM_OP_RELEASE; + args.gpu_id = gpu_id; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_RLC_SPM, &args); + + return ret; +} + + From 089fdeb1fe754b7b093ebc07e8867d6cc83fd962 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 12 Nov 2020 12:29:17 -0500 Subject: [PATCH 0746/1247] CMakeLists: Fix RPM dependency declaration RPM needs _REQUIRES at the end, not _DEPENDS, and also requires a space before the version of the required package. Change-Id: I9dd70bd92fc2407b7e8b31e4d46df43c52438a65 Signed-off-by: Kent Russell --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01b02dbf8d..1558f16095 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,7 +285,7 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE})") -set(CPACK_RPM_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} =${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") +set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "${HSAKMT_PACKAGE} = ${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From bedecc5957e247ec76f74fe23511c62867b013ae Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Wed, 4 Nov 2020 19:10:48 -0500 Subject: [PATCH 0747/1247] libhsakmt: Create P2P links 1. Create P2P links 2. Determine FRAMEBUFFER_PUBLIC/PRIVATE only based host-accessibility, not peer-accesssibility Signed-off-by: Gang Ba Change-Id: I15fccdc60386b453e2a47849a16df15157324b21 --- src/fmm.c | 2 +- src/libhsakmt.h | 3 ++- src/topology.c | 68 +++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 57 insertions(+), 16 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 19fe48e0f9..172c1dc46b 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2178,7 +2178,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); - ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc); + ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc, NULL, NULL); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 1cae43a12a..c2fb51ba9a 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -157,7 +157,8 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - uint32_t *gpu_id, struct pci_ids pacc); + uint32_t *gpu_id, struct pci_ids pacc, + bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); bool topology_is_svm_needed(uint16_t device_id); diff --git a/src/topology.c b/src/topology.c index c080cb4b35..00132c4239 100644 --- a/src/topology.c +++ b/src/topology.c @@ -648,7 +648,6 @@ HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props) bool is_node_supported = true; uint32_t num_supported_nodes = 0; - assert(props); fd = fopen(KFD_SYSFS_PATH_SYSTEM_PROPERTIES, "r"); if (!fd) @@ -912,7 +911,9 @@ exit: HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, - struct pci_ids pacc) + struct pci_ids pacc, + bool *p2p_links, + uint32_t *num_p2pLinks) { FILE *fd; char *read_buf, *p, *envvar, dummy; @@ -974,7 +975,13 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->NumCaches = (uint32_t)prop_val; else if (strcmp(prop_name, "io_links_count") == 0) props->NumIOLinks = (uint32_t)prop_val; - else if (strcmp(prop_name, "cpu_core_id_base") == 0) + else if (strcmp(prop_name, "p2p_links_count") == 0) { + props->NumIOLinks += (uint32_t)prop_val; + if (num_p2pLinks) + *num_p2pLinks = (uint32_t)prop_val; + if (p2p_links) + *p2p_links = true; + } else if (strcmp(prop_name, "cpu_core_id_base") == 0) props->CComputeIdLo = (uint32_t)prop_val; else if (strcmp(prop_name, "simd_id_base") == 0) props->FComputeIdLo = (uint32_t)prop_val; @@ -1411,7 +1418,7 @@ static HSAKMT_STATUS topology_map_sysfs_to_user_node_id(uint32_t sys_node_id, ui */ static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, uint32_t iolink_id, - HsaIoLinkProperties *props) + HsaIoLinkProperties *props, bool p2pLink) { FILE *fd; char *read_buf, *p; @@ -1428,7 +1435,11 @@ static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, if (ret != HSAKMT_STATUS_SUCCESS) return ret; - snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, iolink_id); + if (p2pLink) + snprintf(path, 256, "%s/%d/p2p_links/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, iolink_id); + else + snprintf(path, 256, "%s/%d/io_links/%d/properties", KFD_SYSFS_PATH_NODES, sys_node_id, iolink_id); + fd = fopen(path, "r"); if (!fd) return HSAKMT_STATUS_ERROR; @@ -1736,6 +1747,9 @@ HSAKMT_STATUS topology_take_snapshot(void) struct pci_ids pacc; struct proc_cpuinfo *cpuinfo; const uint32_t num_procs = get_nprocs(); + uint32_t num_ioLinks; + bool p2p_links = false; + uint32_t num_p2pLinks = 0; cpuinfo = calloc(num_procs, sizeof(struct proc_cpuinfo)); if (!cpuinfo) { @@ -1761,7 +1775,8 @@ retry: for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_props[i].node, - &temp_props[i].gpu_id, pacc); + &temp_props[i].gpu_id, + pacc, &p2p_links, &num_p2pLinks); if (ret != HSAKMT_STATUS_SUCCESS) { free_properties(temp_props, i); goto err; @@ -1819,17 +1834,19 @@ retry: free_properties(temp_props, i + 1); goto err; } + num_ioLinks = temp_props[i].node.NumIOLinks - num_p2pLinks; + uint32_t link_id = 0; - if (temp_props[i].node.NumIOLinks) { - uint32_t sys_link_id = 0, link_id = 0; + if (num_ioLinks) { + uint32_t sys_link_id = 0; /* Parse all the sysfs specified io links. Skip the ones where the * remote node (node_to) is not accessible */ - while (sys_link_id < temp_props[i].node.NumIOLinks && + while (sys_link_id < num_ioLinks && link_id < sys_props.NumNodes - 1) { ret = topology_sysfs_get_iolink_props(i, sys_link_id++, - &temp_props[i].link[link_id]); + &temp_props[i].link[link_id], false); if (ret == HSAKMT_STATUS_NOT_SUPPORTED) { ret = HSAKMT_STATUS_SUCCESS; continue; @@ -1840,16 +1857,39 @@ retry: link_id++; } /* sysfs specifies all the io links. Limit the number to valid ones */ + num_ioLinks = link_id; + } + + if (num_p2pLinks) { + uint32_t sys_link_id = 0; + + /* Parse all the sysfs specified p2p links. + */ + while (sys_link_id < num_p2pLinks && + link_id < sys_props.NumNodes - 1) { + ret = topology_sysfs_get_iolink_props(i, sys_link_id++, + &temp_props[i].link[link_id], true); + if (ret == HSAKMT_STATUS_NOT_SUPPORTED) { + ret = HSAKMT_STATUS_SUCCESS; + continue; + } else if (ret != HSAKMT_STATUS_SUCCESS) { + free_properties(temp_props, i + 1); + goto err; + } + link_id++; + } temp_props[i].node.NumIOLinks = link_id; } } pci_ids_destroy(pacc); } - /* All direct IO links are created in the kernel. Here we need to - * connect GPU<->GPU or GPU<->CPU indirect IO links. - */ - topology_create_indirect_gpu_links(&sys_props, temp_props); + if (!p2p_links) { + /* All direct IO links are created in the kernel. Here we need to + * connect GPU<->GPU or GPU<->CPU indirect IO links. + */ + topology_create_indirect_gpu_links(&sys_props, temp_props); + } ret = topology_sysfs_get_generation(&gen_end); if (ret != HSAKMT_STATUS_SUCCESS) { From ac6e96e7a33549fc307d174a498ee63438eed641 Mon Sep 17 00:00:00 2001 From: Aakash Sudhanwa Date: Tue, 17 Nov 2020 19:51:31 -0500 Subject: [PATCH 0748/1247] Revert "CMakeLists: Fix RPM dependency declaration" This reverts commit 089fdeb1fe754b7b093ebc07e8867d6cc83fd962. Reason for revert: Change-Id: Ieaf2da0067c3e89577569c5d478c70b97ca5f5ca --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1558f16095..01b02dbf8d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,7 +285,7 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE})") -set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "${HSAKMT_PACKAGE} = ${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") +set(CPACK_RPM_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} =${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From 2651ce37d8388b302bd19da96e7ed2481d008304 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 23 Nov 2020 08:38:02 -0500 Subject: [PATCH 0749/1247] Look in /opt/rocm* for SMI for setting clocks to high Now that symlinks aren't necessarily guaranteed, use "find" to try to find the rocm-smi, and clarify the error message if it is not found Also tie in a fix for parsing the output now that the output has changed Change-Id: I2081442a71731c186c3ad00585a2ba6e8a8e5a28 --- tests/kfdtest/scripts/run_kfdtest.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 12ef8af51d..5aadce3dfd 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -244,16 +244,17 @@ while [ "$1" != "" ]; do done # If the SMI is missing, just report and continue +SMI="$(find /opt/rocm* -type l -name rocm-smi | tail -1)" if [ "$FORCE_HIGH" == "true" ]; then - if [ -e "/opt/rocm/bin/rocm-smi" ]; then - OLDPERF=$(/opt/rocm/bin/rocm-smi -p | awk '/Performance Level/ {print $NF; exit}') - $(/opt/rocm/bin/rocm-smi --setperflevel high &> /dev/null) + if [ -e "$SMI" ]; then + OLDPERF=$($SMI -p | awk '/Performance Level:/ {print $NF; exit}') + $($SMI --setperflevel high &> /dev/null) if [ $? != 0 ]; then echo "SMI failed to set perf level" OLDPERF="" fi else - echo "Unable to set clocks to high" + echo "Unable to set clocks to high, cannot find rocm-smi" fi fi @@ -263,5 +264,5 @@ runKfdTest # OLDPERF is only set if FORCE_HIGH and SMI both exist if [ -n "$OLDPERF" ]; then - /opt/rocm/bin/rocm-smi --setperflevel $OLDPERF &> /dev/null + $SMI --setperflevel $OLDPERF &> /dev/null fi From 8e94dde685a6bb3e93c4840fd6f4e18afdea6d88 Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Wed, 4 Nov 2020 19:14:44 -0500 Subject: [PATCH 0750/1247] kfdtest: check peer accessible with new function check GPU peer accessible with p2p_links in system Signed-off-by: Gang Ba Change-Id: I026f16564303b687811d6648f0b7f84be6819979 --- tests/kfdtest/src/KFDMemoryTest.cpp | 7 ++-- tests/kfdtest/src/KFDPerformanceTest.cpp | 49 ++++++++++++++++-------- tests/kfdtest/src/KFDQMTest.cpp | 15 +++++--- tests/kfdtest/src/KFDTestUtil.cpp | 44 ++++++++++++--------- tests/kfdtest/src/KFDTestUtil.hpp | 5 ++- 5 files changed, 73 insertions(+), 47 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index f5e102be55..ef58813329 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1992,7 +1992,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { const HsaNodeProperties *pNodeProperties; HSAuint32 *mmioBase = NULL; unsigned int *nullPtr = NULL; - std::vector nodes; + std::vector nodes; int numPeers; const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); @@ -2006,8 +2006,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && - !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + if (!m_NodeInfo.IsPeerAccessibleByNode(g_TestDstNodeId, g_TestNodeId)) { LOG() << "Skipping test: first GPU specified is not peer-accessible." << std::endl; return; } @@ -2018,7 +2017,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { } } else { HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, false); + m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU); if (nodes.size() < 2) { LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; LOG() << " or two GPUs are XGMI connected." << std::endl; diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index e80e4e71d6..32f342bd4d 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -145,7 +145,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { } const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); - std::vector nodes; + std::vector nodes; const bool isSpecified = g_TestDstNodeId != -1 && g_TestNodeId != -1; int numPeers = 0; @@ -153,14 +153,13 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { if (g_TestNodeId != g_TestDstNodeId) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if ((m_NodeInfo.IsGPUNodeLargeBar(g_TestNodeId) && - m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId)) || - m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) + if ((m_NodeInfo.IsPeerAccessibleByNode(g_TestNodeId, g_TestDstNodeId) && + m_NodeInfo.IsPeerAccessibleByNode(g_TestDstNodeId, g_TestNodeId))) numPeers = 2; } } else { - HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + nodes = m_NodeInfo.GetNodesWithGPU(); + numPeers = nodes.size(); } if (numPeers < 2) { @@ -168,7 +167,7 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { return; } - std::vector sysNodes(nodes); // include sysMem node 0... + std::vector sysNodes(nodes); // include sysMem node 0... sysNodes.insert(sysNodes.begin(),0); const int total_tests = 7; @@ -230,6 +229,9 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { if (n1 == n2) continue; + if (!m_NodeInfo.IsPeerAccessibleByNode(n2, n1)) + continue; + snprintf(str, sizeof(str), "[%d -> %d] ", n1, n2); msg << str << std::endl; testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); @@ -251,6 +253,10 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { for (unsigned j = i + 1; j < nodes.size(); j++) { HSAuint32 n2 = nodes[j]; + if (!m_NodeInfo.IsPeerAccessibleByNode(n2, n1) || + !m_NodeInfo.IsPeerAccessibleByNode(n1, n2)) + continue; + snprintf(str, sizeof(str), "[%d <-> %d] ", n1, n2); msg << str << std::endl; testNodeToNodes(n1, &n2, 1, test_suits[s][0], test_suits[s][1], size, &speed, &speed2, &msg); @@ -265,8 +271,8 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { LOG() << test_suits_string[s] << std::endl; msg << test_suits_string[s] << std::endl; /* Just use GPU nodes to do copy.*/ - std::vector &src = test_suits[s][0] != NONE ? nodes : sysNodes; - std::vector &dst = test_suits[s][1] != NONE ? nodes : sysNodes; + std::vector &src = test_suits[s][0] != NONE ? nodes : sysNodes; + std::vector &dst = test_suits[s][1] != NONE ? nodes : sysNodes; for (unsigned i = 0; i < src.size(); i++) { HSAuint32 n1 = src[i]; @@ -275,9 +281,18 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { int n = 0; char str[64]; - for (unsigned j = 0; j < dst.size(); j++) - if (dst[j] != n1) + for (unsigned j = 0; j < dst.size(); j++) { + if (dst[j] != n1) { + if (test_suits[s][0] != NONE && + !m_NodeInfo.IsPeerAccessibleByNode(dst[j], n1)) + continue; + if (test_suits[s][1] != NONE && + !m_NodeInfo.IsPeerAccessibleByNode(n1, dst[j])) + continue; n2[n++] = dst[j]; + } + } + /* At least 2 dst GPUs.*/ if (n < 2) continue; @@ -293,7 +308,6 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { (float)speed2 / 1024 << " GB/s" << std::endl; } } - exit: /* New line.*/ LOG() << std::endl << msg.str() << std::endl; @@ -309,17 +323,17 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) { } const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); - std::vector nodes; + std::vector nodes; - HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - int numPeers = m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + nodes = m_NodeInfo.GetNodesWithGPU(); + int numPeers = nodes.size(); if (numPeers < 2) { LOG() << "Skipping test: Need at least two large bar GPU or XGMI connected." << std::endl; return; } - std::vector sysNodes(nodes); // include sysMem node 0... + std::vector sysNodes(nodes); // include sysMem node 0... sysNodes.insert(sysNodes.begin(),0); /* size should be small.*/ @@ -352,6 +366,9 @@ TEST_F(KFDPerformanceTest, P2POverheadTest) { HSAuint32 n2 = sysNodes[j]; std::stringstream msg; + if (n1 != n2 && !m_NodeInfo.IsPeerAccessibleByNode(n2, n1)) + continue; + msg << test_suits_string[s] << "[" << n1 << " -> " << n2 << "]"; for (auto &size : sizeArray) { testNodeToNodes(n1, &n2, 1, test_suits[s], NONE, size, 0, 0, 0, 1, &time); diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index eab1b54885..ae561fccb2 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1625,7 +1625,7 @@ TEST_F(KFDQMTest, P2PTest) { LOG() << "Skipping test: At least two GPUs are required." << std::endl; return; } - std::vector nodes; + std::vector nodes; /* This test simulates RT team's P2P part in IPCtest: * @@ -1647,8 +1647,7 @@ TEST_F(KFDQMTest, P2PTest) { nodes.push_back(g_TestNodeId); nodes.push_back(g_TestDstNodeId); - if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId) && - !m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { + if (!m_NodeInfo.IsPeerAccessibleByNode(g_TestNodeId, g_TestDstNodeId)) { LOG() << "Skipping test: Dst GPU specified is not peer-accessible." << std::endl; return; } @@ -1657,8 +1656,7 @@ TEST_F(KFDQMTest, P2PTest) { return; } } else { - HSAint32 defaultGPU = m_NodeInfo.HsaDefaultGPUNode(); - m_NodeInfo.FindAccessiblePeers(&nodes, defaultGPU, true); + nodes = m_NodeInfo.GetNodesWithGPU(); if (nodes.size() < 2) { LOG() << "Skipping test: Test requires at least one large bar GPU." << std::endl; LOG() << " or two GPUs are XGMI connected." << std::endl; @@ -1685,7 +1683,7 @@ TEST_F(KFDQMTest, P2PTest) { EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, reinterpret_cast(&sysBuf))); EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL, - mapFlags, nodes.size(), &nodes[0])); + mapFlags, nodes.size(), (HSAuint32 *)&nodes[0])); #define MAGIC_NUM 0xdeadbeaf /* First GPU fills mem with MAGIC_NUM */ @@ -1707,6 +1705,11 @@ TEST_F(KFDQMTest, P2PTest) { } else { n = 2; next = nodes[i]; + + /* check if cur access next node */ + if (!m_NodeInfo.IsPeerAccessibleByNode(next, cur)) + continue; + ASSERT_SUCCESS(hsaKmtAllocMemory(next, size, memFlags, reinterpret_cast(&dst))); ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(dst, size, NULL)); } diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index b55cd80247..1e1b85abda 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -616,6 +616,23 @@ const bool HsaNodeInfo::IsGPUNodeLargeBar(int node) const { return false; } +const bool HsaNodeInfo::IsPeerAccessibleByNode(int peer, int node) const { + const HsaNodeProperties *pNodeProperties; + + pNodeProperties = GetNodeProperties(node); + if (pNodeProperties) { + HsaIoLinkProperties p2pLinksProperties[pNodeProperties->NumIOLinks]; + EXPECT_SUCCESS(hsaKmtGetNodeIoLinkProperties(node, + pNodeProperties->NumIOLinks, p2pLinksProperties)); + + for (unsigned link = 0; link < pNodeProperties->NumIOLinks; link++) + if (p2pLinksProperties[link].NodeTo == peer) + return true; + } + + return false; +} + const int HsaNodeInfo::FindLargeBarGPUNode() const { const std::vector gpuNodes = GetNodesWithGPU(); @@ -637,27 +654,16 @@ const bool HsaNodeInfo::AreGPUNodesXGMI(int node0, int node1) const { return false; } -int HsaNodeInfo::FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, - bool bidirectional) const { - peers->push_back(dstNode); - if (IsGPUNodeLargeBar(dstNode)) { - for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { - if (m_NodesWithGPU.at(i) == dstNode) - continue; +int HsaNodeInfo::FindAccessiblePeers(std::vector *peers, + HSAuint32 node) const { + peers->push_back(node); - if (!bidirectional || IsGPUNodeLargeBar(m_NodesWithGPU.at(i)) || - AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) - peers->push_back(m_NodesWithGPU.at(i)); - } - } else { - for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { - if (m_NodesWithGPU.at(i) == dstNode) - continue; + for (unsigned i = 0; i < m_NodesWithGPU.size(); i++) { + if (m_NodesWithGPU.at(i) == node) + continue; - if (AreGPUNodesXGMI(dstNode, m_NodesWithGPU.at(i))) - peers->push_back(m_NodesWithGPU.at(i)); - } + if (IsPeerAccessibleByNode(m_NodesWithGPU.at(i), node)) + peers->push_back(m_NodesWithGPU.at(i)); } - return peers->size(); } diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 28847f370e..e55ca95062 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -190,12 +190,13 @@ class HsaNodeInfo { void PrintNodeInfo() const; const bool IsGPUNodeLargeBar(int node) const; + const bool IsPeerAccessibleByNode(int peer, int node) const; // @brief Find the first available Large-BAR GPU node // @return: Node ID if successful or -1 const int FindLargeBarGPUNode() const; const bool AreGPUNodesXGMI(int node0, int node1) const; - int FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, - bool bidirectional) const; + int FindAccessiblePeers(std::vector *peers, + HSAuint32 node) const; }; #endif // __KFD__TEST__UTIL__H__ From 4bbfbe7789ce764128be3461ea06149529f3c22d Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Fri, 27 Nov 2020 15:04:49 -0500 Subject: [PATCH 0751/1247] kfdtest: increase default timeout to 10,000 The default kfdtest timeout is not enough for certain platforms, and tests are failing. Change-Id: I2027eadcbeb12a2fbbc9c55f92f31869fa13dbcb Signed-off-by: Philip Cox --- tests/kfdtest/src/KFDTestMain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index a8837f95c4..2b33ac153d 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -27,7 +27,7 @@ #include "GoogleTestExtension.hpp" #include "OSWrapper.hpp" -#define KFD_TEST_DEFAULT_TIMEOUT 2000 +#define KFD_TEST_DEFAULT_TIMEOUT 10000 std::ostream& operator << (std::ostream& out, TESTPROFILE profile) { switch (profile) { From 39386c03bf51e49c339b9c39bf9850f694a01fd1 Mon Sep 17 00:00:00 2001 From: changzhu Date: Thu, 26 Nov 2020 17:13:14 +0800 Subject: [PATCH 0752/1247] Add distinguish for iommuv2/dgpu_fallback when getting gpuName The memory tests between iommuv2 and dgpu_fallback are different.So it needs to ditinguish them. Change-Id: Icc64e9ae0fc1638c3d148795a5f247d9e5e8e503 Signed-off-by: changzhu --- tests/kfdtest/scripts/kfdtest.exclude | 10 ++++++++-- tests/kfdtest/scripts/run_kfdtest.sh | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 40183489ee..6a8d6e9b4f 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -173,10 +173,16 @@ FILTER[vega20]=\ "KFDEvictTest.BurstyTest:"\ "KFDQMTest.GPUDoorbellWrite" +FILTER[raven_dgpuFallback]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDMemoryTest.MMBench:"\ +"KFDMemoryTest.MemoryRegister:"\ +"KFDQMTest.SdmaConcurrentCopies" + FILTER[raven]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.QueueLatency:"\ -"KFDQMTest.SdmaEventInterrupt" +"KFDQMTest.SdmaConcurrentCopies:"\ +"KFDQMTest.MultipleCpQueuesStressDispatch" FILTER[renoir]=\ "$BLACKLIST_ALL_ASICS:"\ diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 5aadce3dfd..6311e1b9a5 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -144,6 +144,13 @@ getHsaNodes() { getNodeName() { local nodeId=$1; shift; local gpuName=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/name) + if [ "$gpuName" == "raven" ]; then + local CpuCoresCount=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep cpu_cores_count | awk '{print $2}') + local SimdCount=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep simd_count | awk '{print $2}') + if [ "$CpuCoresCount" -eq 0 ] && [ "$SimdCount" -gt 0 ]; then + gpuName="raven_dgpuFallback" + fi + fi echo "$gpuName" } From 9600760ff7f8e086645cce7da8abd94ec05c7e6a Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Tue, 3 Dec 2019 15:08:48 +0800 Subject: [PATCH 0753/1247] libhsakmt: add gfx1033 support This patch is to add Van Gogh support on thunk. Change-Id: I75819329b865e4c38c097e83e3a0cb4e4f566fa2 Signed-off-by: Huang Rui --- src/libhsakmt.h | 1 + src/queues.c | 7 +++++++ src/topology.c | 2 ++ 3 files changed, 10 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index c2fb51ba9a..34e30ec3ec 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -131,6 +131,7 @@ enum asic_family_type { CHIP_SIENNA_CICHLID, /* 18 */ CHIP_NAVY_FLOUNDER, /* 19 */ CHIP_DIMGREY_CAVEFISH, /* 20 */ + CHIP_VANGOGH, /* 21 */ CHIP_LAST }; diff --git a/src/queues.c b/src/queues.c index 29949c9092..e24e74aba7 100644 --- a/src/queues.c +++ b/src/queues.c @@ -183,6 +183,12 @@ const struct device_info dimgrey_cavefish_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info vangogh_device_info = { + .asic_family = CHIP_VANGOGH, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -205,6 +211,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_SIENNA_CICHLID] = &sienna_cichlid_device_info, [CHIP_NAVY_FLOUNDER] = &navy_flounder_device_info, [CHIP_DIMGREY_CAVEFISH] = &dimgrey_cavefish_device_info, + [CHIP_VANGOGH] = &vangogh_device_info, }; struct queue { diff --git a/src/topology.c b/src/topology.c index 00132c4239..e04468ec79 100644 --- a/src/topology.c +++ b/src/topology.c @@ -243,6 +243,8 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73AB, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, { 0x73AE, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, { 0x73BF, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + /* VanGogh */ + { 0x163F, 10, 3, 3, "VanGogh", CHIP_VANGOGH }, }; /* information from /proc/cpuinfo */ From 3ed8b96bf023ae16f660d6f0839328166f7d3e95 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Thu, 10 Dec 2020 18:02:41 +0800 Subject: [PATCH 0754/1247] kfdtest: remove unsupported modifier 'offset' fix v2: fix VGPR conflict v3: use s_addc_u32 to replace s_add_u32 Signed-off-by: Chengming Gui Change-Id: I8fe6bf1f5bf99544038ad16128c2bebd559d3da9 --- tests/kfdtest/src/KFDMemoryTest.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index ef58813329..286df56e28 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -185,9 +185,13 @@ shader CopyOnSignal\n\ wave_size(32)\n\ type(CS)\n\ /* Assume input buffer in s0, s1 */\n\ + s_add_u32 s2, s0, 0x8\n\ + s_addc_u32 s3, s1, 0x0\n\ s_mov_b32 s18, 0xcafe\n\ v_mov_b32 v0, s0\n\ v_mov_b32 v1, s1\n\ + v_mov_b32 v4, s2\n\ + v_mov_b32 v5, s3\n\ POLLSIGNAL:\n\ s_load_dword s16, s[0:1], 0x0 glc\n\ s_cmp_eq_i32 s16, s18\n\ @@ -195,7 +199,7 @@ POLLSIGNAL:\n\ s_load_dword s17, s[0:1], 0x4 glc\n\ s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ v_mov_b32 v2, s17\n\ - flat_store_dword v[0,1], v2 offset:0x8 glc\n\ + flat_store_dword v[4,5], v2 glc\n\ s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ s_endpgm\n\ end\n\ @@ -228,16 +232,20 @@ shader WriteAndSignal\n\ wave_size(32)\n\ type(CS)\n\ /* Assume input buffer in s0, s1 */\n\ + s_add_u32 s4, s0, 0x4\n\ + s_addc_u32 s5, s1, 0x0\n\ v_mov_b32 v0, s0\n\ v_mov_b32 v1, s1\n\ v_mov_b32 v2, s2\n\ v_mov_b32 v3, s3\n\ + v_mov_b32 v4, s4\n\ + v_mov_b32 v5, s5\n\ v_mov_b32 v18, 0xbeef\n\ - flat_store_dword v18, v[0:1] offset:0x4 glc\n\ + flat_store_dword v[4:5], v18 glc\n\ v_mov_b32 v18, 0x1\n\ - flat_store_dword v18, v[2:3] offset:0x0 glc\n\ + flat_store_dword v[2:3], v18 glc\n\ v_mov_b32 v18, 0xcafe\n\ - flat_store_dword v18, v[0:1] offset:0x0 glc\n\ + flat_store_dword v[0:1], v18 glc\n\ s_endpgm\n\ end\n\ "; From 742f718722a23ba9a226dcbe10354e59f268d289 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Tue, 15 Dec 2020 16:10:10 +0800 Subject: [PATCH 0755/1247] kfdtest: Take vram size into account when calculate buffer number. Vram size is relatively smaller in APU, e.g. 512MB. Current MMBench doesn't support small vram system. Running MMBench may have below errors: [ RUN ] KFDMemoryTest.MMBench [ ] Found VRAM of 512MB. [ ] Test (avg. ns) alloc mapOne umapOne mapAll umapAll free [ ] -------------------------------------------------------------------------- [ ] 4K-SysMem-noSDMA 4569 20098 1292 18835 926 2218 [ ] 64K-SysMem-noSDMA 12738 20469 1030 19201 1293 4560 [ ] 2M-SysMem-noSDMA 256384 21020 1022 20568 1196 36294 [ ] 32M-SysMem-noSDMA 4031812 83750 5406 61156 4312 535656 [ ] 1G-SysMem-noSDMA 129260000 427000 34000 390000 30000 18548000 [ ] -------------------------------------------------------------------------- [ ] 4K-VRAM-noSDMA 3594 19637 979 19624 1357 2829 [ ] 64K-VRAM-noSDMA 3540 21062 1407 19614 1654 3024 /home/foreman/build/hsakmt-roct-amdgpu-1.0.9/sources/libhsakmt/tests/kfdtest/src/KFDMemoryTest.cpp:1119: Failure Value of: (hsaKmtAllocMemory(allocNode, bufSize, memFlags, &bufs[i])) Actual: 6 Expected: HSAKMT_STATUS_SUCCESS Which is: 0 [ FAILED ] KFDMemoryTest.MMBench (723 ms) Fix this issue by changing buffer number calculation in MMBench. Change-Id: I5cce95707a048248f1e825c807586818619eddaf Signed-off-by: Yifan Zhang --- tests/kfdtest/src/KFDMemoryTest.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 286df56e28..d27d796eb7 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1099,6 +1099,7 @@ TEST_F(KFDMemoryTest, MMBench) { unsigned nBufs = TEST_NBUFS(testIndex); unsigned memType = TEST_MEMTYPE(testIndex); bool interleaveSDMA = TEST_SDMA(testIndex); + unsigned bufLimit; HSAuint64 allocTime, map1Time, unmap1Time, mapAllTime, unmapAllTime, freeTime; HSAuint32 allocNode; @@ -1119,6 +1120,13 @@ TEST_F(KFDMemoryTest, MMBench) { memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 0; memFlags.ui32.NonPaged = 1; + /* Upper limit of buffer number to fit 90% vram size */ + bufLimit = ((vramSizeMB << 20) * 9 / 10) / bufSize ; + if (bufLimit == 0) + continue; // skip when bufSize > vram + + /* When vram is too small to fit all the buffers, fill 90% vram size*/ + nBufs = (nBufs < bufLimit) ? nBufs : bufLimit; } /* Allocation */ From 4cf50fdeaabe32d6895896a04e0c3d80c340cf42 Mon Sep 17 00:00:00 2001 From: Chen Gong Date: Mon, 21 Dec 2020 18:50:22 +0800 Subject: [PATCH 0756/1247] libhsakmt: enhancing support to gfx1033 This patch make get_block_properties() function work on gfx1033 platform Change-Id: Ie5be7dfb38575eec8b39b91f3ee5b3a31abe8bd1 Signed-off-by: Chen Gong --- src/pmc_table.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pmc_table.c b/src/pmc_table.c index 5d30391497..6e76842cda 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -2132,6 +2132,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, break; case CHIP_NAVI10: case CHIP_NAVI14: + case CHIP_VANGOGH: *block = navi_blocks[block_id]; break; default: From f0e6e7ae17a5c1ccbebafea0e800d6eafc796f49 Mon Sep 17 00:00:00 2001 From: Gefei Jiang Date: Fri, 9 Oct 2020 18:00:02 -0700 Subject: [PATCH 0757/1247] CMakeList.txt -- Support Address Sanitize ROCMOPS-1249 append address sanitize flag Change-Id: Ie5d1e5b8b93022b80e0ca74106a16d53d52e41af Signed-off-by: Gefei Jiang --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01b02dbf8d..c868d8f9ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,11 @@ else () set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -g ) endif () +## Address Sanitize Flag +if (DEFINED $ENV{ADDRESS_SANITIZER}) + set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitizer=address ) +endif () + set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) ## Linker Flags From b92d28bd71d7427768fa7cdcae0125ebdf3eb37e Mon Sep 17 00:00:00 2001 From: Gefei Jiang Date: Wed, 14 Oct 2020 06:13:39 -0700 Subject: [PATCH 0758/1247] CMakeLists.txt: Address Sanitize Support ROCMOPS-1249 correct if statement and -f flag name Change-Id: I92e9aa30b1c81f855ad269c0c686ec1e136a85fd Signed-off-by: Gefei Jiang --- CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c868d8f9ff..58f8f2ee9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,8 +91,11 @@ else () endif () ## Address Sanitize Flag -if (DEFINED $ENV{ADDRESS_SANITIZER}) - set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitizer=address ) +if (DEFINED ADDRESS_SANITIZER) + set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) + # for debugging + # toDo: remove this line when merging to mainline + message( STATUS "ADDRESS_SANITIZE: HSAKMT_C_FLAGS=: ${HSAKMT_C_FLAGS}" ) endif () set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) From d3bc75d22968f5c12dfc988b8b1a5e7ebb3bc3c8 Mon Sep 17 00:00:00 2001 From: Gefei Jiang Date: Fri, 16 Oct 2020 12:49:26 -0700 Subject: [PATCH 0759/1247] CMakeLists.txt: Address Sanitizer Support 1. add sanitize flag to link flags 2. use ${ADDRESS_SANITIZER} as a condition to turn on/off instead of (DEFINED ADDRESS_SANITIZER). The latter will always turn the feature on regardless of the value as long as there is "-DADDRESS_SANITIZER in cmake command line,which will be an issue when merging to the mainline Amended: put -fsanitize=address at the begining in link flags Change-Id: I84df0e5b6d7fb8f02f18bf7961f25f15cac10443 Signed-off-by: Gefei Jiang --- CMakeLists.txt | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 58f8f2ee9f..b397ead855 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,19 +90,21 @@ else () set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -g ) endif () -## Address Sanitize Flag -if (DEFINED ADDRESS_SANITIZER) - set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) - # for debugging - # toDo: remove this line when merging to mainline - message( STATUS "ADDRESS_SANITIZE: HSAKMT_C_FLAGS=: ${HSAKMT_C_FLAGS}" ) -endif () - set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) +## Address Sanitize Flag +if ( ${ADDRESS_SANITIZER} ) + set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) + message( STATUS "ADDRESS_SANITIZE: HSAKMT_C_FLAGS=: ${HSAKMT_C_FLAGS}" ) + set ( HSAKMT_LINK_FLAGS "-fsanitize=address" ) +endif () + ## Linker Flags ## Add --enable-new-dtags to generate DT_RUNPATH -set (HSAKMT_LINK_FLAGS "-Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) +set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) + +# for debug +message( STATUS "ADDRESS_SANITIZE: HSAKMT_LINK_FLAGS=: ${HSAKMT_LINK_FLAGS}" ) ## Source files set ( HSAKMT_SRC "src/debug.c" From 3d9f60d7fe3f918685e04a5e8b10ec5016810176 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 23 Oct 2020 13:43:43 -0400 Subject: [PATCH 0760/1247] CMakeLists: Address-sanitizer fix and cleanup Move all the logic into 1 spot, and make sure -fsanitize=address is also passed to the library flags Change-Id: I7b60629d32df6436b5c7ad37997fe14ea48f5d72 --- CMakeLists.txt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b397ead855..7970114c2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,20 +92,18 @@ endif () set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) +## Linker Flags +## Add --enable-new-dtags to generate DT_RUNPATH +set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined") + ## Address Sanitize Flag if ( ${ADDRESS_SANITIZER} ) set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -fsanitize=address ) message( STATUS "ADDRESS_SANITIZE: HSAKMT_C_FLAGS=: ${HSAKMT_C_FLAGS}" ) - set ( HSAKMT_LINK_FLAGS "-fsanitize=address" ) + message( STATUS "ADDRESS_SANITIZE: HSAKMT_LINK_FLAGS=: ${HSAKMT_LINK_FLAGS}" ) endif () -## Linker Flags -## Add --enable-new-dtags to generate DT_RUNPATH -set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined" ) - -# for debug -message( STATUS "ADDRESS_SANITIZE: HSAKMT_LINK_FLAGS=: ${HSAKMT_LINK_FLAGS}" ) - ## Source files set ( HSAKMT_SRC "src/debug.c" "src/events.c" From 8e0a9aa417f119d62d21ba270fde014d982cd747 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 26 Oct 2020 13:49:01 -0400 Subject: [PATCH 0761/1247] Set -no-undefined properly if it's CC Address-sanitizer doesn't like it at all. And it's called differently under clang than gcc, so adjust accordingly Change-Id: Iebe8cd68618d3f7a4c310419c64b4f73d7ecfda4 --- CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7970114c2d..398cbfe272 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,7 +94,7 @@ set ( HSAKMT_LINKER_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/src/libhsakmt.ver" ) ## Linker Flags ## Add --enable-new-dtags to generate DT_RUNPATH -set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete -Wl,-no-undefined") +set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--version-script=${HSAKMT_LINKER_SCRIPT} -Wl,-soname=${HSAKMT_COMPONENT}.so.${LIB_VERSION_MAJOR} -Wl,-z,nodelete") ## Address Sanitize Flag if ( ${ADDRESS_SANITIZER} ) @@ -102,6 +102,12 @@ if ( ${ADDRESS_SANITIZER} ) set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -fsanitize=address ) message( STATUS "ADDRESS_SANITIZE: HSAKMT_C_FLAGS=: ${HSAKMT_C_FLAGS}" ) message( STATUS "ADDRESS_SANITIZE: HSAKMT_LINK_FLAGS=: ${HSAKMT_LINK_FLAGS}" ) +else () + if ( CMAKE_COMPILER_IS_GNUCC ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -Wl,-no-undefined ) + else () + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -Wl,-undefined,error ) + endif () endif () ## Source files From 9cca1216e9f00649e1848044ddae0aadf0949dcb Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 27 Oct 2020 12:15:16 -0400 Subject: [PATCH 0762/1247] kfdtest: Support address sanitizer in KFDTest Change-Id: Iee1182608ddc9896c82feb5004b3fe078d3d3223 --- tests/kfdtest/CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 9a0ac90960..c8b2823c45 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -24,7 +24,7 @@ # If environment variable DRM_DIR or LIBHSAKMT_PATH is set, the script # will pick up the corresponding libraries from those pathes. -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) project(KFDTest) @@ -100,6 +100,15 @@ include_directories(${SP3_DIR}) include_directories(${DRM_INCLUDE_DIRS}) +## Address Sanitize Flag +if ( ${ADDRESS_SANITIZER} ) + set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" -fsanitize=address ) + set ( CMAKE_EXE_LINKER_FLAGS -fsanitize=address ) + message( STATUS "ADDRESS_SANITIZE: CMAKE_CXX_FLAGS=: ${CMAKE_CXX_FLAGS}" ) + message( STATUS "ADDRESS_SANITIZE: CMAKE_EXE_LINKER_FLAGS=: ${CMAKE_EXE_LINKER_FLAGS}" ) +endif () + + set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/AqlQueue.cpp From f6f47aa43d803c9d8f86af6c24757604449d5c82 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 30 Oct 2020 07:16:49 -0400 Subject: [PATCH 0763/1247] Remove address-sanitizer debug messages Change-Id: I08509aaed36459329f0a65264e42f287c27f4a18 --- CMakeLists.txt | 2 -- tests/kfdtest/CMakeLists.txt | 2 -- 2 files changed, 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 398cbfe272..9cf0994b2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,8 +100,6 @@ set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--versio if ( ${ADDRESS_SANITIZER} ) set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -fsanitize=address ) - message( STATUS "ADDRESS_SANITIZE: HSAKMT_C_FLAGS=: ${HSAKMT_C_FLAGS}" ) - message( STATUS "ADDRESS_SANITIZE: HSAKMT_LINK_FLAGS=: ${HSAKMT_LINK_FLAGS}" ) else () if ( CMAKE_COMPILER_IS_GNUCC ) set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -Wl,-no-undefined ) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index c8b2823c45..9b1624148e 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -104,8 +104,6 @@ include_directories(${DRM_INCLUDE_DIRS}) if ( ${ADDRESS_SANITIZER} ) set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" -fsanitize=address ) set ( CMAKE_EXE_LINKER_FLAGS -fsanitize=address ) - message( STATUS "ADDRESS_SANITIZE: CMAKE_CXX_FLAGS=: ${CMAKE_CXX_FLAGS}" ) - message( STATUS "ADDRESS_SANITIZE: CMAKE_EXE_LINKER_FLAGS=: ${CMAKE_EXE_LINKER_FLAGS}" ) endif () From 92ad039915a807c80edc812bad007097fafaa23e Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 2 Nov 2020 06:30:51 -0500 Subject: [PATCH 0764/1247] kfdtest: Add sanitizer flags after C flags are set Otherwise they get overwritten Change-Id: I9042422d4515e7ac812ed34779906b0b2c44545c --- tests/kfdtest/CMakeLists.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 9b1624148e..ae96fdc0cc 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -100,13 +100,6 @@ include_directories(${SP3_DIR}) include_directories(${DRM_INCLUDE_DIRS}) -## Address Sanitize Flag -if ( ${ADDRESS_SANITIZER} ) - set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" -fsanitize=address ) - set ( CMAKE_EXE_LINKER_FLAGS -fsanitize=address ) -endif () - - set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/AqlQueue.cpp @@ -172,6 +165,12 @@ else () set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g" ) endif () +## Address Sanitize Flag +if ( ${ADDRESS_SANITIZER} ) + set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" -fsanitize=address ) + set ( CMAKE_EXE_LINKER_FLAGS -fsanitize=address ) +endif () + # link_directories() has to be put before add_executable() # The modules found by pkg_check_modules() in the default pkg config # path do not need to use link_directories() here. From 323bab073492d9c5bd39a9cc93eac067139ba05a Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 3 Nov 2020 09:24:14 -0500 Subject: [PATCH 0765/1247] kfdtest: Quote all CXX flags Otherwise it doesn't play nicely with -O2 Change-Id: I2e5b60c73ee1ec668b186088a4e2e3a03af65033 --- tests/kfdtest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index ae96fdc0cc..17d4b9bf4a 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -167,7 +167,7 @@ endif () ## Address Sanitize Flag if ( ${ADDRESS_SANITIZER} ) - set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" -fsanitize=address ) + set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address" ) set ( CMAKE_EXE_LINKER_FLAGS -fsanitize=address ) endif () From 3c8273c57bff9b9b081d690f75038e8369edeb5a Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 10 Nov 2020 07:26:51 -0500 Subject: [PATCH 0766/1247] libhsakmt: Explicitly set shared/static sanitizer flags Don't rely on default values for static/shared sanitizer flags, set them explicitly based on whether BUILD_SHARED_LIBS is defined or not Change-Id: Ifbfe95269d1cf184237643176a033a3ce98b62f9 --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9cf0994b2c..02e9e59715 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,11 @@ set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--versio if ( ${ADDRESS_SANITIZER} ) set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -fsanitize=address ) + if ( BUILD_SHARED_LIBS} ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -shared-libasan ) + else () + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -static-libasan ) + endif () else () if ( CMAKE_COMPILER_IS_GNUCC ) set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -Wl,-no-undefined ) From 1290d4d56cfd32d075ea041fc1447fb1b90b3aa6 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 18 Nov 2020 13:22:05 -0500 Subject: [PATCH 0767/1247] CMakeLists.txt: Use %{dist} in RPM naming The %{dist} suffix is part of the package name due to CPACK_RPM_PACKAGE_RELEASE_DIST, but the string provided to the "REQUIRES" field lacks it. Add it in here so the devel package can reference the thunk package correctly. Use a nice function suggested by Cole since CPACK_RPM_PACKAGE_RELEASE_DIST has caused some infra issues in the past This works for packages build in both Ubuntu and CentOS Also fix a mistake in the naming for DEBIAN packages, which should be a no-op since both the DEBIAN and RPM PACKAGE_RELEASE variables are the same right now Change-Id: I70659d2e1b6ff9027b8564ca4366d81b0c164760 Signed-off-by: Kent Russell --- CMakeLists.txt | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01b02dbf8d..1e8f204c02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -276,16 +276,25 @@ if(DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE}) else() set(CPACK_RPM_PACKAGE_RELEASE "local") endif() -# Set rpm distro -set(CPACK_RPM_PACKAGE_RELEASE_DIST ON) + +# 'dist' breaks manual builds on debian systems due to empty Provides +execute_process( COMMAND rpm --eval %{?dist} + RESULT_VARIABLE PROC_RESULT + OUTPUT_VARIABLE EVAL_RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE ) +message("RESULT_VARIABLE ${PROC_RESULT} OUTPUT_VARIABLE: ${EVAL_RESULT}") + +if ( PROC_RESULT EQUAL "0" AND NOT EVAL_RESULT STREQUAL "" ) + string ( APPEND CPACK_RPM_PACKAGE_RELEASE "%{?dist}" ) +endif() # Prepare final version for the CPACK use set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${ROCM_VERSION_FOR_PACKAGE}") set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version -set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE})") -set(CPACK_RPM_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} =${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") +set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_DEBIAN_PACKAGE_RELEASE})") +set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "${HSAKMT_PACKAGE} = ${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From bb7e7df02abdcd653abf69011cb881bdff74ffa4 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 6 Jan 2021 14:20:19 -0500 Subject: [PATCH 0768/1247] Remove extra brace, use libsan vs libasan Change-Id: I82e0d4fc8ea7dc292def7485bcf53c3849442c47 --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed8a5ea09e..a3a4d999c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,10 +100,10 @@ set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--versio if ( ${ADDRESS_SANITIZER} ) set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -fsanitize=address ) - if ( BUILD_SHARED_LIBS} ) - set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -shared-libasan ) + if ( BUILD_SHARED_LIBS ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -shared-libsan ) else () - set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -static-libasan ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -static-libsan ) endif () else () if ( CMAKE_COMPILER_IS_GNUCC ) From 7e184ebb3a8c69361034fa4e4d2b1d7295dc099d Mon Sep 17 00:00:00 2001 From: "Prike.Liang" Date: Fri, 15 Jan 2021 09:39:41 +0800 Subject: [PATCH 0769/1247] libhsakmt: add more gfx90c family device support This patch is to add Cezanne/Lucienne support on thunk. Change-Id: Icd9b9913fa87bbfe6c71b36a2892d6ddb73e3ddd Signed-off-by: Prike.Liang --- src/topology.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/topology.c b/src/topology.c index e04468ec79..7c3f47dc3f 100644 --- a/src/topology.c +++ b/src/topology.c @@ -209,6 +209,8 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x15D8, 9, 0, 2, "Raven", CHIP_RAVEN }, /* Renoir */ { 0x1636, 9, 0, 2, "Renoir", CHIP_RENOIR }, + { 0x1638, 9, 0, 2, "Renoir", CHIP_RENOIR }, + { 0x164C, 9, 0, 2, "Renoir", CHIP_RENOIR }, /* Vega20 */ { 0x66A0, 9, 0, 6, "Vega20", CHIP_VEGA20 }, { 0x66A1, 9, 0, 6, "Vega20", CHIP_VEGA20 }, From 18d9cca879c7e9774e301b37b643ee5df5f3fb5b Mon Sep 17 00:00:00 2001 From: changzhu Date: Mon, 18 Jan 2021 14:52:54 +0800 Subject: [PATCH 0770/1247] Remove MMBench test from kfdtest blacklist for gfx90c and gfx902 The MMBench issue has fixed by patch: kfdtest: Take vram size into account when calculate buffer number So it can remove it from kfdtest blacklist now. Change-Id: Ib918bca72adf28f4082248fae1e3287d395c32bf Signed-off-by: changzhu --- tests/kfdtest/scripts/kfdtest.exclude | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 6a8d6e9b4f..f526986c87 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -175,7 +175,6 @@ FILTER[vega20]=\ FILTER[raven_dgpuFallback]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDMemoryTest.MMBench:"\ "KFDMemoryTest.MemoryRegister:"\ "KFDQMTest.SdmaConcurrentCopies" @@ -187,10 +186,8 @@ FILTER[raven]=\ FILTER[renoir]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDEvictTest.*:"\ -"KFDQMTest.SdmaEventInterrupt:"\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.MMBench:"\ "KFDMemoryTest.SignalHandling" # KFDExceptionTest.* (KFD-435) From 7652932c383d611761ba6c07d4982229b87c7c8f Mon Sep 17 00:00:00 2001 From: Gang Ba Date: Tue, 19 Jan 2021 19:46:25 -0500 Subject: [PATCH 0771/1247] libhsakmt: Correct number of io_links Inside Docker, when limit GPU number to one, it may cause node numIOLinks bigger than total node number. Signed-off-by: Gang Ba Change-Id: Ib84f2f05f8e0c70e48b9043b79aec02b5a214bbe --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 7c3f47dc3f..1f93a31723 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1861,7 +1861,7 @@ retry: link_id++; } /* sysfs specifies all the io links. Limit the number to valid ones */ - num_ioLinks = link_id; + temp_props[i].node.NumIOLinks = link_id; } if (num_p2pLinks) { From 46c94662b7a371ca0e914bda85d9ba0e7de415bd Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Fri, 29 Jan 2021 07:03:21 -0500 Subject: [PATCH 0772/1247] libhsakmt: Add Missing gfx1030 DID The purpose of this patch is to add a missing device ID for gfx1030. The missing ID "0x73A1" is now added to the "topology.c" file. Signed-off-by: Ori Messinger Change-Id: I05a8a55e2c46f941a039fa72a6a5e76bf2a52736 --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index 1f93a31723..043189bb29 100644 --- a/src/topology.c +++ b/src/topology.c @@ -240,6 +240,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x7362, 10, 1, 1, "Navi12", CHIP_NAVI12 }, /* SIENNA_CICHLID */ { 0x73A0, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + { 0x73A1, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, { 0x73A2, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, { 0x73A3, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, { 0x73AB, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, From 526afa8394db3b84b6865a68aafa0b86184882fd Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Wed, 27 Jan 2021 13:53:44 -0600 Subject: [PATCH 0773/1247] kfdtest: Add KFDCWSRTest.InterruptRestore Test for HQD preemption during stalled context restore. Added for regression testing against new microcode. Change-Id: I13eb7d1c598062390e12cf8a5237e53b6489f232 Signed-off-by: Jay Cornwall --- tests/kfdtest/src/KFDCWSRTest.cpp | 68 +++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 5a4dbb4340..8306d05fff 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -91,6 +91,16 @@ LOOP: \n\ end \n\ "; +static const char* infinite_isa = \ +"\ +shader infinite_isa \n\ +wave_size(32) \n\ +type(CS) \n\ +LOOP: \n\ + s_branch LOOP \n\ +end \n\ +"; + void KFDCWSRTest::SetUp() { ROUTINE_START @@ -210,3 +220,61 @@ TEST_F(KFDCWSRTest, BasicTest) { TEST_END } + +/** + * KFDCWSRTest.InterruptRestore + * + * This test verifies that CP can preempt an HQD while it is restoring a dispatch. + * Create queue 1. + * Start a dispatch on queue 1 which runs indefinitely and fills all CU wave slots. + * Create queue 2, triggering context save on queue 1. + * Start a dispatch on queue 2 which runs indefinitely and fills all CU wave slots. + * Create queue 3, triggering context save and restore on queues 1 and 2. + * Preempt runlist. One or both queues must interrupt context restore to preempt. + */ + +TEST_F(KFDCWSRTest, InterruptRestore) { + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + if (m_FamilyId >= FAMILY_VI) { + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + + m_pIsaGen->CompileShader(infinite_isa, "infinite_isa", isaBuffer); + + PM4Queue queue1, queue2, queue3; + + ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); + + Dispatch *dispatch1, *dispatch2; + + dispatch1 = new Dispatch(isaBuffer); + dispatch2 = new Dispatch(isaBuffer); + + dispatch1->SetDim(0x10000, 1, 1); + dispatch2->SetDim(0x10000, 1, 1); + + dispatch1->Submit(queue1); + + ASSERT_SUCCESS(queue2.Create(defaultGPUNode)); + + dispatch2->Submit(queue2); + + // Give waves time to launch. + Delay(1); + + ASSERT_SUCCESS(queue3.Create(defaultGPUNode)); + + EXPECT_SUCCESS(queue1.Destroy()); + EXPECT_SUCCESS(queue2.Destroy()); + EXPECT_SUCCESS(queue3.Destroy()); + + delete dispatch1; + delete dispatch2; + } else { + LOG() << "Skipping test: No CWSR present for family ID 0x" << m_FamilyId << "." << std::endl; + } + + TEST_END +} From 0fc623e8730d929206024b43c5581517997524cf Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Wed, 21 Oct 2020 09:09:27 -0500 Subject: [PATCH 0774/1247] libhsakmt: Limit control stack size on gfx1032 Add to workaround list. Change-Id: I01855d3404203760507879db5af23455407ac450 Signed-off-by: Jay Cornwall --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index e24e74aba7..eb8e061673 100644 --- a/src/queues.c +++ b/src/queues.c @@ -449,7 +449,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + ctl_stack_size); if (q->dev_info->asic_family >= CHIP_NAVI10 && - q->dev_info->asic_family <= CHIP_NAVY_FLOUNDER) { + q->dev_info->asic_family <= CHIP_DIMGREY_CAVEFISH) { /* HW design limits control stack size to 0x7000. * This is insufficient for theoretical PM4 cases * but sufficient for AQL, limited by SPI events. From be096582474982946e7f76f042ea5c6950282f92 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 23 Mar 2020 21:00:07 -0400 Subject: [PATCH 0775/1247] Program mmCOMPUTE_PGM_RSRC3 on gfx90a Change-Id: If387d137ebd388f5aea930a5f7bca3413dcbfcce Signed-off-by: Yong Zhao --- tests/kfdtest/src/Dispatch.cpp | 8 ++++++++ tests/kfdtest/src/IsaGenerator.cpp | 1 + tests/kfdtest/src/KFDTestFlags.hpp | 1 + tests/kfdtest/src/KFDTestUtil.cpp | 4 +++- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index 6ed67ce39b..3aa3892f38 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -30,6 +30,8 @@ #include "KFDBaseComponentTest.hpp" +#define mmCOMPUTE_PGM_RSRC3 0x2e2d + Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset) :m_IsaBuf(isaBuf), m_IndirectBuf(PACKETTYPE_PM4, PAGE_SIZE / sizeof(unsigned int), isaBuf.Node()), m_DimX(1), m_DimY(1), m_DimZ(1), m_pArg1(NULL), m_pArg2(NULL), m_pEop(NULL), m_ScratchEn(false), @@ -218,6 +220,12 @@ void Dispatch::BuildIb() { m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC, ARRAY_SIZE(COMPUTE_PGM_RSRC))); + if (m_FamilyId == FAMILY_AL) { + const unsigned int COMPUTE_PGM_RSRC3[] = {9}; + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC3, COMPUTE_PGM_RSRC3, + ARRAY_SIZE(COMPUTE_PGM_RSRC3))); + } + m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_RESOURCE_LIMITS, COMPUTE_RESOURCE_LIMITS, ARRAY_SIZE(COMPUTE_RESOURCE_LIMITS))); m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_TMPRING_SIZE, COMPUTE_TMPRING_SIZE, diff --git a/tests/kfdtest/src/IsaGenerator.cpp b/tests/kfdtest/src/IsaGenerator.cpp index 9c7376a0a4..3534e02159 100644 --- a/tests/kfdtest/src/IsaGenerator.cpp +++ b/tests/kfdtest/src/IsaGenerator.cpp @@ -92,6 +92,7 @@ IsaGenerator* IsaGenerator::Create(unsigned int familyId) { case FAMILY_AI: case FAMILY_RV: case FAMILY_AR: + case FAMILY_AL: return new IsaGenerator_Gfx9; case FAMILY_NV: return new IsaGenerator_Gfx10; diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index 921b8bc832..9087ba23f8 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -59,6 +59,7 @@ enum KfdFamilyId { FAMILY_AI, // Arctic Islands FAMILY_RV, // Raven FAMILY_AR, // Arcturus + FAMILY_AL, // Aldebaran FAMILY_NV, // Navi10 }; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index b55cd80247..c4ff186686 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -149,8 +149,10 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { familyId = FAMILY_AI; if (props->EngineId.ui32.Stepping == 2) familyId = FAMILY_RV; - if (props->EngineId.ui32.Stepping == 8) + else if (props->EngineId.ui32.Stepping == 8) familyId = FAMILY_AR; + else if (props->EngineId.ui32.Stepping == 10) + familyId = FAMILY_AL; break; case 10: familyId = FAMILY_NV; From 51b6bcf40d39918fc935274a436ffa0fa69fd05d Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 12 Dec 2019 20:23:10 -0500 Subject: [PATCH 0776/1247] kfdtest: Support gfx90a Change-Id: I879ea534729e7adca4892be897dc86f6153aa190 Signed-off-by: Yong Zhao --- tests/kfdtest/scripts/kfdtest.exclude | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 9ecbcc1b55..cc307c1300 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -192,6 +192,11 @@ FILTER[arcturus]=\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDEvictTest.BurstyTest" +FILTER[aldebaran]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDExceptionTest.FaultStorm:"\ +"KFDEvictTest.BurstyTest" + FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDMemoryTest.MMBench" From 8881075ab2a3d547474dd92687e640a492d23029 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Mon, 20 Apr 2020 18:09:16 -0400 Subject: [PATCH 0777/1247] kfdtest: Improve the message when CWSR basic test does not pass This will give out more info. Change-Id: I407422b84bebdf39a886c57736093a035ff02118 Signed-off-by: Yong Zhao --- tests/kfdtest/src/KFDCWSRTest.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 5a4dbb4340..da2ad9bbd8 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -188,11 +188,13 @@ TEST_F(KFDCWSRTest, BasicTest) { int i; for (i = 0 ; i < wave_number; ++i) { if (result1[i] != count1) { - LOG() << "Dispatch 1, work item " << i << ' ' << result1[i] << std::endl; + LOG() << "Dispatch 1, work item [" << std::dec << i << "] " + << result1[i] << " != " << count1 << std::endl; break; } if (result2[i] != count2) { - LOG() << "Dispatch 2, work item " << i << ' ' << result2[i] << std::endl; + LOG() << "Dispatch 2, work item [" << std::dec << i << "] " + << result2[i] << " != " << count2 << std::endl; break; } } From 2464bfc7149b4003be849c85bffb71fb74893658 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Tue, 12 May 2020 15:12:52 -0400 Subject: [PATCH 0778/1247] libhsakmt: add new flag for memory mapped as uncached It is to provide an option to map specific memory as uncached on A+A HW platform. Signed-off-by: Eric Huang Change-Id: Ib665cb306a0e78aba3ea5ee2f0e46cb62ae139f8 --- include/hsakmttypes.h | 3 ++- include/linux/kfd_ioctl.h | 1 + src/fmm.c | 10 ++++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 7a7dc2ad10..b821268c50 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -541,7 +541,8 @@ typedef struct _HsaMemFlags // and optimal alignment requirements unsigned int FixedAddress : 1; // Allocate memory at specified virtual address. Fail if address is not free. unsigned int NoNUMABind: 1; // Don't bind system memory to a specific NUMA node - unsigned int Reserved : 15; + unsigned int Uncached: 1; // Caching flag for fine-grained memory on A+A HW platform + unsigned int Reserved : 14; } ui32; HSAuint32 Value; diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 6a7dd1f44a..aa00e919dd 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -479,6 +479,7 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE (1 << 28) #define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) #define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26) +#define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25) /* Allocate memory for later SVM (shared virtual memory) mapping. * diff --git a/src/fmm.c b/src/fmm.c index 6867c69599..5fe6518ea6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1334,8 +1334,11 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } - if (!flags.ui32.CoarseGrain || svm.disable_cache) + if (!flags.ui32.CoarseGrain || svm.disable_cache) { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + if (flags.ui32.Uncached) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; + } mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); @@ -1540,8 +1543,11 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, else aperture = svm.dgpu_alt_aperture; /* always coherent */ - if (!flags.ui32.CoarseGrain || svm.disable_cache) + if (!flags.ui32.CoarseGrain || svm.disable_cache) { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + if (flags.ui32.Uncached) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; + } ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); if (flags.ui32.AQLQueueMemory) From 87f62056f1b613e4e017ebe2480bb3d2d478573e Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Sun, 26 Apr 2020 18:17:32 -0400 Subject: [PATCH 0779/1247] kfdtest: Move the package definitions to the beginning in CMakeLists.txt This ensures that similiar logic stays together. Signed-off-by: Yong Zhao Change-Id: I32695d7d6a7366bcbf4169e22119d768d111c633 --- tests/kfdtest/CMakeLists.txt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index d92a6154ad..06eb821b17 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -38,6 +38,16 @@ set ( CPACK_PACKAGE_VERSION_MINOR "0" ) set ( CPACK_PACKAGE_VERSION_PATCH "0" ) set ( CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) +## Define default variable and variables for the optional build target hsakmt-dev +set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) +set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) +set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "Default packaging prefix." ) +set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) + +# Debian package specific variables +set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) + + #set ( CMAKE_VERBOSE_MAKEFILE on ) find_package(PkgConfig) @@ -57,15 +67,6 @@ else() include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) endif() -## Define default variable and variables for the optional build target hsakmt-dev -set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) -set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) -set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "Default packaging prefix." ) -set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) - -# Debian package specific variables -set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) - if( DEFINED ENV{LIBHSAKMT_PATH} ) set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} ) message ( "LIBHSAKMT_PATH environment variable is set" ) From 33c34506fa3a94c253502028a23c8f78dc318c66 Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Sun, 26 Apr 2020 18:21:01 -0400 Subject: [PATCH 0780/1247] kfdtest: Fix a path error in CMakeLists.txt PKG_CONFIG_PATH environment variable should be set to /lib/pkgconfig, because the *.pc file is located there. Signed-off-by: Yong Zhao Change-Id: Iec503b1c2409987e52fd88fea160c70762686a28 --- tests/kfdtest/CMakeLists.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 06eb821b17..63627ef04b 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -72,15 +72,14 @@ if( DEFINED ENV{LIBHSAKMT_PATH} ) message ( "LIBHSAKMT_PATH environment variable is set" ) else() if ( ${ROCM_INSTALL_PATH} ) - set ( ENV{PKG_CONFIG_PATH} ${ROCM_INSTALL_PATH} ) - pkg_check_modules(HSAKMT libhsakmt) + set ( ENV{PKG_CONFIG_PATH} ${ROCM_INSTALL_PATH}/share/pkgconfig ) else() - set ( ENV{PKG_CONFIG_PATH} /opt/rocm/libhsakmt/ ) - pkg_check_modules(HSAKMT libhsakmt) + set ( ENV{PKG_CONFIG_PATH} /opt/rocm/share/pkgconfig ) endif() + + pkg_check_modules(HSAKMT libhsakmt) + if( NOT HSAKMT_FOUND ) - set ( ENV{PKG_CONFIG_PATH} /opt/rocm/libhsakmt/ ) - pkg_check_modules(HSAKMT libhsakmt) set ( LIBHSAKMT_PATH $ENV{OUT_DIR} ) endif() endif() From 690a1484274a87d56f36cb849500d72a3153b31a Mon Sep 17 00:00:00 2001 From: Yong Zhao Date: Thu, 21 May 2020 23:29:41 -0400 Subject: [PATCH 0781/1247] kfdtest: Add a simple test case to test local memory Given the chance of local memory breakage is so high on emulators, we should use this simple test to check the local memory function. Signed-off-by: Yong Zhao Change-Id: Ifc48c12e11d75cc777ed7ea13e03bf54c2458e12 --- tests/kfdtest/scripts/kfdtest.exclude | 1 + tests/kfdtest/src/KFDLocalMemoryTest.cpp | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index cc307c1300..e20c5ad73f 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -20,6 +20,7 @@ FILTER[core_sws]=\ "KFDQMTest.AllSdmaQueues:"\ "KFDQMTest.AllXgmiSdmaQueues:"\ "KFDQMTest.AllQueues:"\ +"KFDLocalMemoryTest.AccessLocalMem:"\ "KFDEventTest.SignalEvent" # HWS mode diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index 4c86594ab9..33f33dc869 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -50,6 +50,28 @@ void KFDLocalMemoryTest::TearDown() { ROUTINE_END } +TEST_F(KFDLocalMemoryTest, AccessLocalMem) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + //local memory + HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false, true); + + PM4Queue queue; + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0, 0)); + + queue.Wait4PacketConsumption(); + + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + TEST_F(KFDLocalMemoryTest, BasicTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); From c861873dae35b31e896ade0c0f52e1be43990d23 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Mon, 8 Jun 2020 21:01:52 -0400 Subject: [PATCH 0782/1247] Add SP3 assembler support for gfx90a. Add updated SP3 static library with support for gfx90a and also add initial corresponding changes in kfdtest. Change-Id: I71bc6404ace7f9bf0dd74e712287136aa2b8a03d --- tests/kfdtest/CMakeLists.txt | 1 + tests/kfdtest/sp3/sp3.h | 2 + tests/kfdtest/src/IsaGenerator.cpp | 4 +- tests/kfdtest/src/IsaGenerator_Aldebaran.cpp | 113 +++++++++++++++++++ tests/kfdtest/src/IsaGenerator_Aldebaran.hpp | 49 ++++++++ tests/kfdtest/src/KFDMemoryTest.cpp | 31 ++++- 6 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 tests/kfdtest/src/IsaGenerator_Aldebaran.cpp create mode 100644 tests/kfdtest/src/IsaGenerator_Aldebaran.hpp diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 63627ef04b..6c12039143 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -109,6 +109,7 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/GoogleTestExtension.cpp src/IndirectBuffer.cpp src/IsaGenerator.cpp + src/IsaGenerator_Aldebaran.cpp src/IsaGenerator_Gfx10.cpp src/IsaGenerator_Gfx72.cpp src/IsaGenerator_Gfx8.cpp diff --git a/tests/kfdtest/sp3/sp3.h b/tests/kfdtest/sp3/sp3.h index d6235be5d8..e44ee406cf 100644 --- a/tests/kfdtest/sp3/sp3.h +++ b/tests/kfdtest/sp3/sp3.h @@ -108,9 +108,11 @@ struct sp3_shader { uint32_t nsgprs; ///< Number of scalar GPRs used. uint32_t nvgprs; ///< Number of vector GPRs used. uint32_t nsvgprs; ///< Number of shared vector GPRs used. + uint32_t naccvgprs; ///< Number of accumulator vector GPRs used (only available in certain projects). uint32_t nsgprs_manual_alloc; uint32_t nvgprs_manual_alloc; uint32_t nsvgprs_manual_alloc; + uint32_t naccvgprs_manual_alloc; uint32_t trap_present; uint32_t user_sgpr_count; uint32_t scratch_en; diff --git a/tests/kfdtest/src/IsaGenerator.cpp b/tests/kfdtest/src/IsaGenerator.cpp index 3534e02159..3e69b5f9df 100644 --- a/tests/kfdtest/src/IsaGenerator.cpp +++ b/tests/kfdtest/src/IsaGenerator.cpp @@ -30,6 +30,7 @@ #include "IsaGenerator_Gfx8.hpp" #include "IsaGenerator_Gfx9.hpp" #include "IsaGenerator_Gfx10.hpp" +#include "IsaGenerator_Aldebaran.hpp" #include "GoogleTestExtension.hpp" @@ -92,8 +93,9 @@ IsaGenerator* IsaGenerator::Create(unsigned int familyId) { case FAMILY_AI: case FAMILY_RV: case FAMILY_AR: - case FAMILY_AL: return new IsaGenerator_Gfx9; + case FAMILY_AL: + return new IsaGenerator_Aldbrn; case FAMILY_NV: return new IsaGenerator_Gfx10; diff --git a/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp b/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp new file mode 100644 index 0000000000..2fcb80fd32 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "IsaGenerator_Aldebaran.hpp" + +#include +#include + +const std::string IsaGenerator_Aldbrn::ASIC_NAME = "ALDEBARAN"; + +/* The binaries are generated from following ISA */ +#if 0 +/* flat_atomic_inc will not support by some PCIE, use flat_atomic_add instead */ +shader atomic_add +asic(ALDEBARAN) +type(CS) + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, 1 + flat_atomic_add v3, v[0:1], v2 slc glc + s_waitcnt 0 + s_endpgm +end + +shader copy_dword +asic(ALDEBARAN) +type(CS) +/* copy the parameters from scalar registers to vector registers */ + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 +/* copy a dword between the passed addresses */ + flat_load_dword v4, v[0:1] slc glc + s_waitcnt 0 + flat_store_dword v[2:3], v4 slc glc + s_endpgm +end + +shader main +asic(ALDEBARAN) +type(CS) +loop: + s_branch loop + s_endpgm +end + + +#endif + +const uint32_t IsaGenerator_Aldbrn::NOOP_ISA[] = { + 0xbf810000 +}; + +const uint32_t IsaGenerator_Aldbrn::COPY_DWORD_ISA[] = { + 0x7e000200, 0x7e020201, + 0x7e040202, 0x7e060203, + 0xdc530000, 0x047f0000, + 0xbf8c0000, 0xdc730000, + 0x007f0402, 0xbf810000 +}; + +const uint32_t IsaGenerator_Aldbrn::INFINITE_LOOP_ISA[] = { + 0xbf82ffff, 0xbf810000 +}; + +const uint32_t IsaGenerator_Aldbrn::ATOMIC_ADD_ISA[] = { + 0x7e000200, 0x7e020201, + 0x7e040281, 0xdd0b0000, + 0x037f0200, 0xbf8c0000, + 0xbf810000, 0x00000000 +}; + +void IsaGenerator_Aldbrn::GetNoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Aldbrn::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { + std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); +} + +void IsaGenerator_Aldbrn::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { + std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); +} + +void IsaGenerator_Aldbrn::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { + std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); +} + +const std::string& IsaGenerator_Aldbrn::GetAsicName() { + return ASIC_NAME; +} + diff --git a/tests/kfdtest/src/IsaGenerator_Aldebaran.hpp b/tests/kfdtest/src/IsaGenerator_Aldebaran.hpp new file mode 100644 index 0000000000..5571b91c26 --- /dev/null +++ b/tests/kfdtest/src/IsaGenerator_Aldebaran.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _ISAGENERATOR_ALDEBARAN_H_ +#define _ISAGENERATOR_ALDEBARAN_H_ + +#include +#include "IsaGenerator.hpp" + +class IsaGenerator_Aldbrn : public IsaGenerator { + public: + virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); + virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); + virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); + + protected: + virtual const std::string& GetAsicName(); + + private: + static const std::string ASIC_NAME; + + static const uint32_t NOOP_ISA[]; + static const uint32_t COPY_DWORD_ISA[]; + static const uint32_t INFINITE_LOOP_ISA[]; + static const uint32_t ATOMIC_ADD_ISA[]; +}; + +#endif // _ISAGENERATOR_ALDEBARAN_H_ diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index aa816b84e4..c5c0c0afde 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -108,6 +108,29 @@ wave_size(32)\n\ end\n\ "; +const char* aldbrn_ScratchCopyDword = +"\ +shader ScratchCopyDword\n\ +asic(ALDEBARAN)\n\ +type(CS)\n\ +/*copy the parameters from scalar registers to vector registers*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v2, s2\n\ + v_mov_b32 v3, s3\n\ +/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ + s_mov_b32 flat_scratch_lo, s4\n\ + s_mov_b32 flat_scratch_hi, s5\n\ +/*copy a dword between the passed addresses*/\n\ + flat_load_dword v4, v[0:1] slc\n\ + s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ + flat_store_dword v[2:3], v4 slc\n\ + \n\ + s_endpgm\n\ + \n\ +end\n\ +"; + /* Continuously poll src buffer and check buffer value @@ -650,8 +673,10 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { const char *pScratchCopyDword; if (m_FamilyId < FAMILY_AI) pScratchCopyDword = gfx8_ScratchCopyDword; - else if (m_FamilyId < FAMILY_NV) + else if (m_FamilyId < FAMILY_AL) pScratchCopyDword = gfx9_ScratchCopyDword; + else if (m_FamilyId == FAMILY_AL) + pScratchCopyDword = aldbrn_ScratchCopyDword; else pScratchCopyDword = gfx10_ScratchCopyDword; m_pIsaGen->CompileShader(pScratchCopyDword, "ScratchCopyDword", isaBuffer); @@ -1508,8 +1533,10 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { const char *pScratchCopyDword; if (m_FamilyId < FAMILY_AI) pScratchCopyDword = gfx8_ScratchCopyDword; - else if (m_FamilyId < FAMILY_NV) + else if (m_FamilyId < FAMILY_AL) pScratchCopyDword = gfx9_ScratchCopyDword; + else if (m_FamilyId == FAMILY_AL) + pScratchCopyDword = aldbrn_ScratchCopyDword; else pScratchCopyDword = gfx10_ScratchCopyDword; From 8c6dd3cbae4756158dd35599ca7585fb18ed651f Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Fri, 19 Jun 2020 15:52:08 -0400 Subject: [PATCH 0783/1247] libhsakmt: Add device ID used in Simnow Simnow simulator uses 0x7400 as gfx90a's device ID Signed-off-by: Amber Lin Change-Id: I0022509ef643760bc906e537b4fc64f1523fd8bf --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index 368be3aaa8..dc3c812378 100644 --- a/src/topology.c +++ b/src/topology.c @@ -228,6 +228,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x52, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x60, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x62, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x7400, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, /* Navi10 */ { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x7312, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, From da3abfb0f85daa494710a4dbd8e3c80598964e96 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Wed, 17 Jun 2020 17:31:03 -0400 Subject: [PATCH 0784/1247] Update build script for SP3 static library Update build script and CMakeLists_sp3.txt file as SP3 directory structure has changed. The SP3 source code with gfx90a suport is merged into a new branch mukjoshi/sp3_gfx90a. Please make sure to checkout this branch before using the build script to generate the static library. Change-Id: I2bf0ade8b2d254cd7648cc8a6d69a83ee51344cd --- .../kfdtest/sp3/lib_helper/CMakeLists_sp3.txt | 24 +++++---- tests/kfdtest/sp3/lib_helper/build_sp3.sh | 2 +- tests/kfdtest/sp3/sp3.h | 53 ++++++++++++++++++- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt index db59e3716e..ce8a3cb33f 100644 --- a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt +++ b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt @@ -39,7 +39,7 @@ set ( SCLIB_SRC ${PROJECT_SOURCE_DIR} ) #endif() include_directories(${SCLIB_SRC}/sp3) -include_directories(${SCLIB_SRC}/sp3/release_headers) +#include_directories(${SCLIB_SRC}/sp3/release_headers) include_directories(${SCLIB_SRC}/sp3/gen) set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-asic.c ) @@ -53,15 +53,17 @@ set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-cipher.c ) set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-vm.c ) aux_source_directory(${SCLIB_SRC}/sp3/gen SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/si SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/ci SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/gfx8 SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/gfx81 SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/gfx9 SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/gfx10 SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx81 SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx9 SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/release_headers/gfx10 SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/si/lib SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/ci/lib SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx8/lib SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx81/lib SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx9/lib SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx10/lib SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/aldbrn/lib SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx81/arch SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx9/arch SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx10/arch SRC_FILES) +aux_source_directory(${SCLIB_SRC}/sp3/backend/aldbrn/arch SRC_FILES) message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) @@ -70,7 +72,7 @@ message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) # message(STATUS "${file}") #endforeach() -set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU -fPIC -DGFX10_BUILD" ) +set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU -fPIC -DGFX101_BUILD -DALDBRN_BUILD" ) add_library(amdsp3 ${SRC_FILES}) diff --git a/tests/kfdtest/sp3/lib_helper/build_sp3.sh b/tests/kfdtest/sp3/lib_helper/build_sp3.sh index 7cd20ccfb5..f93f145da6 100755 --- a/tests/kfdtest/sp3/lib_helper/build_sp3.sh +++ b/tests/kfdtest/sp3/lib_helper/build_sp3.sh @@ -44,7 +44,7 @@ popd rsync --progress -a build/libamdsp3.a $LIB_OUTPUT # Put the intermediate header files in the current folder for further processing -rsync --progress -a $SP3_PROJECT/sp3/sp3.h . +rsync --progress -a $SP3_PROJECT/sp3/public/lib/sp3.h . # Remove the build folder and CMakeLists.txt put into SP source folder rm -r build diff --git a/tests/kfdtest/sp3/sp3.h b/tests/kfdtest/sp3/sp3.h index e44ee406cf..513167d595 100644 --- a/tests/kfdtest/sp3/sp3.h +++ b/tests/kfdtest/sp3/sp3.h @@ -54,7 +54,9 @@ enum sp3_shtype { SP3_SHTYPE_HS = 4, SP3_SHTYPE_LS = 5, SP3_SHTYPE_CS = 6, +#ifdef NAVI10LITE_BUILD SP3_SHTYPE_ACV = 7, +#endif }; /// Assorted constants used by sp3 API. @@ -107,7 +109,7 @@ struct sp3_shader { uint32_t size; ///< Size of the compiled shader, in 32-bit words. uint32_t nsgprs; ///< Number of scalar GPRs used. uint32_t nvgprs; ///< Number of vector GPRs used. - uint32_t nsvgprs; ///< Number of shared vector GPRs used. + uint32_t nsvgprs; ///< Number of shared vector GPRs used (only available in certain projects). uint32_t naccvgprs; ///< Number of accumulator vector GPRs used (only available in certain projects). uint32_t nsgprs_manual_alloc; uint32_t nvgprs_manual_alloc; @@ -211,6 +213,13 @@ SP3_EXPORT struct sp3_context *sp3_new(void); /// /// Currently supported options: /// +/// stdlib (string) -- absolute path to standard library files. May be a colon-separated list +/// of paths that will be used to search for stdlib files. Used by sp3_parse_library(). +/// +/// The following options are deprecated because they take integer arguments; you should use +/// sp3_set_option_int() for these settings going forward. They will continue to be accepted by +/// this API to support legacy users. +/// /// Werror (boolean) -- indicates whether warnings should be treated as errors. /// /// wave_size (integer) -- sets the wave size being used by the draw calls that will be using @@ -222,11 +231,53 @@ SP3_EXPORT struct sp3_context *sp3_new(void); /// /// omit_code_end (boolean) -- omit generation of the S_CODE_END footer. /// +/// allow_raw_bits (boolean) -- allow use of the raw_bits() function in sp3 shaders. This is a +/// dangerous option to allow in general so you must explicitly enable this option, otherwise +/// the raw_bits() function will always error out. +/// SP3_EXPORT void sp3_set_option( struct sp3_context *state, const char *option, const char *value); +/// Set option for sp3. +/// +/// @param state sp3 context. +/// @param option Option name. Unknown options will raise an error. +/// @param value Option value. +/// +/// Currently supported options: +/// +/// Werror (boolean) -- indicates whether warnings should be treated as errors. +/// +/// wave_size (integer) -- sets the wave size being used by the draw calls that will be using +/// this shader. Ignored in certain ASICs. You may set this to 32, 64 or the special value 0 +/// to indicate no preference on wave size. The shader will be checked to ensure it is +/// compatible with the size specified here. +/// +/// omit_version (boolean) -- omit generation of the S_VERSION opcode. +/// +/// omit_code_end (boolean) -- omit generation of the S_CODE_END footer. +/// +/// allow_raw_bits (boolean) -- allow use of the raw_bits() function in sp3 shaders. This is a +/// dangerous option to allow in general so you must explicitly enable this option, otherwise +/// the raw_bits() function will always error out. +/// +/// secure_mode (boolean) -- run in secure mode. Disables macro language features in assembly +/// path including calls to custom functions. Useful if sp3 is used as a backend to a web-based +/// assembly tool. +/// +/// debug_encoding (boolean) -- if true, debug encoding selection logic for assembly. Only +/// supported in 10.4+ backends. +/// +/// no_vs_export_check (boolean) -- if true, disable VS export sanity check. Only supported in +/// 10.4+ backends. +/// +SP3_EXPORT void sp3_set_option_int( + struct sp3_context *state, + const char *option, + int32_t value); + /// Parse a file into a context. /// /// Use sp3_compile to generate binary microcode after the shader is parsed. From 97ae33f9de9b3ecca7398291f1a7ca276fd9f7c8 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 9 Jul 2020 11:38:57 -0500 Subject: [PATCH 0785/1247] Add gfx90a Gopher LSE DID (0x54) Change-Id: Ic0a1e3d01373e0d6ba58e42188dced394423de82 Signed-off-by: Oak Zeng --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index dc3c812378..eb212f6c5c 100644 --- a/src/topology.c +++ b/src/topology.c @@ -226,6 +226,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x50, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x51, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x52, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x54, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x60, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x62, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x7400, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, From f398d6d20408c3d23974b66b89db719a03ba152c Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 16 Jul 2020 15:28:38 -0400 Subject: [PATCH 0786/1247] libhsakmt: add host trap send Adding host trap send command. Signed-off-by: Jonathan Kim Change-Id: I291c13f5905e00bc6685a980284a6abd0c98da78 --- include/hsakmt.h | 10 ++++++++++ include/linux/kfd_ioctl.h | 9 +++++++++ src/debug.c | 21 +++++++++++++++++++++ src/libhsakmt.ver | 1 + 4 files changed, 41 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index 237f80bbc7..dd59962bca 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -844,6 +844,16 @@ hsaKmtGetQueueSnapshot( HSAuint32 *QssEntries // IN/OUT ); +/** + Send the host trap +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSendHostTrap( + HSAuint32 NodeId, //IN + HSAuint32 Pid //IN + ); + /** Set the trap override mask. When debug trap is enabled by hsaKmtEnableDebugTrap() each wave launched has its initial diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index aa00e919dd..e2c4467cbd 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -224,6 +224,7 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_DBG_EV_STATUS_VMFAULT 2 #define KFD_DBG_EV_STATUS_SUSPENDED 4 #define KFD_DBG_EV_STATUS_NEW_QUEUE 8 +#define KFD_DBG_EV_STATUS_HOST_TRAP_TIMEDOUT 16 #define KFD_DBG_EV_FLAG_CLEAR_STATUS 1 #define KFD_INVALID_QUEUEID 0xffffffff @@ -309,6 +310,14 @@ struct kfd_ioctl_dbg_wave_control_args { */ #define KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH 9 +/* KFD_IOC_DBG_SEND_HOST_TRAP: + * ptr: unused + * data1: unused + * data2: unused + * data3: unused + */ +#define KFD_IOC_DBG_TRAP_SEND_HOST_TRAP 10 + struct kfd_ioctl_dbg_trap_args { __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ __u32 pid; /* to KFD */ diff --git a/src/debug.c b/src/debug.c index caddc79a8e..5ed6631932 100644 --- a/src/debug.c +++ b/src/debug.c @@ -731,3 +731,24 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtClearAddressWatch( NULL); return result; } + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSendHostTrap( + HSAuint32 NodeId, //IN + HSAuint32 Pid //IN + ) +{ + int result; + + result = debug_trap(NodeId, + KFD_IOC_DBG_TRAP_SEND_HOST_TRAP, + 0, + 0, + 0, + Pid, + 0, + NULL); + + return result; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index d47f68d2d0..a3203e58c3 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -61,6 +61,7 @@ hsaKmtEnableDebugTrap; hsaKmtEnableDebugTrapWithPollFd; hsaKmtDisableDebugTrap; hsaKmtQueryDebugEvent; +hsaKmtSendHostTrap; hsaKmtGetQueueSnapshot; hsaKmtSetWaveLaunchTrapOverride; hsaKmtSetWaveLaunchMode; From 198b5bd450328477ea43d5692180aafd4d66ebb9 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Thu, 27 Aug 2020 17:17:43 -0400 Subject: [PATCH 0787/1247] kfdtest: add function to determine XGMI link to cpu Signed-off-by: Eric Huang Change-Id: I7650f7857f0eecd2ad587634ae11c1cf5116bd97 --- tests/kfdtest/src/KFDTestUtil.cpp | 23 +++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 5 +++++ 2 files changed, 28 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index c4ff186686..c3a528d9c5 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -663,3 +663,26 @@ int HsaNodeInfo::FindAccessiblePeers(std::vector *peers, HSAuint32 ds return peers->size(); } + +const bool HsaNodeInfo::IsNodeXGMItoCPU(int node) const { + const HsaNodeProperties *pNodeProperties; + bool ret = false; + + pNodeProperties = GetNodeProperties(node); + if (pNodeProperties && pNodeProperties->NumIOLinks) { + HsaIoLinkProperties *IolinkProperties = new HsaIoLinkProperties[pNodeProperties->NumIOLinks]; + EXPECT_SUCCESS(hsaKmtGetNodeIoLinkProperties(node, pNodeProperties->NumIOLinks, IolinkProperties)); + + for (int linkId = 0; linkId < pNodeProperties->NumIOLinks; linkId++) { + EXPECT_EQ(node, IolinkProperties[linkId].NodeFrom); + const HsaNodeProperties *pNodeProperties0 = + GetNodeProperties(IolinkProperties[linkId].NodeTo); + if (pNodeProperties0->NumFComputeCores == 0 && + IolinkProperties[linkId].IoLinkType == HSA_IOLINK_TYPE_XGMI) + ret = true; + } + delete [] IolinkProperties; + } + + return ret; +} diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 28847f370e..e640d588fc 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -196,6 +196,11 @@ class HsaNodeInfo { const bool AreGPUNodesXGMI(int node0, int node1) const; int FindAccessiblePeers(std::vector *peers, HSAuint32 dstNode, bool bidirectional) const; + /* @brief: to determine if the node is XGMI-linked to CPU + * @param: node index of the node we are looking at + * @return: bool true or false + */ + const bool IsNodeXGMItoCPU(int node) const; }; #endif // __KFD__TEST__UTIL__H__ From 4b3b941bb36b4a35006c00c59a2488d60447c13a Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Thu, 8 Oct 2020 16:18:11 -0400 Subject: [PATCH 0788/1247] libhsakmt: add device id(0x46) for gfx90a mGPU model in topology Signed-off-by: Eric Huang Change-Id: I43f7c12906c408576e1eb55871d51e7a30569ede --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index eb212f6c5c..a884e41c67 100644 --- a/src/topology.c +++ b/src/topology.c @@ -230,6 +230,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x60, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x62, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x7400, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x46, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, /* Navi10 */ { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, { 0x7312, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, From 18ead8815c3eaf673a0b543ec23629254510b436 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 14 Oct 2020 11:07:15 -0400 Subject: [PATCH 0789/1247] KFDTest: fix an exception bug in P2PTest The largebar check will exit exceptionally from test when destination node is not set. Signed-off-by: Eric Huang Change-Id: I8bf0fed613250cc71468208e645fc562fb1a8757 --- tests/kfdtest/src/KFDQMTest.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index eab1b54885..71768cc2cf 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1671,18 +1671,13 @@ TEST_F(KFDQMTest, P2PTest) { HsaMemFlags memFlags = {0}; HsaMemMapFlags mapFlags = {0}; memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - memFlags.ui32.HostAccess = 1; + memFlags.ui32.HostAccess = 0; memFlags.ui32.NonPaged = 1; memFlags.ui32.NoNUMABind = 1; unsigned int end = size / sizeof(HSAuint32) - 1; - if (!m_NodeInfo.IsGPUNodeLargeBar(g_TestDstNodeId) && - m_NodeInfo.AreGPUNodesXGMI(g_TestNodeId, g_TestDstNodeId)) { - memFlags.ui32.HostAccess = 0; - } - /* 1. Allocate a system buffer and allow the access to GPUs */ - EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, memFlags, + EXPECT_SUCCESS(hsaKmtAllocMemory(0, size, m_MemoryFlags, reinterpret_cast(&sysBuf))); EXPECT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sysBuf, size, NULL, mapFlags, nodes.size(), &nodes[0])); From 731a06c7047a67f996b3e1d951b3607d43fdf313 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 13 Aug 2020 11:01:21 -0400 Subject: [PATCH 0790/1247] Fix GCC warning regarding strncpy in CPU info strlen(src) should not be used as the length in strncpy. Use memcpy since we know the length of the string, and ensure that we NULL-terminate regardless of length Signed-off-by: Kent Russell Change-Id: I21cc6d106510c69464e7ac9d3fc7da3a1e6d1a68 --- src/topology.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/topology.c b/src/topology.c index a884e41c67..e5fb85e406 100644 --- a/src/topology.c +++ b/src/topology.c @@ -888,13 +888,10 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, if (!strncmp("model name", read_buf, sizeof("model name") - 1)) { p = strchr(read_buf, ':'); p += 2; /* remove ": " */ - p_len = strlen(p); - if (p_len < HSA_PUBLIC_NAME_SIZE) { - /* -1 to remove \n from p */ - strncpy(cpuinfo[proc].model_name, p, p_len - 1); - cpuinfo[proc].model_name[p_len - 1] = '\0'; - } else - strncpy(cpuinfo[proc].model_name, p, HSA_PUBLIC_NAME_SIZE); + p_len = (strlen(p) > HSA_PUBLIC_NAME_SIZE ? + HSA_PUBLIC_NAME_SIZE : strlen(p)); + memcpy(cpuinfo[proc].model_name, p, p_len); + cpuinfo[proc].model_name[p_len - 1] = '\0'; continue; } From 5ae49f2321ae2287aeae8339941997a20fed8241 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 25 Nov 2019 13:24:36 -0500 Subject: [PATCH 0791/1247] libhsakmt: add kfd_ioctl.h svm and xnack support Add svm (shared virtual memory) range and xnack mode APIs. Change-Id: Ibd8d7fe566dc200730da0c892caa71aad7589ebd Signed-off-by: Philip Yang Signed-off-by: Alex Sierra --- include/linux/kfd_ioctl.h | 167 +++++++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index e2c4467cbd..a8be1df07d 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -638,6 +638,166 @@ struct kfd_ioctl_cross_memory_copy_args { __u64 bytes_copied; }; + +/* Guarantee host access to memory */ +#define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001 +/* Fine grained coherency between all devices with access */ +#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002 +/* Use any GPU in same hive as preferred device */ +#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004 +/* GPUs only read, allows replication */ +#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008 +/* Allow execution on GPU */ +#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010 + +/** + * kfd_ioctl_svm_op - SVM ioctl operations + * + * @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes + * @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes + */ +enum kfd_ioctl_svm_op { + KFD_IOCTL_SVM_OP_SET_ATTR, + KFD_IOCTL_SVM_OP_GET_ATTR +}; + +/** + * kfd_ioctl_svm_attr_type - SVM attribute types + * + * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: gpuid of the preferred location, 0 for + * system memory + * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: gpuid of the prefetch location, 0 for + * system memory. Setting this triggers an + * immediate prefetch (migration). + * @KFD_IOCTL_SVM_ATTR_ACCESS: + * @KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: + * @KFD_IOCTL_SVM_ATTR_NO_ACCESS: specify memory access for the gpuid given + * by the attribute value + * @KFD_IOCTL_SVM_ATTR_SET_FLAGS: bitmask of flags to set (see + * KFD_IOCTL_SVM_FLAG_...) + * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS: bitmask of flags to clear + * @KFD_IOCTL_SVM_ATTR_GRANULARITY: migration granularity + * (log2 num pages) + */ +enum kfd_ioctl_svm_attr_type { + KFD_IOCTL_SVM_ATTR_PREFERRED_LOC, + KFD_IOCTL_SVM_ATTR_PREFETCH_LOC, + KFD_IOCTL_SVM_ATTR_ACCESS, + KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE, + KFD_IOCTL_SVM_ATTR_NO_ACCESS, + KFD_IOCTL_SVM_ATTR_SET_FLAGS, + KFD_IOCTL_SVM_ATTR_CLR_FLAGS, + KFD_IOCTL_SVM_ATTR_GRANULARITY +}; + +/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations + * + * GPU IDs are used to specify GPUs as preferred and prefetch locations. + * Below definitions are used for system memory or for leaving the preferred + * location unspecified. + */ +enum kfd_ioctl_svm_location { + KFD_IOCTL_SVM_LOCATION_SYSMEM = 0, + KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff +}; + +/** + * kfd_ioctl_svm_attribute - Attributes as pairs of type and value + * + * The meaning of the @value depends on the attribute type. + * + * @type: attribute type (see enum @kfd_ioctl_svm_attr_type) + * @value: attribute value + */ +struct kfd_ioctl_svm_attribute { + __u32 type; + __u32 value; +}; + +/** + * kfd_ioctl_svm_args - Arguments for SVM ioctl + * + * @op specifies the operation to perform (see enum + * @kfd_ioctl_svm_op). @start_addr and @size are common for all + * operations. + * + * A variable number of attributes can be given in @attrs. + * @nattr specifies the number of attributes. New attributes can be + * added in the future without breaking the ABI. If unknown attributes + * are given, the function returns -EINVAL. + * + * @KFD_IOCTL_SVM_OP_SET_ATTR sets attributes for a virtual address + * range. It may overlap existing virtual address ranges. If it does, + * the existing ranges will be split such that the attribute changes + * only apply to the specified address range. + * + * @KFD_IOCTL_SVM_OP_GET_ATTR returns the intersection of attributes + * over all memory in the given range and returns the result as the + * attribute value. If different pages have different preferred or + * prefetch locations, 0xffffffff will be returned for + * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or + * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For + * @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be + * aggregated by bitwise AND. The minimum migration granularity + * throughout the range will be returned for + * @KFD_IOCTL_SVM_ATTR_GRANULARITY. + * + * Querying of accessibility attributes works by initializing the + * attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the + * GPUID being queried. Multiple attributes can be given to allow + * querying multiple GPUIDs. The ioctl function overwrites the + * attribute type to indicate the access for the specified GPU. + * + * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS is invalid for + * @KFD_IOCTL_SVM_OP_GET_ATTR. + */ +struct kfd_ioctl_svm_args { + __u64 start_addr; + __u64 size; + __u32 op; + __u32 nattr; + /* Variable length array of attributes */ + struct kfd_ioctl_svm_attribute attrs[0]; +}; + +/** + * kfd_ioctl_set_xnack_mode_args - Arguments for set_xnack_mode + * + * @xnack_enabled: [in/out] Whether to enable XNACK mode for this process + * + * @xnack_enabled indicates whether recoverable page faults should be + * enabled for the current process. 0 means disabled, positive means + * enabled, negative means leave unchanged. If enabled, virtual address + * translations on GFXv9 and later AMD GPUs can return XNACK and retry + * the access until a valid PTE is available. This is used to implement + * device page faults. + * + * On output, @xnack_enabled returns the (new) current mode (0 or + * positive). Therefore, a negative input value can be used to query + * the current mode without changing it. + * + * The XNACK mode fundamentally changes the way SVM managed memory works + * in the driver, with subtle effects on application performance and + * functionality. + * + * Enabling XNACK mode requires shader programs to be compiled + * differently. Furthermore, not all GPUs support changing the mode + * per-process. Therefore changing the mode is only allowed while no + * user mode queues exist in the process. This ensure that no shader + * code is running that may be compiled for the wrong mode. And GPUs + * that cannot change to the requested mode will be disabled by + * failing subsequent requests to create user mode queues. + * + * This ioctl returns the status of the requested xnack mode. + * + * GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM. + * + * Return: 0 on success, -errno on failure + */ +struct kfd_ioctl_set_xnack_mode_args { + __s32 xnack_enabled; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -735,8 +895,13 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) +#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args) + +#define AMDKFD_IOC_SET_XNACK_MODE \ + AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1F +#define AMDKFD_COMMAND_END 0x22 /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ From a352639df5b281100ad4beed24756312d449c458 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Mon, 23 Nov 2020 20:55:30 -0600 Subject: [PATCH 0792/1247] libhsakmt: add API to support svm and xnack Add function definitions to support SVM (shared virtual memory) and xnack set. Change-Id: Ia97ad9d0c449d8d500d799f702e1a58e87d65a56 Signed-off-by: Alex Sierra Signed-off-by: Philip Yang Signed-off-by: Felix Kuehling --- include/hsakmt.h | 31 +++++++++++++++++++++++++++++++ include/hsakmttypes.h | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index dd59962bca..51f2467d29 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -1219,6 +1219,37 @@ hsaKmtSetMemoryUserData( void * UserData //IN ); +/* Helper functions for calling KFD SVM ioctl */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSVMSetAttr( + void *start_addr, // IN: Start of the virtual address range (page-aligned) + HSAuint64 size, // IN: size (page-aligned) + unsigned int nattr, // IN: number of attributes + HSA_SVM_ATTRIBUTE *attrs // IN: array of attributes +); + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSVMGetAttr( + void *start_addr, // IN: Start of the virtual address range (page-aligned) + HSAuint64 size, // IN: size (page aligned) + unsigned int nattr, // IN: number of attributes + HSA_SVM_ATTRIBUTE *attrs // IN/OUT: array of attributes +); + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetXNACKMode( + HSAint32 enable // IN: enable/disable XNACK node. +); + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetXNACKMode( + HSAint32 * enable // OUT: returns XNACK value. +); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index b821268c50..4d7b7d8d73 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1291,6 +1291,35 @@ typedef struct _HsaMemoryRange { HSAuint64 SizeInBytes; // Size of above memory } HsaMemoryRange; +typedef enum _HSA_SVM_FLAGS { + HSA_SVM_FLAG_HOST_ACCESS = 0x00000001, // Guarantee host access to memory + HSA_SVM_FLAG_COHERENT = 0x00000002, // Fine grained coherency between all devices with access + HSA_SVM_FLAG_HIVE_LOCAL = 0x00000004, // Use any GPU in same hive as preferred device + HSA_SVM_FLAG_GPU_RO = 0x00000008, // GPUs only read, allows replication + HSA_SVM_FLAG_GPU_EXEC = 0x00000010, // Allow execution on GPU +} HSA_SVM_FLAGS; + +typedef enum _HSA_SVM_ATTR_TYPE { + HSA_SVM_ATTR_PREFERRED_LOC, // gpuid of the preferred location, 0 for + // system memory, INVALID_NODEID for + // "don't care" + HSA_SVM_ATTR_PREFETCH_LOC, // gpuid of the prefetch location, 0 for + // system memory. Setting this triggers an + // immediate prefetch (migration) + HSA_SVM_ATTR_ACCESS, + HSA_SVM_ATTR_ACCESS_IN_PLACE, + HSA_SVM_ATTR_NO_ACCESS, // specify memory access for the gpuid given + // by the attribute value + HSA_SVM_ATTR_SET_FLAGS, // bitmask of flags to set (see HSA_SVM_FLAGS) + HSA_SVM_ATTR_CLR_FLAGS, // bitmask of flags to clear + HSA_SVM_ATTR_GRANULARITY // migration granularity (log2 num pages) +} HSA_SVM_ATTR_TYPE; + +typedef struct _HSA_SVM_ATTRIBUTE { + HSAuint32 type; // attribute type (see enum HSA_SVM_ATTR_TYPE) + HSAuint32 value; // attribute value +} HSA_SVM_ATTRIBUTE; + #pragma pack(pop, hsakmttypes_h) From 75e8fe383f269ed8aac28c3e7904d234d9417ea2 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Mon, 23 Nov 2020 20:56:02 -0600 Subject: [PATCH 0793/1247] libhsakmt: add SVM thunk implementation Implement SVM (Shared Virtual Memory) in the thunk. Change-Id: I0380150d1d3da48070f9389a06f416d6059d6948 Signed-off-by: Philip Yang Signed-off-by: Sean Keely Signed-off-by: Felix Kuehling Signed-off-by: Alex Sierra --- CMakeLists.txt | 1 + src/libhsakmt.h | 1 + src/libhsakmt.ver | 2 + src/svm.c | 185 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+) create mode 100644 src/svm.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0bfe9e1d73..b31f09c1c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -123,6 +123,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/time.c" "src/topology.c" "src/rbtree.c" + "src/svm.c" "src/version.c") ## Declare the library target name diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 4fa6f9c6d3..86ec0cb9c0 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -26,6 +26,7 @@ #ifndef LIBHSAKMT_H_INCLUDED #define LIBHSAKMT_H_INCLUDED +#include "linux/kfd_ioctl.h" #include "hsakmt.h" #include "pci_ids.h" #include diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index a3203e58c3..25b3a0b725 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -72,6 +72,8 @@ hsaKmtGetKernelDebugTrapVersionInfo; hsaKmtGetThunkDebugTrapVersionInfo; hsaKmtSetAddressWatch; hsaKmtClearAddressWatch; +hsaKmtSVMSetAttr; +hsaKmtSVMGetAttr; local: *; }; diff --git a/src/svm.c b/src/svm.c new file mode 100644 index 0000000000..d4bf896069 --- /dev/null +++ b/src/svm.c @@ -0,0 +1,185 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include "libhsakmt.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Helper functions for calling KFD SVM ioctl */ + +HSAKMT_STATUS HSAKMTAPI +hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, + HSA_SVM_ATTRIBUTE *attrs) +{ + struct kfd_ioctl_svm_args *args; + HSAuint64 s_attr; + HSAKMT_STATUS r; + HSAuint32 i; + + CHECK_KFD_OPEN(); + + pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size); + + if (!start_addr || !size) + return HSAKMT_STATUS_INVALID_PARAMETER; + if ((uint64_t)start_addr & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + if (size & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + + s_attr = sizeof(*attrs) * nattr; + args = alloca(sizeof(*args) + s_attr); + + args->start_addr = (uint64_t)start_addr; + args->size = size; + args->op = KFD_IOCTL_SVM_OP_SET_ATTR; + args->nattr = nattr; + memcpy(args->attrs, attrs, s_attr); + + for (i = 0; i < nattr; i++) { + if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && + attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) + continue; + + if (attrs[i].type == KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && + attrs[i].value == INVALID_NODEID) { + args->attrs[i].value = KFD_IOCTL_SVM_LOCATION_UNDEFINED; + continue; + } + + r = validate_nodeid(attrs[i].value, &args->attrs[i].value); + if (r != HSAKMT_STATUS_SUCCESS) { + pr_debug("invalid node ID: %d\n", attrs[i].value); + return r; + } else if (!args->attrs[i].value && + (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS || + attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE || + attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS)) { + pr_debug("CPU node invalid for access attribute\n"); + return HSAKMT_STATUS_INVALID_NODE_UNIT; + } + } + + /* Driver does one copy_from_user, with extra attrs size */ + r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args); + if (r) { + pr_debug("op set range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, + HSA_SVM_ATTRIBUTE *attrs) +{ + struct kfd_ioctl_svm_args *args; + HSAuint64 s_attr; + HSAKMT_STATUS r; + HSAuint32 i; + + CHECK_KFD_OPEN(); + + pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size); + + if (!start_addr || !size) + return HSAKMT_STATUS_INVALID_PARAMETER; + if ((uint64_t)start_addr & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + if (size & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + + s_attr = sizeof(*attrs) * nattr; + args = alloca(sizeof(*args) + s_attr); + + args->start_addr = (uint64_t)start_addr; + args->size = size; + args->op = KFD_IOCTL_SVM_OP_GET_ATTR; + args->nattr = nattr; + memcpy(args->attrs, attrs, s_attr); + + for (i = 0; i < nattr; i++) { + if (attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && + attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) + continue; + + r = validate_nodeid(attrs[i].value, &args->attrs[i].value); + if (r != HSAKMT_STATUS_SUCCESS) { + pr_debug("invalid node ID: %d\n", attrs[i].value); + return r; + } else if (!args->attrs[i].value) { + pr_debug("CPU node invalid for access attribute\n"); + return HSAKMT_STATUS_INVALID_NODE_UNIT; + } + } + + /* Driver does one copy_from_user, with extra attrs size */ + r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args); + if (r) { + pr_debug("op get range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + memcpy(attrs, args->attrs, s_attr); + + for (i = 0; i < nattr; i++) { + if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && + attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) + continue; + + switch (attrs[i].value) { + case KFD_IOCTL_SVM_LOCATION_SYSMEM: + attrs[i].value = 0; + break; + case KFD_IOCTL_SVM_LOCATION_UNDEFINED: + attrs[i].value = INVALID_NODEID; + break; + default: + r = gpuid_to_nodeid(attrs[i].value, &attrs[i].value); + if (r != HSAKMT_STATUS_SUCCESS) { + pr_debug("invalid GPU ID: %d\n", + attrs[i].value); + return r; + } + } + } + + return HSAKMT_STATUS_SUCCESS; +} From 3f45f602d404c77eb0b9e1234e73144bba69c910 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Mon, 23 Nov 2020 21:00:49 -0600 Subject: [PATCH 0794/1247] libhsakmt: add XNACK API set/get mode XNACK API for GPUs that support this mode. This API makes calls to amdgpu driver to configure xnack mode. It supports set xnack mode and query the current mode used. Change-Id: If865fd0e3f900f008243dc49504e1a0694e1791a Signed-off-by: Alex Sierra --- src/libhsakmt.ver | 3 ++- src/svm.c | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 25b3a0b725..8a695a82fd 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -74,7 +74,8 @@ hsaKmtSetAddressWatch; hsaKmtClearAddressWatch; hsaKmtSVMSetAttr; hsaKmtSVMGetAttr; - +hsaKmtSetXNACKMode; +hsaKmtGetXNACKMode; local: *; }; diff --git a/src/svm.c b/src/svm.c index d4bf896069..478217259e 100644 --- a/src/svm.c +++ b/src/svm.c @@ -183,3 +183,42 @@ hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, return HSAKMT_STATUS_SUCCESS; } + +static HSAKMT_STATUS +hsaKmtSetGetXNACKMode(HSAint32 * enable) +{ + struct kfd_ioctl_set_xnack_mode_args args; + + CHECK_KFD_OPEN(); + + args.xnack_enabled = *enable; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_XNACK_MODE, &args)) { + if (errno == EPERM) { + pr_debug("set mode not supported %s\n", + strerror(errno)); + return HSAKMT_STATUS_NOT_SUPPORTED; + } else if (errno == EBUSY) { + pr_debug("kmtIoctl queues not empty %s\n", + strerror(errno)); + } + return HSAKMT_STATUS_ERROR; + } + + *enable = args.xnack_enabled; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtSetXNACKMode(HSAint32 enable) +{ + return hsaKmtSetGetXNACKMode(&enable); +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtGetXNACKMode(HSAint32 * enable) +{ + *enable = -1; + return hsaKmtSetGetXNACKMode(enable); +} From 50debca7e9f52bfbceb6f8d77f164d964b0bbae8 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Wed, 25 Nov 2020 10:06:53 -0600 Subject: [PATCH 0795/1247] Support gfx90a real asic device id Change-Id: Ib223b4e890899c3c4e468993a88f849bccc5d182 Signed-off-by: Oak Zeng --- src/topology.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/topology.c b/src/topology.c index e5fb85e406..5be3c9af5a 100644 --- a/src/topology.c +++ b/src/topology.c @@ -230,6 +230,9 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x60, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x62, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x7400, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x7408, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x740C, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, + { 0x740F, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, { 0x46, 9, 0, 10, 1, "Aldebaran", CHIP_ALDEBARAN }, /* Navi10 */ { 0x7310, 10, 1, 0, 1, "Navi10", CHIP_NAVI10 }, From ec7ba38b230e81eb7c52e8cbcdb38cfa55be9256 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Mon, 30 Nov 2020 15:49:23 -0500 Subject: [PATCH 0796/1247] kfdtest: blacklist KFDMemoryTest.DeviceHdpFlush on gfx90a Due to cache coherence change, the remote vram mapping is changed to cached, the written value by remote shader will not be read by local shader. So the test will fail. Signed-off-by: Eric Huang Change-Id: I2b64e8a30bed0066e159bad9bb7febae5ebe84aa --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index e20c5ad73f..7f6694dd93 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -196,7 +196,8 @@ FILTER[arcturus]=\ FILTER[aldebaran]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm:"\ -"KFDEvictTest.BurstyTest" +"KFDEvictTest.BurstyTest:"\ +"KFDMemoryTest.DeviceHdpFlush" FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From 1f05b54dc9b835ac370d1d0918d359d157700e8d Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Sun, 29 Nov 2020 20:58:56 -0600 Subject: [PATCH 0797/1247] Delete device stepping check On every new asic with new stepping, we need to manually relax this checking. This check is not very helpful. Delete it. Change-Id: I11f813023ca2566d82f6d11121d4be38c296674b Signed-off-by: Oak Zeng --- tests/kfdtest/src/KFDTopologyTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index 334317e943..c675e4ca88 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -58,7 +58,6 @@ TEST_F(KFDTopologyTest , BasicTest) { EXPECT_GT(pNodeProperties->EngineId.ui32.uCode, 0) << "uCode version is 0"; EXPECT_GE(pNodeProperties->EngineId.ui32.Major, 7) << "Major Version is less than 7"; EXPECT_LT(pNodeProperties->EngineId.ui32.Minor, 10) << "Minor Version is greater than 9"; - EXPECT_LT(pNodeProperties->EngineId.ui32.Stepping, 10) << "Stepping is greater than 9"; EXPECT_GT(pNodeProperties->uCodeEngineVersions.uCodeSDMA, 0) << "sDMA firmware version is 0"; } EXPECT_GT(pNodeProperties->NumMemoryBanks, HSAuint32(0)) << "Node index: " << node << "No MemoryBanks."; From e342c9c890eb57ac9238b6b6df08bbd05ba20af1 Mon Sep 17 00:00:00 2001 From: Amber Lin Date: Tue, 8 Dec 2020 13:44:13 -0500 Subject: [PATCH 0798/1247] kfdtest: Temporarily blacklist some tests Temporarily blacklist some tests on gfx90a until they are solved. Signed-off-by: Amber Lin Change-Id: I87cc3a996ea7d55ed8f20f5b4eecfd8bb691effd --- tests/kfdtest/scripts/kfdtest.exclude | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 7f6694dd93..d2357b241c 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -193,11 +193,16 @@ FILTER[arcturus]=\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDEvictTest.BurstyTest" +# KFDCWSRTest.BasicTest and KFDEvictTest.QueueTest (SWDEV-263604) +# KFDPerformanceTest.P2PBandWidthTest (SWDEV-262388) FILTER[aldebaran]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm:"\ "KFDEvictTest.BurstyTest:"\ -"KFDMemoryTest.DeviceHdpFlush" +"KFDMemoryTest.DeviceHdpFlush:"\ +"KFDCWSRTest.BasicTest:"\ +"KFDEvictTest.QueueTest:"\ +"KFDPerformanceTest.P2PBandWidthTest"\ FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From a83f9b67ce33d8196bd48fdf28da0f07bedfbe9b Mon Sep 17 00:00:00 2001 From: Laurent Morichetti Date: Thu, 10 Sep 2020 11:39:15 -0700 Subject: [PATCH 0799/1247] Update the context save area size Reserve some space in the context save area for the debugger's use. There should be 32 bytes per wave for a given queue. Change-Id: I65ddb6123d0f6afd3149844617ad19023009101d --- include/hsakmttypes.h | 24 +++++++++++++++--------- src/queues.c | 29 ++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 4d7b7d8d73..7c2ed115a6 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -653,15 +653,16 @@ typedef enum _HSA_QUEUE_TYPE } HSA_QUEUE_TYPE; /** - The user context save area starts at offset 0 with the - HsaUserContextSaveAreaHeader header followed by the space for a - user space copy of the control stack and the user space wave save - state. The area must be dword aligned. The context save area is - valid for the duration that the associated queue exists. When a - context save occurs, the HsaUserContextSaveAreaHeader header will - be updated with information about the context save. The context save - area is not modified by any other operation, including a context - resume. + The user context save area is page aligned. The HsaUserContextSaveAreaHeader + header starts at offset 0. Space for a user space copy of the control stack + comes next and is immediately followed by the user space wave save state. The + start of the user space wave save state is page aligned. The debugger reserved + area comes next and is 64 byte aligned. + + The user context save area is valid for the duration that the associated + queue exists. When a context save occurs, the HsaUserContextSaveAreaHeader + header will be updated with information about the context save. The context + save area is not modified by any other operation, including a context resume. */ typedef struct @@ -677,6 +678,11 @@ typedef struct // of wave state data. Must be 4 byte aligned. HSAuint32 WaveStateSize; // Byte size of the last saved wave state data. // Must be 4 byte aligned. + HSAuint32 DebugOffset; // Byte offset from start of the user context + // save area to the memory reserved for the + // debugger. Must be 64 byte aligned. + HSAuint32 DebugSize; // Byte size of the memory reserved for the + // debugger. Must be 64 byte aligned. } HsaUserContextSaveAreaHeader; diff --git a/src/queues.c b/src/queues.c index aaf667c296..d8ce23e8a4 100644 --- a/src/queues.c +++ b/src/queues.c @@ -49,7 +49,9 @@ #define HWREG_SIZE_PER_CU 0x1000 #define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) #define WAVES_PER_CU 32 -#define CNTL_STACK_BYTES_PER_WAVE 8 +#define CNTL_STACK_BYTES_PER_CU(asic_family) (WAVES_PER_CU * (asic_family >= CHIP_NAVI10 ? 12 : 8)) +#define DEBUGGER_BYTES_ALIGN 64 +#define DEBUGGER_BYTES_PER_CU(asic_family) (WAVES_PER_CU * 32) struct device_info { enum asic_family_type asic_family; @@ -208,6 +210,7 @@ struct queue { void *ctx_save_restore; uint32_t ctx_save_restore_size; uint32_t ctl_stack_size; + uint32_t debug_memory_size; const struct device_info *dev_info; bool use_ats; /* This queue structure is allocated from GPU with page aligned size @@ -426,13 +429,23 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) uint32_t ctl_stack_size, wg_data_size; uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; - ctl_stack_size = cu_num * WAVES_PER_CU * CNTL_STACK_BYTES_PER_WAVE + 8; + ctl_stack_size = cu_num * CNTL_STACK_BYTES_PER_CU(q->dev_info->asic_family) + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->dev_info->asic_family); - q->ctl_stack_size = PAGE_ALIGN_UP(ctl_stack_size - + sizeof(HsaUserContextSaveAreaHeader)); + q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + + ctl_stack_size); + if (q->dev_info->asic_family >= CHIP_NAVI10) { + /* HW design limits control stack size to 0x7000. + * This is insufficient for theoretical PM4 cases + * but sufficient for AQL, limited by SPI events. + */ + q->ctl_stack_size = MIN(q->ctl_stack_size, 0x7000); + } + + q->debug_memory_size = + ALIGN_UP(cu_num * DEBUGGER_BYTES_PER_CU(q->dev_info->asic_family), DEBUGGER_BYTES_ALIGN); q->ctx_save_restore_size = q->ctl_stack_size - + PAGE_ALIGN_UP(wg_data_size); + + PAGE_ALIGN_UP(wg_data_size + q->debug_memory_size); return true; } return false; @@ -560,6 +573,8 @@ static int handle_concrete_asic(struct queue *q, ret = update_ctx_save_restore_size(NodeId, q); if (ret) { + HsaUserContextSaveAreaHeader *header; + args->ctx_save_restore_size = q->ctx_save_restore_size; args->ctl_stack_size = q->ctl_stack_size; q->ctx_save_restore = @@ -570,6 +585,10 @@ static int handle_concrete_asic(struct queue *q, return HSAKMT_STATUS_NO_MEMORY; args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; + + header = (HsaUserContextSaveAreaHeader *)q->ctx_save_restore; + header->DebugOffset = q->ctx_save_restore_size - q->debug_memory_size; + header->DebugSize = q->debug_memory_size; } return HSAKMT_STATUS_SUCCESS; From 4cf11c3a7ed9602655268a28de1ee94e6416a881 Mon Sep 17 00:00:00 2001 From: Laurent Morichetti Date: Fri, 16 Oct 2020 00:12:38 -0700 Subject: [PATCH 0800/1247] libhsakmt: Fix the ctrl stack size calculation On gfx9, the maximum number of wavefronts per queue is the minimum of 40 waves per compute units, or 512 waves per shader engine. On gfx10, there can only be 32 waves per compute units. Signed-off-by: Laurent Morichetti Change-Id: I148d1a4fe6c07cdbfaa1f77939eb29311c81c008 --- src/queues.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/queues.c b/src/queues.c index d8ce23e8a4..8d3bd02a4e 100644 --- a/src/queues.c +++ b/src/queues.c @@ -48,10 +48,9 @@ #define LDS_SIZE_PER_CU 0x10000 #define HWREG_SIZE_PER_CU 0x1000 #define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) -#define WAVES_PER_CU 32 -#define CNTL_STACK_BYTES_PER_CU(asic_family) (WAVES_PER_CU * (asic_family >= CHIP_NAVI10 ? 12 : 8)) +#define CNTL_STACK_BYTES_PER_WAVE(asic_family) (asic_family >= CHIP_NAVI10 ? 12 : 8) #define DEBUGGER_BYTES_ALIGN 64 -#define DEBUGGER_BYTES_PER_CU(asic_family) (WAVES_PER_CU * 32) +#define DEBUGGER_BYTES_PER_WAVE(asic_family) 32 struct device_info { enum asic_family_type asic_family; @@ -428,8 +427,11 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) if (node.NumFComputeCores && node.NumSIMDPerCU) { uint32_t ctl_stack_size, wg_data_size; uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; + uint32_t wave_num = (q->dev_info->asic_family < CHIP_NAVI10) + ? MIN(cu_num * 40, node.NumShaderBanks / node.NumArrays * 512) + : cu_num * 32; - ctl_stack_size = cu_num * CNTL_STACK_BYTES_PER_CU(q->dev_info->asic_family) + 8; + ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(q->dev_info->asic_family) + 8; wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->dev_info->asic_family); q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + ctl_stack_size); @@ -442,7 +444,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) } q->debug_memory_size = - ALIGN_UP(cu_num * DEBUGGER_BYTES_PER_CU(q->dev_info->asic_family), DEBUGGER_BYTES_ALIGN); + ALIGN_UP(wave_num * DEBUGGER_BYTES_PER_WAVE(q->dev_info->asic_family), DEBUGGER_BYTES_ALIGN); q->ctx_save_restore_size = q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size + q->debug_memory_size); From 7c05c5240ff085ae8b7d48df5ab0c577651dd07f Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 30 Dec 2020 18:47:48 -0500 Subject: [PATCH 0801/1247] libhsakmt: A+A: Mark buffers accessed by CP as UC This change is for the A+A bring-up branch as it needs to made more generic to handle all ASICs. For A+A all the system buffers are mapped as NC (non coherent) unless explicitly marked as UC (uncached). The coherency is then expected to be handled by shader by explicitly using acquire/release instructions. However, CP doesn't have same feature. The buffers used by CP thus have to UC. For now queue buffer and Signal handler memory is marked as UC. This change shouldn't affect other ASICs since Uncached flag is not used in those. However, this change still need to be made more generic. Signed-off-by: Harish Kasiviswanathan Change-Id: I56c37a809913f7f08c94d01b0572d0f4864939aa --- src/events.c | 2 +- src/libhsakmt.h | 2 +- src/queues.c | 16 ++++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/events.c b/src/events.c index 23fb710523..d4c751c0cc 100644 --- a/src/events.c +++ b/src/events.c @@ -76,7 +76,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, if (is_dgpu && !events_page) { events_page = allocate_exec_aligned_memory_gpu( - KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true, false); + KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true, false, true); if (!events_page) { pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 86ec0cb9c0..dee5929614 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -169,7 +169,7 @@ HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId, bool NonPaged, - bool DeviceLocal); + bool DeviceLocal, bool Uncached); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes); void destroy_process_doorbells(void); diff --git a/src/queues.c b/src/queues.c index 8d3bd02a4e..b8572b6458 100644 --- a/src/queues.c +++ b/src/queues.c @@ -455,7 +455,8 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t NodeId, bool nonPaged, - bool DeviceLocal) + bool DeviceLocal, + bool Uncached) { void *mem; HSAuint64 gpu_va; @@ -469,6 +470,7 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, flags.ui32.NonPaged = nonPaged; flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; flags.ui32.CoarseGrain = DeviceLocal; + flags.ui32.Uncached = Uncached; /* Get the closest cpu_id to GPU NodeId for system memory allocation * nonPaged=1 system memory allocation uses GTT path @@ -518,11 +520,13 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) static void *allocate_exec_aligned_memory(uint32_t size, bool use_ats, uint32_t NodeId, - bool DeviceLocal) + bool DeviceLocal, + bool Uncached) { if (!use_ats) return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId, - DeviceLocal, DeviceLocal); + DeviceLocal, DeviceLocal, + Uncached); return allocate_exec_aligned_memory_cpu(size); } @@ -564,7 +568,7 @@ static int handle_concrete_asic(struct queue *q, q->eop_buffer = allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, q->use_ats, - NodeId, true); + NodeId, true, /* Unused for VRAM */false); if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; @@ -582,7 +586,7 @@ static int handle_concrete_asic(struct queue *q, q->ctx_save_restore = allocate_exec_aligned_memory(q->ctx_save_restore_size, q->use_ats, - NodeId, false); + NodeId, false, false); if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; @@ -639,7 +643,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, struct queue *q = allocate_exec_aligned_memory(sizeof(*q), use_ats, - NodeId, false); + NodeId, false, true); if (!q) return HSAKMT_STATUS_NO_MEMORY; From 10674916e4d0a8c66b50c55171e8ecd6957957f1 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 30 Dec 2020 18:49:42 -0500 Subject: [PATCH 0802/1247] libhsakmt: Explicitly mark AQL buffers as UC This change might be redundant if ROCr takes care of it Signed-off-by: Harish Kasiviswanathan Change-Id: I7b67143a8ad21baa61b7eda7b8e5fe0ac1e33830 --- src/fmm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 5fe6518ea6..a3f4822297 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1186,7 +1186,8 @@ static uint32_t fmm_translate_hsa_to_ioc_flags(HsaMemFlags flags) uint32_t ioc_flags = 0; if (flags.ui32.AQLQueueMemory) - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM; + ioc_flags |= (KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM | + KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED); if (!flags.ui32.ReadOnly) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE; /* TODO: Since, ROCr interfaces doesn't allow caller to set page From 44adc3dafde7d8c2ae07ddfc6d8abc216d9247ef Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Sun, 27 Dec 2020 10:59:07 -0500 Subject: [PATCH 0803/1247] kfdtest: Add Uncached flag to HsaMemoryBuffer constructor Signed-off-by: Harish Kasiviswanathan Change-Id: I14b0a73ffb04f4798547fe7003de1440736b413d --- tests/kfdtest/src/KFDTestUtil.cpp | 4 +++- tests/kfdtest/src/KFDTestUtil.hpp | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index c3a528d9c5..21675dfdf9 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -203,7 +203,7 @@ HSAuint64 GetSystemTickCountInMicroSec() { const HsaMemoryBuffer HsaMemoryBuffer::Null; HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, - bool isScratch, bool isReadOnly) + bool isScratch, bool isReadOnly, bool isUncached) :m_Size(size), m_pUser(NULL), m_pBuf(NULL), @@ -224,11 +224,13 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b m_Flags.ui32.HostAccess = 0; m_Flags.ui32.NonPaged = 1; m_Flags.ui32.CoarseGrain = 1; + EXPECT_EQ(isUncached, 0) << "Uncached flag is relevant only for system or host memory"; } else { m_Flags.ui32.HostAccess = 1; m_Flags.ui32.NonPaged = 0; m_Flags.ui32.CoarseGrain = 0; m_Flags.ui32.NoNUMABind = 1; + m_Flags.ui32.Uncached = isUncached; } if (isExec) diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index e640d588fc..ef2dd57008 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -66,7 +66,7 @@ class HsaMemoryBuffer { public: HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero = true, bool isLocal = false, - bool isExec = false, bool isScratch = false, bool isReadOnly = false); + bool isExec = false, bool isScratch = false, bool isReadOnly = false, bool isUncached = false); HsaMemoryBuffer(void *addr, HSAuint64 size); template RetType As() { From 0e8500b886ba1a53a40ce50e0a2a8526333c7180 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Sun, 27 Dec 2020 11:01:01 -0500 Subject: [PATCH 0804/1247] kfdtest: A+A: Mark queue address as UC Refer to commit: " Mark buffers accessed by CP as UC" Signed-off-by: Harish Kasiviswanathan Change-Id: I1816e035dbb3178f28f5e34b050c20ecca282060 --- tests/kfdtest/src/BaseQueue.cpp | 3 ++- tests/kfdtest/src/IndirectBuffer.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 835ab4c71d..56ebf6fd18 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -48,7 +48,8 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 memset(&m_Resources, 0, sizeof(m_Resources)); - m_QueueBuf = new HsaMemoryBuffer(size, NodeId, true/*zero*/, false/*local*/, true/*exec*/); + m_QueueBuf = new HsaMemoryBuffer(size, NodeId, true/*zero*/, false/*local*/, true/*exec*/, + /*isScratch */ false, /* isReadOnly */false, /* isUncached */true); if (type == HSA_QUEUE_COMPUTE_AQL) { m_Resources.Queue_read_ptr_aql = &pointers[0]; diff --git a/tests/kfdtest/src/IndirectBuffer.cpp b/tests/kfdtest/src/IndirectBuffer.cpp index b820230b0d..4e3907cc5b 100644 --- a/tests/kfdtest/src/IndirectBuffer.cpp +++ b/tests/kfdtest/src/IndirectBuffer.cpp @@ -30,7 +30,8 @@ IndirectBuffer::IndirectBuffer(PACKETTYPE type, unsigned int sizeInDWords, unsigned int NodeId) :m_NumOfPackets(0), m_MaxSize(sizeInDWords), m_ActualSize(0), m_PacketTypeAllowed(type) { m_IndirectBuf = new HsaMemoryBuffer(sizeInDWords*sizeof(unsigned int), NodeId, true/*zero*/, - false/*local*/, true/*exec*/); + false/*local*/, true/*exec*/, false/*isScratch*/, + false/*isReadOnly*/, true/*isUncached*/); } IndirectBuffer::~IndirectBuffer(void) { From 57f46b53ecc8cefe4d84747736077e86465b92f6 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Sun, 27 Dec 2020 11:05:19 -0500 Subject: [PATCH 0805/1247] kfdtest: A+A: CP writes to NC mem need flush Refer to commit "Mark buffers accessed by CP as UC" A+A buffers are mapped as NC. CP (PM4Writes) need ReleaseMem function to ensure the write go through to the memory Signed-off-by: Harish Kasiviswanathan Change-Id: I4ee55a6e40fba078f5950d95c8fee7ee076260bf --- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 6 +++- tests/kfdtest/src/KFDMemoryTest.cpp | 7 +++- tests/kfdtest/src/KFDPMTest.cpp | 8 +++-- tests/kfdtest/src/KFDQMTest.cpp | 42 +++++++++++++++++------- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index 33f33dc869..6af6765ac3 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -58,6 +58,8 @@ TEST_F(KFDLocalMemoryTest, AccessLocalMem) { //local memory HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false, true); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); PM4Queue queue; @@ -65,10 +67,12 @@ TEST_F(KFDLocalMemoryTest, AccessLocalMem) { queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0, 0)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); + hsaKmtDestroyEvent(event); EXPECT_SUCCESS(queue.Destroy()); + TEST_END } diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index c5c0c0afde..ea24938116 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -432,14 +432,18 @@ TEST_F(KFDMemoryTest, AccessPPRMem) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf, 0xABCDEF09, 0x12345678)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf, 0xABCDEF09); WaitOnValue(destBuf + 1, 0x12345678); + hsaKmtDestroyEvent(event); EXPECT_SUCCESS(queue.Destroy()); /* This sleep hides the dmesg PPR message storm on Raven, which happens @@ -1455,6 +1459,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { mem1 = reinterpret_cast(reinterpret_cast(mem) + VRAM_OFFSET + sizeof(HSAuint64)); PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)mem0, data0[0], data0[1])); queue.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)mem1, diff --git a/tests/kfdtest/src/KFDPMTest.cpp b/tests/kfdtest/src/KFDPMTest.cpp index 79b385cf72..98c2348a8c 100644 --- a/tests/kfdtest/src/KFDPMTest.cpp +++ b/tests/kfdtest/src/KFDPMTest.cpp @@ -78,8 +78,11 @@ TEST_F(KFDPMTest, SuspendWithIdleQueueAfterWork) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuffer.As(), 0x1, 0x2)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(&(destBuffer.As()[0]), 0x1); WaitOnValue(&(destBuffer.As()[1]), 0x2); @@ -88,7 +91,7 @@ TEST_F(KFDPMTest, SuspendWithIdleQueueAfterWork) { EXPECT_EQ(true, SuspendAndWakeUp()); queue.PlaceAndSubmitPacket(PM4WriteDataPacket(&(destBuffer.As()[2]), 0x3, 0x4)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); EXPECT_EQ(destBuffer.As()[0], 0); EXPECT_EQ(destBuffer.As()[1], 0); @@ -96,6 +99,7 @@ TEST_F(KFDPMTest, SuspendWithIdleQueueAfterWork) { WaitOnValue(&(destBuffer.As()[2]), 0x3); WaitOnValue(&(destBuffer.As()[3]), 0x4); + hsaKmtDestroyEvent(event); EXPECT_SUCCESS(queue.Destroy()); TEST_END diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 71768cc2cf..086d5e537b 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -78,13 +78,16 @@ TEST_F(KFDQMTest, SubmitNopCpQueue) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; PM4Queue queue; + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.PlaceAndSubmitPacket(PM4NopPacket()); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); + hsaKmtDestroyEvent(event); EXPECT_SUCCESS(queue.Destroy()); TEST_END @@ -99,17 +102,19 @@ TEST_F(KFDQMTest, SubmitPacketCpQueue) { HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false); destBuf.Fill(0xFF); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); PM4Queue queue; - ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0, 0)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); EXPECT_TRUE(WaitOnValue(destBuf.As(), 0)); + hsaKmtDestroyEvent(event); EXPECT_SUCCESS(queue.Destroy()); TEST_END @@ -132,7 +137,7 @@ TEST_F(KFDQMTest, AllCpQueues) { for (unsigned int qidx = 0; qidx < m_numCpQueues; ++qidx) { queues[qidx].PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As()+qidx*2, qidx, qidx)); - + queues[qidx].PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, true, 0, 0)); queues[qidx].Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(destBuf.As()+qidx*2, qidx)); @@ -330,6 +335,7 @@ TEST_F(KFDQMTest, AllQueues) { for (i = 0; i < numCpQueues; ++i) { cpQueues[i].PlaceAndSubmitPacket(PM4WriteDataPacket(destBufCp.As()+i*2, i, i)); + cpQueues[i].PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, true, 0, 0)); cpQueues[i].Wait4PacketConsumption(); @@ -460,9 +466,12 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); + queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0, 0)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf.As(), 0); @@ -480,10 +489,11 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithNullAddress) { EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf.As(), 1); + hsaKmtDestroyEvent(event); EXPECT_SUCCESS(queue.Destroy()); TEST_END @@ -544,13 +554,16 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); + PM4WriteDataPacket packet1, packet2; packet1.InitPacket(destBuf.As(), 0, 0); packet2.InitPacket(destBuf.As(), 1, 1); queue.PlaceAndSubmitPacket(packet1); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf.As(), 0); @@ -568,7 +581,7 @@ TEST_F(KFDQMTest, DisableCpQueueByUpdateWithZeroPercentage) { EXPECT_SUCCESS(queue.Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, BaseQueue::DEFAULT_PRIORITY, false)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf.As(), 1); @@ -1228,6 +1241,8 @@ TEST_F(KFDQMTest, CpuWriteCoherence) { HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode); ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); /* The queue might be full and we fail to submit. There is always one word space unused in queue. * So let rptr one step ahead then we continually submit packet. @@ -1249,10 +1264,11 @@ TEST_F(KFDQMTest, CpuWriteCoherence) { */ queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), 0x42, 0x42)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf.As(), 0x42); + hsaKmtDestroyEvent(event); TEST_END } @@ -1420,18 +1436,22 @@ TEST_F(KFDQMTest, CpQueueWraparound) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaEvent *event; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event)); + for (unsigned int pktIdx = 0; pktIdx <= PAGE_SIZE/sizeof(PM4WRITE_DATA_CI); ++pktIdx) { queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), pktIdx, pktIdx)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf.As(), pktIdx); } for (unsigned int pktIdx = 0; pktIdx <= PAGE_SIZE/sizeof(PM4WRITE_DATA_CI); ++pktIdx) { queue.PlaceAndSubmitPacket(PM4WriteDataPacket(destBuf.As(), pktIdx, pktIdx)); - queue.Wait4PacketConsumption(); + queue.Wait4PacketConsumption(event); WaitOnValue(destBuf.As(), pktIdx); } + hsaKmtDestroyEvent(event); EXPECT_SUCCESS(queue.Destroy()); TEST_END From f132fb2cd0d4368336b990a9a61990fbc9d7cdcd Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Tue, 5 Jan 2021 13:13:24 -0600 Subject: [PATCH 0806/1247] Make GPU mapping of memory as uncached if HSA_DISABLE_CACHE is set Before gfx90a, coherent memory is uncached. So it was reasonable when environment variable HSA_DISABLE_CACHE is set, memory is mapped as coherent. On gfx90a, coherent memory can be cached, so mapping memory as coherent can't guarantee memory is uncached. When HSA_DISABLE_CACHE is set, we have to map memory as uncached. Change-Id: Ia5ed4cf0ad6aef5644dc8c9e6632b52d606f06f4 Signed-off-by: Oak Zeng --- src/fmm.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index a3f4822297..463a3e2fd1 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1335,11 +1335,11 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } - if (!flags.ui32.CoarseGrain || svm.disable_cache) { + if (!flags.ui32.CoarseGrain) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - if (flags.ui32.Uncached) - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; - } + + if (flags.ui32.Uncached || svm.disable_cache) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); @@ -1544,11 +1544,12 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, else aperture = svm.dgpu_alt_aperture; /* always coherent */ - if (!flags.ui32.CoarseGrain || svm.disable_cache) { + if (!flags.ui32.CoarseGrain) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - if (flags.ui32.Uncached) - ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; - } + + if (flags.ui32.Uncached || svm.disable_cache) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; + ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); if (flags.ui32.AQLQueueMemory) From 3a378fcf0b081c2e7cda56db0357159085c3000c Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 6 Jan 2021 10:26:34 -0500 Subject: [PATCH 0807/1247] kdftest: remove some kfdtests filtered for gfx90a The three kfdtests have been fixed, so remove them from filter list. Signed-off-by: Eric Huang Change-Id: I101a72476970a9d105e8c0b5c022847757fdd316 --- tests/kfdtest/scripts/kfdtest.exclude | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index d2357b241c..7f6694dd93 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -193,16 +193,11 @@ FILTER[arcturus]=\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDEvictTest.BurstyTest" -# KFDCWSRTest.BasicTest and KFDEvictTest.QueueTest (SWDEV-263604) -# KFDPerformanceTest.P2PBandWidthTest (SWDEV-262388) FILTER[aldebaran]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm:"\ "KFDEvictTest.BurstyTest:"\ -"KFDMemoryTest.DeviceHdpFlush:"\ -"KFDCWSRTest.BasicTest:"\ -"KFDEvictTest.QueueTest:"\ -"KFDPerformanceTest.P2PBandWidthTest"\ +"KFDMemoryTest.DeviceHdpFlush" FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From f7759df6e044bd7cad03b4c4a30fb880f5a0fda7 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Thu, 7 Jan 2021 13:54:19 -0500 Subject: [PATCH 0808/1247] kfdtest: fix KFDQMTest.Atomics test failure on A+A destBuf is mapped as cached, the intruction flat_atomic_add operates on cache that cause test failed. Adding scc modifier in the instruction will fix the issue. Signed-off-by: Eric Huang Change-Id: I8e138f93ae4f5e23020e3ac1549ef924968a74c5 --- tests/kfdtest/src/IsaGenerator_Aldebaran.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp b/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp index 2fcb80fd32..2c377f9111 100644 --- a/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp +++ b/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp @@ -37,7 +37,7 @@ type(CS) v_mov_b32 v0, s0 v_mov_b32 v1, s1 v_mov_b32 v2, 1 - flat_atomic_add v3, v[0:1], v2 slc glc + flat_atomic_add v3, v[0:1], v2 slc glc scc s_waitcnt 0 s_endpgm end @@ -86,7 +86,7 @@ const uint32_t IsaGenerator_Aldbrn::INFINITE_LOOP_ISA[] = { const uint32_t IsaGenerator_Aldbrn::ATOMIC_ADD_ISA[] = { 0x7e000200, 0x7e020201, - 0x7e040281, 0xdd0b0000, + 0x7e040281, 0xdf0b0000, 0x037f0200, 0xbf8c0000, 0xbf810000, 0x00000000 }; From 085005f07b6a959699baea84b1da1fda0258e93f Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 13 Jan 2021 15:54:03 -0500 Subject: [PATCH 0809/1247] kfdtest: Add gfx9_PollNCMemory function to support NC memory In A+A all system memory is mapped as NC. So add a new function gfx9_PollNCMemory which will support NC memory. Signed-off-by: Harish Kasiviswanathan Change-Id: I097b95fb156f73d6f480cd4fd262cc6fa5933f69 --- tests/kfdtest/src/KFDMemoryTest.cpp | 32 ++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index ea24938116..54ab73d1c0 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -154,6 +154,32 @@ type(CS)\n\ end\n\ "; +/* Similar to gfx9_PollMemory except that the buffer + * polled can be Non-coherant memory. SCC system-level + * cache coherence is not supported in scalar (smem) path. + * Use vmem operations with scc + */ +const char* gfx9_PollNCMemory = +"\ +shader ReadMemory\n\ +asic(ALDEBARAN)\n\ +wave_size(32)\n\ +type(CS)\n\ +/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ + v_mov_b32 v6, 0x5678\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + LOOP:\n\ + flat_load_dword v4, v[0:1] scc\n\ + v_cmp_eq_u32 vcc, v4, v6\n\ + s_cbranch_vccz LOOP\n\ + v_mov_b32 v0, s2\n\ + v_mov_b32 v1, s3\n\ + flat_store_dword v[0:1], v6 scc\n\ + s_endpgm\n\ + end\n\ +"; + /* Input: A buffer of at least 3 dwords. * DW0: used as a signal. 0xcafe means it is signaled * DW1: Input buffer for device to read. @@ -336,7 +362,11 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) + /* On A+A system memory is mapped as NC */ + m_pIsaGen->CompileShader(gfx9_PollNCMemory, "ReadMemory", isaBuffer); + else + m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); From 9aa521d1ffa58c93abcdd187b6c83af9c0eed29d Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 15 Apr 2020 16:13:52 -0400 Subject: [PATCH 0810/1247] KFDTest: add cache coherence tests for gfx90a Three kfd subtests are added to verify new XGMI connection with cache coherence HW link on A+A. Signed-off-by: Eric Huang Change-Id: I6960ec91cbfb696c4e6acb3b79fd83107003acdd --- tests/kfdtest/src/KFDMemoryTest.cpp | 283 ++++++++++++++++++++++++++++ 1 file changed, 283 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 54ab73d1c0..95997c0641 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -229,6 +229,81 @@ type(CS)\n\ end\n\ "; +/* Continuously poll the flag at src buffer + * After the flag of s[0:1] is 1 filled, + * copy the value from s[0:1]+4 to dst buffer + */ +const char* gfx9_PollAndCopy = +"\ +shader CopyMemory\n\ +wave_size(32)\n\ +type(CS)\n\ +/* Assume src buffer in s[0:1] and dst buffer in s[2:3]*/\n\ + s_movk_i32 s18, 0x1\n\ + LOOP:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_cmp_eq_i32 s16, s18\n\ + s_cbranch_scc0 LOOP\n\ + s_load_dword s17, s[0:1], 0x4 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_store_dword s17, s[2:3], 0x0 glc:1\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_endpgm\n\ + end\n\ +"; + +const char* gfx9aldbrn_PollAndCopy = +"\ +shader CopyMemory\n\ +wave_size(32)\n\ +type(CS)\n\ +/* Assume src buffer in s[0:1] and dst buffer in s[2:3]*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + v_mov_b32 v18, 0x1\n\ + LOOP:\n\ + flat_load_dword v16, v[0:1] scc:1\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + v_cmp_eq_i32 vcc, v16, v18\n\ + s_cbranch_vccz LOOP\n\ + buffer_invl2\n\ + s_load_dword s17, s[0:1], 0x4 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_store_dword s17, s[2:3], 0x0 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + buffer_wbl2\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_endpgm\n\ + end\n\ +"; + +/* Input0: A buffer of at least 2 dwords. + * DW0: used as a signal. Write 0x1 to signal + * DW1: Write the value from 2nd input buffer + * for other device to read. + * Input1: A buffer of at least 2 dwords. + * DW0: used as the value to be written. + */ +const char* gfx9aldbrn_WriteFlagAndValue = +"\ +shader WriteMemory\n\ +wave_size(32)\n\ +type(CS)\n\ +/* Assume two inputs buffer in s[0:1] and s[2:3]*/\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + s_load_dword s18, s[2:3], 0x0 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + s_store_dword s18, s[0:1], 0x4 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + buffer_wbl2\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ + v_mov_b32 v16, 0x1\n\ + flat_store_dword v[0:1], v16 scc:1\n\ + s_endpgm\n\ + end\n\ +"; + //These gfx9_PullMemory, gfx9_CopyOnSignal, gfx9_WriteAndSignal shaders can be used by both gfx9 and gfx10 void KFDMemoryTest::SetUp() { @@ -2258,3 +2333,211 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnRemoteWrite) { TEST_END } + +/* Test is for new cache coherence on Aldebaran. It is to verify + * two GPUs can coherently share a fine grain FB. + */ +TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); + volatile HSAuint32 *tmp = tmpBuffer.As(); + const int dwSource = 0x40 * sizeof(int); /* At 3rd cache line */ + const int dwLocation = 0x80 * sizeof(int); /* At 5th cache line */ + + if (m_FamilyId != FAMILY_AL) { + LOG() << "Skipping test: Test requires aldebaran series asics." << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: At least two GPUs are required." << std::endl; + return; + } + + HSAuint32 nondefaultNode; + for (unsigned i = 0; i < gpuNodes.size(); i++) { + if (gpuNodes.at(i) != defaultGPUNode) { + nondefaultNode = gpuNodes.at(i); + break; + } + } + + unsigned int nodes[2] = {defaultGPUNode, nondefaultNode}; + + /* Allocate a local FB */ + HsaMemoryBuffer buffer(PAGE_SIZE, defaultGPUNode, false/*zero*/, true/*local*/, false/*exec*/); + buffer.MapMemToNodes(&nodes[0], 2); + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + buffer.Fill(0, sdmaQueue, 0, PAGE_SIZE); + buffer.Fill(0x5678, sdmaQueue, dwSource, 4); + + /* Read buffer[0] as flag from local shader to fill cache line (64 dws) + * which should has 0 at buffer[1] + */ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer); + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); + dispatch.Submit(queue); + + /* Delay 100ms to make sure shader executed*/ + Delay(100); + + /* Using remote shader to write the flag and copy value from dwSource + * to dwLocation in buffer. + * Local shader should get the flag and execute CopyMemory + */ + PM4Queue queue1; + ASSERT_SUCCESS(queue1.Create(nondefaultNode)); + HsaMemoryBuffer isaBuffer1(PAGE_SIZE, nondefaultNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9aldbrn_WriteFlagAndValue, "WriteMemory", isaBuffer1); + Dispatch dispatch1(isaBuffer1); + dispatch1.SetArgs(buffer.As(), buffer.As()+dwSource); + dispatch1.Submit(queue1); + dispatch1.Sync(g_TestTimeOut); + + /* Check test result*/ + dispatch.Sync(g_TestTimeOut); + EXPECT_EQ(buffer.IsPattern(dwLocation, 0x5678, sdmaQueue, tmp), true); + + // Clean up + EXPECT_SUCCESS(queue.Destroy()); + EXPECT_SUCCESS(queue1.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); + + TEST_END +} + +/* Test is for new cache coherence on A+A(Aldebaran). It is to verify + * new XGMI coherence HW link in caches between CPU and GPUs + * in local FB with fine grain mode. + */ +TEST_F(KFDMemoryTest, VramCacheCoherenceWithCPU) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (m_FamilyId != FAMILY_AL) { + LOG() << "Skipping test: Test requires aldebaran series asics." << std::endl; + return; + } + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + const int dwLocation = 0x80; + + if (!m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) { + LOG() << "Skipping test: XGMI link to CPU is required." << std::endl; + return; + } + + unsigned int *buffer; + HsaMemFlags memFlags = {0}; + /* Allocate a fine grain local FB accessed by CPU */ + memFlags.ui32.HostAccess = 1; + memFlags.ui32.NonPaged = 1; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, memFlags, + reinterpret_cast(&buffer))); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(buffer, PAGE_SIZE, NULL)); + buffer[0] = 0; + buffer[dwLocation] = 0; + + /* Read buffer from shader to fill cache */ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer); + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(buffer, buffer+dwLocation); + dispatch.Submit(queue); + + /* Delay 100ms to make sure shader executed*/ + Delay(100); + + /* CPU writes to buffer. Shader should get 0x5678 CPU writes + * after cache invalidating(buffer_invl2) and quits + */ + buffer[1] = 0x5678; + buffer[0] = 1; + + /* Check test result*/ + dispatch.Sync(g_TestTimeOut); + EXPECT_EQ(buffer[dwLocation], 0x5678); + + // Clean up + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(buffer)); + EXPECT_SUCCESS(hsaKmtFreeMemory(buffer, PAGE_SIZE)); + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} + +/* Test is for new cache coherence on Aldebaran. It is to verify + * new XGMI coherence HW link in caches between CPU and GPUs + * in system RAM. + */ +TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (m_FamilyId != FAMILY_AL) { + LOG() << "Skipping test: Test requires aldebaran series asics." << std::endl; + return; + } + + unsigned int *fineBuffer = NULL; + unsigned int tmp; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + const int dwLocation = 0x80; + + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode /* system */, PAGE_SIZE, m_MemoryFlags, + reinterpret_cast(&fineBuffer))); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(fineBuffer, PAGE_SIZE, NULL)); + fineBuffer[0] = 0; + fineBuffer[1] = 0; + /* Read buffer from CPU to fill cache */ + tmp = fineBuffer[dwLocation]; + + /* Read fine grain buffer from shader to fill cache */ + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + + if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) + m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer); + else + m_pIsaGen->CompileShader(gfx9_PollAndCopy, "CopyMemory", isaBuffer); + + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(fineBuffer, fineBuffer+dwLocation); + dispatch.Submit(queue); + + /* Delay 100ms to make sure shader executed*/ + Delay(100); + + /* CPU writes to buffer. Shader should get what CPU writes and quits*/ + fineBuffer[1] = 0x5678; + fineBuffer[0] = 1; + + /* Check test result, based on KFDEventTest.SignalEvent passed. + * if Sync times out, + * it means coherence issue that GPU doesn't read what CPU wrote. + * if buffer value is not expected, + * it means coherence issue that CPU doesn't read what GPU wrote. + */ + dispatch.Sync(g_TestTimeOut); + EXPECT_EQ(fineBuffer[dwLocation], 0x5678); + + // Clean up + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(fineBuffer)); + EXPECT_SUCCESS(hsaKmtFreeMemory(fineBuffer, PAGE_SIZE)); + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END +} From c21466d7355924fec74a8416dd98a0abf3e11b46 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Fri, 26 Feb 2021 17:40:13 +0800 Subject: [PATCH 0811/1247] libhsakmt: add DID for gfx1031 Signed-off-by: Chengming Gui Change-Id: I1b890dda0ef9ee53c3950c17c106197167f210b9 --- src/topology.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/topology.c b/src/topology.c index 043189bb29..d5c054d24a 100644 --- a/src/topology.c +++ b/src/topology.c @@ -246,6 +246,11 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73AB, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, { 0x73AE, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, { 0x73BF, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + /* NAVY_FLOUNDER */ + { 0x73C0, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, + { 0x73C1, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, + { 0x73C3, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, + { 0x73DF, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, /* VanGogh */ { 0x163F, 10, 3, 3, "VanGogh", CHIP_VANGOGH }, }; From ae0e74095ec03e279c46e714b4887b3a67984475 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Wed, 24 Feb 2021 19:03:35 -0600 Subject: [PATCH 0812/1247] Allocate coherent uncached memory when HSA_DISABLE_CACHE is set Set the KFD_IOC_ALLOC_MEM_FLAGS_COHERENT flag and KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED flag to allocate uncached coherent memory when HSA_DISABLE_CACHE environment variable is set. At KFD driver, Single KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED flag is not sufficient to allocate uncached memory. We have to use both two flags to allocate uncached memory. Change-Id: Ie490f37b2e696314e60048f5b1b57442431696e9 Signed-off-by: Oak Zeng --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 463a3e2fd1..fc588a1617 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1335,7 +1335,7 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } - if (!flags.ui32.CoarseGrain) + if (!flags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; if (flags.ui32.Uncached || svm.disable_cache) @@ -1544,7 +1544,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, else aperture = svm.dgpu_alt_aperture; /* always coherent */ - if (!flags.ui32.CoarseGrain) + if (!flags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; if (flags.ui32.Uncached || svm.disable_cache) From e35778ed4dcc83ba001f0a25b1319cb59716ee1a Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 25 Feb 2021 14:10:49 -0500 Subject: [PATCH 0813/1247] kfdtest: Temporarily blacklist KFDMemoryTest.PtraceAccess Possibly because of moving to gart table for vram access from Kernel. This test failure shouldn't be a blocker. Temporarily blacklist till a solution is found. Signed-off-by: Harish Kasiviswanathan Change-Id: I99725f368aced863188e30f619288ad4d033b9a6 --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 7f6694dd93..e30ba99958 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -197,6 +197,7 @@ FILTER[aldebaran]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm:"\ "KFDEvictTest.BurstyTest:"\ +"KFDMemoryTest.PtraceAccess:"\ "KFDMemoryTest.DeviceHdpFlush" FILTER[navi10]=\ From 95d58346c5b4ad8093510248b84992c26e354694 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 4 Mar 2021 21:16:06 -0500 Subject: [PATCH 0814/1247] kfdtest: Temporarily disable shader tests on gfx10 Temporarily disable shader related tests until SP3 compiler is fixed Signed-off-by: Harish Kasiviswanathan Change-Id: I0468d82f845d1d69841ad8fdbd037761b8d4d9af --- tests/kfdtest/scripts/kfdtest.exclude | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index f2b605cf84..2dc5bbd1a0 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -207,26 +207,42 @@ FILTER[aldebaran]=\ "KFDMemoryTest.PtraceAccess:"\ "KFDMemoryTest.DeviceHdpFlush" +# SP3 Compiler needs to be updated for GFX10. Temporarily disable all tests +# that require shader compiler +TEMP_GFX10_BLACKLIST=\ +"KFDMemoryTest.FlatScratchAccess:"\ +"KFDMemoryTest.PtraceAccessInvisibleVram:"\ +"KFDQMTest.QueuePriorityOnDifferentPipe:"\ +"KFDQMTest.QueuePriorityOnSamePipe:"\ +"KFDCWSRTest.BasicTest:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDEvictTest.QueueTest" + FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ +"$TEMP_GFX10_BLACKLIST:"\ "KFDMemoryTest.MMBench" FILTER[navi14]=\ -"$BLACKLIST_ALL_ASICS" +"$BLACKLIST_ALL_ASICS:"\ +"$TEMP_GFX10_BLACKLIST" FILTER[sienna_cichlid]=\ "$BLACKLIST_ALL_ASICS:"\ +"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ FILTER[navy_flounder]=\ "$BLACKLIST_ALL_ASICS:"\ +"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ FILTER[dimgrey_cavefish]=\ +"$TEMP_GFX10_BLACKLIST:"\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ From 0f16e6f35f4a81fd520e0c473ac9c1735c7a9d0e Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 8 Mar 2021 21:12:55 -0500 Subject: [PATCH 0815/1247] kfdtest: Fix gfx1032 blacklist BLACKLIST_ALL_ASICS has to the first in the list otherwise "-" negative flag won't be inserted Signed-off-by: Harish Kasiviswanathan Change-Id: I9ee150d7f793809641b16012929c4e157595d37f --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 2dc5bbd1a0..7be47cac9b 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -242,8 +242,8 @@ FILTER[navy_flounder]=\ "KFDPerfCountersTest.*:"\ FILTER[dimgrey_cavefish]=\ -"$TEMP_GFX10_BLACKLIST:"\ "$BLACKLIST_ALL_ASICS:"\ +"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ From a8f4c43fef32318708301fff1ab5cb0d1a30e056 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 10 Mar 2021 20:34:08 -0500 Subject: [PATCH 0816/1247] Revert "libhsakmt: add XNACK API set/get mode" This reverts commit 3f45f602d404c77eb0b9e1234e73144bba69c910. SVM is not ready yet. This was merged by accident. Change-Id: I7c0d835a0d3a448f2ac1094f818601e5d6363045 Signed-off-by: Felix Kuehling --- src/libhsakmt.ver | 2 -- src/svm.c | 39 --------------------------------------- 2 files changed, 41 deletions(-) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 66ca6ea5b3..3992e6535a 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -77,8 +77,6 @@ hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; hsaKmtSVMSetAttr; hsaKmtSVMGetAttr; -hsaKmtSetXNACKMode; -hsaKmtGetXNACKMode; local: *; }; diff --git a/src/svm.c b/src/svm.c index 478217259e..d4bf896069 100644 --- a/src/svm.c +++ b/src/svm.c @@ -183,42 +183,3 @@ hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, return HSAKMT_STATUS_SUCCESS; } - -static HSAKMT_STATUS -hsaKmtSetGetXNACKMode(HSAint32 * enable) -{ - struct kfd_ioctl_set_xnack_mode_args args; - - CHECK_KFD_OPEN(); - - args.xnack_enabled = *enable; - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_XNACK_MODE, &args)) { - if (errno == EPERM) { - pr_debug("set mode not supported %s\n", - strerror(errno)); - return HSAKMT_STATUS_NOT_SUPPORTED; - } else if (errno == EBUSY) { - pr_debug("kmtIoctl queues not empty %s\n", - strerror(errno)); - } - return HSAKMT_STATUS_ERROR; - } - - *enable = args.xnack_enabled; - - return HSAKMT_STATUS_SUCCESS; -} - -HSAKMT_STATUS HSAKMTAPI -hsaKmtSetXNACKMode(HSAint32 enable) -{ - return hsaKmtSetGetXNACKMode(&enable); -} - -HSAKMT_STATUS HSAKMTAPI -hsaKmtGetXNACKMode(HSAint32 * enable) -{ - *enable = -1; - return hsaKmtSetGetXNACKMode(enable); -} From 4ebda913cd4046337a5d6f9a4741256e7bc3628c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 10 Mar 2021 20:36:41 -0500 Subject: [PATCH 0817/1247] Revert "libhsakmt: add SVM thunk implementation" This reverts commit 75e8fe383f269ed8aac28c3e7904d234d9417ea2. SVM is not ready yet. This was merged by accident. Change-Id: I372f7d293fd38429ec570bc0e0add7e612871594 Signed-off-by: Felix Kuehling --- CMakeLists.txt | 3 +- src/libhsakmt.h | 1 - src/libhsakmt.ver | 2 - src/svm.c | 185 ---------------------------------------------- 4 files changed, 1 insertion(+), 190 deletions(-) delete mode 100644 src/svm.c diff --git a/CMakeLists.txt b/CMakeLists.txt index ccf39e8d82..a3a4d999c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,8 +129,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/topology.c" "src/rbtree.c" "src/spm.c" - "src/version.c" - "src/svm.c") + "src/version.c") ## Declare the library target name add_library ( ${HSAKMT_TARGET} "") diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 9c5b91f7b6..f455699c6d 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -26,7 +26,6 @@ #ifndef LIBHSAKMT_H_INCLUDED #define LIBHSAKMT_H_INCLUDED -#include "linux/kfd_ioctl.h" #include "hsakmt.h" #include "pci_ids.h" #include diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 3992e6535a..4383695e98 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -75,8 +75,6 @@ hsaKmtClearAddressWatch; hsaKmtSPMAcquire; hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; -hsaKmtSVMSetAttr; -hsaKmtSVMGetAttr; local: *; }; diff --git a/src/svm.c b/src/svm.c deleted file mode 100644 index d4bf896069..0000000000 --- a/src/svm.c +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright © 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including - * the next paragraph) shall be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#include "libhsakmt.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Helper functions for calling KFD SVM ioctl */ - -HSAKMT_STATUS HSAKMTAPI -hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, - HSA_SVM_ATTRIBUTE *attrs) -{ - struct kfd_ioctl_svm_args *args; - HSAuint64 s_attr; - HSAKMT_STATUS r; - HSAuint32 i; - - CHECK_KFD_OPEN(); - - pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size); - - if (!start_addr || !size) - return HSAKMT_STATUS_INVALID_PARAMETER; - if ((uint64_t)start_addr & (PAGE_SIZE - 1)) - return HSAKMT_STATUS_INVALID_PARAMETER; - if (size & (PAGE_SIZE - 1)) - return HSAKMT_STATUS_INVALID_PARAMETER; - - s_attr = sizeof(*attrs) * nattr; - args = alloca(sizeof(*args) + s_attr); - - args->start_addr = (uint64_t)start_addr; - args->size = size; - args->op = KFD_IOCTL_SVM_OP_SET_ATTR; - args->nattr = nattr; - memcpy(args->attrs, attrs, s_attr); - - for (i = 0; i < nattr; i++) { - if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && - attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC && - attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && - attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && - attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) - continue; - - if (attrs[i].type == KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && - attrs[i].value == INVALID_NODEID) { - args->attrs[i].value = KFD_IOCTL_SVM_LOCATION_UNDEFINED; - continue; - } - - r = validate_nodeid(attrs[i].value, &args->attrs[i].value); - if (r != HSAKMT_STATUS_SUCCESS) { - pr_debug("invalid node ID: %d\n", attrs[i].value); - return r; - } else if (!args->attrs[i].value && - (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS || - attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE || - attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS)) { - pr_debug("CPU node invalid for access attribute\n"); - return HSAKMT_STATUS_INVALID_NODE_UNIT; - } - } - - /* Driver does one copy_from_user, with extra attrs size */ - r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args); - if (r) { - pr_debug("op set range attrs failed %s\n", strerror(errno)); - return HSAKMT_STATUS_ERROR; - } - - return HSAKMT_STATUS_SUCCESS; -} - -HSAKMT_STATUS HSAKMTAPI -hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, - HSA_SVM_ATTRIBUTE *attrs) -{ - struct kfd_ioctl_svm_args *args; - HSAuint64 s_attr; - HSAKMT_STATUS r; - HSAuint32 i; - - CHECK_KFD_OPEN(); - - pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size); - - if (!start_addr || !size) - return HSAKMT_STATUS_INVALID_PARAMETER; - if ((uint64_t)start_addr & (PAGE_SIZE - 1)) - return HSAKMT_STATUS_INVALID_PARAMETER; - if (size & (PAGE_SIZE - 1)) - return HSAKMT_STATUS_INVALID_PARAMETER; - - s_attr = sizeof(*attrs) * nattr; - args = alloca(sizeof(*args) + s_attr); - - args->start_addr = (uint64_t)start_addr; - args->size = size; - args->op = KFD_IOCTL_SVM_OP_GET_ATTR; - args->nattr = nattr; - memcpy(args->attrs, attrs, s_attr); - - for (i = 0; i < nattr; i++) { - if (attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && - attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && - attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) - continue; - - r = validate_nodeid(attrs[i].value, &args->attrs[i].value); - if (r != HSAKMT_STATUS_SUCCESS) { - pr_debug("invalid node ID: %d\n", attrs[i].value); - return r; - } else if (!args->attrs[i].value) { - pr_debug("CPU node invalid for access attribute\n"); - return HSAKMT_STATUS_INVALID_NODE_UNIT; - } - } - - /* Driver does one copy_from_user, with extra attrs size */ - r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args); - if (r) { - pr_debug("op get range attrs failed %s\n", strerror(errno)); - return HSAKMT_STATUS_ERROR; - } - - memcpy(attrs, args->attrs, s_attr); - - for (i = 0; i < nattr; i++) { - if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && - attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC && - attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && - attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && - attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) - continue; - - switch (attrs[i].value) { - case KFD_IOCTL_SVM_LOCATION_SYSMEM: - attrs[i].value = 0; - break; - case KFD_IOCTL_SVM_LOCATION_UNDEFINED: - attrs[i].value = INVALID_NODEID; - break; - default: - r = gpuid_to_nodeid(attrs[i].value, &attrs[i].value); - if (r != HSAKMT_STATUS_SUCCESS) { - pr_debug("invalid GPU ID: %d\n", - attrs[i].value); - return r; - } - } - } - - return HSAKMT_STATUS_SUCCESS; -} From 5edd00136dbf3308b05813489a236d9dc1d3ed9c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 10 Mar 2021 20:38:51 -0500 Subject: [PATCH 0818/1247] Revert "libhsakmt: add API to support svm and xnack" This reverts commit a352639df5b281100ad4beed24756312d449c458. SVM is not ready yet. This was merged by accident. Change-Id: I1bee102823e7e612be8e8f2e0f50580e8692cc80 Signed-off-by: Felix Kuehling --- include/hsakmt.h | 31 ------------------------------- include/hsakmttypes.h | 29 ----------------------------- 2 files changed, 60 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 39b1a0c792..c311f34b2e 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -1254,37 +1254,6 @@ hsaKmtSPMSetDestBuffer( bool *isSPMDataLoss //OUT ); -/* Helper functions for calling KFD SVM ioctl */ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSVMSetAttr( - void *start_addr, // IN: Start of the virtual address range (page-aligned) - HSAuint64 size, // IN: size (page-aligned) - unsigned int nattr, // IN: number of attributes - HSA_SVM_ATTRIBUTE *attrs // IN: array of attributes -); - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSVMGetAttr( - void *start_addr, // IN: Start of the virtual address range (page-aligned) - HSAuint64 size, // IN: size (page aligned) - unsigned int nattr, // IN: number of attributes - HSA_SVM_ATTRIBUTE *attrs // IN/OUT: array of attributes -); - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetXNACKMode( - HSAint32 enable // IN: enable/disable XNACK node. -); - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetXNACKMode( - HSAint32 * enable // OUT: returns XNACK value. -); - #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index afc811d2c9..a12c5c1200 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1297,35 +1297,6 @@ typedef struct _HsaMemoryRange { HSAuint64 SizeInBytes; // Size of above memory } HsaMemoryRange; -typedef enum _HSA_SVM_FLAGS { - HSA_SVM_FLAG_HOST_ACCESS = 0x00000001, // Guarantee host access to memory - HSA_SVM_FLAG_COHERENT = 0x00000002, // Fine grained coherency between all devices with access - HSA_SVM_FLAG_HIVE_LOCAL = 0x00000004, // Use any GPU in same hive as preferred device - HSA_SVM_FLAG_GPU_RO = 0x00000008, // GPUs only read, allows replication - HSA_SVM_FLAG_GPU_EXEC = 0x00000010, // Allow execution on GPU -} HSA_SVM_FLAGS; - -typedef enum _HSA_SVM_ATTR_TYPE { - HSA_SVM_ATTR_PREFERRED_LOC, // gpuid of the preferred location, 0 for - // system memory, INVALID_NODEID for - // "don't care" - HSA_SVM_ATTR_PREFETCH_LOC, // gpuid of the prefetch location, 0 for - // system memory. Setting this triggers an - // immediate prefetch (migration) - HSA_SVM_ATTR_ACCESS, - HSA_SVM_ATTR_ACCESS_IN_PLACE, - HSA_SVM_ATTR_NO_ACCESS, // specify memory access for the gpuid given - // by the attribute value - HSA_SVM_ATTR_SET_FLAGS, // bitmask of flags to set (see HSA_SVM_FLAGS) - HSA_SVM_ATTR_CLR_FLAGS, // bitmask of flags to clear - HSA_SVM_ATTR_GRANULARITY // migration granularity (log2 num pages) -} HSA_SVM_ATTR_TYPE; - -typedef struct _HSA_SVM_ATTRIBUTE { - HSAuint32 type; // attribute type (see enum HSA_SVM_ATTR_TYPE) - HSAuint32 value; // attribute value -} HSA_SVM_ATTRIBUTE; - #pragma pack(pop, hsakmttypes_h) From 41cd7aea2f1d6705c4c232d3b8f21d5004cd1e8f Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 10 Mar 2021 20:39:33 -0500 Subject: [PATCH 0819/1247] Revert "libhsakmt: add kfd_ioctl.h svm and xnack support" This reverts commit 5ae49f2321ae2287aeae8339941997a20fed8241. SVM is not ready yet. This was merged by accident. Change-Id: I8901594a72e785ba5d25a6448718a570e76fe117 Signed-off-by: Felix Kuehling --- include/linux/kfd_ioctl.h | 167 +------------------------------------- 1 file changed, 1 insertion(+), 166 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 2d948a68cd..80f78d8578 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -731,166 +731,6 @@ struct kfd_ioctl_cross_memory_copy_args { __u64 bytes_copied; }; - -/* Guarantee host access to memory */ -#define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001 -/* Fine grained coherency between all devices with access */ -#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002 -/* Use any GPU in same hive as preferred device */ -#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004 -/* GPUs only read, allows replication */ -#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008 -/* Allow execution on GPU */ -#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010 - -/** - * kfd_ioctl_svm_op - SVM ioctl operations - * - * @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes - * @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes - */ -enum kfd_ioctl_svm_op { - KFD_IOCTL_SVM_OP_SET_ATTR, - KFD_IOCTL_SVM_OP_GET_ATTR -}; - -/** - * kfd_ioctl_svm_attr_type - SVM attribute types - * - * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: gpuid of the preferred location, 0 for - * system memory - * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: gpuid of the prefetch location, 0 for - * system memory. Setting this triggers an - * immediate prefetch (migration). - * @KFD_IOCTL_SVM_ATTR_ACCESS: - * @KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: - * @KFD_IOCTL_SVM_ATTR_NO_ACCESS: specify memory access for the gpuid given - * by the attribute value - * @KFD_IOCTL_SVM_ATTR_SET_FLAGS: bitmask of flags to set (see - * KFD_IOCTL_SVM_FLAG_...) - * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS: bitmask of flags to clear - * @KFD_IOCTL_SVM_ATTR_GRANULARITY: migration granularity - * (log2 num pages) - */ -enum kfd_ioctl_svm_attr_type { - KFD_IOCTL_SVM_ATTR_PREFERRED_LOC, - KFD_IOCTL_SVM_ATTR_PREFETCH_LOC, - KFD_IOCTL_SVM_ATTR_ACCESS, - KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE, - KFD_IOCTL_SVM_ATTR_NO_ACCESS, - KFD_IOCTL_SVM_ATTR_SET_FLAGS, - KFD_IOCTL_SVM_ATTR_CLR_FLAGS, - KFD_IOCTL_SVM_ATTR_GRANULARITY -}; - -/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations - * - * GPU IDs are used to specify GPUs as preferred and prefetch locations. - * Below definitions are used for system memory or for leaving the preferred - * location unspecified. - */ -enum kfd_ioctl_svm_location { - KFD_IOCTL_SVM_LOCATION_SYSMEM = 0, - KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff -}; - -/** - * kfd_ioctl_svm_attribute - Attributes as pairs of type and value - * - * The meaning of the @value depends on the attribute type. - * - * @type: attribute type (see enum @kfd_ioctl_svm_attr_type) - * @value: attribute value - */ -struct kfd_ioctl_svm_attribute { - __u32 type; - __u32 value; -}; - -/** - * kfd_ioctl_svm_args - Arguments for SVM ioctl - * - * @op specifies the operation to perform (see enum - * @kfd_ioctl_svm_op). @start_addr and @size are common for all - * operations. - * - * A variable number of attributes can be given in @attrs. - * @nattr specifies the number of attributes. New attributes can be - * added in the future without breaking the ABI. If unknown attributes - * are given, the function returns -EINVAL. - * - * @KFD_IOCTL_SVM_OP_SET_ATTR sets attributes for a virtual address - * range. It may overlap existing virtual address ranges. If it does, - * the existing ranges will be split such that the attribute changes - * only apply to the specified address range. - * - * @KFD_IOCTL_SVM_OP_GET_ATTR returns the intersection of attributes - * over all memory in the given range and returns the result as the - * attribute value. If different pages have different preferred or - * prefetch locations, 0xffffffff will be returned for - * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or - * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For - * @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be - * aggregated by bitwise AND. The minimum migration granularity - * throughout the range will be returned for - * @KFD_IOCTL_SVM_ATTR_GRANULARITY. - * - * Querying of accessibility attributes works by initializing the - * attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the - * GPUID being queried. Multiple attributes can be given to allow - * querying multiple GPUIDs. The ioctl function overwrites the - * attribute type to indicate the access for the specified GPU. - * - * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS is invalid for - * @KFD_IOCTL_SVM_OP_GET_ATTR. - */ -struct kfd_ioctl_svm_args { - __u64 start_addr; - __u64 size; - __u32 op; - __u32 nattr; - /* Variable length array of attributes */ - struct kfd_ioctl_svm_attribute attrs[0]; -}; - -/** - * kfd_ioctl_set_xnack_mode_args - Arguments for set_xnack_mode - * - * @xnack_enabled: [in/out] Whether to enable XNACK mode for this process - * - * @xnack_enabled indicates whether recoverable page faults should be - * enabled for the current process. 0 means disabled, positive means - * enabled, negative means leave unchanged. If enabled, virtual address - * translations on GFXv9 and later AMD GPUs can return XNACK and retry - * the access until a valid PTE is available. This is used to implement - * device page faults. - * - * On output, @xnack_enabled returns the (new) current mode (0 or - * positive). Therefore, a negative input value can be used to query - * the current mode without changing it. - * - * The XNACK mode fundamentally changes the way SVM managed memory works - * in the driver, with subtle effects on application performance and - * functionality. - * - * Enabling XNACK mode requires shader programs to be compiled - * differently. Furthermore, not all GPUs support changing the mode - * per-process. Therefore changing the mode is only allowed while no - * user mode queues exist in the process. This ensure that no shader - * code is running that may be compiled for the wrong mode. And GPUs - * that cannot change to the requested mode will be disabled by - * failing subsequent requests to create user mode queues. - * - * This ioctl returns the status of the requested xnack mode. - * - * GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM. - * - * Return: 0 on success, -errno on failure - */ -struct kfd_ioctl_set_xnack_mode_args { - __s32 xnack_enabled; -}; - #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -988,13 +828,8 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) -#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args) - -#define AMDKFD_IOC_SET_XNACK_MODE \ - AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args) - #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x22 +#define AMDKFD_COMMAND_END 0x1F /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ From d287c60246026ccb6db5208afccc895520fd24cf Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 26 Feb 2021 17:25:56 -0500 Subject: [PATCH 0820/1247] libhsakmt: New SRAM EDC support bit The old bit was deprecated, because old buggy user mode depends on it being always 0. The correct value is now reported in a new bit. New user mode handles the reported EDC setting correctly, so we can report the correct value in a new bit. Signed-off-by: Felix Kuehling Change-Id: Ib5d5ed2519810e650458c6b69c97670dab435ddb --- include/hsakmttypes.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index a12c5c1200..0ae74d9400 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -209,11 +209,12 @@ typedef union unsigned int WaveLaunchTrapOverrideSupported: 1; // Indicates if Wave Launch Trap Override is supported on the node. unsigned int WaveLaunchModeSupported: 1; // Indicates if Wave Launch Mode is supported on the node. unsigned int PreciseMemoryOperationsSupported: 1; // Indicates if Precise Memory Operations are supported on the node. - unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active + unsigned int DEPRECATED_SRAM_EDCSupport: 1; // Old buggy user mode depends on this being 0 unsigned int Mem_EDCSupport: 1; // Indicates if GFX internal DRAM/HBM EDC/ECC functionality is active unsigned int RASEventNotify: 1; // Indicates if GFX extended RASFeatures and RAS EventNotify status is available unsigned int ASICRevision: 4; // Indicates the ASIC revision of the chip on this node. - unsigned int Reserved : 6; + unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active + unsigned int Reserved : 5; } ui32; } HSA_CAPABILITY; From a9ce8683eb7d8817567ccaf2ea4a6f34d1e329ad Mon Sep 17 00:00:00 2001 From: Cole Nelson Date: Mon, 22 Mar 2021 13:09:46 -0700 Subject: [PATCH 0821/1247] CMakeLists.txt: add ENABLE_LDCONFIG to support multi-version install Depends-On: I58fdf1d0b4e864b5a61ffe8e335d430d424811ab Change-Id: I8e3e873fde99eaec79651ce6c3581870e9c2112d Signed-off-by: Cole Nelson --- CMakeLists.txt | 16 +++++++++++++--- DEBIAN/postinst | 19 ------------------- DEBIAN/postinst.in | 23 +++++++++++++++++++++++ DEBIAN/prerm | 19 ------------------- DEBIAN/prerm.in | 21 +++++++++++++++++++++ RPM/post.in | 5 +++++ RPM/postun.in | 5 +++++ RPM/rpm_post | 1 - RPM/rpm_postun | 3 --- 9 files changed, 67 insertions(+), 45 deletions(-) delete mode 100755 DEBIAN/postinst create mode 100755 DEBIAN/postinst.in delete mode 100755 DEBIAN/prerm create mode 100755 DEBIAN/prerm.in create mode 100644 RPM/post.in create mode 100644 RPM/postun.in delete mode 100644 RPM/rpm_post delete mode 100644 RPM/rpm_postun diff --git a/CMakeLists.txt b/CMakeLists.txt index a3a4d999c1..3eed035b5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -248,6 +248,7 @@ set(CPACK_PACKAGE_VERSION_PATCH ${VERSION_PATCH}) set(CPACK_PACKAGE_CONTACT "AMD GFX mailing list ") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md") set(CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support\n This repository includes the user-mode API interfaces\n used to interact with the ROCk driver.") +set ( ENABLE_LDCONFIG ON CACHE BOOL "Set library links and caches using ldconfig.") ## Component packaging, DEVEL set(CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package.\n This package contains the headers, pkgonfig and\n cmake files for the ROCT package") @@ -261,7 +262,6 @@ endif() # Debian package specific variables set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface") -set(CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst;${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm") set(CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1") set(CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev") if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) @@ -269,6 +269,10 @@ if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) else() set(CPACK_DEBIAN_PACKAGE_RELEASE "local") endif() +## Process the Debian install/remove scripts to update the CPACK variables +configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst.in DEBIAN/postinst @ONLY ) +configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm.in DEBIAN/prerm @ONLY ) +set(CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "DEBIAN/postinst;DEBIAN/prerm") # RPM package specific variables # get OS-info @@ -280,8 +284,14 @@ elseif(DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6") else() set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc") endif() -set(CPACK_RPM_BINARY_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_post") -set(CPACK_RPM_BINARY_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/rpm_postun") + +## Process the Rpm install/remove scripts to update the CPACK variables +configure_file ( "${CMAKE_CURRENT_SOURCE_DIR}/RPM/post.in" RPM/post @ONLY ) +configure_file ( "${CMAKE_CURRENT_SOURCE_DIR}/RPM/postun.in" RPM/postun @ONLY ) +## RPM package specific variables +set ( CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/RPM/post" ) +set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/RPM/postun" ) + if(DEFINED CPACK_PACKAGING_INSTALL_PREFIX) set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX} ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}") endif() diff --git a/DEBIAN/postinst b/DEBIAN/postinst deleted file mode 100755 index 9510db5d1e..0000000000 --- a/DEBIAN/postinst +++ /dev/null @@ -1,19 +0,0 @@ -#/bin/bash - -set -e - -do_ldconfig() { - echo /opt/rocm/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig -} - -case "$1" in - configure) - do_ldconfig - ;; - abort-upgrade|abort-remove|abort-deconfigure) - echo "$1" - ;; - *) - exit 0 - ;; -esac diff --git a/DEBIAN/postinst.in b/DEBIAN/postinst.in new file mode 100755 index 0000000000..04198dc7bb --- /dev/null +++ b/DEBIAN/postinst.in @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +# left-hand term originates from ENABLE_LDCONFIG = ON/OFF at package build +do_ldconfig() { + if [ "@ENABLE_LDCONFIG@" == "ON" ]; then + echo @CPACK_PACKAGING_INSTALL_PREFIX@/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf + ldconfig + fi +} + +case "$1" in + ( configure ) + do_ldconfig + ;; + ( abort-upgrade | abort-remove | abort-deconfigure ) + echo "$1" + ;; + ( * ) + exit 0 + ;; +esac diff --git a/DEBIAN/prerm b/DEBIAN/prerm deleted file mode 100755 index f8aa5a1993..0000000000 --- a/DEBIAN/prerm +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -e - -rm_ldconfig() { - rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig -} - -case "$1" in - remove) - rm_ldconfig - ;; - purge) - ;; - *) - exit 0 - ;; -esac - diff --git a/DEBIAN/prerm.in b/DEBIAN/prerm.in new file mode 100755 index 0000000000..5cc004f399 --- /dev/null +++ b/DEBIAN/prerm.in @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# left-hand term originates from ENABLE_LDCONFIG = ON/OFF at package build +rm_ldconfig() { + if [ "@ENABLE_LDCONFIG@" == "ON" ]; then + rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig + fi +} + +case "$1" in + ( remove ) + rm_ldconfig + ;; + ( purge ) + ;; + ( * ) + exit 0 + ;; +esac diff --git a/RPM/post.in b/RPM/post.in new file mode 100644 index 0000000000..5fc16b48a2 --- /dev/null +++ b/RPM/post.in @@ -0,0 +1,5 @@ +# left-hand term originates from ENABLE_LDCONFIG = ON/OFF at package build +if [ "@ENABLE_LDCONFIG@" == "ON" ]; then + echo -e "@CPACK_PACKAGING_INSTALL_PREFIX@/lib\n@CPACK_PACKAGING_INSTALL_PREFIX@/lib64" > /etc/ld.so.conf.d/x86_64-libhsakmt.conf + ldconfig +fi diff --git a/RPM/postun.in b/RPM/postun.in new file mode 100644 index 0000000000..8e8eb1fea4 --- /dev/null +++ b/RPM/postun.in @@ -0,0 +1,5 @@ +# second term originates from ENABLE_LDCONFIG = ON/OFF at package build +if [ $1 -eq 0 ] && [ "@ENABLE_LDCONFIG@" == "ON" ]; then + rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf + ldconfig +fi diff --git a/RPM/rpm_post b/RPM/rpm_post deleted file mode 100644 index fb4436af29..0000000000 --- a/RPM/rpm_post +++ /dev/null @@ -1 +0,0 @@ -echo -e "/opt/rocm/lib\n/opt/rocm/lib64" > /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig diff --git a/RPM/rpm_postun b/RPM/rpm_postun deleted file mode 100644 index fb6028fd85..0000000000 --- a/RPM/rpm_postun +++ /dev/null @@ -1,3 +0,0 @@ -if [ $1 -eq 0 ]; then - rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig -fi From b6d613d284247cb6215fc6ba8b3c9f78dcd4a773 Mon Sep 17 00:00:00 2001 From: Roberto Di Remigio Date: Wed, 24 Mar 2021 14:58:39 +0100 Subject: [PATCH 0822/1247] Treat link flags as a string, rather than a list Fixes linking failure with gold Change-Id: Ie9d2a7f68e71a0a793b041d57b8078ead22a93c8 Signed-off-by: Roberto Di Remigio Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling --- CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3eed035b5e..63983d9293 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,17 +99,17 @@ set (HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,--enable-new-dtags -Wl,--versio ## Address Sanitize Flag if ( ${ADDRESS_SANITIZER} ) set ( HSAKMT_C_FLAGS "${HSAKMT_C_FLAGS}" -fsanitize=address ) - set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -fsanitize=address ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -fsanitize=address" ) if ( BUILD_SHARED_LIBS ) - set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -shared-libsan ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -shared-libsan" ) else () - set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -static-libsan ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -static-libsan" ) endif () else () if ( CMAKE_COMPILER_IS_GNUCC ) - set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -Wl,-no-undefined ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,-no-undefined" ) else () - set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS}" -Wl,-undefined,error ) + set ( HSAKMT_LINK_FLAGS "${HSAKMT_LINK_FLAGS} -Wl,-undefined,error" ) endif () endif () From 66da1c9cd2987c061cb1abba6c1315eaebf2b4e2 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 5 Apr 2021 13:50:15 -0400 Subject: [PATCH 0823/1247] kfdtest.exclude: Add some more SP3-shader tests to GFX10 exclude These are failing as well, due to the SP3 shader merge. Blacklist them as well to avoid more segfaults Signed-off-by: Kent Russell Change-Id: I07e142a1aad9b2a5304230f333eeaf4392bea4b7 --- tests/kfdtest/scripts/kfdtest.exclude | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 7be47cac9b..da01a4c1f6 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -216,7 +216,10 @@ TEMP_GFX10_BLACKLIST=\ "KFDQMTest.QueuePriorityOnSamePipe:"\ "KFDCWSRTest.BasicTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDEvictTest.QueueTest" +"KFDEvictTest.QueueTest:"\ +"KFDMemoryTest.MapUnmapToNodes:"\ +"KFDMemoryTest.HostHdpFlush:"\ +"KFDMemoryTest.DeviceHdpFlush" FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From d748d6dce744fb8778369b03238033ccbdd16bcb Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 22 Mar 2021 09:59:41 -0400 Subject: [PATCH 0824/1247] CMakelists: Build thunk as static by default This can be overwritten by changing BUILD_SHARED_LIBS=true, but we default it to static to allow for merging into ROCr Change-Id: Ic286ef7903a5bc788fe3b84bb13b15bdd3a6f60b --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 63983d9293..26c8528016 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,8 +32,9 @@ set ( HSAKMT_TARGET "${HSAKMT}" ) project ( ${HSAKMT_TARGET} VERSION 1.9.0) +# Build thunk as static by default if ( NOT DEFINED BUILD_SHARED_LIBS ) - set ( BUILD_SHARED_LIBS "on" ) + set ( BUILD_SHARED_LIBS "off" ) endif() set ( BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS} CACHE BOOL "Build shared library (.so) or not.") From 43ce63b68b5950ad1109870886b82a7372fce9a3 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Sat, 11 Jul 2020 23:37:32 -0400 Subject: [PATCH 0825/1247] libhsakmt: Query KFD version once after opening /dev/kfd Query the KFD interface version once and store it in a global variable. This makes it more efficient for KFD APIs to query the API version later. Signed-off-by: Felix Kuehling Change-Id: I267f3465f754e78fb21a7c42c5877cd68eaa9d05 --- src/libhsakmt.h | 8 ++++++++ src/openclose.c | 5 +++++ src/version.c | 16 ++++++++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index f455699c6d..9c7047caa2 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -38,6 +38,8 @@ extern bool hsakmt_forked; extern pthread_mutex_t hsakmt_mutex; extern bool is_dgpu; +extern HsaVersionInfo kfd_version_info; + extern int force_asic; extern char force_asic_name[HSA_PUBLIC_NAME_SIZE]; extern struct hsa_gfxip_table force_asic_entry; @@ -54,6 +56,10 @@ extern struct hsa_gfxip_table force_asic_entry; #define CHECK_KFD_OPEN() \ do { if (kfd_open_count == 0 || hsakmt_forked) return HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; } while (0) +#define CHECK_KFD_MINOR_VERSION(minor) \ + do { if ((minor) > kfd_version_info.KernelInterfaceMinorVersion)\ + return HSAKMT_STATUS_NOT_SUPPORTED; } while (0) + extern int PAGE_SIZE; extern int PAGE_SHIFT; @@ -145,6 +151,8 @@ struct hsa_gfxip_table { enum asic_family_type asic_family; // Device family id }; +HSAKMT_STATUS init_kfd_version(void); + #define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); diff --git a/src/openclose.c b/src/openclose.c index 784ee51f2a..55ba5f23f2 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -196,6 +196,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) init_page_size(); + result = init_kfd_version(); + if (result != HSAKMT_STATUS_SUCCESS) + goto kfd_version_failed; + result = topology_sysfs_get_system_props(&sys_props); if (result != HSAKMT_STATUS_SUCCESS) goto topology_sysfs_failed; @@ -238,6 +242,7 @@ init_doorbell_failed: fmm_destroy_process_apertures(); init_process_aperture_failed: topology_sysfs_failed: +kfd_version_failed: close(fd); open_failed: pthread_mutex_unlock(&hsakmt_mutex); diff --git a/src/version.c b/src/version.c index 2de79d0b08..34e51e0811 100644 --- a/src/version.c +++ b/src/version.c @@ -28,17 +28,29 @@ #include #include "linux/kfd_ioctl.h" +HsaVersionInfo kfd_version_info; + HSAKMT_STATUS HSAKMTAPI hsaKmtGetVersion(HsaVersionInfo *VersionInfo) { CHECK_KFD_OPEN(); + *VersionInfo = kfd_version_info; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS init_kfd_version(void) +{ struct kfd_ioctl_get_version_args args = {0}; if (kmtIoctl(kfd_fd, AMDKFD_IOC_GET_VERSION, &args) == -1) return HSAKMT_STATUS_ERROR; - VersionInfo->KernelInterfaceMajorVersion = args.major_version; - VersionInfo->KernelInterfaceMinorVersion = args.minor_version; + kfd_version_info.KernelInterfaceMajorVersion = args.major_version; + kfd_version_info.KernelInterfaceMinorVersion = args.minor_version; + + if (args.major_version != 1) + return HSAKMT_STATUS_DRIVER_MISMATCH; return HSAKMT_STATUS_SUCCESS; } From ce26348f3a301bd7bd7b5c0a63a09d478193ba90 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 19 Mar 2021 15:33:56 -0400 Subject: [PATCH 0826/1247] libhsakmt: add kfd_ioctl.h svm and xnack support Add svm (shared virtual memory) range and xnack mode APIs. Change-Id: Ibd8d7fe566dc200730da0c892caa71aad7589ebd Signed-off-by: Philip Yang Signed-off-by: Alex Sierra --- include/linux/kfd_ioctl.h | 177 +++++++++++++++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 2 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 80f78d8578..868fbdd370 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -26,8 +26,14 @@ #include #include +/* + * - 1.1 - initial version + * - 1.3 - Add SMI events support + * - 1.4 - Indicate new SRAM EDC bit in device properties + * - 1.5 - Add SVM API + */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 2 +#define KFD_IOCTL_MINOR_VERSION 5 /* * Debug revision change log @@ -731,6 +737,168 @@ struct kfd_ioctl_cross_memory_copy_args { __u64 bytes_copied; }; + +/* Guarantee host access to memory */ +#define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001 +/* Fine grained coherency between all devices with access */ +#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002 +/* Use any GPU in same hive as preferred device */ +#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004 +/* GPUs only read, allows replication */ +#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008 +/* Allow execution on GPU */ +#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010 +/* GPUs mostly read, may allow similar optimizations as RO, but writes fault */ +#define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020 + +/** + * kfd_ioctl_svm_op - SVM ioctl operations + * + * @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes + * @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes + */ +enum kfd_ioctl_svm_op { + KFD_IOCTL_SVM_OP_SET_ATTR, + KFD_IOCTL_SVM_OP_GET_ATTR +}; + +/** + * kfd_ioctl_svm_attr_type - SVM attribute types + * + * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: gpuid of the preferred location, 0 for + * system memory + * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: gpuid of the prefetch location, 0 for + * system memory. Setting this triggers an + * immediate prefetch (migration). + * @KFD_IOCTL_SVM_ATTR_ACCESS: + * @KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: + * @KFD_IOCTL_SVM_ATTR_NO_ACCESS: specify memory access for the gpuid given + * by the attribute value + * @KFD_IOCTL_SVM_ATTR_SET_FLAGS: bitmask of flags to set (see + * KFD_IOCTL_SVM_FLAG_...) + * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS: bitmask of flags to clear + * @KFD_IOCTL_SVM_ATTR_GRANULARITY: migration granularity + * (log2 num pages) + */ +enum kfd_ioctl_svm_attr_type { + KFD_IOCTL_SVM_ATTR_PREFERRED_LOC, + KFD_IOCTL_SVM_ATTR_PREFETCH_LOC, + KFD_IOCTL_SVM_ATTR_ACCESS, + KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE, + KFD_IOCTL_SVM_ATTR_NO_ACCESS, + KFD_IOCTL_SVM_ATTR_SET_FLAGS, + KFD_IOCTL_SVM_ATTR_CLR_FLAGS, + KFD_IOCTL_SVM_ATTR_GRANULARITY +}; + +/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations + * + * GPU IDs are used to specify GPUs as preferred and prefetch locations. + * Below definitions are used for system memory or for leaving the preferred + * location unspecified. + */ +enum kfd_ioctl_svm_location { + KFD_IOCTL_SVM_LOCATION_SYSMEM = 0, + KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff +}; + +/** + * kfd_ioctl_svm_attribute - Attributes as pairs of type and value + * + * The meaning of the @value depends on the attribute type. + * + * @type: attribute type (see enum @kfd_ioctl_svm_attr_type) + * @value: attribute value + */ +struct kfd_ioctl_svm_attribute { + __u32 type; + __u32 value; +}; + +/** + * kfd_ioctl_svm_args - Arguments for SVM ioctl + * + * @op specifies the operation to perform (see enum + * @kfd_ioctl_svm_op). @start_addr and @size are common for all + * operations. + * + * A variable number of attributes can be given in @attrs. + * @nattr specifies the number of attributes. New attributes can be + * added in the future without breaking the ABI. If unknown attributes + * are given, the function returns -EINVAL. + * + * @KFD_IOCTL_SVM_OP_SET_ATTR sets attributes for a virtual address + * range. It may overlap existing virtual address ranges. If it does, + * the existing ranges will be split such that the attribute changes + * only apply to the specified address range. + * + * @KFD_IOCTL_SVM_OP_GET_ATTR returns the intersection of attributes + * over all memory in the given range and returns the result as the + * attribute value. If different pages have different preferred or + * prefetch locations, 0xffffffff will be returned for + * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or + * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For + * @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be + * aggregated by bitwise AND. The minimum migration granularity + * throughout the range will be returned for + * @KFD_IOCTL_SVM_ATTR_GRANULARITY. + * + * Querying of accessibility attributes works by initializing the + * attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the + * GPUID being queried. Multiple attributes can be given to allow + * querying multiple GPUIDs. The ioctl function overwrites the + * attribute type to indicate the access for the specified GPU. + * + * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS is invalid for + * @KFD_IOCTL_SVM_OP_GET_ATTR. + */ +struct kfd_ioctl_svm_args { + __u64 start_addr; + __u64 size; + __u32 op; + __u32 nattr; + /* Variable length array of attributes */ + struct kfd_ioctl_svm_attribute attrs[0]; +}; + +/** + * kfd_ioctl_set_xnack_mode_args - Arguments for set_xnack_mode + * + * @xnack_enabled: [in/out] Whether to enable XNACK mode for this process + * + * @xnack_enabled indicates whether recoverable page faults should be + * enabled for the current process. 0 means disabled, positive means + * enabled, negative means leave unchanged. If enabled, virtual address + * translations on GFXv9 and later AMD GPUs can return XNACK and retry + * the access until a valid PTE is available. This is used to implement + * device page faults. + * + * On output, @xnack_enabled returns the (new) current mode (0 or + * positive). Therefore, a negative input value can be used to query + * the current mode without changing it. + * + * The XNACK mode fundamentally changes the way SVM managed memory works + * in the driver, with subtle effects on application performance and + * functionality. + * + * Enabling XNACK mode requires shader programs to be compiled + * differently. Furthermore, not all GPUs support changing the mode + * per-process. Therefore changing the mode is only allowed while no + * user mode queues exist in the process. This ensure that no shader + * code is running that may be compiled for the wrong mode. And GPUs + * that cannot change to the requested mode will be disabled by + * failing subsequent requests to create user mode queues. + * + * This ioctl returns the status of the requested xnack mode. + * + * GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM. + * + * Return: 0 on success, -errno on failure + */ +struct kfd_ioctl_set_xnack_mode_args { + __s32 xnack_enabled; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -828,8 +996,13 @@ struct kfd_ioctl_cross_memory_copy_args { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) +#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args) + +#define AMDKFD_IOC_SET_XNACK_MODE \ + AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1F +#define AMDKFD_COMMAND_END 0x22 /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ From c44a4be77648cda5fe066caa04be9f43566031c3 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 19 Mar 2021 15:34:05 -0400 Subject: [PATCH 0827/1247] libhsakmt: add API to support svm and xnack Add function definitions to support SVM (shared virtual memory) and xnack set. Change-Id: Ia97ad9d0c449d8d500d799f702e1a58e87d65a56 Signed-off-by: Alex Sierra Signed-off-by: Philip Yang Signed-off-by: Felix Kuehling --- include/hsakmt.h | 31 +++++++++++++++++++++++++++++++ include/hsakmttypes.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index c311f34b2e..39b1a0c792 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -1254,6 +1254,37 @@ hsaKmtSPMSetDestBuffer( bool *isSPMDataLoss //OUT ); +/* Helper functions for calling KFD SVM ioctl */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSVMSetAttr( + void *start_addr, // IN: Start of the virtual address range (page-aligned) + HSAuint64 size, // IN: size (page-aligned) + unsigned int nattr, // IN: number of attributes + HSA_SVM_ATTRIBUTE *attrs // IN: array of attributes +); + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSVMGetAttr( + void *start_addr, // IN: Start of the virtual address range (page-aligned) + HSAuint64 size, // IN: size (page aligned) + unsigned int nattr, // IN: number of attributes + HSA_SVM_ATTRIBUTE *attrs // IN/OUT: array of attributes +); + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtSetXNACKMode( + HSAint32 enable // IN: enable/disable XNACK node. +); + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetXNACKMode( + HSAint32 * enable // OUT: returns XNACK value. +); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 0ae74d9400..172643770e 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1298,6 +1298,36 @@ typedef struct _HsaMemoryRange { HSAuint64 SizeInBytes; // Size of above memory } HsaMemoryRange; +typedef enum _HSA_SVM_FLAGS { + HSA_SVM_FLAG_HOST_ACCESS = 0x00000001, // Guarantee host access to memory + HSA_SVM_FLAG_COHERENT = 0x00000002, // Fine grained coherency between all devices with access + HSA_SVM_FLAG_HIVE_LOCAL = 0x00000004, // Use any GPU in same hive as preferred device + HSA_SVM_FLAG_GPU_RO = 0x00000008, // GPUs only read, allows replication + HSA_SVM_FLAG_GPU_EXEC = 0x00000010, // Allow execution on GPU + HSA_SVM_FLAG_GPU_READ_MOSTLY = 0x00000020, // GPUs mostly read, may allow similar optimizations as RO, but writes fault +} HSA_SVM_FLAGS; + +typedef enum _HSA_SVM_ATTR_TYPE { + HSA_SVM_ATTR_PREFERRED_LOC, // gpuid of the preferred location, 0 for + // system memory, INVALID_NODEID for + // "don't care" + HSA_SVM_ATTR_PREFETCH_LOC, // gpuid of the prefetch location, 0 for + // system memory. Setting this triggers an + // immediate prefetch (migration) + HSA_SVM_ATTR_ACCESS, + HSA_SVM_ATTR_ACCESS_IN_PLACE, + HSA_SVM_ATTR_NO_ACCESS, // specify memory access for the gpuid given + // by the attribute value + HSA_SVM_ATTR_SET_FLAGS, // bitmask of flags to set (see HSA_SVM_FLAGS) + HSA_SVM_ATTR_CLR_FLAGS, // bitmask of flags to clear + HSA_SVM_ATTR_GRANULARITY // migration granularity (log2 num pages) +} HSA_SVM_ATTR_TYPE; + +typedef struct _HSA_SVM_ATTRIBUTE { + HSAuint32 type; // attribute type (see enum HSA_SVM_ATTR_TYPE) + HSAuint32 value; // attribute value +} HSA_SVM_ATTRIBUTE; + #pragma pack(pop, hsakmttypes_h) From dd72f236c1275ad9748e97c0104c66a5fa8e8c37 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 19 Mar 2021 15:34:13 -0400 Subject: [PATCH 0828/1247] libhsakmt: add SVM thunk implementation Implement SVM (Shared Virtual Memory) in the thunk. Change-Id: I0380150d1d3da48070f9389a06f416d6059d6948 Signed-off-by: Philip Yang Signed-off-by: Sean Keely Signed-off-by: Felix Kuehling Signed-off-by: Alex Sierra --- CMakeLists.txt | 3 +- src/libhsakmt.h | 1 + src/libhsakmt.ver | 2 + src/svm.c | 187 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 src/svm.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 26c8528016..bd6dbbfa82 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,7 +130,8 @@ set ( HSAKMT_SRC "src/debug.c" "src/topology.c" "src/rbtree.c" "src/spm.c" - "src/version.c") + "src/version.c" + "src/svm.c") ## Declare the library target name add_library ( ${HSAKMT_TARGET} "") diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 9c7047caa2..6be639261b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -26,6 +26,7 @@ #ifndef LIBHSAKMT_H_INCLUDED #define LIBHSAKMT_H_INCLUDED +#include "linux/kfd_ioctl.h" #include "hsakmt.h" #include "pci_ids.h" #include diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 4383695e98..3992e6535a 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -75,6 +75,8 @@ hsaKmtClearAddressWatch; hsaKmtSPMAcquire; hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; +hsaKmtSVMSetAttr; +hsaKmtSVMGetAttr; local: *; }; diff --git a/src/svm.c b/src/svm.c new file mode 100644 index 0000000000..8f20fc0378 --- /dev/null +++ b/src/svm.c @@ -0,0 +1,187 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ +#include "libhsakmt.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Helper functions for calling KFD SVM ioctl */ + +HSAKMT_STATUS HSAKMTAPI +hsaKmtSVMSetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, + HSA_SVM_ATTRIBUTE *attrs) +{ + struct kfd_ioctl_svm_args *args; + HSAuint64 s_attr; + HSAKMT_STATUS r; + HSAuint32 i; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(5); + + pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size); + + if (!start_addr || !size) + return HSAKMT_STATUS_INVALID_PARAMETER; + if ((uint64_t)start_addr & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + if (size & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + + s_attr = sizeof(*attrs) * nattr; + args = alloca(sizeof(*args) + s_attr); + + args->start_addr = (uint64_t)start_addr; + args->size = size; + args->op = KFD_IOCTL_SVM_OP_SET_ATTR; + args->nattr = nattr; + memcpy(args->attrs, attrs, s_attr); + + for (i = 0; i < nattr; i++) { + if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && + attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) + continue; + + if (attrs[i].type == KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && + attrs[i].value == INVALID_NODEID) { + args->attrs[i].value = KFD_IOCTL_SVM_LOCATION_UNDEFINED; + continue; + } + + r = validate_nodeid(attrs[i].value, &args->attrs[i].value); + if (r != HSAKMT_STATUS_SUCCESS) { + pr_debug("invalid node ID: %d\n", attrs[i].value); + return r; + } else if (!args->attrs[i].value && + (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS || + attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE || + attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS)) { + pr_debug("CPU node invalid for access attribute\n"); + return HSAKMT_STATUS_INVALID_NODE_UNIT; + } + } + + /* Driver does one copy_from_user, with extra attrs size */ + r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args); + if (r) { + pr_debug("op set range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, + HSA_SVM_ATTRIBUTE *attrs) +{ + struct kfd_ioctl_svm_args *args; + HSAuint64 s_attr; + HSAKMT_STATUS r; + HSAuint32 i; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(5); + + pr_debug("%s: address 0x%p size 0x%lx\n", __func__, start_addr, size); + + if (!start_addr || !size) + return HSAKMT_STATUS_INVALID_PARAMETER; + if ((uint64_t)start_addr & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + if (size & (PAGE_SIZE - 1)) + return HSAKMT_STATUS_INVALID_PARAMETER; + + s_attr = sizeof(*attrs) * nattr; + args = alloca(sizeof(*args) + s_attr); + + args->start_addr = (uint64_t)start_addr; + args->size = size; + args->op = KFD_IOCTL_SVM_OP_GET_ATTR; + args->nattr = nattr; + memcpy(args->attrs, attrs, s_attr); + + for (i = 0; i < nattr; i++) { + if (attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && + attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) + continue; + + r = validate_nodeid(attrs[i].value, &args->attrs[i].value); + if (r != HSAKMT_STATUS_SUCCESS) { + pr_debug("invalid node ID: %d\n", attrs[i].value); + return r; + } else if (!args->attrs[i].value) { + pr_debug("CPU node invalid for access attribute\n"); + return HSAKMT_STATUS_INVALID_NODE_UNIT; + } + } + + /* Driver does one copy_from_user, with extra attrs size */ + r = kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args); + if (r) { + pr_debug("op get range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + memcpy(attrs, args->attrs, s_attr); + + for (i = 0; i < nattr; i++) { + if (attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFERRED_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_PREFETCH_LOC && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS && + attrs[i].type != KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE && + attrs[i].type != KFD_IOCTL_SVM_ATTR_NO_ACCESS) + continue; + + switch (attrs[i].value) { + case KFD_IOCTL_SVM_LOCATION_SYSMEM: + attrs[i].value = 0; + break; + case KFD_IOCTL_SVM_LOCATION_UNDEFINED: + attrs[i].value = INVALID_NODEID; + break; + default: + r = gpuid_to_nodeid(attrs[i].value, &attrs[i].value); + if (r != HSAKMT_STATUS_SUCCESS) { + pr_debug("invalid GPU ID: %d\n", + attrs[i].value); + return r; + } + } + } + + return HSAKMT_STATUS_SUCCESS; +} From bb441d0bdde9b99c93662266eb37711baae43ff0 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 19 Mar 2021 15:34:20 -0400 Subject: [PATCH 0829/1247] libhsakmt: add XNACK API set/get mode XNACK API for GPUs that support this mode. This API makes calls to amdgpu driver to configure xnack mode. It supports set xnack mode and query the current mode used. Change-Id: If865fd0e3f900f008243dc49504e1a0694e1791a Signed-off-by: Alex Sierra --- src/libhsakmt.ver | 2 ++ src/svm.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 3992e6535a..66ca6ea5b3 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -77,6 +77,8 @@ hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; hsaKmtSVMSetAttr; hsaKmtSVMGetAttr; +hsaKmtSetXNACKMode; +hsaKmtGetXNACKMode; local: *; }; diff --git a/src/svm.c b/src/svm.c index 8f20fc0378..edb73bbd18 100644 --- a/src/svm.c +++ b/src/svm.c @@ -185,3 +185,43 @@ hsaKmtSVMGetAttr(void *start_addr, HSAuint64 size, unsigned int nattr, return HSAKMT_STATUS_SUCCESS; } + +static HSAKMT_STATUS +hsaKmtSetGetXNACKMode(HSAint32 * enable) +{ + struct kfd_ioctl_set_xnack_mode_args args; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(5); + + args.xnack_enabled = *enable; + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SET_XNACK_MODE, &args)) { + if (errno == EPERM) { + pr_debug("set mode not supported %s\n", + strerror(errno)); + return HSAKMT_STATUS_NOT_SUPPORTED; + } else if (errno == EBUSY) { + pr_debug("kmtIoctl queues not empty %s\n", + strerror(errno)); + } + return HSAKMT_STATUS_ERROR; + } + + *enable = args.xnack_enabled; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtSetXNACKMode(HSAint32 enable) +{ + return hsaKmtSetGetXNACKMode(&enable); +} + +HSAKMT_STATUS HSAKMTAPI +hsaKmtGetXNACKMode(HSAint32 * enable) +{ + *enable = -1; + return hsaKmtSetGetXNACKMode(enable); +} From e8f369b385039baede98e2d1fb1cce7527d87870 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 9 Dec 2020 22:04:30 -0500 Subject: [PATCH 0830/1247] libhsakmt: dynamic HMM and xnack detection New properties SVMAPISupported added in Thunk spec HSA_CAPABILITY, read from sysfs from KFD topology. New local memory property flag CoherentHostAccess added to Thunk HSA_MEMORYPROPERTY, read from sysfs from KFD topology. Change-Id: I83933f0e5a61508508168873209dba4af0b77295 Signed-off-by: Philip Yang --- include/hsakmttypes.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 172643770e..efff03d912 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -214,7 +214,8 @@ typedef union unsigned int RASEventNotify: 1; // Indicates if GFX extended RASFeatures and RAS EventNotify status is available unsigned int ASICRevision: 4; // Indicates the ASIC revision of the chip on this node. unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active - unsigned int Reserved : 5; + unsigned int SVMAPISupported : 1; // Whether or not the SVM API is supported + unsigned int Reserved : 4; } ui32; } HSA_CAPABILITY; @@ -340,7 +341,8 @@ typedef union unsigned int HotPluggable : 1; // the memory may be removed by some system action, // memory should be used for temporary data unsigned int NonVolatile : 1; // memory content is preserved across a power-off cycle. - unsigned int Reserved :30; + unsigned int CoherentHostAccess: 1; // Whether or not device memory can be coherently accessed by the host CPU + unsigned int Reserved :29; } ui32; } HSA_MEMORYPROPERTY; From e8990cf830c043bf8f81704df4270c950d8f5bec Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 19 Mar 2021 16:10:16 -0400 Subject: [PATCH 0831/1247] kfdtest: add SVM tests KFD changes are ready, all SVM tests should pass now. Skip SVM tests if the SVM API is not supported. Change-Id: I5e358565a0458eea45eae0aaf4969ce3a36574a7 Signed-off-by: Philip Yang Signed-off-by: Alex Sierra Signed-off-by: Felix Kuehling --- tests/kfdtest/CMakeLists.txt | 3 +- tests/kfdtest/src/KFDBaseComponentTest.hpp | 8 + tests/kfdtest/src/KFDSVMEvictTest.cpp | 452 ++++++++ tests/kfdtest/src/KFDSVMEvictTest.hpp | 70 ++ tests/kfdtest/src/KFDSVMRangeTest.cpp | 1139 ++++++++++++++++++++ tests/kfdtest/src/KFDSVMRangeTest.hpp | 46 + tests/kfdtest/src/KFDTestUtil.cpp | 180 +++- tests/kfdtest/src/KFDTestUtil.hpp | 41 + 8 files changed, 1937 insertions(+), 2 deletions(-) create mode 100644 tests/kfdtest/src/KFDSVMEvictTest.cpp create mode 100644 tests/kfdtest/src/KFDSVMEvictTest.hpp create mode 100644 tests/kfdtest/src/KFDSVMRangeTest.cpp create mode 100644 tests/kfdtest/src/KFDSVMRangeTest.hpp diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index a8c9f30299..04e797e7d5 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -144,7 +144,8 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/KFDHWSTest.cpp src/KFDPerformanceTest.cpp src/KFDPMTest.cpp - + src/KFDSVMRangeTest.cpp + src/KFDSVMEvictTest.cpp src/KFDRASTest.cpp src/RDMATest.cpp) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 5fc61015e4..187d9fbd73 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -78,6 +78,14 @@ class KFDBaseComponentTest : public testing::Test { virtual void SetUp(); // @brief Executed after every test that uses KFDBaseComponentTest class. virtual void TearDown(); + + bool SVMAPISupported() { + bool supported = m_NodeInfo.HsaDefaultGPUNodeProperties() + ->Capability.ui32.SVMAPISupported; + if (!supported) + LOG() << "SVM API not supported" << std::endl; + return supported; + } }; extern KFDBaseComponentTest* g_baseTest; diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp new file mode 100644 index 0000000000..c05e460c3d --- /dev/null +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -0,0 +1,452 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDSVMEvictTest.hpp" +#include +#include +#include +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "Dispatch.hpp" + +#define N_PROCESSES (4) /* number of processes running in parallel, at least 2 */ +#define ALLOCATE_BUF_SIZE_MB (64) +#define ALLOCATE_RETRY_TIMES (3) + +HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) { + HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT; + HSAuint64 sysMemSize = GetSysMemSize(); + HSAuint64 size, sizeInPages; + HSAuint32 count; + + LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; + + /* use one third of total system memory for eviction buffer to test + * limit max allocate size to duoble of vramSize + * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) + */ + size = sysMemSize / 3 + vramSize; + size = size > vramSize << 1 ? vramSize << 1 : size; + sizeInPages = size >> PAGE_SHIFT; + count = sizeInPages / (vramBufSizeInPages * N_PROCESSES); + + return count; +} + +void KFDSVMEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, + std::vector &pBuffers) { + HSAuint64 totalMB; + + totalMB = N_PROCESSES * count * (vramBufSize >> 20); + if (m_IsParent) { + LOG() << "Testing " << N_PROCESSES << "*" << count << "*" << (vramBufSize>>20) << "(="<< totalMB << ")MB" << std::endl; + } + HSAKMT_STATUS ret; + HSAuint32 retry = 0; + + for (HSAuint32 i = 0; i < count; i++) { + m_pBuf = mmap(0, vramBufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + EXPECT_NOTNULL(m_pBuf); + + m_Flags = (HSA_SVM_FLAGS)0; +retry: + ret = RegisterSVMRange(defaultGPUNode, m_pBuf, vramBufSize, defaultGPUNode, m_Flags); + if (ret == HSAKMT_STATUS_SUCCESS) { + pBuffers.push_back(m_pBuf); + retry = 0; + } else { + if (retry++ > ALLOCATE_RETRY_TIMES) { + munmap(m_pBuf, vramBufSize); + break; + } + printf("retry %d allocate vram\n", retry); + + /* wait for 1 second to try allocate again */ + sleep(1); + goto retry; + } + } +} + +void KFDSVMEvictTest::FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize) { + for (HSAuint32 i = 0; i < pBuffers.size(); i++) { + m_pBuf = pBuffers[i]; + if (m_pBuf != NULL) + munmap(m_pBuf, vramBufSize); + } +} + +void KFDSVMEvictTest::ForkChildProcesses(int nprocesses) { + int i; + + for (i = 0; i < nprocesses - 1; ++i) { + pid_t pid = fork(); + ASSERT_GE(pid, 0); + + if (pid == 0) { + /* Child process */ + /* Cleanup file descriptors copied from parent process + * then call SetUp->hsaKmtOpenKFD to create new process + */ + m_psName = "Test process " + std::to_string(i) + " "; + TearDown(); + SetUp(); + m_ChildPids.clear(); + m_IsParent = false; + return; + } + + /* Parent process */ + m_ChildPids.push_back(pid); + } + + m_psName = "Test process " + std::to_string(i) + " "; +} + +void KFDSVMEvictTest::WaitChildProcesses() { + if (m_IsParent) { + /* only run by parent process */ + int childStatus; + int childExitOkNum = 0; + int size = m_ChildPids.size(); + + for (HSAuint32 i = 0; i < size; i++) { + pid_t pid = m_ChildPids.front(); + + waitpid(pid, &childStatus, 0); + if (WIFEXITED(childStatus) == 1 && WEXITSTATUS(childStatus) == 0) + childExitOkNum++; + + m_ChildPids.erase(m_ChildPids.begin()); + } + + ASSERT_EQ(childExitOkNum, size); + } + + /* child process or parent process finished successfullly */ + m_ChildStatus = HSAKMT_STATUS_SUCCESS; +} + +/* Evict and restore procedure basic test + * + * Use N_PROCESSES processes to allocate vram buf size larger than total vram size + * + * ALLOCATE_BUF_SIZE_MB buf allocation size + * + * number of buf is equal to (vramSizeMB / (vramBufSizeMB * N_PROCESSES) ) + 8 + * Total vram all processes allocated: 8GB for 4GB Fiji, and 20GB for 16GB Vega10 + * + * many times of eviction and restore will happen: + * ttm will evict buffers of another process if not enough free vram + * process restore will evict buffers of another process + * + * Sometimes the allocate may fail (maybe that is normal) + * ALLOCATE_RETRY_TIMES max retry times to allocate + * + * This is basic test, no queue so vram are not used by GPU during test + * + * Todo: + * - Synchronization between the processes, so they know for sure when + * they are done allocating memory + */ +TEST_F(KFDSVMEvictTest, BasicTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + + if (!vramSize) { + LOG() << "No VRAM found, skipping the test" << std::endl; + return; + } else { + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; + } + + HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); + if (count == 0) { + LOG() << "Not enough system memory, skipping the test" << std::endl; + return; + } + + /* Fork the child processes */ + ForkChildProcesses(N_PROCESSES); + + std::vector pBuffers; + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + + /* wait for other processes to finish allocation, then free buffer */ + sleep(ALLOCATE_RETRY_TIMES); + + LOG() << m_psName << "free buffer" << std::endl; + FreeBuffers(pBuffers, vramBufSize); + + WaitChildProcesses(); + + TEST_END +} + +/* Shader to read local buffers using multiple wavefronts in parallel + * until address buffer is filled with specific value 0x5678 by host program, + * then each wavefront fills value 0x5678 at corresponding result buffer and quit + * + * initial state: + * s[0:1] - address buffer base address + * s[2:3] - result buffer base address + * s4 - workgroup id + * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 + * registers: + * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X + * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 + * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 + * v[6:7] - local buf address used for read test + */ +static const char* gfx9_ReadMemory = +"\ + shader ReadMemory\n\ + asic(GFX9)\n\ + type(CS)\n\ + \n\ + // compute address of corresponding output buffer\n\ + v_mov_b32 v0, s4 // use workgroup id as index\n\ + v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ + v_mov_b32 v5, s3\n\ + v_add_u32 v5, vcc_lo, v5\n\ + \n\ + // compute input buffer offset used to store corresponding local buffer address\n\ + v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ + v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ + v_mov_b32 v3, s1\n\ + v_add_u32 v3, vcc_lo, v3\n\ + \n\ + // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ + flat_load_dwordx2 v[6:7], v[2:3] slc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + \n\ + v_mov_b32 v8, 0x5678\n\ + s_movk_i32 s8, 0x5678\n\ +L_REPEAT:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + s_cmp_eq_i32 s16, s8\n\ + s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ + // loop read 64M local buffer starting at v[6:7]\n\ + // every 4k page only read once\n\ + v_mov_b32 v9, 0\n\ + v_mov_b32 v10, 0x1000 // 4k page\n\ + v_mov_b32 v11, 0x4000000 // 64M size\n\ + v_mov_b32 v12, v6\n\ + v_mov_b32 v13, v7\n\ +L_LOOP_READ:\n\ + flat_load_dwordx2 v[14:15], v[12:13] slc\n\ + v_add_u32 v9, v9, v10 \n\ + v_add_co_u32 v12, vcc, v12, v10\n\ + v_add_u32 v13, vcc_lo, v13\n\ + v_cmp_lt_u32 vcc, v9, v11\n\ + s_cbranch_vccnz L_LOOP_READ\n\ + s_branch L_REPEAT\n\ +L_QUIT:\n\ + flat_store_dword v[4:5], v8\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ + s_endpgm\n\ + end\n\ +"; + +static const char* gfx8_ReadMemory = +"\ + shader ReadMemory\n\ + asic(VI)\n\ + type(CS)\n\ + \n\ + // compute address of corresponding output buffer\n\ + v_mov_b32 v0, s4 // use workgroup id as index\n\ + v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ + v_mov_b32 v5, s3\n\ + v_addc_u32 v5, vcc, v5, 0, vcc\n\ + \n\ + // compute input buffer offset used to store corresponding local buffer address\n\ + v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ + v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ + v_mov_b32 v3, s1\n\ + v_addc_u32 v3, vcc, v3, 0, vcc\n\ + \n\ + // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ + flat_load_dwordx2 v[6:7], v[2:3] slc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + \n\ + v_mov_b32 v8, 0x5678\n\ + s_movk_i32 s8, 0x5678\n\ +L_REPEAT:\n\ + s_load_dword s16, s[0:1], 0x0 glc\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ + s_cmp_eq_i32 s16, s8\n\ + s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ + // loop read 64M local buffer starting at v[6:7]\n\ + // every 4k page only read once\n\ + v_mov_b32 v9, 0\n\ + v_mov_b32 v10, 0x1000 // 4k page\n\ + v_mov_b32 v11, 0x4000000 // 64M size\n\ + v_mov_b32 v12, v6\n\ + v_mov_b32 v13, v7\n\ +L_LOOP_READ:\n\ + flat_load_dwordx2 v[14:15], v[12:13] slc\n\ + v_add_u32 v9, vcc, v9, v10 \n\ + v_add_u32 v12, vcc, v12, v10\n\ + v_addc_u32 v13, vcc, v13, 0, vcc\n\ + v_cmp_lt_u32 vcc, v9, v11\n\ + s_cbranch_vccnz L_LOOP_READ\n\ + s_branch L_REPEAT\n\ +L_QUIT:\n\ + flat_store_dword v[4:5], v8\n\ + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ + s_endpgm\n\ + end\n\ +"; + +std::string KFDSVMEvictTest::CreateShader() { + if (m_FamilyId >= FAMILY_AI) + return gfx9_ReadMemory; + else + return gfx8_ReadMemory; +} + +/* Evict and restore queue test + * + * N_PROCESSES processes read all local buffers in parallel while buffers are evicted and restored + * If GPU vm page fault happens, then test shader will stop and failed to write specific value + * at dest buffer. Test will report failed. + * + * Steps: + * - fork N_PROCESSES processes, each process does the same below + * - allocate local buffers, each buffer size is 64MB + * - allocate zero initialized host access address buffer and result buffer + * address buffer to pass address of local buffers to shader + * result buffer to store shader output result + * - submit queue to run ReadMemory shader + * - shader start m_DimX wavefronts, each wavefront keep reading one local buffer + * - notify shader to quit + * - check result buffer with specific value to confirm all wavefronts quit normally + */ +TEST_F(KFDSVMEvictTest, QueueTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + if (!SVMAPISupported()) + return; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + + const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + + /* Skip test for chip it doesn't have CWSR, which the test depends on */ + if (m_FamilyId < FAMILY_VI || isTonga(pNodeProperties)) { + LOG() << std::hex << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; + return; + } + + HSAuint32 i; + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + + if (!vramSize) { + LOG() << "No VRAM found, skipping the test" << std::endl; + return; + } else { + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; + } + + HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); + if (count == 0) { + LOG() << "Not enough system memory, skipping the test" << std::endl; + return; + } + /* assert all buffer address can be stored within one page + * because only one page host memory srcBuf is allocated + */ + ASSERT_LE(count, PAGE_SIZE/sizeof(unsigned int *)); + + /* Fork the child processes */ + ForkChildProcesses(N_PROCESSES); + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + HsaMemoryBuffer addrBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer resultBuffer(PAGE_SIZE, defaultGPUNode); + + std::vector pBuffers; + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + + unsigned int wavefront_num = pBuffers.size(); + LOG() << m_psName << "wavefront number " << wavefront_num << std::endl; + + void **localBufAddr = addrBuffer.As(); + unsigned int *result = resultBuffer.As(); + + for (i = 0; i < wavefront_num; i++) + *(localBufAddr + i) = pBuffers[i]; + + m_pIsaGen->CompileShader(CreateShader().c_str(), "ReadMemory", isaBuffer); + + PM4Queue pm4Queue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + + Dispatch dispatch0(isaBuffer); + dispatch0.SetArgs(localBufAddr, result); + dispatch0.SetDim(wavefront_num, 1, 1); + /* submit the packet and start shader */ + dispatch0.Submit(pm4Queue); + + /* doing evict/restore queue test for 5 seconds while queue is running */ + sleep(5); + + /* LOG() << m_psName << "notify shader to quit" << std::endl; */ + /* fill address buffer so shader quits */ + addrBuffer.Fill(0x5678); + + /* wait for shader to finish or timeout if shade has vm page fault */ + dispatch0.SyncWithStatus(120000); + + ASSERT_SUCCESS(pm4Queue.Destroy()); + /* LOG() << m_psName << "free buffer" << std::endl; */ + /* cleanup */ + FreeBuffers(pBuffers, vramBufSize); + + /* check if all wavefronts finish successfully */ + for (i = 0; i < wavefront_num; i++) + ASSERT_EQ(0x5678, *(result + i)); + + WaitChildProcesses(); + + TEST_END +} + diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/tests/kfdtest/src/KFDSVMEvictTest.hpp new file mode 100644 index 0000000000..7749ac2dee --- /dev/null +++ b/tests/kfdtest/src/KFDSVMEvictTest.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_SVM_EVICT_TEST__H__ +#define __KFD_SVM_EVICT_TEST__H__ + +#include +#include +#include "KFDLocalMemoryTest.hpp" +#include "KFDBaseComponentTest.hpp" +#include "IsaGenerator.hpp" + +// @class KFDEvictTest +// Test eviction and restore procedure using two processes +class KFDSVMEvictTest : public KFDLocalMemoryTest { + public: + KFDSVMEvictTest(void): m_ChildStatus(HSAKMT_STATUS_ERROR), m_IsParent(true) {} + + ~KFDSVMEvictTest(void) { + if (!m_IsParent) { + /* child process has to exit + * otherwise gtest will continue other tests + */ + exit(m_ChildStatus); + } + + try { + WaitChildProcesses(); + } catch (...) {} + } + + protected: + std::string CreateShader(); + void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, + std::vector &pBuffers); + void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); + void ForkChildProcesses(int nprocesses); + void WaitChildProcesses(); + HSAint32 GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize); + + protected: // members + std::string m_psName; + std::vector m_ChildPids; + HSA_SVM_FLAGS m_Flags; + void* m_pBuf; + HSAKMT_STATUS m_ChildStatus; + bool m_IsParent; +}; + +#endif // __KFD_SVM_EVICT_TEST__H__ diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp new file mode 100644 index 0000000000..01243f8156 --- /dev/null +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -0,0 +1,1139 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "KFDSVMRangeTest.hpp" +#include +#include +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAPacket.hpp" +#include "SDMAQueue.hpp" +#include "Dispatch.hpp" + +void KFDSVMRangeTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + m_pIsaGen = IsaGenerator::Create(m_FamilyId); + + ROUTINE_END +} + +void KFDSVMRangeTest::TearDown() { + ROUTINE_START + + if (m_pIsaGen) + delete m_pIsaGen; + m_pIsaGen = NULL; + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDSVMRangeTest, BasicSystemMemTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + PM4Queue queue; + HSAuint64 AlternateVAGPU; + unsigned int BufferSize = PAGE_SIZE; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + HsaSVMRange srcSysBuffer(BufferSize, defaultGPUNode); + HsaSVMRange destSysBuffer(BufferSize, defaultGPUNode); + + srcSysBuffer.Fill(0x01010101); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + queue.SetSkipWaitConsump(0); + + Dispatch dispatch(isaBuffer); + + dispatch.SetArgs(srcSysBuffer.As(), destSysBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + EXPECT_SUCCESS(queue.Destroy()); + + EXPECT_EQ(destSysBuffer.As()[0], 0x01010101); + + TEST_END +} + +TEST_F(KFDSVMRangeTest, SetGetAttributesTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + int i; + unsigned int BufSize = PAGE_SIZE; + HsaSVMRange *sysBuffer; + HSAuint32 nAttributes = 5; + HSA_SVM_ATTRIBUTE outputAttributes[nAttributes]; + HSA_SVM_ATTRIBUTE inputAttributes[] = { + {HSA_SVM_ATTR_PREFETCH_LOC, (HSAuint32)defaultGPUNode}, + {HSA_SVM_ATTR_PREFERRED_LOC, (HSAuint32)defaultGPUNode}, + {HSA_SVM_ATTR_SET_FLAGS, + HSA_SVM_FLAG_HOST_ACCESS | HSA_SVM_FLAG_GPU_EXEC | HSA_SVM_FLAG_COHERENT}, + {HSA_SVM_ATTR_GRANULARITY, 0xFF}, + {HSA_SVM_ATTR_ACCESS, (HSAuint32)defaultGPUNode}, + }; + + HSAuint32 expectedDefaultResults[] = { + INVALID_NODEID, + INVALID_NODEID, + HSA_SVM_FLAG_HOST_ACCESS | HSA_SVM_FLAG_COHERENT, + 9, + 0, + }; + HSAint32 enable = -1; + EXPECT_SUCCESS(hsaKmtGetXNACKMode(&enable)); + //expectedDefaultResults[4] = (enable)?HSA_SVM_ATTR_ACCESS:HSA_SVM_ATTR_NO_ACCESS; + // FIXME: Waiting for KFD to implement retry faults on unregistered addresses + expectedDefaultResults[4] = HSA_SVM_ATTR_NO_ACCESS; + sysBuffer = new HsaSVMRange(BufSize); + char *pBuf = sysBuffer->As(); + + LOG() << "Get default atrributes" << std::endl; + memcpy(outputAttributes, inputAttributes, nAttributes * sizeof(HSA_SVM_ATTRIBUTE)); + EXPECT_SUCCESS(hsaKmtSVMGetAttr(pBuf, BufSize, + nAttributes, outputAttributes)); + + for (i = 0; i < nAttributes; i++) { + if (outputAttributes[i].type == HSA_SVM_ATTR_ACCESS || + outputAttributes[i].type == HSA_SVM_ATTR_ACCESS_IN_PLACE || + outputAttributes[i].type == HSA_SVM_ATTR_NO_ACCESS) + EXPECT_EQ(outputAttributes[i].type, expectedDefaultResults[i]); + else + EXPECT_EQ(outputAttributes[i].value, expectedDefaultResults[i]); + } + LOG() << "Setting/Getting atrributes" << std::endl; + memcpy(outputAttributes, inputAttributes, nAttributes * sizeof(HSA_SVM_ATTRIBUTE)); + EXPECT_SUCCESS(hsaKmtSVMSetAttr(pBuf, BufSize, + nAttributes, inputAttributes)); + EXPECT_SUCCESS(hsaKmtSVMGetAttr(pBuf, BufSize, + nAttributes, outputAttributes)); + for (i = 0; i < nAttributes; i++) { + if (outputAttributes[i].type == HSA_SVM_ATTR_ACCESS || + outputAttributes[i].type == HSA_SVM_ATTR_ACCESS_IN_PLACE || + outputAttributes[i].type == HSA_SVM_ATTR_NO_ACCESS) + EXPECT_EQ(inputAttributes[i].type, outputAttributes[i].type); + else + EXPECT_EQ(inputAttributes[i].value, outputAttributes[i].value); + } + + TEST_END +} + +TEST_F(KFDSVMRangeTest, XNACKModeTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + HSAuint32 i, j; + HSAint32 r; + PM4Queue queue; + HSAint32 enable = 0; + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + + EXPECT_SUCCESS(hsaKmtGetXNACKMode(&enable)); + for (i = 0; i < 2; i++) { + enable = !enable; + r = hsaKmtSetXNACKMode(enable); + if (r == HSAKMT_STATUS_SUCCESS) { + LOG() << "XNACK mode: " << std::boolalpha << enable << + " supported" << std::endl; + + for (j = 0; j < gpuNodes.size(); j++) { + LOG() << "Creating queue and try to set xnack mode on node: " + << gpuNodes.at(j) << std::endl; + ASSERT_SUCCESS(queue.Create(gpuNodes.at(j))); + EXPECT_EQ(HSAKMT_STATUS_ERROR, + hsaKmtSetXNACKMode(enable)); + EXPECT_SUCCESS(queue.Destroy()); + } + } else if (r == HSAKMT_STATUS_NOT_SUPPORTED) { + LOG() << "XNACK mode: " << std::boolalpha << enable << + " NOT supported" << std::endl; + } + } + TEST_END +} + +TEST_F(KFDSVMRangeTest, InvalidRangeTest) { + TEST_START(TESTPROFILE_RUNALL) + + if (!SVMAPISupported()) + return; + + HSAuint32 Flags;; + HSAKMT_STATUS ret; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + Flags = HSA_SVM_FLAG_HOST_ACCESS | HSA_SVM_FLAG_COHERENT; + + ret = RegisterSVMRange(defaultGPUNode, reinterpret_cast(0x10000), 0x1000, 0, Flags); + EXPECT_NE(ret, HSAKMT_STATUS_SUCCESS); + + TEST_END +} + +void KFDSVMRangeTest::SplitRangeTest(int defaultGPUNode, int prefetch_location) { + unsigned int BufSize = 16 * PAGE_SIZE; + + if (!SVMAPISupported()) + return; + + HsaSVMRange *sysBuffer; + HsaSVMRange *sysBuffer2; + HsaSVMRange *sysBuffer3; + HsaSVMRange *sysBuffer4; + + void *pBuf; + + // case 1 + pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, prefetch_location); + sysBuffer2 = new HsaSVMRange(reinterpret_cast(pBuf) + 8192, PAGE_SIZE, defaultGPUNode, prefetch_location); + delete sysBuffer2; + delete sysBuffer; + munmap(pBuf, BufSize); + + // case 2.1 + pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, prefetch_location); + sysBuffer2 = new HsaSVMRange(reinterpret_cast(pBuf) + 4096, BufSize - 4096, defaultGPUNode, + prefetch_location); + delete sysBuffer2; + delete sysBuffer; + munmap(pBuf, BufSize); + + // case 2.2 + pBuf = mmap(0, BufSize + 8192, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, prefetch_location); + sysBuffer2 = new HsaSVMRange(reinterpret_cast(pBuf) + 8192, BufSize, defaultGPUNode, prefetch_location); + delete sysBuffer2; + delete sysBuffer; + munmap(pBuf, BufSize + 8192); + + // case 3 + pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, prefetch_location); + sysBuffer2 = new HsaSVMRange(reinterpret_cast(pBuf), BufSize - 8192, defaultGPUNode, prefetch_location); + delete sysBuffer2; + delete sysBuffer; + munmap(pBuf, BufSize); + + // case 4.1 + pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, prefetch_location); + sysBuffer2 = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, prefetch_location); + delete sysBuffer2; + delete sysBuffer; + munmap(pBuf, BufSize); + + // case 4.2 + pBuf = mmap(0, BufSize + 8192, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, prefetch_location); + sysBuffer2 = new HsaSVMRange(pBuf, BufSize + 8192, defaultGPUNode, prefetch_location); + delete sysBuffer2; + delete sysBuffer; + munmap(pBuf, BufSize + 8192); + + // case 5 + pBuf = mmap(0, BufSize + 65536, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(reinterpret_cast(pBuf) + 8192, 8192, defaultGPUNode, prefetch_location); + sysBuffer2 = new HsaSVMRange(reinterpret_cast(pBuf) + 32768, 8192, defaultGPUNode, prefetch_location); + sysBuffer3 = new HsaSVMRange(pBuf, BufSize + 65536, defaultGPUNode, prefetch_location); + delete sysBuffer2; + delete sysBuffer3; + delete sysBuffer; + munmap(pBuf, BufSize + 65536); + + // case 6, unregister after free + pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(reinterpret_cast(pBuf) + 8192, 8192, defaultGPUNode, prefetch_location); + munmap(pBuf, BufSize); + delete sysBuffer; +} + +TEST_F(KFDSVMRangeTest, SplitSystemRangeTest) { + const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + TEST_START(TESTPROFILE_RUNALL) + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + SplitRangeTest(defaultGPUNode, 0); + + TEST_END +} + +TEST_F(KFDSVMRangeTest, EvictSystemRangeTest) { + const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); + TEST_START(TESTPROFILE_RUNALL) + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + HSAuint32 stackData[2 * PAGE_SIZE] = {0}; + char *pBuf = reinterpret_cast(((uint64_t)stackData + PAGE_SIZE) & ~(PAGE_SIZE - 1)); + HSAuint32 *globalData = reinterpret_cast(pBuf); + const unsigned dstOffset = ((uint64_t)pBuf + 2 * PAGE_SIZE - (uint64_t)stackData) / 4; + const unsigned sdmaOffset = dstOffset + PAGE_SIZE; + + *globalData = 0xdeadbeef; + + HsaSVMRange srcBuffer((globalData), PAGE_SIZE, defaultGPUNode); + HsaSVMRange dstBuffer(&stackData[dstOffset], PAGE_SIZE, defaultGPUNode); + HsaSVMRange sdmaBuffer(&stackData[sdmaOffset], PAGE_SIZE, defaultGPUNode); + + /* Create PM4 and SDMA queues before fork+COW to test queue + * eviction and restore + */ + PM4Queue pm4Queue; + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + Dispatch dispatch0(isaBuffer); + dispatch0.SetArgs(srcBuffer.As(), dstBuffer.As()); + dispatch0.Submit(pm4Queue); + dispatch0.Sync(g_TestTimeOut); + + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaQueue.GetFamilyId(), + sdmaBuffer.As(), 0x12345678)); + + sdmaQueue.Wait4PacketConsumption(); + EXPECT_TRUE(WaitOnValue(&stackData[sdmaOffset], 0x12345678)); + + /* Fork a child process to mark pages as COW */ + pid_t pid = fork(); + ASSERT_GE(pid, 0); + if (pid == 0) { + /* Child process waits for a SIGTERM from the parent. It can't + * make any write access to the stack because we want the + * parent to make the first write access and get a new copy. A + * busy loop is the safest way to do that, since any function + * call (e.g. sleep) would write to the stack. + */ + while (1) + {} + WARN() << "Shouldn't get here!" << std::endl; + exit(0); + } + + /* Parent process writes to COW page(s) and gets a new copy. MMU + * notifier needs to update the GPU mapping(s) for the test to + * pass. + */ + *globalData = 0xD00BED00; + stackData[dstOffset] = 0xdeadbeef; + stackData[sdmaOffset] = 0xdeadbeef; + + /* Terminate the child process before a possible test failure that + * would leave it spinning in the background indefinitely. + */ + int status; + EXPECT_EQ(0, kill(pid, SIGTERM)); + EXPECT_EQ(pid, waitpid(pid, &status, 0)); + EXPECT_NE(0, WIFSIGNALED(status)); + EXPECT_EQ(SIGTERM, WTERMSIG(status)); + + /* Now check that the GPU is accessing the correct page */ + Dispatch dispatch1(isaBuffer); + dispatch1.SetArgs(srcBuffer.As(), dstBuffer.As()); + dispatch1.Submit(pm4Queue); + dispatch1.Sync(g_TestTimeOut); + + sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaQueue.GetFamilyId(), + sdmaBuffer.As(), 0xD0BED0BE)); + sdmaQueue.Wait4PacketConsumption(); + + EXPECT_SUCCESS(pm4Queue.Destroy()); + EXPECT_SUCCESS(sdmaQueue.Destroy()); + + EXPECT_EQ(0xD00BED00, *globalData); + EXPECT_EQ(0xD00BED00, stackData[dstOffset]); + EXPECT_EQ(0xD0BED0BE, stackData[sdmaOffset]); + + TEST_END +} + +TEST_F(KFDSVMRangeTest, PartialUnmapSysMemTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + unsigned int BufSize = 16 * PAGE_SIZE; + void *pBuf; + + PM4Queue queue; + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + HsaSVMRange *sysBuffer; + HsaSVMRange destSysBuffer(BufSize, defaultGPUNode); + + pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + sysBuffer = new HsaSVMRange(pBuf, BufSize, defaultGPUNode, 0); + sysBuffer->Fill(0x01010101); + + char *pBuf2 = reinterpret_cast(pBuf) + 8192; + unsigned int Buf2Size = 4 * PAGE_SIZE; + char *pBuf3 = pBuf2 + Buf2Size; + + munmap(pBuf2, Buf2Size); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + Dispatch dispatch(isaBuffer); + Dispatch dispatch2(isaBuffer); + + dispatch.SetArgs(pBuf3, destSysBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + EXPECT_EQ(destSysBuffer.As()[0], 0x01010101); + + dispatch2.SetArgs(pBuf, destSysBuffer.As()); + dispatch2.Submit(queue); + dispatch2.Sync(g_TestTimeOut); + + EXPECT_EQ(destSysBuffer.As()[0], 0x01010101); + + EXPECT_SUCCESS(queue.Destroy()); + munmap(pBuf, BufSize); + + TEST_END +} + +TEST_F(KFDSVMRangeTest, BasicVramTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + PM4Queue queue; + HSAuint64 AlternateVAGPU; + unsigned int BufferSize = PAGE_SIZE; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + HsaSVMRange srcSysBuffer(BufferSize, defaultGPUNode); + HsaSVMRange locBuffer(BufferSize, defaultGPUNode, defaultGPUNode); + HsaSVMRange destSysBuffer(BufferSize, defaultGPUNode); + + srcSysBuffer.Fill(0x01010101); + + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + queue.SetSkipWaitConsump(0); + + Dispatch dispatch(isaBuffer); + Dispatch dispatch2(isaBuffer); + + dispatch.SetArgs(srcSysBuffer.As(), locBuffer.As()); + dispatch.Submit(queue); + dispatch.Sync(g_TestTimeOut); + + dispatch2.SetArgs(locBuffer.As(), destSysBuffer.As()); + dispatch2.Submit(queue); + dispatch2.Sync(g_TestTimeOut); + + EXPECT_SUCCESS(queue.Destroy()); + + EXPECT_EQ(destSysBuffer.As()[0], 0x01010101); + TEST_END +} + +TEST_F(KFDSVMRangeTest, SplitVramRangeTest) { + TEST_START(TESTPROFILE_RUNALL) + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + SplitRangeTest(defaultGPUNode, defaultGPUNode); + TEST_END +} + +TEST_F(KFDSVMRangeTest, PrefetchTest) { + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + unsigned int BufSize = 16 << 10; + HsaSVMRange *sysBuffer; + uint32_t node_id; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + sysBuffer = new HsaSVMRange(BufSize, defaultGPUNode); + char *pBuf = sysBuffer->As(); + /* Using invalid svm range to get prefetch node should return failed */ + delete sysBuffer; + EXPECT_SUCCESS(!SVMRangeGetPrefetchNode(pBuf, BufSize, &node_id)); + + sysBuffer = new HsaSVMRange(BufSize, defaultGPUNode); + pBuf = sysBuffer->As(); + char *pLocBuf = pBuf + BufSize / 2; + + EXPECT_SUCCESS(SVMRangeGetPrefetchNode(pBuf, BufSize, &node_id)); + EXPECT_EQ(node_id, 0); + + EXPECT_SUCCESS(SVMRangePrefetchToNode(pLocBuf, BufSize / 2, defaultGPUNode)); + + EXPECT_SUCCESS(SVMRangeGetPrefetchNode(pLocBuf, BufSize / 2, &node_id)); + EXPECT_EQ(node_id, defaultGPUNode); + + EXPECT_SUCCESS(SVMRangeGetPrefetchNode(pBuf, BufSize, &node_id)); + EXPECT_EQ(node_id, 0xffffffff); + delete sysBuffer; + + TEST_END +} + +TEST_F(KFDSVMRangeTest, MigrateTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + HSAuint32 migrateRepeat = 8; + unsigned int BufferSize = 16 << 20; + + HsaSVMRange DataBuffer(BufferSize, defaultGPUNode); + HSAuint32 *pData = DataBuffer.As(); + + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); + HSAuint32 *pBuf = SysBuffer.As(); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, 0)); + + HsaSVMRange SysBuffer2(BufferSize, defaultGPUNode); + HSAuint32 *pBuf2 = SysBuffer2.As(); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf2, BufferSize, 0)); + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + for (HSAuint32 i = 0; i < BufferSize / 4; i++) + pData[i] = i; + + while (migrateRepeat--) { + /* Migrate from ram to vram */ + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, defaultGPUNode)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf2, BufferSize, defaultGPUNode)); + /* Update content in migrated buffer in vram */ + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pBuf, pData, BufferSize)); + sdmaQueue.Wait4PacketConsumption(); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pBuf2, pData, BufferSize)); + sdmaQueue.Wait4PacketConsumption(); + + /* Migrate from vram to ram + * CPU access the buffer migrated to vram have page fault + * page fault trigger migration from vram back to ram + * so SysBuffer should have same value as in vram + */ + for (HSAuint32 i = 0; i < BufferSize / 4; i++) { + ASSERT_EQ(i, pBuf[i]); + ASSERT_EQ(i, pBuf2[i]); + } + } + + /* If xnack off, after migrating back to ram, GPU mapping should be updated to ram + * test if shade can read from ram + * If xnack on, GPU mapping should be cleared, test if GPU vm fault can update + * page table and shade can read from ram. + */ + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pBuf, pData, BufferSize)); + sdmaQueue.Wait4PacketConsumption(); + for (HSAuint32 i = 0; i < BufferSize / 4; i++) + ASSERT_EQ(i, pBuf[i]); + + TEST_END +} + +/* + * The test changes migration granularity, then trigger CPU page fault to migrate + * the svm range from vram to ram. + * Check the dmesg driver output to confirm the number of CPU page fault is correct + * based on granularity. + * + * For example, this is BufferPages = 5, while granularity change from 2 to 0 + * [ 292.623498] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597ee000 + * [ 292.623727] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f0000 + * [ 292.724414] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597ee000 + * [ 292.724824] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f0000 + * [ 292.725094] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f2000 + * [ 292.728186] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597ee000 + * [ 292.729171] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597ef000 + * [ 292.729576] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f0000 + * [ 292.730010] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f1000 + * [ 292.730931] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f2000 + */ +TEST_F(KFDSVMRangeTest, MigrateGranularityTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + HSAuint64 BufferPages = 16384; + HSAuint64 BufferSize = BufferPages * PAGE_SIZE; + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); + HSAint32 *pBuf = SysBuffer.As(); + + HsaSVMRange SysBuffer2(BufferSize, defaultGPUNode); + HSAint32 *pBuf2 = SysBuffer2.As(); + + HSAint32 Granularity; + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + for (Granularity = 0; (1ULL << Granularity) <= BufferPages; Granularity++); + for (HSAuint32 i = 0; i < BufferPages; i++) + pBuf2[i * PAGE_SIZE / 4] = i; + + while (Granularity--) { + /* Prefetch the entire range to vram */ + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, defaultGPUNode)); + EXPECT_SUCCESS(SVMRangSetGranularity(pBuf, BufferSize, Granularity)); + + /* Change Buffer content in vram, then migrate it back to ram */ + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pBuf, pBuf2, BufferSize)); + sdmaQueue.Wait4PacketConsumption(); + + /* Migrate from vram to ram */ + for (HSAuint32 i = 0; i < BufferPages; i++) + ASSERT_EQ(i, pBuf[i * PAGE_SIZE / 4]); + } + TEST_END +} + +TEST_F(KFDSVMRangeTest, MigrateLargeBufTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + PM4Queue queue; + HSAuint64 AlternateVAGPU; + unsigned long BufferSize = 1L << 30; + unsigned long maxSDMASize = 128L << 20; /* IB size is 4K */ + unsigned long Size, i; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); + SysBuffer.Fill(0x1); + + HsaSVMRange SysBuffer2(BufferSize, defaultGPUNode); + SysBuffer2.Fill(0x2); + + /* Migrate from ram to vram + * using same address to register to GPU to trigger migration + * so LocalBuffer will have same value as SysBuffer + */ + HsaSVMRange LocalBuffer(SysBuffer.As(), BufferSize, defaultGPUNode, defaultGPUNode); + + SDMAQueue sdmaQueue; + + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + for (i = 0; i < BufferSize; i += Size) { + Size = (BufferSize - i) > maxSDMASize ? maxSDMASize : (BufferSize - i); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + SysBuffer2.As() + i, LocalBuffer.As() + i, Size)); + sdmaQueue.Wait4PacketConsumption(); + } + + /* Check content in migrated buffer in vram */ + for (i = 0; i < BufferSize / 4; i += 1024) + ASSERT_EQ(0x1, SysBuffer2.As()[i]); + + /* Change LocalBuffer content in vram, then migrate it back to ram */ + SysBuffer2.Fill(0x3); + + for (i = 0; i < BufferSize; i += Size) { + Size = (BufferSize - i) > maxSDMASize ? maxSDMASize : (BufferSize - i); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + LocalBuffer.As() + i, SysBuffer2.As() + i, Size)); + sdmaQueue.Wait4PacketConsumption(); + } + + /* Migrate from vram to ram + * CPU access the buffer migrated to vram have page fault + * page fault trigger migration from vram back to ram + * so SysBuffer should have same value as in LocalBuffer + */ + EXPECT_SUCCESS(SVMRangSetGranularity(SysBuffer.As(), BufferSize, 30)); + for (i = 0; i < BufferSize / 4; i += 1024) + ASSERT_EQ(0x3, SysBuffer.As()[i]); + + /* After migrating back to ram, GPU mapping should be updated to ram + * test if shade can read from ram + */ + SysBuffer.Fill(0x4); + + for (i = 0; i < BufferSize; i += Size) { + Size = (BufferSize - i) > maxSDMASize ? maxSDMASize : (BufferSize - i); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + SysBuffer2.As() + i, LocalBuffer.As() + i, Size)); + sdmaQueue.Wait4PacketConsumption(); + } + + for (i = 0; i < BufferSize / 4; i += 1024) + ASSERT_EQ(0x4, SysBuffer2.As()[i]); + + TEST_END +} + +TEST_F(KFDSVMRangeTest, MigratePolicyTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + unsigned long BufferSize = 1UL << 20; + + HsaSVMRange DataBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pData = DataBuffer.As(); + + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pBuf = SysBuffer.As(); + + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + for (HSAuint64 i = 0; i < BufferSize / 8; i++) + pData[i] = i; + + /* Prefetch to migrate from ram to vram */ + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, defaultGPUNode)); + + /* Update content in migrated buffer in vram */ + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pBuf, pData, BufferSize)); + sdmaQueue.Wait4PacketConsumption(NULL, HSA_EVENTTIMEOUT_INFINITE); + + /* Migrate from vram to ram + * CPU access the buffer migrated to vram have page fault + * page fault trigger migration from vram back to ram + * so SysBuffer should have same value as in vram + */ + for (HSAuint64 i = 0; i < BufferSize / 8; i++) { + ASSERT_EQ(i, pBuf[i]); + /* Update buf */ + pBuf[i] = i + 1; + } + + /* Migrate from ram to vram if xnack on + * If xnack off, after migrating back to ram, GPU mapping should be updated to ram + * test if shade can read from ram + * If xnack on, GPU mapping should be cleared, test if GPU vm fault can update + * page table and shade can read from ram. + */ +//#define USE_PM4_QUEUE_TRIGGER_VM_FAULT +#ifdef USE_PM4_QUEUE_TRIGGER_VM_FAULT + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); + PM4Queue queue; + m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + for (HSAuint64 i = 0; i < BufferSize / 8; i += 512) { + Dispatch dispatch(isaBuffer); + + dispatch.SetArgs(pBuf + i, pData + i); + dispatch.Submit(queue); + dispatch.Sync(HSA_EVENTTIMEOUT_INFINITE); + } +#else + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pData, pBuf, BufferSize)); + sdmaQueue.Wait4PacketConsumption(NULL, HSA_EVENTTIMEOUT_INFINITE); +#endif + + for (HSAuint64 i = 0; i < BufferSize / 8; i += 512) + ASSERT_EQ(i + 1, pData[i]); + + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + TEST_END +} + +/* Multiple GPU migration test + * + * Steps: + * 1. Prefetch pBuf, pData to all GPUs, to test migration from GPU to GPU + * 2. Use sdma queue on all GPUs, to copy data from pBuf to pData + * 3. Check pData data + * + * Notes: + * With xnack on, step 2 will have retry fault on pBuf, to migrate from GPU to GPU, + * retry fault on pData, to migrate from CPU to GPU + * + * With xnack off, pBuf and pData should prefetch to CPU to ensure multiple GPU access + * + * step3 migrate pData from GPU to CPU + * + * Test will skip if only one GPU found + */ +TEST_F(KFDSVMRangeTest, MultiGPUMigrationTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: at least two GPUs needed." << std::endl; + return; + } + + unsigned long BufferSize = 1UL << 20; + + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pBuf = SysBuffer.As(); + HsaSVMRange DataBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pData = DataBuffer.As(); + + SDMAQueue sdmaQueue; + + for (HSAuint64 i = 0; i < BufferSize / 8; i++) + pBuf[i] = i; + + for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { + EXPECT_SUCCESS(SVMRangeMapToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); + + EXPECT_SUCCESS(SVMRangeMapToNode(pData, BufferSize, gpuNodes.at(gpuidx))); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pData, BufferSize, gpuNodes.at(gpuidx))); + } + + for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { + ASSERT_SUCCESS(sdmaQueue.Create(gpuNodes.at(gpuidx))); + + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pData, pBuf, BufferSize)); + sdmaQueue.Wait4PacketConsumption(); + + for (HSAuint64 i = 0; i < BufferSize / 8; i += 512) + ASSERT_EQ(i, pData[i]); + + EXPECT_SUCCESS(sdmaQueue.Destroy()); + } + + TEST_END +} + +/* Multiple GPU access in place test + * + * Steps: + * 1. Prefetch pBuf, pData to all GPUs, with ACCESS_IN_PLACE on GPUs + * 2. Use sdma queue on all GPUs, to copy data from pBuf to pData + * 3. Prefetch pData to CPU, check pData data + * + * Notes: + * With xnack on, step 2 will have retry fault on pBuf, to migrate from GPU to GPU. + * If multiple GPU on xGMI same hive, there should not have retry fault on pBuf + * because mapping should update to another GPU vram through xGMI + * + * With xnack off, pBuf and pData should prefetch to CPU to ensure multiple GPU access + * + * step3 migrate pData from GPU to CPU, should not have retry fault on GPUs. + * + * Test will skip if only one GPU found + */ +TEST_F(KFDSVMRangeTest, MultiGPUAccessInPlaceTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + if (gpuNodes.size() < 2) { + LOG() << "Skipping test: at least two GPUs needed." << std::endl; + return; + } + + unsigned long BufferSize = 1UL << 20; + + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pBuf = SysBuffer.As(); + HsaSVMRange DataBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pData = DataBuffer.As(); + + SDMAQueue sdmaQueue; + + for (HSAuint64 i = 0; i < BufferSize / 8; i++) + pBuf[i] = i; + + for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { + EXPECT_SUCCESS(SVMRangeMapInPlaceToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); + + EXPECT_SUCCESS(SVMRangeMapInPlaceToNode(pData, BufferSize, gpuNodes.at(gpuidx))); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pData, BufferSize, gpuNodes.at(gpuidx))); + } + + for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { + ASSERT_SUCCESS(sdmaQueue.Create(gpuNodes.at(gpuidx))); + + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pData, pBuf, BufferSize)); + sdmaQueue.Wait4PacketConsumption(); + + for (HSAuint64 i = 0; i < BufferSize / 8; i += 512) + ASSERT_EQ(i, pData[i]); + + EXPECT_SUCCESS(sdmaQueue.Destroy()); + } + + TEST_END +} + +/* Multiple thread migration test + * + * 2 threads do migration at same time to test range migration race conditon handle. + * + * Steps: + * 1. register 128MB range on system memory, don't map to GPU, 128MB is max size to put in + * sdma queue 4KB IB buffer. + * 2. one thread prefetch range to GPU, another thread use sdma queue to access range at same + * time to generate retry vm fault to migrate range to GPU + * 3. one thread prefetch range to CPU, another thread read range to generate CPU page fault + * to migrate range to CPU at same time + * 4. loop test step 2 and 3 twice, to random CPU/GPU fault and prefetch migration order + */ +struct ReadThreadParams { + HSAuint64* pBuf; + HSAint64 BufferSize; + int defaultGPUNode; +}; + +unsigned int CpuReadThread(void* p) { + struct ReadThreadParams* pArgs = reinterpret_cast(p); + + for (HSAuint64 i = 0; i < pArgs->BufferSize / 8; i += 512) + EXPECT_EQ(i, pArgs->pBuf[i]); + return 0; +} + +unsigned int GpuReadThread(void* p) { + struct ReadThreadParams* pArgs = reinterpret_cast(p); + + EXPECT_SUCCESS(SVMRangePrefetchToNode(pArgs->pBuf, pArgs->BufferSize, pArgs->defaultGPUNode)); + return 0; +} + +TEST_F(KFDSVMRangeTest, MultiThreadMigrationTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + unsigned long test_loops = 2; + unsigned long BufferSize = 1UL << 27; + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pBuf = SysBuffer.As(); + HsaSVMRange DataBuffer(BufferSize, defaultGPUNode); + HSAuint64 *pData = DataBuffer.As(); + SDMAQueue sdmaQueue; + uint64_t threadId; + struct ReadThreadParams params; + + params.pBuf = pBuf; + params.BufferSize = BufferSize; + params.defaultGPUNode = defaultGPUNode; + + EXPECT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + for (HSAuint64 i = 0; i < BufferSize / 8; i++) + pBuf[i] = i; + + for (HSAuint64 i = 0; i < test_loops; i++) { + /* 2 threads migrate to GPU */ + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pData, pBuf, BufferSize)); + ASSERT_EQ(true, StartThread(&GpuReadThread, ¶ms, threadId)); + sdmaQueue.Wait4PacketConsumption(); + WaitForThread(threadId); + + /* 2 threads migrate to cpu */ + ASSERT_EQ(true, StartThread(&CpuReadThread, ¶ms, threadId)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, 0)); + WaitForThread(threadId); + } + + EXPECT_SUCCESS(sdmaQueue.Destroy()); + + TEST_END +} diff --git a/tests/kfdtest/src/KFDSVMRangeTest.hpp b/tests/kfdtest/src/KFDSVMRangeTest.hpp new file mode 100644 index 0000000000..88bddd94a8 --- /dev/null +++ b/tests/kfdtest/src/KFDSVMRangeTest.hpp @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_SVMRANGE_TEST__H__ +#define __KFD_SVMRANGE_TEST__H__ + +#include + +#include "IsaGenerator.hpp" +#include "KFDBaseComponentTest.hpp" + +class KFDSVMRangeTest : public KFDBaseComponentTest { + public: + KFDSVMRangeTest() :m_pIsaGen(NULL) {} + ~KFDSVMRangeTest() {} + void SplitRangeTest(int defaultGPUNode, int prefetch_location); + + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: // Members + IsaGenerator* m_pIsaGen; +}; + +#endif // __KFD_LOCALMEMORY_TEST__H__ diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 8651eaa51d..e52e7b496b 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -22,9 +22,9 @@ */ #include "KFDTestUtil.hpp" - #include #include +#include #include #include #include "BaseQueue.hpp" @@ -694,3 +694,181 @@ const bool HsaNodeInfo::IsNodeXGMItoCPU(int node) const { return ret; } + +HSAKMT_STATUS RegisterSVMRange(HSAuint32 GPUNode, void *MemoryAddress, + HSAuint64 SizeInBytes, HSAuint32 PrefetchNode, + HSAuint32 SVMFlags) { + HSA_SVM_ATTRIBUTE *attrs; + HSAuint64 s_attr; + HSAuint32 nattr; + HSAKMT_STATUS r; + + nattr = 4; + s_attr = sizeof(*attrs) * nattr; + attrs = (HSA_SVM_ATTRIBUTE *)alloca(s_attr); + + attrs[0].type = HSA_SVM_ATTR_PREFETCH_LOC; + attrs[0].value = PrefetchNode; + attrs[1].type = HSA_SVM_ATTR_PREFERRED_LOC; + attrs[1].value = PrefetchNode; + attrs[2].type = HSA_SVM_ATTR_SET_FLAGS; + attrs[2].value = SVMFlags; + attrs[3].type = HSA_SVM_ATTR_ACCESS; + attrs[3].value = GPUNode; + + r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, nattr, attrs); + if (r) { + LOG() << "set range attrs failed" << std::endl; + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS SVMRangeGetPrefetchNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 *PrefetchNode) { + HSA_SVM_ATTRIBUTE attr; + int r; + + attr.type = HSA_SVM_ATTR_PREFETCH_LOC; + attr.value = 0; + + r = hsaKmtSVMGetAttr(MemoryAddress, SizeInBytes, 1, &attr); + if (r) { + LOG() << "get prefetch node failed" << std::endl; + return HSAKMT_STATUS_ERROR; + } + + *PrefetchNode = attr.value; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS SVMRangePrefetchToNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 PrefetchNode) { + HSA_SVM_ATTRIBUTE attr; + int r; + + attr.type = HSA_SVM_ATTR_PREFETCH_LOC; + attr.value = PrefetchNode; + + r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); + if (r) { + LOG() << "set prefetch node failed" << std::endl; + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS SVMRangeMapToNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 NodeID) { + HSA_SVM_ATTRIBUTE attr; + int r; + + attr.type = HSA_SVM_ATTR_ACCESS; + attr.value = NodeID; + + r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); + if (r) { + LOG() << "set map to node failed" << std::endl; + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS SVMRangeMapInPlaceToNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 NodeID) { + HSA_SVM_ATTRIBUTE attr; + int r; + + attr.type = HSA_SVM_ATTR_ACCESS_IN_PLACE; + attr.value = NodeID; + + r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); + if (r) { + LOG() << "set map in place to node failed" << std::endl; + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS SVMRangSetGranularity(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 Granularity) { + HSA_SVM_ATTRIBUTE attr; + int r; + + attr.type = HSA_SVM_ATTR_GRANULARITY; + attr.value = Granularity; + + r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); + if (r) { + LOG() << "set granularity failed" << std::endl; + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HsaSVMRange::HsaSVMRange(HSAuint64 size, HSAuint32 GPUNode) : + HsaSVMRange(NULL, size, GPUNode, 0) {} + +HsaSVMRange::HsaSVMRange(HSAuint64 size) : + HsaSVMRange(NULL, size, 0, 0, true) {} + +HsaSVMRange::HsaSVMRange(HSAuint64 size, HSAuint32 GPUNode, HSAuint32 PrefetchNode) : + HsaSVMRange(NULL, size, GPUNode, PrefetchNode) {} + +HsaSVMRange::HsaSVMRange(void *addr, HSAuint64 size, HSAuint32 GPUNode, HSAuint32 PrefetchNode, + bool noRegister, bool isLocal, bool isExec, bool isReadOnly): + m_Size(size), + m_pUser(addr), + m_Local(isLocal), + m_Node(PrefetchNode), + m_SelfAllocated(false) { + if (!m_pUser) { + m_pUser = mmap(0, m_Size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + EXPECT_NOTNULL(m_pUser); + m_SelfAllocated = true; + } + + if (m_Local) + m_Flags = HSA_SVM_FLAG_HOST_ACCESS; + else + m_Flags = HSA_SVM_FLAG_HOST_ACCESS | HSA_SVM_FLAG_COHERENT; + + if (isReadOnly) + m_Flags |= HSA_SVM_FLAG_GPU_RO; + if (isExec) + m_Flags |= HSA_SVM_FLAG_GPU_EXEC; + + if (!noRegister) + EXPECT_SUCCESS(RegisterSVMRange(GPUNode, m_pUser, m_Size, PrefetchNode, m_Flags)); +} + +HsaSVMRange::~HsaSVMRange() { + if (m_pUser != NULL) { + if (m_SelfAllocated) + munmap(m_pUser, m_Size); + m_pUser = NULL; + } +} + +void HsaSVMRange::Fill(HSAuint32 value, HSAuint64 offset, HSAuint64 size) { + HSAuint64 i; + HSAuint32 *ptr = NULL; + + size = size ? size : m_Size; + EXPECT_EQ((size & (sizeof(HSAuint32) - 1)), 0) << "Not word aligned. Call Fill(unsigned char)"; + ASSERT_TRUE(size + offset <= m_Size) << "Buffer Overflow" << std::endl; + + if (m_pUser != NULL) + ptr = reinterpret_cast(reinterpret_cast(m_pUser) + offset); + + ASSERT_NOTNULL(ptr); + + for (i = 0; i < size / sizeof(HSAuint32); i++) + ptr[i] = value; +} diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 2076e27df0..ee88ed3909 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -121,8 +121,49 @@ class HsaMemoryBuffer { unsigned int m_Node; HSAuint64 m_MappedNodes; }; +HSAKMT_STATUS RegisterSVMRange(HSAuint32 GPUNode, void *MemoryAddress, + HSAuint64 SizeInBytes, HSAuint32 PrefetchNode, + HSAuint32 SVMFlags); +HSAKMT_STATUS SVMRangeGetPrefetchNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 *PrefetchNode); +HSAKMT_STATUS SVMRangePrefetchToNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 PrefetchNode); +HSAKMT_STATUS SVMRangeMapToNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 NodeID); +HSAKMT_STATUS SVMRangeMapInPlaceToNode(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 NodeID); +HSAKMT_STATUS SVMRangSetGranularity(void *MemoryAddress, HSAuint64 SizeInBytes, + HSAuint32 Granularity); +class HsaSVMRange { + public: + HsaSVMRange(HSAuint64 size, HSAuint32 GPUNode); + HsaSVMRange(HSAuint64 size, HSAuint32 GPUNode, HSAuint32 PreferredNode); + HsaSVMRange(HSAuint64 size); + HsaSVMRange(void *addr, HSAuint64 size, HSAuint32 GPUNode, HSAuint32 PreferredNode = 0, + bool noRegister = false, bool isLocal = false, bool isExec = false, + bool isReadOnly = false); + template + RetType As() { + return reinterpret_cast(m_pUser); + } + template + const RetType As() const { + return reinterpret_cast(m_pUser); + } + ~HsaSVMRange(); + + void Fill(HSAuint32 value, HSAuint64 offset = 0, HSAuint64 size = 0); + + private: + HSAuint32 m_Flags; + HSAuint64 m_Size; + void* m_pUser; + bool m_SelfAllocated; + bool m_Local; + unsigned int m_Node; +}; class HsaInteropMemoryBuffer { public: From 77f1bfa27795fdbb1a3fb1de6453c45e7d2823dc Mon Sep 17 00:00:00 2001 From: Mike Li Date: Mon, 12 Apr 2021 10:15:23 -0400 Subject: [PATCH 0832/1247] Add cache information for GPU Signed-off-by: Mike Li Change-Id: I93606e676ae944fa3d72886654566c75ab8f9806 --- tests/kfdtest/src/KFDTopologyTest.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index c675e4ca88..05a57fb411 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -175,6 +175,27 @@ TEST_F(KFDTopologyTest, GetNodeCacheProperties) { LOG() << " ProcIdLow " << cacheProperties[n].ProcessorIdLow << " SiblingMap " << string << std::endl; } + } else { // this is a GPU node + LOG() << "GPU Node " << std::dec << node << ": " << pNodeProperties->NumCaches << " caches" + << std::endl; + for (unsigned n = 0; n < pNodeProperties->NumCaches; n++) { + LOG()<< n << " - Level " << cacheProperties[n].CacheLevel << + " Type " << cacheProperties[n].CacheType.Value << + " Size " << cacheProperties[n].CacheSize << "K " << + " Associativity " << cacheProperties[n].CacheAssociativity << + " LineSize " << cacheProperties[n].CacheLineSize << + " LinesPerTag " << cacheProperties[n].CacheLinesPerTag << std::endl; + char string[1024] = ""; + char sibling[5] = ""; + for (unsigned i = 0; i < 256; i++) { + if (cacheProperties[n].SiblingMap[i]) { + snprintf(sibling, 5, "%d,", i); + strcat(string, sibling); + } + } + LOG() << " ProcIdLow " << cacheProperties[n].ProcessorIdLow << + " SiblingMap " << string << std::endl; + } } delete [] cacheProperties; } From c1c46d9c9702446cfa07aaa21e1d668ebd83b5b0 Mon Sep 17 00:00:00 2001 From: Joseph Greathouse Date: Tue, 13 Apr 2021 21:38:32 -0500 Subject: [PATCH 0833/1247] Update GWS tests for gfx1030 gfx10 GPUs such as gfx1030 need new assembly code to test the GWS. Removed scalar stores and added proper usage of DLC and VSCNT waits. Removed gfx9-specific assembler meta-values. Change-Id: I2bbdb77692ace2dba10997f721ba9decaa9be82a --- tests/kfdtest/src/KFDGWSTest.cpp | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDGWSTest.cpp b/tests/kfdtest/src/KFDGWSTest.cpp index 111e5e186d..12ed9a16cf 100644 --- a/tests/kfdtest/src/KFDGWSTest.cpp +++ b/tests/kfdtest/src/KFDGWSTest.cpp @@ -27,11 +27,11 @@ #include "Dispatch.hpp" /* Shader to initialize gws counter to 1*/ -const char* gfx9_GwsInit = +const char* gfx9_10_GwsInit = "\ shader GwsInit\n\ -asic(GFX9)\n\ type(CS)\n\ +wave_size(32)\n\ s_mov_b32 m0, 0\n\ s_nop 0\n\ s_load_dword s16, s[0:1], 0x0 glc\n\ @@ -71,6 +71,30 @@ type(CS)\n\ end\n\ "; +const char* gfx10_AtomicIncrease = +"\ +shader AtomicIncrease\n\ +asic(GFX10)\n\ +type(CS)\n\ +wave_size(32)\n\ +/* Assume src address in s0, s1 */\n\ + s_mov_b32 m0, 0\n\ + s_mov_b32 exec_lo, 0x1\n\ + v_mov_b32 v0, s0\n\ + v_mov_b32 v1, s1\n\ + ds_gws_sema_p gds:1 offset0:0\n\ + s_waitcnt 0\n\ + flat_load_dword v2, v[0:1] glc:1 dlc:1\n\ + s_waitcnt 0\n\ + v_add_nc_u32 v2, v2, 1\n\ + flat_store_dword v[0:1], v2\n\ + s_waitcnt_vscnt null, 0\n\ + ds_gws_sema_v gds:1 offset0:0\n\ + s_waitcnt 0\n\ + s_endpgm\n\ + end\n\ +"; + void KFDGWSTest::SetUp() { ROUTINE_START @@ -138,14 +162,20 @@ TEST_F(KFDGWSTest, Semaphore) { EXPECT_EQ(0, firstGWS); m_pIsaGen = IsaGenerator::Create(m_FamilyId); - m_pIsaGen->CompileShader(gfx9_GwsInit, "GwsInit", isaBuffer); + m_pIsaGen->CompileShader(gfx9_10_GwsInit, "GwsInit", isaBuffer); Dispatch dispatch0(isaBuffer); buffer.Fill(numResources, 0, 4); dispatch0.SetArgs(buffer.As(), NULL); dispatch0.Submit(queue); dispatch0.Sync(); - m_pIsaGen->CompileShader(gfx9_AtomicIncrease, "AtomicIncrease", isaBuffer); + const char *pAtomicIncrease; + if (m_FamilyId <= FAMILY_AL) + pAtomicIncrease = gfx9_AtomicIncrease; + else + pAtomicIncrease = gfx10_AtomicIncrease; + + m_pIsaGen->CompileShader(pAtomicIncrease, "AtomicIncrease", isaBuffer); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), NULL); From e06d54933772c72054c32424b20c45bef7f9ebad Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Wed, 21 Apr 2021 14:26:48 -0400 Subject: [PATCH 0834/1247] kfdtest: Remove GFX9 assembler meta information Fixes assembler error. The SP3 backend if already set to FamilyId Signed-off-by: Harish Kasiviswanathan Change-Id: If127a71693b293e2748b06efb668a359b939cd14 --- tests/kfdtest/src/KFDGWSTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kfdtest/src/KFDGWSTest.cpp b/tests/kfdtest/src/KFDGWSTest.cpp index 12ed9a16cf..4c8aefc447 100644 --- a/tests/kfdtest/src/KFDGWSTest.cpp +++ b/tests/kfdtest/src/KFDGWSTest.cpp @@ -53,7 +53,6 @@ wave_size(32)\n\ const char* gfx9_AtomicIncrease = "\ shader AtomicIncrease\n\ -asic(GFX9)\n\ type(CS)\n\ /* Assume src address in s0, s1 */\n\ s_mov_b32 m0, 0\n\ From 7d53e947504097c8fdfef6206eab0c0c99209b9c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 26 Apr 2021 10:31:37 -0400 Subject: [PATCH 0835/1247] kfdtest: skip KFDSVMEvictTest.QueueTest on gfx10 KFDSVMEvictTest.QueueTest shader asm code need update to support gfx10 and gfx9, skip the test to unblock CI test. Change-Id: Id2842127cf5fc98a652afa82035a4b3603bf5c33 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index c05e460c3d..beedbe7b24 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -371,7 +371,7 @@ TEST_F(KFDSVMEvictTest, QueueTest) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); /* Skip test for chip it doesn't have CWSR, which the test depends on */ - if (m_FamilyId < FAMILY_VI || isTonga(pNodeProperties)) { + if (m_FamilyId < FAMILY_VI || isTonga(pNodeProperties) || m_FamilyId >= FAMILY_NV) { LOG() << std::hex << "Test is skipped for family ID 0x" << m_FamilyId << std::endl; return; } From a6703395f6572f70ba99e2af86e80c945c3dbaae Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 7 Apr 2021 10:17:05 -0400 Subject: [PATCH 0836/1247] kfdtest: remove scc bit for cache coherence tests It is to address gfx90a HW memory model changes. Signed-off-by: Eric Huang Change-Id: Ie5c5c5ee5ddfb75c0b4f625baf59ce37b4cc7c31 --- tests/kfdtest/src/KFDMemoryTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 7b1e18a5f9..fdbd5539c1 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -308,7 +308,7 @@ type(CS)\n\ v_mov_b32 v1, s1\n\ v_mov_b32 v18, 0x1\n\ LOOP:\n\ - flat_load_dword v16, v[0:1] scc:1\n\ + flat_load_dword v16, v[0:1] glc\n\ s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ v_cmp_eq_i32 vcc, v16, v18\n\ s_cbranch_vccz LOOP\n\ @@ -345,7 +345,7 @@ type(CS)\n\ buffer_wbl2\n\ s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ v_mov_b32 v16, 0x1\n\ - flat_store_dword v[0:1], v16 scc:1\n\ + flat_store_dword v[0:1], v16 glc\n\ s_endpgm\n\ end\n\ "; From 9b95185a614f01575c6f4bdd953aa68f1cb215c7 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Thu, 22 Apr 2021 15:56:29 -0400 Subject: [PATCH 0837/1247] libhsakmt: Add DIDs for gfx1032 Signed-off-by: Harish Kasiviswanathan Change-Id: I61e938db3763bc2cdb4e0ea74f9aaae810b5d27b --- src/topology.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/topology.c b/src/topology.c index 8a8cc96272..c3d1cb1662 100644 --- a/src/topology.c +++ b/src/topology.c @@ -255,6 +255,11 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73C1, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, { 0x73C3, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, { 0x73DF, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, + /* DIMGREY_CAVEFISH */ + { 0x73E0, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, + { 0x73E1, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, + { 0x73E2, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, + { 0x73FF, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, /* VanGogh */ { 0x163F, 10, 3, 3, "VanGogh", CHIP_VANGOGH }, }; From 0a2d7d831926635f1fdf9c17b15c145205d3af84 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Wed, 21 Apr 2021 14:28:42 -0500 Subject: [PATCH 0838/1247] kfdtest: SetGetAttributes default access attr returned based on xnack After unregistered memory is added, now default access attribute is returned based on xnack configuration. Change-Id: I8ef44fe1e165ba009622e8112436c1f7a683f6cb Signed-off-by: Alex Sierra --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 01243f8156..72a344b904 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -132,9 +132,7 @@ TEST_F(KFDSVMRangeTest, SetGetAttributesTest) { }; HSAint32 enable = -1; EXPECT_SUCCESS(hsaKmtGetXNACKMode(&enable)); - //expectedDefaultResults[4] = (enable)?HSA_SVM_ATTR_ACCESS:HSA_SVM_ATTR_NO_ACCESS; - // FIXME: Waiting for KFD to implement retry faults on unregistered addresses - expectedDefaultResults[4] = HSA_SVM_ATTR_NO_ACCESS; + expectedDefaultResults[4] = (enable)?HSA_SVM_ATTR_ACCESS:HSA_SVM_ATTR_NO_ACCESS; sysBuffer = new HsaSVMRange(BufSize); char *pBuf = sysBuffer->As(); From d8d8e3ddd6508aa8f465f321af39b0c88c475cad Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 30 Apr 2021 13:42:27 -0400 Subject: [PATCH 0839/1247] libhsakmt: Add a new device ID for gfx90a It is gfx90a VF device ID, for virtualization support. Signed-off-by: Felix Kuehling Change-Id: I9e51d6b58c702d185e6758a9c511e9b8bc72c2f5 --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index c3d1cb1662..0a0131e9b1 100644 --- a/src/topology.c +++ b/src/topology.c @@ -228,6 +228,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x7408, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, { 0x740C, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, { 0x740F, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, + { 0x7410, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, /* Navi10 */ { 0x7310, 10, 1, 0, "Navi10", CHIP_NAVI10 }, { 0x7312, 10, 1, 0, "Navi10", CHIP_NAVI10 }, From 25288e07dc16a698393bd50e224342d5c7ac3084 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 26 Feb 2021 22:31:08 -0500 Subject: [PATCH 0840/1247] kfdtest: Handle EINTR in waitpid If the signal arrives too late, it interrupts waitpid. Handle this situation gracefully. Signed-off-by: Felix Kuehling Change-Id: If4925c352c81ba7fef8a940460b91f5e720b451e --- tests/kfdtest/src/KFDMemoryTest.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index fdbd5539c1..70c8033fcb 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1812,9 +1812,13 @@ TEST_F(KFDMemoryTest, SignalHandling) { ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(pDb, size, NULL)); LOG() << "Mapping finished" << std::endl; int childStatus; + pid_t pid; // Parent process, just wait for the child to finish - EXPECT_EQ(childPid, waitpid(childPid, &childStatus, 0)); + do { + pid = waitpid(childPid, &childStatus, 0); + } while(pid == -1 && errno == EINTR); + EXPECT_EQ(childPid, pid); EXPECT_NE(0, WIFEXITED(childStatus)); EXPECT_EQ(0, WEXITSTATUS(childStatus)); } From bd68646772751dfd6b79dbb3c9d0458d79ec7625 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 26 Feb 2021 22:31:50 -0500 Subject: [PATCH 0841/1247] kfdtest: Workaround delayed page faults Delayed page faults from a terminated process can be attributed to the next process with the same PASID. Work around that by adding a delay after the Exception tests to allow the kernel to clean up any fault storms before the next test. Signed-off-by: Felix Kuehling Change-Id: Id310c13ea9eb92b04d37b95d91a0dd60bd9954e5 --- tests/kfdtest/src/KFDExceptionTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 4976d0e319..11df6279c9 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -47,6 +47,11 @@ void KFDExceptionTest::TearDown() { KFDBaseComponentTest::TearDown(); + // WORKAROUND: This needs to be fixed in the kernel + // Wait 500ms for the kernel to process any fault storms before the + // next test to avoid reporting incorrect faults in the next test. + Delay(500); + ROUTINE_END } From 8baf02e80baae9c844a2df9c859eb44a074de32d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 26 Feb 2021 22:35:00 -0500 Subject: [PATCH 0842/1247] kfdtest: Allow some CS to fail in EvictTest amdgpu_cs_submit can fail intermittently if another process has too much memory reserved at the time. Allow a small percental of command submissions to fail to make the test more robust. Signed-off-by: Felix Kuehling Change-Id: If9f62b2b6f67be71420016d4e38d4dd6b6bca9a5 --- tests/kfdtest/src/KFDEvictTest.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index adeb847f73..0fa45d8fa8 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -213,6 +213,7 @@ void KFDEvictTest::AmdgpuCommandSubmissionSdmaNop(int rn, amdgpu_bo_handle handl amdgpu_va_handle vaHandle; uint32_t *ptr; uint32_t expired; + unsigned failCount = 0; ASSERT_EQ(0, amdgpu_cs_ctx_create(m_RenderNodes[rn].device_handle, &contextHandle)); @@ -244,8 +245,14 @@ void KFDEvictTest::AmdgpuCommandSubmissionSdmaNop(int rn, amdgpu_bo_handle handl memset(&fenceStatus, 0, sizeof(struct amdgpu_cs_fence)); for (int i = 0; i < 100; i++) { - ASSERT_EQ(0, amdgpu_cs_submit(contextHandle, 0, &ibsRequest, 1)); + int r = amdgpu_cs_submit(contextHandle, 0, &ibsRequest, 1); + Delay(50); + if (r) { + failCount++; + ASSERT_LE(failCount, 2); + continue; + } fenceStatus.context = contextHandle; fenceStatus.ip_type = AMDGPU_HW_IP_DMA; From 47ccc6604d086443cbd43810582c243d14750a98 Mon Sep 17 00:00:00 2001 From: Mike Li Date: Fri, 23 Apr 2021 09:47:48 -0400 Subject: [PATCH 0843/1247] Add Size of VGPR and SGPR to HsaNodeProperties Signed-off-by: Mike Li Change-Id: I7e6c0c5b9fd90c0bb5f3b7d35362a073afdcf9b8 --- include/hsakmttypes.h | 5 ++++- src/libhsakmt.h | 5 +++++ src/queues.c | 3 --- src/topology.c | 5 +++++ tests/kfdtest/src/KFDTopologyTest.cpp | 7 +++++++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index efff03d912..bbe24d286b 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -312,7 +312,10 @@ typedef struct _HsaNodeProperties HSAuint32 Domain; // PCI domain of the GPU HSAuint64 UniqueID; // Globally unique immutable id - HSAuint8 Reserved[20]; + + HSAuint32 VGPRSizePerCU; // VGPR size in bytes per CU + HSAuint32 SGPRSizePerCU; // SGPR size in bytes per CU + HSAuint8 Reserved[12]; } HsaNodeProperties; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 6be639261b..f248c0eaee 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -212,4 +212,9 @@ void clear_process_doorbells(void); uint32_t get_num_sysfs_nodes(void); bool is_forked_child(void); + +/* Calculate VGPR and SGPR register file size per CU */ +#define VGPR_SIZE_PER_CU(asic_family) ((asic_family == CHIP_ARCTURUS || \ + asic_family == CHIP_ALDEBARAN) ? 0x80000 : 0x40000) +#define SGPR_SIZE_PER_CU 0x4000 #endif diff --git a/src/queues.c b/src/queues.c index 1e4188382a..2cd48af5f3 100644 --- a/src/queues.c +++ b/src/queues.c @@ -42,9 +42,6 @@ #define DOORBELL_SIZE_GFX9 8 #define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) -#define VGPR_SIZE_PER_CU(asic_family) ((asic_family == CHIP_ARCTURUS || \ - asic_family == CHIP_ALDEBARAN) ? 0x80000 : 0x40000) -#define SGPR_SIZE_PER_CU 0x4000 #define LDS_SIZE_PER_CU 0x10000 #define HWREG_SIZE_PER_CU 0x1000 #define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) diff --git a/src/topology.c b/src/topology.c index 0a0131e9b1..8f1712102f 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1103,6 +1103,11 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->MarketingName[i] = name[i]; props->MarketingName[i] = '\0'; } + + /* Get VGPR/SGPR size in byte per CU */ + props->VGPRSizePerCU = VGPR_SIZE_PER_CU(hsa_gfxip->asic_family); + props->SGPRSizePerCU = SGPR_SIZE_PER_CU; + } else if (props->DeviceId) /* still return success */ pr_err("device ID 0x%x is not supported in libhsakmt\n", diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index 05a57fb411..bad2533d49 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -59,6 +59,13 @@ TEST_F(KFDTopologyTest , BasicTest) { EXPECT_GE(pNodeProperties->EngineId.ui32.Major, 7) << "Major Version is less than 7"; EXPECT_LT(pNodeProperties->EngineId.ui32.Minor, 10) << "Minor Version is greater than 9"; EXPECT_GT(pNodeProperties->uCodeEngineVersions.uCodeSDMA, 0) << "sDMA firmware version is 0"; + HSAuint32 VGPRSize = (FamilyIdFromNode(pNodeProperties) == FAMILY_AR || + FamilyIdFromNode(pNodeProperties) == FAMILY_AL) ? + 0x80000 : 0x40000; + EXPECT_EQ(pNodeProperties->VGPRSizePerCU, VGPRSize) << "VGPR Size Per CU is not correct"; + EXPECT_EQ(pNodeProperties->SGPRSizePerCU, 0x4000) << "SGPR Size Per CU is not correct"; + LOG() << "VGPR Size is " << pNodeProperties->VGPRSizePerCU << + " SGPR Size is " << pNodeProperties->SGPRSizePerCU << std::endl; } EXPECT_GT(pNodeProperties->NumMemoryBanks, HSAuint32(0)) << "Node index: " << node << "No MemoryBanks."; EXPECT_GT(pNodeProperties->NumCaches, HSAuint32(0)) << "Node index: " << node << "No Caches."; From 9168dfe041b91b18732fdc83c2dabd9d1570500d Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 12 May 2021 14:06:03 -0400 Subject: [PATCH 0844/1247] kfdtest: Increase timeout in EvictTest Increasing the timeout will avoid some test failures. This shouldn't mask any issues as any incomplete shaders should still hang and would just time out at 180 sec instead of 120 sec. Signed-off-by: Kent Russell Change-Id: If4e893ab80d9d159bd0b8b112aa7574abc5e4f44 --- tests/kfdtest/src/KFDEvictTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 0fa45d8fa8..126541abcf 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -603,7 +603,7 @@ TEST_F(KFDEvictTest, QueueTest) { addrBuffer.Fill(0x5678); /* Wait for shader to finish or timeout if shader has vm page fault */ - EXPECT_EQ(0, dispatch0.SyncWithStatus(120000)); + EXPECT_EQ(0, dispatch0.SyncWithStatus(180000)); EXPECT_SUCCESS(pm4Queue.Destroy()); From 86a68b2774d8f12d0d7e0f11431b133244ca0ffe Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 13 May 2021 19:34:25 -0400 Subject: [PATCH 0845/1247] kfdtest: Remove KFDSVMEvictTest.QueueTest GFX9 assembler meta Fixes assembler error. The SP3 backend if already set to FamilyId. Change-Id: I7721a555b05688b16993a03242a765694594825a Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index beedbe7b24..f326d32093 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -232,7 +232,6 @@ TEST_F(KFDSVMEvictTest, BasicTest) { static const char* gfx9_ReadMemory = "\ shader ReadMemory\n\ - asic(GFX9)\n\ type(CS)\n\ \n\ // compute address of corresponding output buffer\n\ From ce995fe48d8348d0235aeb3988ef60d3f97fd4c3 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Fri, 6 Nov 2020 17:27:47 +0800 Subject: [PATCH 0846/1247] libhsakmt: Prepare Beige_goby support PCI IDs have yet to be added later. Signed-off-by: Chengming Gui Change-Id: Ia0cbda17469b13fca807ce4eb74deae6f0d1eeac --- src/libhsakmt.h | 1 + src/queues.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index f248c0eaee..9fcd6d66fa 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -140,6 +140,7 @@ enum asic_family_type { CHIP_DIMGREY_CAVEFISH, /* 20 */ CHIP_VANGOGH, /* 21 */ CHIP_ALDEBARAN, /* 22 */ + CHIP_BEIGE_GOBY, /* 23 */ CHIP_LAST }; diff --git a/src/queues.c b/src/queues.c index 2cd48af5f3..eda01e1cca 100644 --- a/src/queues.c +++ b/src/queues.c @@ -193,6 +193,12 @@ const struct device_info vangogh_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info beige_goby_device_info = { + .asic_family = CHIP_BEIGE_GOBY, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -217,6 +223,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_NAVY_FLOUNDER] = &navy_flounder_device_info, [CHIP_DIMGREY_CAVEFISH] = &dimgrey_cavefish_device_info, [CHIP_VANGOGH] = &vangogh_device_info, + [CHIP_BEIGE_GOBY] = &beige_goby_device_info, }; struct queue { From f28dbdf7bf2c23a090dfcf972c8431d27c135f94 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Fri, 6 Nov 2020 17:30:04 +0800 Subject: [PATCH 0847/1247] kfdtest: Add Beige_Goby support Signed-off-by: Chengming Gui Change-Id: I3c9d4f8af1dbb4fd7ce7ff238426a4af61fd771f --- tests/kfdtest/scripts/kfdtest.exclude | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index da01a4c1f6..aebefa2ec1 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -250,3 +250,9 @@ FILTER[dimgrey_cavefish]=\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ + +FILTER[beige_goby]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDDBGTest.*:"\ +"KFDPerfCountersTest.*:" From b8ef20e35c5cf91bdc4e727e6cc34de24a4f2b90 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Fri, 9 Apr 2021 11:54:06 +0800 Subject: [PATCH 0848/1247] kfdtest: Temp disable all shader test related cases due to sp3 compiler update The updated sp3 compiler does not support GFX10 temperaly. Signed-off-by: Chengming Gui Change-Id: Idd9336663814b7925d9742eee0bd310d00945d3e --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index aebefa2ec1..8106c905e5 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -253,6 +253,7 @@ FILTER[dimgrey_cavefish]=\ FILTER[beige_goby]=\ "$BLACKLIST_ALL_ASICS:"\ +"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:" From 55cb03dbae269c6543d7b02dafdd37f69ffc70ef Mon Sep 17 00:00:00 2001 From: changzhu Date: Mon, 17 May 2021 13:23:20 +0800 Subject: [PATCH 0849/1247] kfdtest: skip KFDLocalMemoryTest.AccessLocalMem if not on dgpu path It needs to skip LocalMemoryTest because it doesn't support local memory with no dgpu path. Change-Id: Iedb6f6deba55e239b21747d933cf2d7005623106 Signed-off-by: changzhu --- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index 6af6765ac3..ffc9eb505d 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -56,6 +56,12 @@ TEST_F(KFDLocalMemoryTest, AccessLocalMem) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + /* Skip test if not on dGPU path, which the test depends on */ + if (!is_dgpu()) { + LOG() << "Not dGPU path, skipping the test" << std::endl; + return; + } + //local memory HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode, false, true); HsaEvent *event; From 9e0fc7f3c6d7abb30573d66304ece8228fc203a3 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Wed, 19 May 2021 16:03:38 +0800 Subject: [PATCH 0850/1247] kfdtest: Temporarily blacklist some svm related test cases. blacklist some svm related test cases until they are solved. Signed-off-by: Yifan Zhang Change-Id: I05e2d965d89bcbf3d43bed2873297e98ad0738ef --- tests/kfdtest/scripts/kfdtest.exclude | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 8106c905e5..d26bb290eb 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -182,14 +182,34 @@ FILTER[raven_dgpuFallback]=\ FILTER[raven]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.SdmaConcurrentCopies:"\ -"KFDQMTest.MultipleCpQueuesStressDispatch" +"KFDQMTest.MultipleCpQueuesStressDispatch:"\ +"KFDSVMRangeTest.BasicSystemMemTest:"\ +"KFDSVMRangeTest.BasicVramTest:"\ +"KFDSVMRangeTest.EvictSystemRangeTest:"\ +"KFDSVMRangeTest.PartialUnmapSysMemTest:"\ +"KFDSVMRangeTest.MigrateTest:"\ +"KFDSVMRangeTest.MigratePolicyTest:"\ +"KFDSVMRangeTest.MigrateGranularityTest:"\ +"KFDSVMRangeTest.MigrateLargeBufTest:"\ +"KFDSVMRangeTest.MultiThreadMigrationTest:"\ +"KFDSVMEvictTest.QueueTest" FILTER[renoir]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDEvictTest.*:"\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.SignalHandling" +"KFDMemoryTest.SignalHandling:"\ +"KFDSVMRangeTest.BasicSystemMemTest:"\ +"KFDSVMRangeTest.BasicVramTest:"\ +"KFDSVMRangeTest.EvictSystemRangeTest:"\ +"KFDSVMRangeTest.PartialUnmapSysMemTest:"\ +"KFDSVMRangeTest.MigrateTest:"\ +"KFDSVMRangeTest.MigratePolicyTest:"\ +"KFDSVMRangeTest.MigrateGranularityTest:"\ +"KFDSVMRangeTest.MigrateLargeBufTest:"\ +"KFDSVMRangeTest.MultiThreadMigrationTest:"\ +"KFDSVMEvictTest.QueueTest" # KFDExceptionTest.* (KFD-435) # KFDEvictTest.BurstyTest (KFD-425) From e28b3fe8b39b95602ce2125ede9309e367c76092 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 17 May 2021 12:40:52 -0400 Subject: [PATCH 0851/1247] libhsakmt: Handle unaccessible p2p_links Device cgroup can limit accessible devices. Handle the cases where p2p_links are not accessible Signed-off-by: Harish Kasiviswanathan Change-Id: I513dc75ad14e4f2d426cf2fbd301bcba12b4ee54 --- src/topology.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 8f1712102f..d24cb73a6d 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1476,7 +1476,8 @@ static HSAKMT_STATUS topology_sysfs_get_iolink_props(uint32_t node_id, read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { - ret = HSAKMT_STATUS_ERROR; + ret = (errno == EPERM) ? HSAKMT_STATUS_NOT_SUPPORTED : + HSAKMT_STATUS_ERROR; goto err2; } From 973b35bc06113986a2f0b972c533334b41ddfa50 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Wed, 9 Sep 2020 15:55:04 -0700 Subject: [PATCH 0852/1247] libhsakmt: change memory allocation alignment it is to optimize memory allocation latency, which changes alignment from 2MB to 1GB. Signed-off-by: Alex Sierra Signed-off-by: Eric Huang Change-Id: I7818e9f13b17e2c0992e75b17f978dc03a018a57 --- src/fmm.c | 6 +++--- src/libhsakmt.h | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index c9a356d0fd..fa874156b0 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -714,10 +714,10 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, return NULL; } - /* Align big buffers to the next power-of-2 up to huge page - * size for flexible fragment size TLB optimizations + /* Align big buffers to the next power-of-2 up to 1GB + * size for memory allocation optimization */ - while (align < GPU_HUGE_PAGE_SIZE && size >= (align << 1)) + while (align < GPU_GIANT_PAGE_SIZE && size >= (align << 1)) align <<= 1; /* Add padding to guarantee proper alignment and leave guard diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 9fcd6d66fa..f815e43e99 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -72,6 +72,7 @@ extern int PAGE_SHIFT; /* 2MB huge page size for 4-level page tables on Vega10 and later GPUs */ #define GPU_HUGE_PAGE_SIZE (2 << 20) +#define GPU_GIANT_PAGE_SIZE (1 << 30) #define CHECK_PAGE_MULTIPLE(x) \ do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) From e72be0e54d41e122e91be8406056fe72caca1d28 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 3 Jun 2021 16:54:33 +0800 Subject: [PATCH 0853/1247] kfdtest: Temporarily blacklist some svm related test cases for gfx902. move blacklisted test case from gfx902 iommuv2 to dgpu path. Signed-off-by: Yifan Zhang Change-Id: I8b101226ca8dcd0c12c484f5f6ce12fe73a75bdc Signed-off-by: Yifan Zhang (cherry picked from commit 9cf4377572321396225950b9a58beb549120c2a3) --- tests/kfdtest/scripts/kfdtest.exclude | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index d26bb290eb..ca898c81a9 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -177,12 +177,7 @@ FILTER[vega20]=\ FILTER[raven_dgpuFallback]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDMemoryTest.MemoryRegister:"\ -"KFDQMTest.SdmaConcurrentCopies" - -FILTER[raven]=\ -"$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.SdmaConcurrentCopies:"\ -"KFDQMTest.MultipleCpQueuesStressDispatch:"\ +"KFDQMTest.SdmaConcurrentCopies"\ "KFDSVMRangeTest.BasicSystemMemTest:"\ "KFDSVMRangeTest.BasicVramTest:"\ "KFDSVMRangeTest.EvictSystemRangeTest:"\ @@ -194,6 +189,12 @@ FILTER[raven]=\ "KFDSVMRangeTest.MultiThreadMigrationTest:"\ "KFDSVMEvictTest.QueueTest" + +FILTER[raven]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDQMTest.SdmaConcurrentCopies:"\ +"KFDQMTest.MultipleCpQueuesStressDispatch" + FILTER[renoir]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDEvictTest.*:"\ From c24ed10dfa098980ae3229e889e627519f0bb059 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Mon, 7 Jun 2021 18:21:59 +0800 Subject: [PATCH 0854/1247] libhsakmt: add colon after KFDQMTest.SdmaConcurrentCopies Signed-off-by: Yifan Zhang Change-Id: Ief14e513e4b09db0607f5533a55f80d3b0be017e --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index ca898c81a9..76bd52e54f 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -177,7 +177,7 @@ FILTER[vega20]=\ FILTER[raven_dgpuFallback]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDMemoryTest.MemoryRegister:"\ -"KFDQMTest.SdmaConcurrentCopies"\ +"KFDQMTest.SdmaConcurrentCopies:"\ "KFDSVMRangeTest.BasicSystemMemTest:"\ "KFDSVMRangeTest.BasicVramTest:"\ "KFDSVMRangeTest.EvictSystemRangeTest:"\ From f85b428265aa37b041218c8217c5f6b9220f7129 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 10 Jun 2021 21:54:42 -0500 Subject: [PATCH 0855/1247] libhsakmt: move CoherentHostAccess prop to HSA_CAPABILITY CoherentHostAccess flag member moved from HSA_MEMORYPROPERTY to HSA_CAPABILITY struct. Now this is reported to the topology as a capability of the device instead of a device memory property. Change-Id: I48e43e4b4a0635b711b62933734587facdfbf88b Signed-off-by: Alex Sierra --- include/hsakmttypes.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index bbe24d286b..c96fd839f0 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -215,7 +215,8 @@ typedef union unsigned int ASICRevision: 4; // Indicates the ASIC revision of the chip on this node. unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active unsigned int SVMAPISupported : 1; // Whether or not the SVM API is supported - unsigned int Reserved : 4; + unsigned int CoherentHostAccess: 1; // Whether or not device memory can be coherently accessed by the host CPU + unsigned int Reserved : 3; } ui32; } HSA_CAPABILITY; @@ -344,8 +345,7 @@ typedef union unsigned int HotPluggable : 1; // the memory may be removed by some system action, // memory should be used for temporary data unsigned int NonVolatile : 1; // memory content is preserved across a power-off cycle. - unsigned int CoherentHostAccess: 1; // Whether or not device memory can be coherently accessed by the host CPU - unsigned int Reserved :29; + unsigned int Reserved :30; } ui32; } HSA_MEMORYPROPERTY; From 351a41ac767df4d5aa729f613f33c4ff3c8f7a9b Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 8 Jun 2021 12:12:42 -0400 Subject: [PATCH 0856/1247] kfdtest: add KFDSVMRangeTest MigrateAccessInPlaceTest To test ACCESS_IN_PLACE GPU mapping update to system memory. Change-Id: I5b990215f39692e829128d848125e1ae0d571e03 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 72a344b904..3bf7d95143 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -657,6 +657,64 @@ TEST_F(KFDSVMRangeTest, MigrateTest) { TEST_END } +/* + * Test if GPU mapping to system memory is correct after range on VRAM split and migrate back + * to system memory. + * + * Steps, it is same for XNACK on or off + * 1. alloc 256MB range on system memory, set ACCESS_IN_PLACE by GPU + * 2. Prefetcg to migrate range to GPU VRAM + * 3. Use CPU to fill the range, range is migrated back to system memory, and split by granularity, + * GPU mapping update to system memory + * 4. Use GPU sdma to fill the range in system memory + * 5. Check if data is correct in system memory + */ +TEST_F(KFDSVMRangeTest, MigrateAccessInPlaceTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + unsigned int BufferSize = 256 << 20; + SDMAQueue sdmaQueue; + ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + HsaSVMRange DataBuffer(BufferSize, defaultGPUNode); + HSAuint32 *pData = DataBuffer.As(); + + EXPECT_SUCCESS(SVMRangeMapInPlaceToNode(pData, BufferSize, defaultGPUNode)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pData, BufferSize, defaultGPUNode)); + + for (HSAuint32 i = 0; i < BufferSize / 4; i += 1024) + pData[i] = i; + + /* GPU/SDMA update content in buffer migrated back to system memory */ + sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(sdmaQueue.GetFamilyId(), + pData, 0x55AAAA55, BufferSize)); + sdmaQueue.Wait4PacketConsumption(); + + for (HSAuint32 i = 0; i < BufferSize / 4; i += 1024) + ASSERT_EQ(0x55AAAA55, pData[i]); + + ASSERT_SUCCESS(sdmaQueue.Destroy()); + + TEST_END +} + /* * The test changes migration granularity, then trigger CPU page fault to migrate * the svm range from vram to ram. From 57962250112337645564ec65c61ec9477dad6009 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 17 Jun 2021 17:06:40 -0400 Subject: [PATCH 0857/1247] kfdtest: Remove EvictTest.BasicTest from gfx906 This is causing PSDB/OSDB failures so disable it until investigation is done Signed-off-by: Kent Russell Change-Id: I666cd45fdf8ae585486adc7cf43eacd1700704bb --- tests/kfdtest/scripts/kfdtest.exclude | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 76bd52e54f..6a28c59fe9 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -168,10 +168,12 @@ FILTER[vega12]=\ "$SDMA_BLACKLIST"\ # KFDEvictTest.BurstyTest (ROCMOPS-464) +# KFDEvictTest.BurstyTest (SWDEV-291256) FILTER[vega20]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ "KFDEvictTest.BurstyTest:"\ +"KFDEvictTest.BasicTest:"\ "KFDQMTest.GPUDoorbellWrite" FILTER[raven_dgpuFallback]=\ From c4d5ee28f09c0471346cedc5f26d2c401f8d32f3 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 17 Jun 2021 09:31:17 -0400 Subject: [PATCH 0858/1247] libhsakmt: fix multiple threads register userptr race Aperture locking is too fine-grained, it has race between find userptr and allocate userptr object. Change _fmm_allocate_device and fmm_allocate_memory_object to not take the aperture lock, the callers take it, this implements an atomic find userptr or allocate a new one. Change-Id: I6773404e22c1f4382a211c5a9817df23c5534a2a Signed-off-by: Philip Yang --- src/fmm.c | 88 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index fa874156b0..132df4f7f8 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -941,12 +941,10 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, return NULL; /* Allocate object */ - pthread_mutex_lock(&aperture->fmm_mutex); vm_obj = aperture_allocate_object(aperture, mem, args.handle, MemorySizeInBytes, flags); if (!vm_obj) goto err_object_allocation_failed; - pthread_mutex_unlock(&aperture->fmm_mutex); if (mmap_offset) *mmap_offset = args.mmap_offset; @@ -954,7 +952,6 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, return vm_obj; err_object_allocation_failed: - pthread_mutex_unlock(&aperture->fmm_mutex); free_args.handle = args.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args); @@ -1278,9 +1275,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo return NULL; /* Allocate address space */ - pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, address, MemorySizeInBytes); - pthread_mutex_unlock(&aperture->fmm_mutex); /* * Now that we have the area reserved, allocate memory in the device @@ -1293,9 +1288,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo * allocation of memory in device failed. * Release region in aperture */ - pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, MemorySizeInBytes); - pthread_mutex_unlock(&aperture->fmm_mutex); /* Assign NULL to mem to indicate failure to calling function */ mem = NULL; @@ -1341,17 +1334,19 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB if (flags.ui32.Uncached || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; + pthread_mutex_lock(&aperture->fmm_mutex); + mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); if (mem && vm_obj) { - pthread_mutex_lock(&aperture->fmm_mutex); /* Store memory allocation flags, not ioc flags */ vm_obj->flags = flags.Value; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); - pthread_mutex_unlock(&aperture->fmm_mutex); } + pthread_mutex_unlock(&aperture->fmm_mutex); + if (mem) { int map_fd = gpu_mem[gpu_mem_id].drm_render_fd; int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : @@ -1397,6 +1392,8 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + pthread_mutex_lock(&aperture->fmm_mutex); + mem = __fmm_allocate_device(gpu_id, NULL, MemorySizeInBytes, aperture, NULL, ioc_flags, &vm_obj); @@ -1409,12 +1406,12 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, flags.ui32.HostAccess = 1; flags.ui32.Reserved = 0xBe1; - pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->flags = flags.Value; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); - pthread_mutex_unlock(&aperture->fmm_mutex); } + pthread_mutex_unlock(&aperture->fmm_mutex); + if (mem) { void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, @@ -1564,13 +1561,13 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, /* Paged memory is allocated as a userptr mapping, non-paged * memory is allocated from KFD */ + pthread_mutex_lock(&aperture->fmm_mutex); + if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { /* Allocate address space */ - pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, address, size); - pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) - return NULL; + goto out_unlock; /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, @@ -1608,7 +1605,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - return NULL; + goto out_unlock; } if (flags.ui32.AQLQueueMemory) { @@ -1624,18 +1621,17 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, if (mem && vm_obj) { /* Store memory allocation flags, not ioc flags */ - pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->flags = flags.Value; vm_obj->node_id = node_id; - pthread_mutex_unlock(&aperture->fmm_mutex); } + pthread_mutex_unlock(&aperture->fmm_mutex); return mem; out_release_area: /* Release address space */ - pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, size); +out_unlock: pthread_mutex_unlock(&aperture->fmm_mutex); return NULL; @@ -2075,6 +2071,9 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; + + pthread_mutex_lock(&aperture->fmm_mutex); + mem = __fmm_allocate_device(gpu_id, NULL, PAGE_SIZE, aperture, &mmap_offset, ioc_flags, &vm_obj); @@ -2085,7 +2084,6 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) flags.ui32.NonPaged = 1; flags.ui32.HostAccess = 1; flags.ui32.Reserved = 0; - pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->flags = flags.Value; vm_obj->node_id = node_id; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -2550,9 +2548,6 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, vm_object_t *object; int ret = 0; - if (!obj) - pthread_mutex_lock(&aperture->fmm_mutex); - object = obj; if (!object) { /* Find the object to retrieve the handle */ @@ -2615,9 +2610,6 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, exit_ok: err_object_not_found: err_map_failed: - if (!obj) - pthread_mutex_unlock(&aperture->fmm_mutex); - return ret; } @@ -2646,22 +2638,23 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert return -1; ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); + /* allocate object within the scratch backing aperture */ + pthread_mutex_lock(&aperture->fmm_mutex); + if (!ret && !is_debugger) { obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); if (!obj) - return -1; + goto out_unlock; /* Create a CPU mapping for the debugger */ map_fd = gpu_mem[gpu_mem_id].drm_render_fd; mmap_ret = mmap(address, size, PROT_NONE, MAP_PRIVATE | MAP_FIXED, map_fd, mmap_offset); - if (mmap_ret == MAP_FAILED) { - __fmm_release(obj, aperture); - return -1; - } + if (mmap_ret == MAP_FAILED) + goto out_release; } else { obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, &mmap_offset, @@ -2671,10 +2664,8 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); - if (mmap_ret == MAP_FAILED) { - __fmm_release(obj, aperture); - return -1; - } + if (mmap_ret == MAP_FAILED) + goto out_release; } @@ -2683,7 +2674,14 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert if (ret != 0) __fmm_release(obj, aperture); + pthread_mutex_unlock(&aperture->fmm_mutex); return ret; + +out_release: + __fmm_release(obj, aperture); +out_unlock: + pthread_mutex_unlock(&aperture->fmm_mutex); + return -1; } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, @@ -3003,29 +3001,45 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, if (svm.check_userptr) fmm_check_user_memory(addr, size); + pthread_mutex_lock(&aperture->fmm_mutex); + + /* catch the race condition where some other thread added the userptr + * object already after the vm_find_object. + */ + obj = vm_find_object_by_userptr(aperture, addr, size); + if (obj) { + ++obj->registration_count; + goto out_found; + } + /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, NULL, aligned_size, aperture, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), &obj); - if (!svm_addr) + if (!svm_addr) { + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; + } if (obj) { - pthread_mutex_lock(&aperture->fmm_mutex); obj->userptr = addr; gpuid_to_nodeid(gpu_id, &obj->node_id); obj->userptr_size = size; obj->registration_count = 1; obj->user_node.key = rbtree_key((unsigned long)addr, size); rbtree_insert(&aperture->user_tree, &obj->user_node); + } else { pthread_mutex_unlock(&aperture->fmm_mutex); - } else return HSAKMT_STATUS_ERROR; + } +out_found: if (obj_ret) *obj_ret = obj; + + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } From 1a9604ad5792545f564f59d7cd4e70b6ee5265f0 Mon Sep 17 00:00:00 2001 From: changzhu Date: Thu, 24 Jun 2021 10:28:45 +0800 Subject: [PATCH 0859/1247] kfdtest: skip KFDSVMRangeTest.MigrateAccessInPlaceTest for gfx902 and gfx90c Change-Id: I671440c212a07fdfdb1c4245b4551c6344eaedc6 Signed-off-by: changzhu --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 6a28c59fe9..cd10253417 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -189,9 +189,9 @@ FILTER[raven_dgpuFallback]=\ "KFDSVMRangeTest.MigrateGranularityTest:"\ "KFDSVMRangeTest.MigrateLargeBufTest:"\ "KFDSVMRangeTest.MultiThreadMigrationTest:"\ +"KFDSVMRangeTest.MigrateAccessInPlaceTest:"\ "KFDSVMEvictTest.QueueTest" - FILTER[raven]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.SdmaConcurrentCopies:"\ @@ -212,6 +212,7 @@ FILTER[renoir]=\ "KFDSVMRangeTest.MigrateGranularityTest:"\ "KFDSVMRangeTest.MigrateLargeBufTest:"\ "KFDSVMRangeTest.MultiThreadMigrationTest:"\ +"KFDSVMRangeTest.MigrateAccessInPlaceTest:"\ "KFDSVMEvictTest.QueueTest" # KFDExceptionTest.* (KFD-435) From 408fca02785fd35bff801e8c355de1cc42e24deb Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Fri, 4 Jun 2021 17:44:15 -0500 Subject: [PATCH 0860/1247] Add error message to assertion. Warn that HSA_FORCE_ASIC_TYPE may be needed if the engine major id assertion fails. Change-Id: I67e01e99c3d1bdc84630ccfae489dce5e77961b5 --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index d24cb73a6d..01cdf61ee2 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1114,7 +1114,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->DeviceId); if (props->NumFComputeCores) - assert(props->EngineId.ui32.Major); + assert(props->EngineId.ui32.Major && "HSA_FORCE_ASIC_TYPE may be needed"); err: free(read_buf); From b2fb2a347071e26f4511b50297d8a31d1bbd496f Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 6 Jul 2021 09:48:02 -0400 Subject: [PATCH 0861/1247] kfdtest.exclude: Add NV12 blacklist Add a blacklist for gfx1xxx12, using the same list as gfx1012 Change-Id: I7e620dba8a36f6f89152a48066234884150a15dd --- tests/kfdtest/scripts/kfdtest.exclude | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index cd10253417..04899e615a 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -250,6 +250,17 @@ FILTER[navi10]=\ "$TEMP_GFX10_BLACKLIST:"\ "KFDMemoryTest.MMBench" +# Need to verify the following failed tests on another machine: +# Exceptions not being received during exception tests +# PerfCounters return HSAKMT_STATUS_INVALID_PARAMETER +# P2PBandwidth failing (wait times out) on node-to-multiple-nodes by [push, NONE] +FILTER[navi12]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDExceptionTest.*:"\ +"KFDPerfCountersTest.*:"\ +"KFDPerformanceTest.P2PBandWidthTest:"\ +"$TEMP_GFX10_BLACKLIST" + FILTER[navi14]=\ "$BLACKLIST_ALL_ASICS:"\ "$TEMP_GFX10_BLACKLIST" From fd131e875eec437ac1a43fa73c355fa6eed641a7 Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Wed, 30 Jun 2021 19:44:53 +0800 Subject: [PATCH 0862/1247] kfdtest: MigrateLargeBufTest support APU Limit test buffer size to 3/4 total VRAM size, and max 1GB. Signed-off-by: Aaron Liu Change-Id: I937e10b0a6bd8215e3865b50f22ce75b3982a6f7 --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 3bf7d95143..ecec01d4ac 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -803,11 +803,15 @@ TEST_F(KFDSVMRangeTest, MigrateLargeBufTest) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - if (!GetVramSize(defaultGPUNode)) { + HSAuint64 vramSize; + vramSize = GetVramSize(defaultGPUNode); + if (!vramSize) { LOG() << "Skipping test: No VRAM found." << std::endl; return; } + BufferSize = MIN(BufferSize, vramSize * 3 / 4); + HsaSVMRange SysBuffer(BufferSize, defaultGPUNode); SysBuffer.Fill(0x1); From a55551309cbfe7d15771722e909bfcd4431e58d7 Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Mon, 25 Jan 2021 18:09:40 +0800 Subject: [PATCH 0863/1247] libhsakmt: add yellow carp support This patch is to add yellow carp support on thunk. Signed-off-by: Aaron Liu Change-Id: Icfecc3fd1f472c9924f934c6a5352448356d83df --- src/libhsakmt.h | 1 + src/pmc_table.c | 1 + src/queues.c | 7 +++++++ 3 files changed, 9 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index f815e43e99..a616c822b9 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -142,6 +142,7 @@ enum asic_family_type { CHIP_VANGOGH, /* 21 */ CHIP_ALDEBARAN, /* 22 */ CHIP_BEIGE_GOBY, /* 23 */ + CHIP_YELLOW_CARP, /* 24 */ CHIP_LAST }; diff --git a/src/pmc_table.c b/src/pmc_table.c index 502804c6cd..8697661f3e 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -2134,6 +2134,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, case CHIP_NAVI10: case CHIP_NAVI14: case CHIP_VANGOGH: + case CHIP_YELLOW_CARP: *block = navi_blocks[block_id]; break; default: diff --git a/src/queues.c b/src/queues.c index eda01e1cca..de408903a5 100644 --- a/src/queues.c +++ b/src/queues.c @@ -199,6 +199,12 @@ const struct device_info beige_goby_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info yellow_carp_device_info = { + .asic_family = CHIP_YELLOW_CARP, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + static const struct device_info *dev_lookup_table[] = { [CHIP_KAVERI] = &kaveri_device_info, [CHIP_HAWAII] = &hawaii_device_info, @@ -224,6 +230,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_DIMGREY_CAVEFISH] = &dimgrey_cavefish_device_info, [CHIP_VANGOGH] = &vangogh_device_info, [CHIP_BEIGE_GOBY] = &beige_goby_device_info, + [CHIP_YELLOW_CARP] = &yellow_carp_device_info, }; struct queue { From ef9c532187d0e8ffb9e50de4b8bdbb939f85702e Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Wed, 30 Jun 2021 11:01:34 +0800 Subject: [PATCH 0864/1247] kfdtest: add yellow_carp blacklist Signed-off-by: Chen Gong Signed-off-by: Aaron Liu Change-Id: Ib3a3172b0ac40109acbe42b9dc92517b3fedc84c --- tests/kfdtest/scripts/kfdtest.exclude | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 04899e615a..1e64dc6af4 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -291,4 +291,10 @@ FILTER[beige_goby]=\ "$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ -"KFDPerfCountersTest.*:" +"KFDPerfCountersTest.*:"\ + +FILTER[yellow_carp]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$TEMP_GFX10_BLACKLIST:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDIPCTest.CMABasicTest" From 92076f6f1b6e5c3609c79539a7de13ef3dd177c6 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 16 Jun 2021 22:05:53 -0400 Subject: [PATCH 0865/1247] kfdtest: add KFDMemoryTest MultiThreadRegisterUserptrTest Test Thunk multiple threads register and deregister same userptr race condition, to emulate application register same userptr to multiple GPUs using multiple threads. Use thread barrier to sync the threads, to start register userptr at same time. Change-Id: I6723dc39f75908026fa14a490e39e1fe49a13a1b Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDMemoryTest.cpp | 78 +++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 70c8033fcb..0e4c58a067 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2643,3 +2643,81 @@ TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) { TEST_END } + +/* Application register same userptr to multiple GPUs using multiple threads + * Test multiple threads register/deregister same userptr, to verify Thunk race handling + */ +struct ThreadParams { + void* pBuf; + HSAuint64 BufferSize; + HSAuint64 VAGPU; + pthread_barrier_t *barrier; +}; +static unsigned int RegisterThread(void* p) { + struct ThreadParams* pArgs = reinterpret_cast(p); + + pthread_barrier_wait(pArgs->barrier); + EXPECT_SUCCESS(hsaKmtRegisterMemory(pArgs->pBuf, pArgs->BufferSize)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(pArgs->pBuf, pArgs->BufferSize, &pArgs->VAGPU)); + + return 0; +} +static unsigned int UnregisterThread(void* p) { + struct ThreadParams* pArgs = reinterpret_cast(p); + + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(pArgs->VAGPU))); + pthread_barrier_wait(pArgs->barrier); + EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(pArgs->VAGPU))); + + return 0; +} + +#define N_THREADS 32 + +TEST_F(KFDMemoryTest, MultiThreadRegisterUserptrTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HSAuint32 test_loops = 1; + HSAuint64 BufferSize = 1UL << 27; + + void *pBuf = mmap(NULL, BufferSize, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + ASSERT_NE(pBuf, MAP_FAILED); + + struct ThreadParams params[N_THREADS]; + HSAuint64 threadId[N_THREADS]; + + pthread_barrier_t barrier; + ASSERT_SUCCESS(pthread_barrier_init(&barrier, NULL, N_THREADS)); + + for (HSAuint32 loop = 0; loop < test_loops; loop++) { + for (HSAuint32 i = 0; i < N_THREADS; i++) { + params[i].pBuf = pBuf; + params[i].BufferSize = BufferSize; + params[i].VAGPU = 0; + params[i].barrier = &barrier; + } + + for (HSAuint32 i = 0; i < N_THREADS; i++) + ASSERT_EQ(true, StartThread(&RegisterThread, ¶ms[i], threadId[i])); + for (HSAuint32 i = 0; i < N_THREADS; i++) + WaitForThread(threadId[i]); + + for (HSAuint32 i = 0; i < N_THREADS; i++) + ASSERT_EQ(params[0].VAGPU, params[i].VAGPU); + + for (HSAuint32 i = 0; i < N_THREADS; i++) + ASSERT_EQ(true, StartThread(&UnregisterThread, ¶ms[i], threadId[i])); + for (HSAuint32 i = 0; i < N_THREADS; i++) + WaitForThread(threadId[i]); + } + + pthread_barrier_destroy(&barrier); + munmap(pBuf, BufferSize); + + TEST_END +} From bdfe3a12a8c788a825b72229a1cddf93f8a7bcc6 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 8 Jul 2021 15:19:27 -0400 Subject: [PATCH 0866/1247] kfdtest: Ensure devices are peer-accessible for peer mapping If the devices aren't peer-accessible, we shouldn't try to run a test that requires that the devices be peer-accessible. Thus, add a check in MapVramToGPUNodesTest to check for peer accessibility before executing the peer mappings. Signed-off-by: Kent Russell Change-Id: Ib79b141f8c1ac6d85f5ab49d62af62ec10b988b7 --- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index ffc9eb505d..a27b502f97 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -516,6 +516,11 @@ TEST_F(KFDLocalMemoryTest, MapVramToGPUNodesTest) { } } + if (!m_NodeInfo.IsPeerAccessibleByNode(dst_node, src_node)) { + LOG() << "Skipping test: GPUs are not peer-accessible" << std::endl; + return; + } + LOG() << "Testing from GPU " << src_node << " to GPU " << dst_node << std::endl; void *shared_addr; From 19536080a87aeb4c903ce766971461302d2ac677 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 7 Jul 2021 21:19:25 -0400 Subject: [PATCH 0867/1247] libhsakmt: Fix deadlock in map_mmio Unlock mutex if MMIO mapping fails. This happens on all GFXv8 GPUs. Signed-off-by: Felix Kuehling Change-Id: I1dee1cbddefd9185c24ea79377f49f8ae2c5ff57 --- src/fmm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 132df4f7f8..f00584b1e5 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2077,8 +2077,10 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) mem = __fmm_allocate_device(gpu_id, NULL, PAGE_SIZE, aperture, &mmap_offset, ioc_flags, &vm_obj); - if (!mem || !vm_obj) + if (!mem || !vm_obj) { + pthread_mutex_unlock(&aperture->fmm_mutex); return NULL; + } flags.Value = 0; flags.ui32.NonPaged = 1; From 5fac7dcc3b7112f933ad07e90a57917b7ae6c1a5 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 9 Jul 2021 20:14:23 -0400 Subject: [PATCH 0868/1247] libhsakmt: Fix deadlocks in __fmm_release __fmm_release is sometimes called with the aperture lock, and sometimes without. Consistently call it with the aperture lock held and remove the lock/unlock calls from this function. Signed-off-by: Felix Kuehling Change-Id: I80dddc64cc0703e5eed8e9f1eb65b75a2c7ae2eb --- src/fmm.c | 53 ++++++++++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index f00584b1e5..43d2a5f44c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -287,6 +287,9 @@ static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture); +static int _fmm_map_to_gpu(manageable_aperture_t *aperture, + void *address, uint64_t size, vm_object_t *obj, + uint32_t *nodes_to_map, uint32_t nodes_array_size); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address); @@ -1345,8 +1348,6 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB gpuid_to_nodeid(gpu_id, &vm_obj->node_id); } - pthread_mutex_unlock(&aperture->fmm_mutex); - if (mem) { int map_fd = gpu_mem[gpu_mem_id].drm_render_fd; int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : @@ -1358,7 +1359,8 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - return NULL; + mem = NULL; + goto out_unlock; } /* * This madvise() call is needed to avoid additional references @@ -1369,6 +1371,9 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB madvise(mem, MemorySizeInBytes, MADV_DONTFORK); } +out_unlock: + pthread_mutex_unlock(&aperture->fmm_mutex); + return mem; } @@ -1410,8 +1415,6 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, gpuid_to_nodeid(gpu_id, &vm_obj->node_id); } - pthread_mutex_unlock(&aperture->fmm_mutex); - if (mem) { void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, @@ -1419,10 +1422,12 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, doorbell_mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - return NULL; + mem = NULL; } } + pthread_mutex_unlock(&aperture->fmm_mutex); + return mem; } @@ -1652,23 +1657,18 @@ static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) if (!object) return -EINVAL; - pthread_mutex_lock(&aperture->fmm_mutex); - /* If memory is user memory and it's still GPU mapped, munmap * would cause an eviction. If the restore happens quickly * enough, restore would also fail with an error message. So * free the BO before unmapping the pages. */ args.handle = object->handle; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) { - pthread_mutex_unlock(&aperture->fmm_mutex); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) return -errno; - } aperture_release_area(aperture, object->start, object->size); vm_remove_object(aperture, object); - pthread_mutex_unlock(&aperture->fmm_mutex); return 0; } @@ -1701,9 +1701,10 @@ HSAKMT_STATUS fmm_release(void *address) pthread_mutex_unlock(&aperture->fmm_mutex); munmap(address, size); } else { - pthread_mutex_unlock(&aperture->fmm_mutex); + int r = __fmm_release(object, aperture); - if (__fmm_release(object, aperture)) + pthread_mutex_unlock(&aperture->fmm_mutex); + if (r) return HSAKMT_STATUS_ERROR; if (!aperture->is_cpu_accessible) @@ -2088,7 +2089,6 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) flags.ui32.Reserved = 0; vm_obj->flags = flags.Value; vm_obj->node_id = node_id; - pthread_mutex_unlock(&aperture->fmm_mutex); /* Map for CPU access*/ ret = mmap(mem, PAGE_SIZE, @@ -2097,14 +2097,14 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - return NULL; + mem = NULL; + /* Map for GPU access*/ + } else if (_fmm_map_to_gpu(aperture, mem, PAGE_SIZE, vm_obj, NULL, 0)) { + __fmm_release(vm_obj, aperture); + mem = NULL; } - /* Map for GPU access*/ - if (fmm_map_to_gpu(mem, PAGE_SIZE, NULL)) { - __fmm_release(vm_obj, aperture); - return NULL; - } + pthread_mutex_unlock(&aperture->fmm_mutex); return mem; } @@ -2888,13 +2888,8 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; - if (ret) - goto err; - - pthread_mutex_unlock(&aperture->fmm_mutex); - - /* free object in scratch backing aperture */ - return __fmm_release(object, aperture); + if (!ret) + ret = __fmm_release(object, aperture); err: pthread_mutex_unlock(&aperture->fmm_mutex); @@ -3398,8 +3393,8 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) * buffer. Deregistering imported graphics buffers or * userptrs means releasing the BO. */ - pthread_mutex_unlock(&aperture->fmm_mutex); __fmm_release(object, aperture); + pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } From 96c7a5c9dcd80155edf2e9f908b082e74a1c6915 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 10 May 2021 11:56:06 -0400 Subject: [PATCH 0869/1247] libhsakmt: update create queue for exception handling Update hsaKmtCreateQueue to initialize the new save area header with the exception payload and event ID. Signed-by-off: Jonathan Kim Reviewed-by: Felix Kuehling Reviewed-by: Sean Keely Change-Id: Icd38062dc982cb29b30644699014eeb0b3e26d00 --- include/hsakmttypes.h | 7 +++++++ src/queues.c | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index c96fd839f0..a8a396946a 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -689,6 +689,12 @@ typedef struct // debugger. Must be 64 byte aligned. HSAuint32 DebugSize; // Byte size of the memory reserved for the // debugger. Must be 64 byte aligned. + volatile HSAint64 *ErrorReason; // Address of the HSA signal payload for + // reporting the error reason bitmask. + // Must be 4 byte aligned. + HSAuint32 ErrorEventId; // Event ID used for exception signalling. + // Must be 4 byte aligned. + HSAuint32 Reserved1; } HsaUserContextSaveAreaHeader; @@ -738,6 +744,7 @@ typedef struct _HsaQueueResource HSAuint64 QueueRptrValue; }; + volatile HSAint64* ErrorReason; /** exception bits signal payload */ } HsaQueueResource; diff --git a/src/queues.c b/src/queues.c index de408903a5..e667f1a1ce 100644 --- a/src/queues.c +++ b/src/queues.c @@ -588,7 +588,9 @@ static void free_queue(struct queue *q) static int handle_concrete_asic(struct queue *q, struct kfd_ioctl_create_queue_args *args, - uint32_t NodeId) + uint32_t NodeId, + HsaEvent *Event, + volatile HSAint64 *ErrPayload) { const struct device_info *dev_info = q->dev_info; bool ret; @@ -626,6 +628,10 @@ static int handle_concrete_asic(struct queue *q, args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; header = (HsaUserContextSaveAreaHeader *)q->ctx_save_restore; + header->ErrorEventId = 0; + if (Event) + header->ErrorEventId = Event->EventId; + header->ErrorReason = ErrPayload; header->DebugOffset = q->ctx_save_restore_size - q->debug_memory_size; header->DebugSize = q->debug_memory_size; } @@ -724,7 +730,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, QueueResource->QueueWptrValue = (uintptr_t)&q->wptr; } - err = handle_concrete_asic(q, &args, NodeId); + err = handle_concrete_asic(q, &args, NodeId, Event, QueueResource->ErrorReason); if (err != HSAKMT_STATUS_SUCCESS) { free_queue(q); return err; From 31ac82617caf446a9bc915a8f74d8b5ba8a36de9 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 25 May 2021 11:14:26 -0400 Subject: [PATCH 0870/1247] libhsakmt: update thunk api for exception handling The debugger and debug agent no longer use the Thunk API. Remove all deprecated functions and keep commented references for future KFD tests. Update and the keep the version checks for future use and hsaKmtRuntimeEnable/Disable. Signed-off-by: Jonathan Kim Signed-off-by: Laurent Morichetti Reviewed-by: Felix Kuehling Change-Id: Ia2f10d82f5ac36d0bd1bda233810f26e8a154d55 --- include/hsakmt.h | 432 +-------------------------- include/linux/kfd_ioctl.h | 354 ++++++++++++++++++---- src/debug.c | 484 +------------------------------ src/libhsakmt.ver | 13 - tests/kfdtest/src/KFDDBGTest.cpp | 15 + 5 files changed, 324 insertions(+), 974 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 39b1a0c792..615ebafef8 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -634,296 +634,7 @@ hsaKmtDbgAddressWatch( ); /** - Suspend the execution of a set of queues. A queue that is suspended - allows the wave context save state to be inspected and modified. If a - queue is already suspended it remains suspended. A suspended queue - can be resumed by hsaKmtDbgQueueResume(). - - For each node that has a queue suspended, a sequentially consistent - system scope release will be performed that synchronizes with a - sequentially consistent system scope acquire performed by this - call. This ensures any memory updates performed by the suspended - queues are visible to the thread calling this operation. - - Pid is the process that owns the queues that are to be supended or - resumed. If the value is -1 then the Pid of the process calling - hsaKmtQueueSuspend or hsaKmtQueueResume is used. - - NumQueues is the number of queues that are being requested to - suspend or resume. - - Queues is a pointer to an array with NumQueues entries of - HSA_QUEUEID. The queues in the list must be for queues that exist - for Pid, and can be a mixture of queues for different nodes. - - GracePeriod to wait after initialiating context save before forcing - waves to context save. A value of 0 indicates no grace period. - It is ignored by hsaKmtQueueResume. - - Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid for Pid. -*/ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueueSuspend( - HSAuint32 Pid, // IN - HSAuint32 NumQueues, // IN - HSA_QUEUEID *Queues, // IN - HSAuint32 GracePeriod, // IN - HSAuint32 Flags); // IN - -/** - Resume the execution of a set of queues. If a queue is not - suspended by hsaKmtDbgQueueSuspend() then it remains executing. Any - changes to the wave state data will be used when the waves are - restored. Changes to the control stack data will have no effect. - - For each node that has a queue resumed, a sequentially consistent - system scope release will be performed that synchronizes with a - sequentially consistent system scope acquire performed by all - queues being resumed. This ensures any memory updates performed by - the thread calling this operation are visible to the resumed - queues. - - For each node that has a queue resumed, the instruction cache will - be invalidated. This ensures any instruction code updates performed - by the thread calling this operation are visible to the resumed - queues. - - Pid is the process that owns the queues that are to be supended or - resumed. If the value is -1 then the Pid of the process calling - hsaKmtQueueSuspend or hsaKmtQueueResume is used. - - NumQueues is the number of queues that are being requested to - suspend or resume. - - Queues is a pointer to an array with NumQueues entries of - HSA_QUEUEID. The queues in the list must be for queues that exist - for Pid, and can be a mixture of queues for different nodes. - - Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. - Returns: - - HSAKMT_STATUS_SUCCESS if successful - - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. -*/ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueueResume( - HSAuint32 Pid, // IN - HSAuint32 NumQueues, // IN - HSA_QUEUEID *Queues, // IN - HSAuint32 Flags); // IN - -/** - Enable debug trap for NodeId. If QueueId is INVALID_QUEUEID then - enable for all queues on NodeId, otherwise enable only for QueueId. - Return file descriptor PollFd where on poll wake, fd has readable - FIFO data for pending debug events. - - When debug trap is enabled the trap handler behavior changes - depending on architecture of the node and can include the following: - - - Initialize Trap Temp Registers: All new waves are launched with - specific trap temp registers initialized with: - - - HSA dispatch packet address of the wave. - - - X, Y, Z grid and work-group position of the wave within the - dispatch. - - - The scratch backing memory address. - - - Enable wave launch trap override. hsaKmtEnableDebugTrap() sets the - TrapMask to 0 and the TrapOverride to HSA_DBG_TRAP_OVERRIDE_OR and - they can be changed by hsaKmtSetWaveLaunchTrapOverride(). - - If debug trap is already enabled for NodeId, any features controlled - by it are still reset to their default values as defined above. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_INVALID_HANDLE if: - - - NodeId is invalid - - - QueueId is not INVALID_QUEUE, or is not a valid queue of - NodeId. - - - HSAKMT_STATUS_UNAVAILABLE if debugging is not available to this - process. For example, there may be a limit on number of - processes that can perform debugging at the same time. - - - HSAKMT_STATUS_NOT_SUPPORTED if debug trap is not supported by - NodeId, or if QueueId is not INVALID_QUEUEID and NodeId does not - support per queue enabling. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtEnableDebugTrap( - HSAuint32 NodeId, //IN - HSA_QUEUEID QueueId //IN - ); - - -/* Similar to EnableDebugTrap with polling fd return*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtEnableDebugTrapWithPollFd( - HSAuint32 NodeId, //IN - HSA_QUEUEID QueueId, //IN - HSAint32 *PollFd //OUT - ); - -/** - Disable debug trap enabled by hsaKmtEnableDebugTrap(). If debug trap - is not currently enabled not action is taken. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. - - - HSAKMT_STATUS_NOT_SUPPORTED if debug trap not supported for NodeId. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDisableDebugTrap( - HSAuint32 NodeId //IN - ); - - -/** - Query pending debug event set by ptrace. - - Can query by target QueueId. If QueueId is INVALID_QUEUEID, return the - first queue id that has a pending event. Option to clear pending event - after query is used by the ClearEvents parameter. - - Pending debug event type will be returned in EventsReceived parameter and is - defined by HSA_DEBUG_EVENT_TYPE. Suspended state of queue is returned in - IsSuspended. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful - */ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueryDebugEvent( - HSAuint32 NodeId, // IN - HSAuint32 Pid, // IN - HSAuint32 *QueueId, // IN/OUT - bool ClearEvents, // IN - HSA_DEBUG_EVENT_TYPE *EventsReceived, // OUT - bool *IsSuspended, // OUT - bool *IsNew //OUT - ); - -/** - Newly created queue snapshot per ptraced process. - - Returns queue snapshot including queue id, gpuid, context save base address, - queue status word, queue address and size, and queue read and write pointer. - - ClearEvents set will clear new queue bit and queue status word bits. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful - */ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetQueueSnapshot( - HSAuint32 NodeId, // IN - HSAuint32 Pid, // IN - bool ClearEvents, // IN - void *SnapshotBuf, // IN - HSAuint32 *QssEntries // IN/OUT - ); - -/** - Send the host trap -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSendHostTrap( - HSAuint32 NodeId, //IN - HSAuint32 Pid //IN - ); - -/** - Set the trap override mask. When debug trap is enabled by - hsaKmtEnableDebugTrap() each wave launched has its initial - MODE.excp_en register overriden by TrapMask as specified by - TrapOverride. - - An error is returned if debug trap is not currently enabled for - NodeId. Debug trap is enabled by hsaKmtEnableDebugTrap() which - initializes TrapMask to 0 and TrapOverride to - HSA_DBG_TRAP_OVERRIDE_OR. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_NOT_SUPPORTED if wave launch trap override is not - supported by NodeId. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. - - - HSAKMT_STATUS_INVALID_PARAMETER if TrapOverride is invalid. - - - HSAKMT_STATUS_ERROR if debug trap is not currently enabled by - hsaKmtEnableDebugTrap() for NodeId. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetWaveLaunchTrapOverride( - HSAuint32 NodeId, //IN - HSA_DBG_TRAP_OVERRIDE TrapOverride, //IN - HSA_DBG_TRAP_MASK TrapMask //IN - ); - -/** - Set the mode in which all future waves will be launched for - NodeId. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_UNAVAILABLE if debugging is not available to this - process. For example, there may be a limit on number of - processes that can perform debugging at the same time. - - - HSAKMT_STATUS_NOT_SUPPORTED if the WaveLaunchMode requested is - not supported by the NodeId. Different implementations and - different nodes within an implementation can support different - sets of launch modes. Only HSA_DBG_WAVE_LAUNCH_MODE_NORMAL mode - is supported by all. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId is not a valid node. - - - HSAKMT_STATUS_INVALID_PARAMETER if WaveLaunchMode is not a valid - value. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetWaveLaunchMode( - HSAuint32 NodeId, //IN - HSA_DBG_WAVE_LAUNCH_MODE WaveLaunchMode //IN - ); - -/** - * Get the major and minor version of the kernel debugger support. - * - * Returns: - * - HSAKMT_STATUS_SUCCESS if successful. - * - * - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. - * - * - HSAKMT_STATUS_NOT_SUPPORTED if debug trap not supported for NodeId. + Get the major and minor version of the kernel debugger support. */ HSAKMT_STATUS HSAKMTAPI @@ -932,147 +643,6 @@ hsaKmtGetKernelDebugTrapVersionInfo( HSAuint32 *Minor //Out ); -/** - * Get the major and minor version of the thunk debugger support. -*/ -void -HSAKMTAPI -hsaKmtGetThunkDebugTrapVersionInfo( - HSAuint32 *Major, //Out - HSAuint32 *Minor //Out - ); - - - - -/** - Set a debug memory access watch point. A memory access of the kind - specified by WatchMode to an matching address will cause the trap - handler to be entered. An address matches if, after ANDing the - watch-addr-mask-lo..watch-addr-mask-hi bits of WatchAddrMask, it - equals the WatchAddress with the bottom watch-addr-mask-lo bits - cleared. - - WatchId will be in the range 0 to watch-count - 1. The WatchId - value will match the address watch exception reported to the trap - handler. - - hsaKmtGetNodeProperties() can be used to obtain HsaNodeProperties. - watch-addr-mask-lo and watch-addr-mask-hi can be obtained from - HsaNodeProperties.Capabilities.WatchAddrMaskLoBit and - HsaNodeProperties.Capabilities.WatchAddrMaskHiBit respectively. - watch-count can be obtained from - 2^HsaNodeProperties.Capabilities.WatchPointsTotalBits. - - To cause debug memory address watch points to be reported to the - trap handler the address watch exception must be enabled. This can - be accomplished by using hsaKmtSetWaveLaunchTrapOverride() with a - TrapMask that includes HSA_DBG_TRAP_MASK_DBG_ADDRESS_WATCH. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_NOT_SUPPORTED if debug memory watch points are - not supported for NodeId. - - - HSAKMT_STATUS_UNAVAILABLE if debugging is not available to this - process. For example, there may be a limit on number of - processes that can perform debugging at the same time. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId or WatchId* is invalid. - - - HSAKMT_STATUS_INVALID_PARAMETER if: - - - WatchAddrMask contains non-0 bits outside the inclusive range - watch-addr-mask-lo to watch-addr-mask-hi. - - - If WatchAddress contain non-0 bits in the inclusive range 0 to - watch-addr-mask-lo. - - - If WatchMode is not one of the values of HSA_DBG_WATCH_MODE. - - - WatchId is NULL. - - - HSAKMT_STATUS_OUT_OF_RESOURCES if no more watch points are - available to set currently. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSetAddressWatch( - HSAuint32 NodeId, //IN - HSAuint32 Pid, //IN - HSA_DBG_WATCH_MODE WatchMode, //IN - void* WatchAddress, //IN - HSAuint64 WatchAddrMask, //IN - HSAuint32* WatchId //OUT - ); - -/** - Clear a debug memory access watch point set by - hsaKmtSetAddressWatch(). - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_NOT_SUPPORTED if debug memory watch points are - not supported for NodeId. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid or WatchId is not valid for this - NodeId. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtClearAddressWatch( - HSAuint32 NodeId, //IN - HSAuint32 Pid, //IN - HSAuint32 WatchId //IN - ); - -/** - Enable precise memory operations. - - When precise memory operations are enabled a wave waits for each - memory operation to complete before executing further - operations. This results in more precise reporting of memory related - events such as memory violation or address watch points. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_UNAVAILABLE if precise memory operations is not - available to this process. For example, the feature may require - specific privileges. - - - HSAKMT_STATUS_NOT_SUPPORTED if precise memory operations is not - supported by NodeId. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtEnablePreciseMemoryOperations( - HSAuint32 NodeId //IN - ); - -/** - Disable precise memory operations enabled by - hsaKmtEnablePreciseMemoryOperations(). If precise memory operations - are not currently enabled no action is taken. - - Returns: - - HSAKMT_STATUS_SUCCESS if successful. - - - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. - - - HSAKMT_STATUS_NOT_SUPPORTED if precise memory operations is not - supported by NodeId. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtDisablePreciseMemoryOperations( - HSAuint32 NodeId //IN - ); - /** Gets GPU and CPU clock counters for particular Node */ diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 868fbdd370..9bb1fb8a64 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -23,7 +23,7 @@ #ifndef KFD_IOCTL_H_INCLUDED #define KFD_IOCTL_H_INCLUDED -#include +#include #include /* @@ -44,15 +44,29 @@ * 1.1 - Allow attaching to processes that have not opened /dev/kfd yet * 1.2 - Allow flag option to clear queue status on queue suspend * 1.3 - Fix race condition between clear on suspend and trap event handling - * 1.3 - Fix race condition between clear on suspend and trap event handling * 1.4 - Fix bad kfifo free * 1.5 - Fix ABA issue between queue snapshot and suspend * 2.0 - Return number of queues suspended/resumed and mask invalid/error - * array slots + * array slots * 2.1 - Add Set Address Watch, and Clear Address Watch support. + * 3.0 - Overhaul set wave launch override API + * 3.1 - Add support for GFX10 + * 3.2 - Add support for GFX10.3 + * 3.3 - Add precise memory operations enable + * 4.0 - Remove gpu_id from api + * 5.0 - Report exception codes to the debugger + * 6.0 - Pass event file descriptor from userspace. + * 6.1 - Add KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO. + * 6.2 - Add KFD_IOC_DBG_TRAP_DEVICE_SNAPSHOT. + * 7.0 - Redefine exception codes + * 7.1 - Add KFD_IOC_DBG_TRAP_RUNTIME_ENABLE + * 7.2 - Add KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT + * 8.0 - Expand runtime information given to the debugger + * 8.1 - Allow the debugger to set the exception mask + * 9.0 - Handle multiple exceptions from single trap signal */ -#define KFD_IOCTL_DBG_MAJOR_VERSION 2 -#define KFD_IOCTL_DBG_MINOR_VERSION 1 +#define KFD_IOCTL_DBG_MAJOR_VERSION 9 +#define KFD_IOCTL_DBG_MINOR_VERSION 0 struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ @@ -117,6 +131,7 @@ struct kfd_ioctl_get_queue_wave_state_args { }; struct kfd_queue_snapshot_entry { + __u64 exception_status; __u64 ring_base_address; __u64 write_pointer_address; __u64 read_pointer_address; @@ -125,8 +140,19 @@ struct kfd_queue_snapshot_entry { __u32 gpu_id; __u32 ring_size; __u32 queue_type; - __u32 queue_status; - __u32 reserved[19]; + __u32 reserved[18]; +}; + +struct kfd_dbg_device_info_entry { + __u64 exception_status; + __u64 lds_base; + __u64 lds_limit; + __u64 scratch_base; + __u64 scratch_limit; + __u64 gpuvm_base; + __u64 gpuvm_limit; + __u32 gpu_id; + __u32 pad; }; /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ @@ -224,114 +250,323 @@ struct kfd_ioctl_dbg_wave_control_args { __u32 gpu_id; /* to KFD */ __u32 buf_size_in_bytes; /*including gpu_id and buf_size */ }; - -/* mapping event types to API spec */ -#define KFD_DBG_EV_STATUS_TRAP 1 -#define KFD_DBG_EV_STATUS_VMFAULT 2 -#define KFD_DBG_EV_STATUS_SUSPENDED 4 -#define KFD_DBG_EV_STATUS_NEW_QUEUE 8 -#define KFD_DBG_EV_STATUS_HOST_TRAP_TIMEDOUT 16 #define KFD_DBG_EV_FLAG_CLEAR_STATUS 1 +/* queue states for suspend/resume */ +#define KFD_DBG_QUEUE_ERROR_BIT 30 +#define KFD_DBG_QUEUE_INVALID_BIT 31 +#define KFD_DBG_QUEUE_ERROR_MASK (1 << KFD_DBG_QUEUE_ERROR_BIT) +#define KFD_DBG_QUEUE_INVALID_MASK (1 << KFD_DBG_QUEUE_INVALID_BIT) + +#define KFD_INVALID_GPUID 0xffffffff #define KFD_INVALID_QUEUEID 0xffffffff +enum kfd_dbg_trap_override_mode { + KFD_DBG_TRAP_OVERRIDE_OR = 0, + KFD_DBG_TRAP_OVERRIDE_REPLACE = 1 +}; +enum kfd_dbg_trap_mask { + KFD_DBG_TRAP_MASK_FP_INVALID = 1, + KFD_DBG_TRAP_MASK_FP_INPUT_DENORMAL = 2, + KFD_DBG_TRAP_MASK_FP_DIVIDE_BY_ZERO = 4, + KFD_DBG_TRAP_MASK_FP_OVERFLOW = 8, + KFD_DBG_TRAP_MASK_FP_UNDERFLOW = 16, + KFD_DBG_TRAP_MASK_FP_INEXACT = 32, + KFD_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO = 64, + KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH = 128, + KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION = 256 +}; + +enum kfd_dbg_trap_exception_code { + EC_NONE = 0, + /* per queue */ + EC_QUEUE_WAVE_ABORT = 1, + EC_QUEUE_WAVE_TRAP = 2, + EC_QUEUE_WAVE_MATH_ERROR = 3, + EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION = 4, + EC_QUEUE_WAVE_MEMORY_VIOLATION = 5, + EC_QUEUE_WAVE_APERTURE_VIOLATION = 6, + EC_QUEUE_PACKET_DISPATCH_DIM_INVALID = 16, + EC_QUEUE_PACKET_DISPATCH_GROUP_SEGMENT_SIZE_INVALID = 17, + EC_QUEUE_PACKET_DISPATCH_CODE_INVALID = 18, + EC_QUEUE_PACKET_RESERVED = 19, + EC_QUEUE_PACKET_UNSUPPORTED = 20, + EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID = 21, + EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID = 22, + EC_QUEUE_PACKET_VENDOR_UNSUPPORTED = 23, + EC_QUEUE_PREEMPTION_ERROR = 30, + EC_QUEUE_NEW = 31, + /* per device */ + EC_DEVICE_QUEUE_DELETE = 32, + EC_DEVICE_MEMORY_VIOLATION = 33, + EC_DEVICE_RAS_ERROR = 34, + EC_DEVICE_FATAL_HALT = 35, + EC_DEVICE_NEW = 36, + /* per process */ + EC_PROCESS_RUNTIME = 48, + EC_PROCESS_DEVICE_REMOVE = 49, + EC_MAX +}; + +/* Mask generated by ecode defined in enum above. */ +#define KFD_EC_MASK(ecode) (1ULL << (ecode - 1)) + +/* Masks for exception code type checks below. */ +#define KFD_EC_MASK_QUEUE (KFD_EC_MASK(EC_QUEUE_WAVE_ABORT) | \ + KFD_EC_MASK(EC_QUEUE_WAVE_TRAP) | \ + KFD_EC_MASK(EC_QUEUE_WAVE_MATH_ERROR) | \ + KFD_EC_MASK(EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION) | \ + KFD_EC_MASK(EC_QUEUE_WAVE_MEMORY_VIOLATION) | \ + KFD_EC_MASK(EC_QUEUE_WAVE_APERTURE_VIOLATION) | \ + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_DIM_INVALID) | \ + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_GROUP_SEGMENT_SIZE_INVALID) | \ + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_CODE_INVALID) | \ + KFD_EC_MASK(EC_QUEUE_PACKET_UNSUPPORTED) | \ + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID) | \ + KFD_EC_MASK(EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID) | \ + KFD_EC_MASK(EC_QUEUE_PACKET_VENDOR_UNSUPPORTED) | \ + KFD_EC_MASK(EC_QUEUE_PREEMPTION_ERROR) | \ + KFD_EC_MASK(EC_QUEUE_NEW)) +#define KFD_EC_MASK_DEVICE (KFD_EC_MASK(EC_DEVICE_QUEUE_DELETE) | \ + KFD_EC_MASK(EC_DEVICE_RAS_ERROR) | \ + KFD_EC_MASK(EC_DEVICE_FATAL_HALT) | \ + KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION) | \ + KFD_EC_MASK(EC_DEVICE_NEW)) +#define KFD_EC_MASK_PROCESS (KFD_EC_MASK(EC_PROCESS_RUNTIME) | \ + KFD_EC_MASK(EC_PROCESS_DEVICE_REMOVE)) + +/* Checks for exception code types for KFD search. */ +#define KFD_DBG_EC_TYPE_IS_QUEUE(ecode) \ + (!!(KFD_EC_MASK(ecode) & KFD_EC_MASK_QUEUE)) +#define KFD_DBG_EC_TYPE_IS_DEVICE(ecode) \ + (!!(KFD_EC_MASK(ecode) & KFD_EC_MASK_DEVICE)) +#define KFD_DBG_EC_TYPE_IS_PROCESS(ecode) \ + (!!(KFD_EC_MASK(ecode) & KFD_EC_MASK_PROCESS)) + +enum kfd_dbg_runtime_state { + DEBUG_RUNTIME_STATE_DISABLED = 0, + DEBUG_RUNTIME_STATE_ENABLED = 1, + DEBUG_RUNTIME_STATE_ENABLED_BUSY = 2, + DEBUG_RUNTIME_STATE_ENABLED_ERROR = 3 +}; + +struct kfd_runtime_info { + __u64 r_debug; + __u32 runtime_state; + __u32 ttmp_setup; +}; + /* KFD_IOC_DBG_TRAP_ENABLE: - * ptr: unused + * exception_mask: exceptions to be reported to the debugger + * ptr: runtime info buffer to copy to (IN) * data1: 0=disable, 1=enable - * data2: queue ID (for future use) - * data3: return value for fd + * data2: return value for fd + * data3: runtime info size + * data4: unused + * */ #define KFD_IOC_DBG_TRAP_ENABLE 0 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE: + * exception_mask: unused * ptr: unused - * data1: override mode: 0=OR, 1=REPLACE - * data2: mask - * data3: unused + * data1: override mode (see enum kfd_dbg_trap_override_mode) + * data2: [in/out] trap mask (see enum kfd_dbg_trap_mask) + * data3: [in] requested mask, [out] supported mask + * data4: unused + * + * May fail with -EPERM if the requested mode is not supported. + * + * May fail with -EACCES if requested trap mask bits are not supported. + * In that case the supported trap mask bits are returned in data3. + * + * If successful, output parameters return the previous trap mask + * value and the hardware-dependent mask of supported trap mask bits. */ #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 1 /* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE: + * exception_mask: unused * ptr: unused * data1: 0=normal, 1=halt, 2=kill, 3=singlestep, 4=disable * data2: unused * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 2 /* KFD_IOC_DBG_TRAP_NODE_SUSPEND: - * ptr: pointer to an array of Queues IDs - * data1: flags - * data2: number of queues - * data3: grace period + * exception_mask: exceptions to clear on suspend + * ptr: pointer to an array of Queues IDs (IN/OUT) + * data1: number of queues (IN) + * data2: grace period (IN) + * data3: unused + * data4: unused + * + * Returns the number of queues suspended from array of Queue IDs (ptr). + * Requested queues that fail to suspend are masked in the array: + * + * KFD_DBG_QUEUE_INVALID_MASK - requested queue does not exist or cannot be + * suspended (new or being destroyed). + * + * KFD_DBG_QUEUE_ERROR_MASK - bad internal operation occurred on requested + * queue. + * + * NOTE! All queue destroy requests will be blocked on a suspended queue. + * Queue resume will unblock. + * + * Grace period (data2) is time allowed for waves to complete before CWSR. + * 0 can be entered for immediate preemption. */ #define KFD_IOC_DBG_TRAP_NODE_SUSPEND 3 /* KFD_IOC_DBG_TRAP_NODE_RESUME: - * ptr: pointer to an array of Queues IDs - * data1: flags - * data2: number of queues + * exception_mask: unused + * ptr: pointer to an array of Queues IDs (IN/OUT) + * data1: number of queues (IN) + * data2: unused * data3: unused + * data4: unused + * + * Returns the number of queues resumed from array of Queue IDs (ptr). + * Requested queues that fail to resume are masked in the array: + * + * KFD_DBG_QUEUE_INVALID_MASK - requested queue does not exist. + * + * KFD_DBG_QUEUE_ERROR_MASK - bad internal operation occurred on requested + * queue. */ #define KFD_IOC_DBG_TRAP_NODE_RESUME 4 /* KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT: + * exception_mask: exception to clear (IN) on query and report (OUT) * ptr: unused - * data1: queue id (IN/OUT) - * data2: flags (IN) - * data3: suspend[2:2], event type [1:0] (OUT) + * data1: source id (OUT) + * data2: unused + * data3: unused + * data4: unused + * + * Source ID (data 1 - OUT) returns the queue_id for a queue event, the gpu_id + * for a device event or 0 for a process event. + * + * Returns EAGAIN if no event is found and 0 otherwise. */ - #define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 5 /* KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT: + * exception_mask: exception to clear on snapshot * ptr: user buffer (IN) - * data1: flags (IN) - * data2: number of queue snapshot entries (IN/OUT) + * data1: number of queue snapshots (IN/OUT) - 0 for IN ignores buffer writes + * data2: unused * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT 6 /* KFD_IOC_DBG_TRAP_GET_VERSION: + * exception_mask: unused * ptr: unsused * data1: major version (OUT) * data2: minor version (OUT) * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_GET_VERSION 7 /* KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH: + * exception_mask: unused * ptr: unused * data1: watch ID * data2: unused * data3: unused + * data4: unused */ #define KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH 8 /* KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH: + * exception_mask: unused * ptr: Watch address * data1: Watch ID (OUT) * data2: watch_mode: 0=read, 1=nonread, 2=atomic, 3=all * data3: watch address mask + * data4: unused */ #define KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH 9 -/* KFD_IOC_DBG_SEND_HOST_TRAP: +/* KFD_IOC_DBG_TRAP_SET_PRECISE_MEM_OPS + * exception_mask: unused + * ptr: unused + * data1: 0=disable, 1=enable (IN) + * data2: unused + * data3: unused + * data4: unused + */ +#define KFD_IOC_DBG_TRAP_SET_PRECISE_MEM_OPS 10 + +/* KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO + * exception_mask: unused + * ptr: exception info pointer to copy to + * data1: source_id + * data2: exception_code + * data3: clear_exception (1 == true, 0 == false) + * data4: exception info data size + * + * NOTE: If data2 == EC_PROCESS_RUNTIME, the saved runtime info will be copied + * to the exception info pointer. + */ +#define KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO 11 + +/* KFD_IOC_DBG_TRAP_DEVICE_SNAPSHOT + * exception_mask: exception to clear on snapshot + * ptr: user buffer for 'struct kfd_dbg_device_info_entry' entries (IN) + * data1: number of devices in snapshot (IN/OUT) + * data2: unused + * data3: unused + * data4: unused + */ +#define KFD_IOC_DBG_TRAP_DEVICE_SNAPSHOT 12 + +/* KFD_IOC_DBG_TRAP_RUNTIME_ENABLE + * exception_mask: unused + * ptr: r_debug info to save + * data1: enable (0=disable, 1=enable) + * data2: enable ttmp save (0=disable, 1=enable) + * data3: unused + * data4: unused + * + * FIXME: This option is temporary. Future upstream will use a separate IOCTL. + */ +#define KFD_IOC_DBG_TRAP_RUNTIME_ENABLE 13 + +/* KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT + * exception_mask: exception to send + * ptr: unused + * data1: destination device id + * data2: destination queue id + * data3: usused + * data4: unused + */ +#define KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT 14 + +/* KFD_IOC_DBG_TRAP_SET_EXCEPTION_ENABLED + * exception_mask: exception to set * ptr: unused * data1: unused * data2: unused - * data3: unused + * data3: usused + * data4: unused */ -#define KFD_IOC_DBG_TRAP_SEND_HOST_TRAP 10 +#define KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED 15 struct kfd_ioctl_dbg_trap_args { - __u64 ptr; /* to KFD -- used for pointer arguments: queue arrays */ + __u64 exception_mask; /* to KFD */ + __u64 ptr; /* to KFD */ __u32 pid; /* to KFD */ - __u32 gpu_id; /* to KFD */ __u32 op; /* to KFD */ __u32 data1; /* to KFD */ __u32 data2; /* to KFD */ __u32 data3; /* to KFD */ + __u32 data4; /* to KFD */ }; /* Matching HSA_EVENTTYPE */ @@ -569,7 +804,7 @@ struct kfd_ioctl_alloc_queue_gws_args { __u32 queue_id; /* to KFD */ __u32 num_gws; /* to KFD */ __u32 first_gws; /* from KFD */ - __u32 pad; /* to KFD */ + __u32 pad; }; struct kfd_ioctl_get_dmabuf_info_args { @@ -737,7 +972,6 @@ struct kfd_ioctl_cross_memory_copy_args { __u64 bytes_copied; }; - /* Guarantee host access to memory */ #define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001 /* Fine grained coherency between all devices with access */ @@ -749,7 +983,7 @@ struct kfd_ioctl_cross_memory_copy_args { /* Allow execution on GPU */ #define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010 /* GPUs mostly read, may allow similar optimizations as RO, but writes fault */ -#define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020 +#define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020 /** * kfd_ioctl_svm_op - SVM ioctl operations @@ -762,6 +996,17 @@ enum kfd_ioctl_svm_op { KFD_IOCTL_SVM_OP_GET_ATTR }; +/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations + * + * GPU IDs are used to specify GPUs as preferred and prefetch locations. + * Below definitions are used for system memory or for leaving the preferred + * location unspecified. + */ +enum kfd_ioctl_svm_location { + KFD_IOCTL_SVM_LOCATION_SYSMEM = 0, + KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff +}; + /** * kfd_ioctl_svm_attr_type - SVM attribute types * @@ -791,17 +1036,6 @@ enum kfd_ioctl_svm_attr_type { KFD_IOCTL_SVM_ATTR_GRANULARITY }; -/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations - * - * GPU IDs are used to specify GPUs as preferred and prefetch locations. - * Below definitions are used for system memory or for leaving the preferred - * location unspecified. - */ -enum kfd_ioctl_svm_location { - KFD_IOCTL_SVM_LOCATION_SYSMEM = 0, - KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff -}; - /** * kfd_ioctl_svm_attribute - Attributes as pairs of type and value * @@ -882,16 +1116,16 @@ struct kfd_ioctl_svm_args { * functionality. * * Enabling XNACK mode requires shader programs to be compiled - * differently. Furthermore, not all GPUs support changing the mode + * differently. Furthermore, not all GPUs support changing the mode * per-process. Therefore changing the mode is only allowed while no * user mode queues exist in the process. This ensure that no shader * code is running that may be compiled for the wrong mode. And GPUs - * that cannot change to the requested mode will be disabled by - * failing subsequent requests to create user mode queues. - * - * This ioctl returns the status of the requested xnack mode. + * that cannot change to the requested mode will prevent the XNACK + * mode from occurring. All GPUs used by the process must be in the + * same XNACK mode. * * GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM. + * Therefore those GPUs are not considered for the XNACK mode switch. * * Return: 0 on success, -errno on failure */ @@ -956,7 +1190,7 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_SET_SCRATCH_BACKING_VA \ AMDKFD_IOWR(0x11, struct kfd_ioctl_set_scratch_backing_va_args) -#define AMDKFD_IOC_GET_TILE_CONFIG \ +#define AMDKFD_IOC_GET_TILE_CONFIG \ AMDKFD_IOWR(0x12, struct kfd_ioctl_get_tile_config_args) #define AMDKFD_IOC_SET_TRAP_HANDLER \ @@ -996,6 +1230,9 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_ALLOC_QUEUE_GWS \ AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) +#define AMDKFD_IOC_SMI_EVENTS \ + AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args) + #define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args) #define AMDKFD_IOC_SET_XNACK_MODE \ @@ -1021,7 +1258,6 @@ struct kfd_ioctl_set_xnack_mode_args { AMDKFD_IOWR(0x84, struct kfd_ioctl_spm_args) - #define AMDKFD_COMMAND_START_2 0x80 #define AMDKFD_COMMAND_END_2 0x85 diff --git a/src/debug.c b/src/debug.c index 5ed6631932..484fd4d16e 100644 --- a/src/debug.c +++ b/src/debug.c @@ -271,335 +271,7 @@ int debug_get_reg_status(uint32_t node_id, bool *is_debugged) return 0; } -static HSAKMT_STATUS debug_trap(HSAuint32 NodeId, - HSAuint32 op, - HSAuint32 data1, - HSAuint32 data2, - HSAuint32 data3, - HSAuint32 pid, - HSAuint64 pointer, - struct kfd_ioctl_dbg_trap_args *argout) -{ - uint32_t gpu_id; - HSAKMT_STATUS result; - HsaNodeProperties NodeProperties = {0}; - struct kfd_ioctl_dbg_trap_args args = {0}; - - CHECK_KFD_OPEN(); - - if (op == KFD_IOC_DBG_TRAP_NODE_SUSPEND || - op == KFD_IOC_DBG_TRAP_NODE_RESUME || - op == KFD_IOC_DBG_TRAP_GET_VERSION || - op == KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT) { - if (NodeId != INVALID_NODEID) - return HSAKMT_STATUS_INVALID_HANDLE; - - // gpu_id is ignored for suspend/resume queues. - gpu_id = INVALID_NODEID; - } else { - if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) - return HSAKMT_STATUS_INVALID_HANDLE; - - result = hsaKmtGetNodeProperties(NodeId, &NodeProperties); - - if (result != HSAKMT_STATUS_SUCCESS) - return result; - - if (!NodeProperties.Capability.ui32.DebugTrapSupported) - return HSAKMT_STATUS_NOT_SUPPORTED; - } - - if (pid == INVALID_PID) { - pid = (HSAuint32) getpid(); - } - - memset(&args, 0x00, sizeof(args)); - args.gpu_id = gpu_id; - args.op = op; - args.data1 = data1; - args.data2 = data2; - args.data3 = data3; - args.pid = pid; - args.ptr = pointer; - - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); - - if (argout) - *argout = args; - - if ((op == KFD_IOC_DBG_TRAP_NODE_SUSPEND || - op == KFD_IOC_DBG_TRAP_NODE_RESUME) && err >= 0 && - err <= args.data2) - result = HSAKMT_STATUS_SUCCESS; - else if (err == 0) - result = HSAKMT_STATUS_SUCCESS; - else - result = HSAKMT_STATUS_ERROR; - - return result; -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrapWithPollFd(HSAuint32 NodeId, - HSA_QUEUEID QueueId, - HSAint32 *PollFd) //OUT -{ - int result; - struct kfd_ioctl_dbg_trap_args argout = {0}; - - if (QueueId != INVALID_QUEUEID) - return HSAKMT_STATUS_NOT_SUPPORTED; - - result = debug_trap(NodeId, - KFD_IOC_DBG_TRAP_ENABLE, - 1, - QueueId, - 0, - INVALID_PID, - 0, - &argout); - - *PollFd = argout.data3; - - return result; -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtEnableDebugTrap(HSAuint32 NodeId, - HSA_QUEUEID QueueId) -{ - HSAint32 PollFd = 0; - HSAKMT_STATUS status = hsaKmtEnableDebugTrapWithPollFd(NodeId, - QueueId, - &PollFd); - - if (status == HSAKMT_STATUS_SUCCESS) - close(PollFd); - return status; -} - - - -HSAKMT_STATUS HSAKMTAPI hsaKmtDisableDebugTrap(HSAuint32 NodeId) -{ - return debug_trap(NodeId, - KFD_IOC_DBG_TRAP_ENABLE, - 0, - 0, - 0, - INVALID_PID, - 0, - NULL); -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchTrapOverride( - HSAuint32 NodeId, - HSA_DBG_TRAP_OVERRIDE TrapOverride, - HSA_DBG_TRAP_MASK TrapMask) -{ - if (TrapOverride >= HSA_DBG_TRAP_OVERRIDE_NUM) - return HSAKMT_STATUS_INVALID_PARAMETER; - - return debug_trap(NodeId, - KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE, - TrapOverride, - TrapMask, - 0, - INVALID_PID, - 0, - NULL); -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtSetWaveLaunchMode( - HSAuint32 NodeId, - HSA_DBG_WAVE_LAUNCH_MODE WaveLaunchMode) -{ - return debug_trap(NodeId, - KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE, - WaveLaunchMode, - 0, - 0, - INVALID_PID, - 0, - NULL); -} - -/** - * Suspend the execution of a set of queues. A queue that is suspended - * allows the wave context save state to be inspected and modified. If a - * queue is already suspended it remains suspended. A suspended queue - * can be resumed by hsaKmtDbgQueueResume(). - * - * For each node that has a queue suspended, a sequentially consistent - * system scope release will be performed that synchronizes with a - * sequentially consistent system scope acquire performed by this - * call. This ensures any memory updates performed by the suspended - * queues are visible to the thread calling this operation. - * - * Pid is the process that owns the queues that are to be supended or - * resumed. If the value is -1 then the Pid of the process calling - * hsaKmtQueueSuspend or hsaKmtQueueResume is used. - * - * NumQueues is the number of queues that are being requested to - * suspend or resume. - * - * Queues is a pointer to an array with NumQueues entries of - * HSA_QUEUEID. The queues in the list must be for queues the exist - * for Pid, and can be a mixture of queues for different nodes. - * - * GracePeriod is the number of milliseconds to wait after - * initialiating context save before forcing waves to context save. A - * value of 0 indicates no grace period. It is ignored by - * hsaKmtQueueResume. - * - * Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. - * Returns: - * - HSAKMT_STATUS_SUCCESS if successful. - * - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid for Pid. - */ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueueSuspend( - HSAuint32 Pid, // IN - HSAuint32 NumQueues, // IN - HSA_QUEUEID *Queues, // IN - HSAuint32 GracePeriod, // IN - HSAuint32 Flags) // IN -{ - HSAKMT_STATUS result; - uint32_t *queue_ids_ptr; - - CHECK_KFD_OPEN(); - - queue_ids_ptr = convert_queue_ids(NumQueues, Queues); - if (!queue_ids_ptr) - return HSAKMT_STATUS_NO_MEMORY; - - result = debug_trap(INVALID_NODEID, - KFD_IOC_DBG_TRAP_NODE_SUSPEND, - Flags, - NumQueues, - GracePeriod, - Pid, - (HSAuint64)queue_ids_ptr, - NULL); - - free(queue_ids_ptr); - return result; -} -/** - * Resume the execution of a set of queues. If a queue is not - * suspended by hsaKmtDbgQueueSuspend() then it remains executing. Any - * changes to the wave state data will be used when the waves are - * restored. Changes to the control stack data will have no effect. - * - * For each node that has a queue resumed, a sequentially consistent - * system scope release will be performed that synchronizes with a - * sequentially consistent system scope acquire performed by all - * queues being resumed. This ensures any memory updates performed by - * the thread calling this operation are visible to the resumed - * queues. - * - * For each node that has a queue resumed, the instruction cache will - * be invalidated. This ensures any instruction code updates performed - * by the thread calling this operation are visible to the resumed - * queues. - * - * Pid is the process that owns the queues that are to be supended or - * resumed. If the value is -1 then the Pid of the process calling - * hsaKmtQueueSuspend or hsaKmtQueueResume is used. - * - * NumQueues is the number of queues that are being requested to - * suspend or resume. - * - * Queues is a pointer to an array with NumQueues entries of - * HSA_QUEUEID. The queues in the list must be for queues the exist - * for Pid, and can be a mixture of queues for different nodes. - * - * Flags is a bit set of the values defined by HSA_DBG_NODE_CONTROL. - * Returns: - * - HSAKMT_STATUS_SUCCESS if successful - * - HSAKMT_STATUS_INVALID_HANDLE if any QueueId is invalid. - */ - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueueResume( - HSAuint32 Pid, // IN - HSAuint32 NumQueues, // IN - HSA_QUEUEID *Queues, // IN - HSAuint32 Flags) // IN -{ - HSAKMT_STATUS result; - uint32_t *queue_ids_ptr; - - CHECK_KFD_OPEN(); - - queue_ids_ptr = convert_queue_ids(NumQueues, Queues); - if (!queue_ids_ptr) - return HSAKMT_STATUS_NO_MEMORY; - - result = debug_trap(INVALID_NODEID, - KFD_IOC_DBG_TRAP_NODE_RESUME, - Flags, - NumQueues, - 0, - Pid, - (HSAuint64)queue_ids_ptr, - NULL); - free(queue_ids_ptr); - return result; -} - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtQueryDebugEvent( - HSAuint32 NodeId, //IN - HSAuint32 Pid, // IN - HSAuint32 *QueueId, // IN/OUT - bool ClearEvents, // IN - HSA_DEBUG_EVENT_TYPE *EventsReceived, // OUT - bool *IsSuspended, // OUT - bool *IsNew // OUT - ) -{ - HSAKMT_STATUS result; - struct kfd_ioctl_dbg_trap_args argout = {0}; - uint32_t flags = 0; - - if (ClearEvents) - flags |= KFD_DBG_EV_FLAG_CLEAR_STATUS; - - result = debug_trap(NodeId, - KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT, - *QueueId, - flags, - 0, - Pid, - 0, - &argout); - - if (result) - return result; - - *QueueId = argout.data1; - *EventsReceived = argout.data3 & - (KFD_DBG_EV_STATUS_TRAP | KFD_DBG_EV_STATUS_VMFAULT); - *IsSuspended = argout.data3 & KFD_DBG_EV_STATUS_SUSPENDED; - *IsNew = argout.data3 & KFD_DBG_EV_STATUS_NEW_QUEUE; - - return result; -} - -/** - * Get the major and minor version of the kernel debugger support. - * - * Returns: - * - HSAKMT_STATUS_SUCCESS if successful. - * - * - HSAKMT_STATUS_INVALID_HANDLE if NodeId is invalid. - * - * - HSAKMT_STATUS_NOT_SUPPORTED if debug trap not supported for NodeId. -*/ +/* Get the major and minor version of the kernel debugger support. */ HSAKMT_STATUS HSAKMTAPI hsaKmtGetKernelDebugTrapVersionInfo( @@ -607,148 +279,18 @@ hsaKmtGetKernelDebugTrapVersionInfo( HSAuint32 *Minor //Out ) { - HSAKMT_STATUS result; - struct kfd_ioctl_dbg_trap_args argout = {0}; + struct kfd_ioctl_dbg_trap_args args = {0}; - result = debug_trap(INVALID_NODEID, - KFD_IOC_DBG_TRAP_GET_VERSION, - 0, - 0, - 0, - INVALID_PID, - 0, - &argout); + memset(&args, 0x00, sizeof(args)); + args.op = KFD_IOC_DBG_TRAP_GET_VERSION; + args.pid = getpid(); - *Major = argout.data1; - *Minor = argout.data2; - return result; -} - -/** - * Get the major and minor version of the Thunk debugger support. -*/ -void -HSAKMTAPI -hsaKmtGetThunkDebugTrapVersionInfo( - HSAuint32 *Major, //Out - HSAuint32 *Minor //Out -) -{ - *Major = KFD_IOCTL_DBG_MAJOR_VERSION; - *Minor = KFD_IOCTL_DBG_MINOR_VERSION; -} - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetQueueSnapshot( - HSAuint32 NodeId, //IN - HSAuint32 Pid, // IN - bool ClearEvents, //IN - void *SnapshotBuf, //IN - HSAuint32 *QssEntries //IN/OUT - ) -{ - HSAKMT_STATUS result; - struct kfd_ioctl_dbg_trap_args argout = {0}; - uint32_t flags = 0; - - if (ClearEvents) - flags |= KFD_DBG_EV_FLAG_CLEAR_STATUS; - - result = debug_trap(NodeId, - KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT, - flags, - *QssEntries, - 0, - Pid, - (HSAuint64)SnapshotBuf, - &argout); - - if (result) - return result; - - *QssEntries = argout.data2; - - return 0; -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtSetAddressWatch( - HSAuint32 NodeId, - HSAuint32 Pid, - HSA_DBG_WATCH_MODE WatchMode, - void *WatchAddress, - HSAuint64 WatchAddrMask, - HSAuint32 *WatchId - ) -{ - - HSAKMT_STATUS result; - HSAuint32 TruncatedWatchAddressMask; - struct kfd_ioctl_dbg_trap_args argout = {0}; - - /* Right now we only support 32 bit watch address masks, so we need - * to check that we aren't losing data when we truncate the mask - * to be passed to the kernel. - */ - if (WatchAddrMask > (HSAuint64) UINT_MAX) - { - return HSAKMT_STATUS_INVALID_PARAMETER; - } - TruncatedWatchAddressMask = (HSAuint32) WatchAddrMask; - - if (WatchId == NULL) - return HSAKMT_STATUS_INVALID_PARAMETER; - - result = debug_trap(NodeId, - KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH, // op - *WatchId, - WatchMode, - TruncatedWatchAddressMask, - Pid, - (HSAuint64) WatchAddress, - &argout); - *WatchId = argout.data1; - - return result; -} - -HSAKMT_STATUS HSAKMTAPI hsaKmtClearAddressWatch( - HSAuint32 NodeId, - HSAuint32 Pid, - HSAuint32 WatchId - ) -{ - - HSAKMT_STATUS result; - - result = debug_trap(NodeId, - KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH, // op - WatchId, - 0, - 0, - Pid, - 0, - NULL); - return result; -} - -HSAKMT_STATUS -HSAKMTAPI -hsaKmtSendHostTrap( - HSAuint32 NodeId, //IN - HSAuint32 Pid //IN - ) -{ - int result; - - result = debug_trap(NodeId, - KFD_IOC_DBG_TRAP_SEND_HOST_TRAP, - 0, - 0, - 0, - Pid, - 0, - NULL); - - return result; + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); + + if (err) + return HSAKMT_STATUS_ERROR; + + *Major = args.data1; + *Minor = args.data2; + return HSAKMT_STATUS_SUCCESS; } diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 66ca6ea5b3..f423e505cf 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -57,21 +57,8 @@ hsaKmtGetTileConfig; hsaKmtQueryPointerInfo; hsaKmtSetMemoryUserData; hsaKmtGetQueueInfo; -hsaKmtEnableDebugTrap; -hsaKmtEnableDebugTrapWithPollFd; -hsaKmtDisableDebugTrap; -hsaKmtQueryDebugEvent; -hsaKmtSendHostTrap; -hsaKmtGetQueueSnapshot; -hsaKmtSetWaveLaunchTrapOverride; -hsaKmtSetWaveLaunchMode; -hsaKmtQueueSuspend; -hsaKmtQueueResume; hsaKmtAllocQueueGWS; hsaKmtGetKernelDebugTrapVersionInfo; -hsaKmtGetThunkDebugTrapVersionInfo; -hsaKmtSetAddressWatch; -hsaKmtClearAddressWatch; hsaKmtSPMAcquire; hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 9e7d93cc4c..b7a38bbd46 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -285,6 +285,8 @@ TEST_F(KFDDBGTest, BasicAddressWatch) { } #endif +#if 0 +// Deprecated - Reference Use Only /** * checkDebugVersion: * Inputs: @@ -333,7 +335,10 @@ static bool checkDebugVersion(HSAuint32 requiredMajor, HSAuint32 requiredMinor) return true; } +#endif +#if 0 +// Deprecated - Reference Use Only TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId >= FAMILY_AI) { @@ -431,7 +436,10 @@ exit: LOG() << std::endl; TEST_END } +#endif +#if 0 +// Deprecated - Reference Use Only TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId >= FAMILY_AI) { @@ -574,8 +582,11 @@ exit: LOG() << std::endl; TEST_END } +#endif +#if 0 // clean up routine +// Deprecated - Reference Use Only static void ExitVMFaultQueryChild(std::string errMsg, int exitStatus, HSAint32 pollFd, @@ -613,7 +624,10 @@ static void ExitVMFaultQueryChild(std::string errMsg, exit(exitStatus); } +#endif +#if 0 +// Deprecated - Reference Use Only TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId >= FAMILY_AI) { @@ -811,3 +825,4 @@ exit: LOG() << std::endl; TEST_END } +#endif From 1ce548829b0db7b0d8f0a8662baf4fe97a322a06 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 25 May 2021 11:13:48 -0400 Subject: [PATCH 0871/1247] libhsakmt: add runtime enable and disable calls Add hsaKmtRuntimeEnable and disable. Signed-off-by: Jonathan Kim Reviewed-by: Felix Kuehling Change-Id: I083f9293948e975546a1b3c1334cb41499b9ab1f --- include/hsakmt.h | 11 ++++++++ src/debug.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++ src/libhsakmt.ver | 2 ++ 3 files changed, 77 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index 615ebafef8..ff2d023760 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -633,6 +633,17 @@ hsaKmtDbgAddressWatch( HsaEvent* WatchEvent[] //IN, optional ); +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRuntimeEnable( + void* rDebug, // IN + bool setupTtmp + ); + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtRuntimeDisable(void); + /** Get the major and minor version of the kernel debugger support. */ diff --git a/src/debug.c b/src/debug.c index 484fd4d16e..e9e6b1406f 100644 --- a/src/debug.c +++ b/src/debug.c @@ -25,6 +25,7 @@ #include "libhsakmt.h" #include "linux/kfd_ioctl.h" +#include #include #include #include @@ -294,3 +295,66 @@ hsaKmtGetKernelDebugTrapVersionInfo( *Minor = args.data2; return HSAKMT_STATUS_SUCCESS; } + +#define HSA_RUNTIME_ENABLE_MIN_MAJOR 9 +#define HSA_RUNTIME_ENABLE_MIN_MINOR 0 +HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, + bool setupTtmp) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + HSAuint32 kMajor, kMinor; + HSAKMT_STATUS result; + + result = hsaKmtGetKernelDebugTrapVersionInfo(&kMajor, &kMinor); + + if (result) + return HSAKMT_STATUS_NOT_SUPPORTED; + + if (kMajor != HSA_RUNTIME_ENABLE_MIN_MAJOR || + (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR) + return HSAKMT_STATUS_NOT_SUPPORTED; + + memset(&args, 0x00, sizeof(args)); + args.op = KFD_IOC_DBG_TRAP_RUNTIME_ENABLE; + args.pid = getpid(); + args.data1 = 1; //enable + args.data2 = setupTtmp; + args.ptr = (HSAuint64)rDebug; + + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); + + if (err) { + if (err == EBUSY) + return HSAKMT_STATUS_UNAVAILABLE; + else + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + HSAuint32 kMajor, kMinor; + HSAKMT_STATUS result; + + result = hsaKmtGetKernelDebugTrapVersionInfo(&kMajor, &kMinor); + + if (result) + return HSAKMT_STATUS_NOT_SUPPORTED; + + if (kMajor != HSA_RUNTIME_ENABLE_MIN_MAJOR || + (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR) + return HSAKMT_STATUS_NOT_SUPPORTED; + + memset(&args, 0x00, sizeof(args)); + args.op = KFD_IOC_DBG_TRAP_RUNTIME_ENABLE; + args.pid = getpid(); + args.data1 = 0; //disable + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index f423e505cf..2d63d22d15 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -59,6 +59,8 @@ hsaKmtSetMemoryUserData; hsaKmtGetQueueInfo; hsaKmtAllocQueueGWS; hsaKmtGetKernelDebugTrapVersionInfo; +hsaKmtRuntimeEnable; +hsaKmtRuntimeDisable; hsaKmtSPMAcquire; hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; From 303c0748ce761dbb2f5bf34fd99b3b6d4bce9bcb Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 19 Jul 2021 12:44:46 -0400 Subject: [PATCH 0872/1247] libhsakmt: add drm.h header dependency for sles Update to thunk API introduced dependency on drm.h in commit 31ac82617caf libhsakmt: update thunk api for exception handling so update dependency list in SLES builds. Signed-off-by: Jonathan Kim Change-Id: I6d987fac07612e3eca7b6087205d76df50dc13d9 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd6dbbfa82..ac6c8ca90f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -280,7 +280,7 @@ set(CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "DEBIAN/postinst;DEBIAN/prerm") # get OS-info get_os_info() if(DISTRO_ID MATCHES "sles" ) - set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1") + set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1, libdrm-devel") elseif(DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6") set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl, libgcc") else() From 4f3440a8ac2737e27ad3d122619d46b8a2c8f652 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 20 Jul 2021 12:45:42 -0400 Subject: [PATCH 0873/1247] Fix drm.h include path kernel-headers provides the drm/drm.h path, while libdrm-dev[el] provides the libdrm/drm.h path, which is what we want to use. Fix the path so we use the newer drm.h header, as well as fixing SLES, which doesn't provide drm.h in their kernel-headers. Change-Id: Icb2b6643698d356169e3baeef17527a1b4e05483 --- include/linux/kfd_ioctl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 9bb1fb8a64..9dfac607a4 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -23,7 +23,7 @@ #ifndef KFD_IOCTL_H_INCLUDED #define KFD_IOCTL_H_INCLUDED -#include +#include #include /* From b63dde24d07f9670f5a5f6823dfc586dcaeaed5c Mon Sep 17 00:00:00 2001 From: Icarus Sparry Date: Mon, 5 Jul 2021 23:01:10 +0000 Subject: [PATCH 0874/1247] Add dependency on rocm-core Signed-off-by: Icarus Sparry Change-Id: I5f99114e9186679585862f05db8a508663b74b0d Signed-off-by: Icarus Sparry --- CMakeLists.txt | 20 ++++++++++++++------ tests/kfdtest/CMakeLists.txt | 9 +++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac6c8ca90f..3bf5a46436 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,7 +264,7 @@ endif() # Debian package specific variables set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface") -set(CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1") +set(CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1, rocm-core") set(CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev") if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) set(CPACK_DEBIAN_PACKAGE_RELEASE $ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) @@ -280,11 +280,11 @@ set(CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "DEBIAN/postinst;DEBIAN/prerm") # get OS-info get_os_info() if(DISTRO_ID MATCHES "sles" ) - set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1, libdrm-devel") + set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1, libdrm-devel, rocm-core") elseif(DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6") - set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl, libgcc") + set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl, libgcc, rocm-core") else() - set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc") + set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc, rocm-core") endif() ## Process the Rpm install/remove scripts to update the CPACK variables @@ -322,11 +322,19 @@ set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version -set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_DEBIAN_PACKAGE_RELEASE})") -set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "${HSAKMT_PACKAGE} = ${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}") +set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_DEBIAN_PACKAGE_RELEASE}), rocm-core") +set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "${HSAKMT_PACKAGE} = ${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}, rocm-core") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") +# Remove dependency on rocm-core if -DROCM_DEP_ROCMCORE=ON not given to cmake +if(NOT ROCM_DEP_ROCMCORE) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_BINARY_PACKAGE_REQUIRES ${CPACK_RPM_BINARY_PACKAGE_REQUIRES}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_DEVEL_PACKAGE_REQUIRES ${CPACK_RPM_DEVEL_PACKAGE_REQUIRES}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS ${CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS ${CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS}) +endif() + include(CPack) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 04e797e7d5..05557ec16d 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -46,6 +46,10 @@ set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) # Debian package specific variables set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) +set (CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core") + +# RPM package specific variables +set (CPACK_RPM_PACKAGE_REQUIRES "rocm-core") #set ( CMAKE_VERBOSE_MAKEFILE on ) @@ -192,4 +196,9 @@ install( FILES ${CMAKE_CURRENT_BINARY_DIR}/kfdtest ${CMAKE_CURRENT_BINARY_DIR}/r install( FILES ${CMAKE_CURRENT_BINARY_DIR}/kfdtest.exclude PERMISSIONS OWNER_WRITE OWNER_READ GROUP_WRITE GROUP_READ WORLD_READ DESTINATION share/kfdtest ) +# Remove dependency on rocm-core if -DROCM_DEP_ROCMCORE=ON not given to cmake +if(NOT ROCM_DEP_ROCMCORE) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_RPM_PACKAGE_REQUIRES}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS}) +endif() include ( CPack ) From dee9c023a2570e338a076f86c09d8f171c042941 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 20 Jul 2021 22:28:55 -0400 Subject: [PATCH 0875/1247] libhsakmt: update to KFD ioctl version 1.6 sync with KFD ioctl version 1.6: 1.6 - Query clear flags in SVM get_attr API Change import export handle args pad field to flags, to pass memory alloc flags from alloc process to import process. Change-Id: I69360b244651947e885c4a8da9f64a1163101d20 Signed-off-by: Philip Yang --- include/linux/kfd_ioctl.h | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 9dfac607a4..039b30b311 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -31,9 +31,10 @@ * - 1.3 - Add SMI events support * - 1.4 - Indicate new SRAM EDC bit in device properties * - 1.5 - Add SVM API + * - 1.6 - Query clear flags in SVM get_attr API */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 5 +#define KFD_IOCTL_MINOR_VERSION 6 /* * Debug revision change log @@ -929,16 +930,16 @@ struct kfd_ioctl_ipc_export_handle_args { __u64 handle; /* to KFD */ __u32 share_handle[4]; /* from KFD */ __u32 gpu_id; /* to KFD */ - __u32 pad; + __u32 flags; /* to KFD */ }; struct kfd_ioctl_ipc_import_handle_args { __u64 handle; /* from KFD */ __u64 va_addr; /* to KFD */ - __u64 mmap_offset; /* from KFD */ + __u64 mmap_offset; /* from KFD */ __u32 share_handle[4]; /* to KFD */ __u32 gpu_id; /* to KFD */ - __u32 pad; + __u32 flags; /* from KFD */ }; struct kfd_memory_range { @@ -1073,18 +1074,19 @@ struct kfd_ioctl_svm_attribute { * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For * @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be - * aggregated by bitwise AND. The minimum migration granularity - * throughout the range will be returned for - * @KFD_IOCTL_SVM_ATTR_GRANULARITY. + * aggregated by bitwise AND. That means, a flag will be set in the + * output, if that flag is set for all pages in the range. For + * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS, flags of all pages will be + * aggregated by bitwise NOR. That means, a flag will be set in the + * output, if that flag is clear for all pages in the range. + * The minimum migration granularity throughout the range will be + * returned for @KFD_IOCTL_SVM_ATTR_GRANULARITY. * * Querying of accessibility attributes works by initializing the * attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the * GPUID being queried. Multiple attributes can be given to allow * querying multiple GPUIDs. The ioctl function overwrites the * attribute type to indicate the access for the specified GPU. - * - * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS is invalid for - * @KFD_IOCTL_SVM_OP_GET_ATTR. */ struct kfd_ioctl_svm_args { __u64 start_addr; From ed62c7aa1cae0db8f6cdce3e6bf2bbfa48d212fa Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 21 Jul 2021 11:30:41 -0400 Subject: [PATCH 0876/1247] libhsakmt: Add gfx1032 DID 0x73E3 DID was missing, add it. Signed-off-by: Kent Russell Change-Id: Id1ae2f268e0e8b5cfec5ae2065153fe73854b93a --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index 01cdf61ee2..9b37f39687 100644 --- a/src/topology.c +++ b/src/topology.c @@ -260,6 +260,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73E0, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, { 0x73E1, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, { 0x73E2, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, + { 0x73E3, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, { 0x73FF, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, /* VanGogh */ { 0x163F, 10, 3, 3, "VanGogh", CHIP_VANGOGH }, From 3d12437f1b387c6c36d21013be7886062b4d1f4e Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Tue, 6 Jul 2021 16:08:41 +0800 Subject: [PATCH 0877/1247] libhsakmt: add yellow carp DID Signed-off-by: Aaron Liu Change-Id: If798dc733bc8d65c67922fb5d2e32aa1dada8381 --- src/topology.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/topology.c b/src/topology.c index 9b37f39687..3761c4f076 100644 --- a/src/topology.c +++ b/src/topology.c @@ -264,6 +264,9 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73FF, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, /* VanGogh */ { 0x163F, 10, 3, 3, "VanGogh", CHIP_VANGOGH }, + /* Yellow_Carp */ + { 0x164D, 10, 3, 5, "YELLOW_CARP", CHIP_YELLOW_CARP }, + { 0x1681, 10, 3, 5, "YELLOW_CARP", CHIP_YELLOW_CARP }, }; /* information from /proc/cpuinfo */ From 2c796e62be3bb3ddd7919b51085e7fbc7d220b55 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 21 Jul 2021 17:19:14 -0400 Subject: [PATCH 0878/1247] libhsakmt: Query pointer info returns HsaMemFlags Query pointer info returns KFD_IOC_ALLOC_MEM_FLAGS_* flags, it should return HsaMemFlags, fix it by renaming vm_obj->flags to mflags and always saving HsaMemFlags. Use consistent function parameter and variable name to avoid confusion: mflags for HsaMemFlags and ioc_flags for KFD_IOC_ALLOC_MEM_FLAGS_* flags. AMDKFD_IOC_GET_DMABUF_INFO return ioc_flags, translate it to mflags using new helper fmm_translate_ioc_to_hsa_flags. Change-Id: If9e117c507139c0166abb1ab0df8c233ef7e48a1 Signed-off-by: Philip Yang --- src/fmm.c | 138 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 78 insertions(+), 60 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 43d2a5f44c..631b521c4d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -88,7 +88,7 @@ struct vm_object { rbtree_node_t node; rbtree_node_t user_node; - uint32_t flags; /* memory allocation flags */ + HsaMemFlags mflags; /* memory allocation flags */ /* Registered nodes to map on SVM mGPU */ uint32_t *registered_device_id_array; uint32_t registered_device_id_array_size; @@ -309,7 +309,7 @@ static vm_area_t *vm_create_and_init_area(void *start, void *end) } static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, - uint64_t handle, uint32_t flags) + uint64_t handle, HsaMemFlags mflags) { vm_object_t *object = (vm_object_t *) malloc(sizeof(vm_object_t)); @@ -327,7 +327,7 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->mapped_node_id_array = NULL; object->registration_count = 0; object->mapping_count = 0; - object->flags = flags; + object->mflags = mflags; object->metadata = NULL; object->user_data = NULL; object->is_imported_kfd_bo = false; @@ -799,14 +799,14 @@ static vm_object_t *aperture_allocate_object(manageable_aperture_t *app, void *new_address, uint64_t handle, uint64_t MemorySizeInBytes, - uint32_t flags) + HsaMemFlags mflags) { vm_object_t *new_object; /* Allocate new object */ new_object = vm_create_and_init_object(new_address, MemorySizeInBytes, - handle, flags); + handle, mflags); if (!new_object) return NULL; @@ -911,6 +911,19 @@ static manageable_aperture_t *fmm_find_aperture(const void *address, return aperture; } +static HsaMemFlags fmm_translate_ioc_to_hsa_flags(uint32_t ioc_flags) +{ + HsaMemFlags mflags = {0}; + + if (!(ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE)) + mflags.ui32.ReadOnly = 1; + if (!(ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)) + mflags.ui32.CoarseGrain = 1; + if (ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) + mflags.ui32.HostAccess = 1; + return mflags; +} + /* After allocating the memory, return the vm_object created for this memory. * Return NULL if any failure. */ @@ -918,11 +931,12 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t *mmap_offset, - uint32_t flags) + uint32_t ioc_flags) { struct kfd_ioctl_alloc_memory_of_gpu_args args = {0}; struct kfd_ioctl_free_memory_of_gpu_args free_args = {0}; vm_object_t *vm_obj = NULL; + HsaMemFlags mflags; if (!mem) return NULL; @@ -931,21 +945,23 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, args.gpu_id = gpu_id; args.size = MemorySizeInBytes; - args.flags = flags | + args.flags = ioc_flags | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE; args.va_addr = (uint64_t)mem; if (!is_dgpu && - (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) + (ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) args.va_addr = VOID_PTRS_SUB(mem, aperture->base); - if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) + if (ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) args.mmap_offset = *mmap_offset; if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) return NULL; + mflags = fmm_translate_ioc_to_hsa_flags(ioc_flags); + /* Allocate object */ vm_obj = aperture_allocate_object(aperture, mem, args.handle, - MemorySizeInBytes, flags); + MemorySizeInBytes, mflags); if (!vm_obj) goto err_object_allocation_failed; @@ -1268,7 +1284,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeIn static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, manageable_aperture_t *aperture, uint64_t *mmap_offset, - uint32_t flags, vm_object_t **vm_obj) + uint32_t ioc_flags, vm_object_t **vm_obj) { void *mem = NULL; vm_object_t *obj; @@ -1285,7 +1301,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo * itself */ obj = fmm_allocate_memory_object(gpu_id, mem, - MemorySizeInBytes, aperture, mmap_offset, flags); + MemorySizeInBytes, aperture, mmap_offset, ioc_flags); if (!obj) { /* * allocation of memory in device failed. @@ -1302,7 +1318,7 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo return mem; } -void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags) +void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags mflags) { manageable_aperture_t *aperture; int32_t gpu_mem_id; @@ -1318,23 +1334,23 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB size = MemorySizeInBytes; - if (flags.ui32.HostAccess) + if (mflags.ui32.HostAccess) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC; - ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); + ioc_flags |= fmm_translate_hsa_to_ioc_flags(mflags); if (topology_is_svm_needed(get_device_id_by_gpu_id(gpu_id))) { aperture = svm.dgpu_aperture; - if (flags.ui32.AQLQueueMemory) + if (mflags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } - if (!flags.ui32.CoarseGrain || svm.disable_cache) + if (!mflags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - if (flags.ui32.Uncached || svm.disable_cache) + if (mflags.ui32.Uncached || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; pthread_mutex_lock(&aperture->fmm_mutex); @@ -1344,15 +1360,15 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB if (mem && vm_obj) { /* Store memory allocation flags, not ioc flags */ - vm_obj->flags = flags.Value; + vm_obj->mflags = mflags; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); } if (mem) { int map_fd = gpu_mem[gpu_mem_id].drm_render_fd; - int prot = flags.ui32.HostAccess ? PROT_READ | PROT_WRITE : + int prot = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; - int flag = flags.ui32.HostAccess ? MAP_SHARED | MAP_FIXED : + int flag = mflags.ui32.HostAccess ? MAP_SHARED | MAP_FIXED : MAP_PRIVATE|MAP_FIXED; void *ret = mmap(mem, MemorySizeInBytes, prot, flag, map_fd, mmap_offset); @@ -1403,15 +1419,15 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, ioc_flags, &vm_obj); if (mem && vm_obj) { - HsaMemFlags flags; + HsaMemFlags mflags; /* Cook up some flags for storing in the VM object */ - flags.Value = 0; - flags.ui32.NonPaged = 1; - flags.ui32.HostAccess = 1; - flags.ui32.Reserved = 0xBe1; + mflags.Value = 0; + mflags.ui32.NonPaged = 1; + mflags.ui32.HostAccess = 1; + mflags.ui32.Reserved = 0xBe1; - vm_obj->flags = flags.Value; + vm_obj->mflags = mflags; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); } @@ -1432,7 +1448,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, } static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, - HsaMemFlags flags) + HsaMemFlags mflags) { void *mem = NULL; vm_object_t *vm_obj; @@ -1441,10 +1457,10 @@ static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, if (address) return NULL; - if (flags.ui32.ExecuteAccess) + if (mflags.ui32.ExecuteAccess) mmap_prot |= PROT_EXEC; - if (!flags.ui32.ReadOnly) + if (!mflags.ui32.ReadOnly) mmap_prot |= PROT_WRITE; /* mmap will return a pointer with alignment equal to @@ -1458,7 +1474,7 @@ static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, pthread_mutex_lock(&cpuvm_aperture.fmm_mutex); vm_obj = aperture_allocate_object(&cpuvm_aperture, mem, 0, - MemorySizeInBytes, flags.Value); + MemorySizeInBytes, mflags); if (vm_obj) vm_obj->node_id = 0; /* APU systems only have one CPU node */ pthread_mutex_unlock(&cpuvm_aperture.fmm_mutex); @@ -1467,7 +1483,7 @@ static void *fmm_allocate_host_cpu(void *address, uint64_t MemorySizeInBytes, } static int bind_mem_to_numa(uint32_t node_id, void *mem, - uint64_t SizeInBytes, HsaMemFlags flags) + uint64_t SizeInBytes, HsaMemFlags mflags) { int mode = MPOL_F_STATIC_NODES; struct bitmask *node_mask; @@ -1475,9 +1491,9 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, long r; pr_debug("%s mem %p flags 0x%x size 0x%lx node_id %d\n", __func__, - mem, flags.Value, SizeInBytes, node_id); + mem, mflags.Value, SizeInBytes, node_id); - if (flags.ui32.NoNUMABind) + if (mflags.ui32.NoNUMABind) return 0; if (numa_available() == -1) @@ -1499,7 +1515,7 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, return -ENOMEM; numa_bitmask_setbit(node_mask, node_id); - mode |= flags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; + mode |= mflags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; r = mbind(mem, SizeInBytes, mode, node_mask->maskp, num_node + 1, 0); numa_bitmask_free(node_mask); @@ -1515,7 +1531,7 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, } /* Ignore mbind failure if no memory available on node */ - if (!flags.ui32.NoSubstitute) + if (!mflags.ui32.NoSubstitute) return 0; pr_warn_once("Failed to set NUMA policy for %p: %s\n", mem, @@ -1528,7 +1544,7 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, } static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, - uint64_t MemorySizeInBytes, HsaMemFlags flags) + uint64_t MemorySizeInBytes, HsaMemFlags mflags) { void *mem; manageable_aperture_t *aperture; @@ -1547,20 +1563,20 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, size = MemorySizeInBytes; ioc_flags = 0; - if (flags.ui32.CoarseGrain) + if (mflags.ui32.CoarseGrain) aperture = svm.dgpu_aperture; else aperture = svm.dgpu_alt_aperture; /* always coherent */ - if (!flags.ui32.CoarseGrain || svm.disable_cache) + if (!mflags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - if (flags.ui32.Uncached || svm.disable_cache) + if (mflags.ui32.Uncached || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; - ioc_flags |= fmm_translate_hsa_to_ioc_flags(flags); + ioc_flags |= fmm_translate_hsa_to_ioc_flags(mflags); - if (flags.ui32.AQLQueueMemory) + if (mflags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; /* Paged memory is allocated as a userptr mapping, non-paged @@ -1568,7 +1584,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, */ pthread_mutex_lock(&aperture->fmm_mutex); - if (!flags.ui32.NonPaged && svm.userptr_for_paged_mem) { + if (!mflags.ui32.NonPaged && svm.userptr_for_paged_mem) { /* Allocate address space */ mem = aperture_allocate_area(aperture, address, size); if (!mem) @@ -1581,7 +1597,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, goto out_release_area; /* Bind to NUMA node */ - if (bind_mem_to_numa(node_id, mem, MemorySizeInBytes, flags)) + if (bind_mem_to_numa(node_id, mem, MemorySizeInBytes, mflags)) goto out_release_area; /* Mappings in the DGPU aperture don't need to be copied on @@ -1603,7 +1619,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); - if (mem && flags.ui32.HostAccess) { + if (mem && mflags.ui32.HostAccess) { int map_fd = gpu_drm_fd; void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, @@ -1613,7 +1629,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, goto out_unlock; } - if (flags.ui32.AQLQueueMemory) { + if (mflags.ui32.AQLQueueMemory) { uint64_t my_buf_size = size / 2; memset(ret, 0, MemorySizeInBytes); @@ -1626,7 +1642,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, if (mem && vm_obj) { /* Store memory allocation flags, not ioc flags */ - vm_obj->flags = flags.Value; + vm_obj->mflags = mflags; vm_obj->node_id = node_id; } @@ -1643,11 +1659,11 @@ out_unlock: } void *fmm_allocate_host(uint32_t node_id, void *address, - uint64_t MemorySizeInBytes, HsaMemFlags flags) + uint64_t MemorySizeInBytes, HsaMemFlags mflags) { if (is_dgpu) - return fmm_allocate_host_gpu(node_id, address, MemorySizeInBytes, flags); - return fmm_allocate_host_cpu(address, MemorySizeInBytes, flags); + return fmm_allocate_host_gpu(node_id, address, MemorySizeInBytes, mflags); + return fmm_allocate_host_cpu(address, MemorySizeInBytes, mflags); } static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) @@ -2064,7 +2080,7 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) manageable_aperture_t *aperture = svm.dgpu_alt_aperture; uint32_t ioc_flags; vm_object_t *vm_obj = NULL; - HsaMemFlags flags; + HsaMemFlags mflags; void *ret; uint64_t mmap_offset; @@ -2083,11 +2099,11 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) return NULL; } - flags.Value = 0; - flags.ui32.NonPaged = 1; - flags.ui32.HostAccess = 1; - flags.ui32.Reserved = 0; - vm_obj->flags = flags.Value; + mflags.Value = 0; + mflags.ui32.NonPaged = 1; + mflags.ui32.HostAccess = 1; + mflags.ui32.Reserved = 0; + vm_obj->mflags = mflags; vm_obj->node_id = node_id; /* Map for CPU access*/ @@ -3118,6 +3134,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, struct kfd_ioctl_import_dmabuf_args importArgs = {0}; struct kfd_ioctl_free_memory_of_gpu_args freeArgs = {0}; manageable_aperture_t *aperture; + HsaMemFlags mflags; vm_object_t *obj; void *metadata; void *mem, *aperture_base; @@ -3178,8 +3195,9 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, goto error_release_aperture; pthread_mutex_lock(&aperture->fmm_mutex); + mflags = fmm_translate_ioc_to_hsa_flags(infoArgs.flags); obj = aperture_allocate_object(aperture, mem, importArgs.handle, - infoArgs.size, infoArgs.flags); + infoArgs.size, mflags); if (obj) { obj->metadata = metadata; obj->registered_device_id_array = gpu_id_array; @@ -3280,6 +3298,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo const HsaSharedMemoryStruct *SharedMemoryStruct = to_const_hsa_shared_memory_struct(SharedMemoryHandle); HSAuint64 SizeInPages = SharedMemoryStruct->SizeInPages; + HsaMemFlags mflags = {0}; if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -3308,8 +3327,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo pthread_mutex_lock(&aperture->fmm_mutex); obj = aperture_allocate_object(aperture, reservedMem, importArgs.handle, - (SizeInPages << PAGE_SHIFT), - 0); + (SizeInPages << PAGE_SHIFT), mflags); if (!obj) { err = HSAKMT_STATUS_NO_MEMORY; goto err_free_mem; @@ -3594,7 +3612,7 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) info->SizeInBytes = vm_obj->userptr_size; info->GPUAddress += ((HSAuint64)info->CPUAddress & (PAGE_SIZE - 1)); } else if (info->Type == HSA_POINTER_ALLOCATED) { - info->MemFlags.Value = vm_obj->flags; + info->MemFlags = vm_obj->mflags; info->CPUAddress = vm_obj->start; } From fa7b4a6268ee8fe6de26e297701eda03022d2e51 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 20 Jul 2021 22:37:20 -0400 Subject: [PATCH 0879/1247] libhsakmt: Qyery pointer info flags for registered memory hsaKmtQueryPointerInfo return vm_obj flags for all below registered memory types other than hsaKmtAllocMemory, and set the CoarseGrain flag correctly for: Graphics: always coarse grain. Shared: hsaKmtShareMemory pass mflags with export handle to KFD to store in KFD objs, hsaKmtRegisterSharedHandle get mflags from KFD with import handle. Userptr: it is already coarse-grain by default, or based on mflags provided in hsaKmtRegisterMemoryWithFlags call. Change-Id: Idc23e8b0cf599b02580737639da2f9ef4ccd0c0d Signed-off-by: Philip Yang --- src/fmm.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 631b521c4d..9bcf9f9115 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3196,6 +3196,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, pthread_mutex_lock(&aperture->fmm_mutex); mflags = fmm_translate_ioc_to_hsa_flags(infoArgs.flags); + mflags.ui32.CoarseGrain = 1; obj = aperture_allocate_object(aperture, mem, importArgs.handle, infoArgs.size, mflags); if (obj) { @@ -3267,7 +3268,7 @@ HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, } exportArgs.handle = obj->handle; exportArgs.gpu_id = gpu_id; - + exportArgs.flags = obj->mflags.Value; r = kmtIoctl(kfd_fd, AMDKFD_IOC_IPC_EXPORT_HANDLE, (void *)&exportArgs); if (r) @@ -3298,7 +3299,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo const HsaSharedMemoryStruct *SharedMemoryStruct = to_const_hsa_shared_memory_struct(SharedMemoryHandle); HSAuint64 SizeInPages = SharedMemoryStruct->SizeInPages; - HsaMemFlags mflags = {0}; + HsaMemFlags mflags; if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -3326,6 +3327,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo } pthread_mutex_lock(&aperture->fmm_mutex); + mflags.Value = importArgs.flags; obj = aperture_allocate_object(aperture, reservedMem, importArgs.handle, (SizeInPages << PAGE_SHIFT), mflags); if (!obj) { @@ -3607,12 +3609,13 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) info->MappedNodes = vm_obj->mapped_node_id_array; info->UserData = vm_obj->user_data; + info->MemFlags = vm_obj->mflags; + if (info->Type == HSA_POINTER_REGISTERED_USER) { info->CPUAddress = vm_obj->userptr; info->SizeInBytes = vm_obj->userptr_size; info->GPUAddress += ((HSAuint64)info->CPUAddress & (PAGE_SIZE - 1)); } else if (info->Type == HSA_POINTER_ALLOCATED) { - info->MemFlags = vm_obj->mflags; info->CPUAddress = vm_obj->start; } From db288df6959e8905b8a588d392e13aed085bb4a5 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 21 Jul 2021 12:45:41 -0400 Subject: [PATCH 0880/1247] kfdtest: RegisterGraphicsHandle check CoarseGrain set Register graphics memory should always coarse grain. Change-Id: I9d056b41287fa63247a042e5530ba29d535173ea Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDGraphicsInterop.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp index b5a408534a..bf2a928db1 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.cpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -124,6 +124,7 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { EXPECT_EQ(ptrInfo.Node, (HSAuint32)defaultGPUNode); EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)info.MemoryAddress); EXPECT_EQ(ptrInfo.SizeInBytes, alloc.alloc_size); + EXPECT_EQ(ptrInfo.MemFlags.ui32.CoarseGrain, 1); // Cleanup EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(info.MemoryAddress)); From c3c1618db727f4a02f3ad599b0e9444c8d38a3e5 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 21 Jul 2021 17:16:01 -0400 Subject: [PATCH 0881/1247] kfdtest: query userptr pointer alloc flags Test if query userptr pointer info return correct alloc flags, CoarseGrain by default. Test if query hsaKmtAllocMemory pointer info return correct alloc CoarseGrain flags. Change-Id: If3a1175645717e5d7c475d6ff35b02d6876a1f7c Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDMemoryTest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 0e4c58a067..9f62727cff 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1389,6 +1389,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As()); EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)hostBuffer.As()); EXPECT_EQ(ptrInfo.SizeInBytes, (HSAuint64)hostBuffer.Size()); + EXPECT_EQ(ptrInfo.MemFlags.ui32.CoarseGrain, 0); if (is_dgpu()) { EXPECT_EQ((HSAuint64)ptrInfo.NMappedNodes, nGPU); // Check NMappedNodes again after unmapping the memory @@ -1407,6 +1408,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { EXPECT_EQ(ptrInfo.CPUAddress, localBuffer.As()); EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)localBuffer.As()); EXPECT_EQ(ptrInfo.SizeInBytes, (HSAuint64)localBuffer.Size()); + EXPECT_EQ(ptrInfo.MemFlags.ui32.CoarseGrain, 1); HSAuint32 *addr = localBuffer.As() + 4; EXPECT_SUCCESS(hsaKmtQueryPointerInfo(reinterpret_cast(addr), &ptrInfo)); @@ -1425,6 +1427,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { EXPECT_EQ(ptrInfo.SizeInBytes, sizeof(HSAuint32)*2); EXPECT_EQ(ptrInfo.NRegisteredNodes, 0); EXPECT_EQ(ptrInfo.NMappedNodes, nGPU); + EXPECT_EQ(ptrInfo.MemFlags.ui32.CoarseGrain, 1); // Register to nodes HSAuint32 nodes[nGPU]; for (unsigned int i = 0; i < nGPU; i++) From 426b94015cd2a4d1c9c81bed88e5794f9967d181 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 21 Jul 2021 21:27:50 -0400 Subject: [PATCH 0882/1247] kfdtest: KFDIPCTest BasicTest query shared memory Parent process alloc memory with specific HsaMemFlags, child process import the memory handle exported by parent process, query shared memory pointer info, verify it has same HsaMemFlags. If child process test failed, exit with error code to notify parent process. Change-Id: I5c9534e931b4565186d27124868799b893504c81 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDIPCTest.cpp | 40 ++++++++++++++++++++++---------- tests/kfdtest/src/KFDIPCTest.hpp | 4 ++-- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index ea5a23a94e..c4a758f6f1 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -57,13 +57,16 @@ KFDIPCTest::~KFDIPCTest(void) { * starts the next test while the parent is still active. */ if (m_ChildPid == 0) - exit(0); + exit(::testing::UnitTest::GetInstance()->failed_test_count()); } /* Import shared Local Memory from parent process. Check for the pattern * filled in by the parent process. Then fill a new pattern. + * + * Check import handle has same HsaMemFlags as export handle to verify thunk and KFD + * import export handle ioctl pass HsaMemFlags correctly. */ -void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { +void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd, HsaMemFlags mflags) { /* Open KFD device for child process. This needs to called before * any memory definitions */ @@ -83,7 +86,7 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { ASSERT_SUCCESS(hsaKmtRegisterSharedHandle(&sharedHandleLM, reinterpret_cast(&sharedLocalBuffer), &sharedSize)); ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(sharedLocalBuffer, sharedSize, NULL, - mapFlags, 1, reinterpret_cast(&defaultGPUNode))); + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); /* Check for pattern in the shared Local Memory */ ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); @@ -97,6 +100,14 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { sdmaQueue.PlaceAndSubmitPacket(SDMAWriteDataPacket(sdmaQueue.GetFamilyId(), sharedLocalBuffer, 0xBBBBBBBB)); sdmaQueue.Wait4PacketConsumption(); + HsaPointerInfo ptrInfo; + EXPECT_SUCCESS(hsaKmtQueryPointerInfo(sharedLocalBuffer, &ptrInfo)); + EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_SHARED); + EXPECT_EQ(ptrInfo.Node, (HSAuint32)defaultGPUNode); + EXPECT_EQ(ptrInfo.GPUAddress, (HSAuint64)sharedLocalBuffer); + EXPECT_EQ(ptrInfo.SizeInBytes, sharedSize); + EXPECT_EQ(ptrInfo.MemFlags.Value, mflags.Value); + /* Clean up */ EXPECT_SUCCESS(sdmaQueue.Destroy()); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(sharedLocalBuffer)); @@ -108,29 +119,30 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd) { * filled in by the child process. */ -void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pipefd) { +void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pipefd, HsaMemFlags mflags) { HSAuint64 size = PAGE_SIZE, sharedSize; int status; HSAuint64 AlternateVAGPU; - HsaMemoryBuffer toShareLocalBuffer(size, defaultGPUNode, false, true); + void *toShareLocalBuffer; HsaMemoryBuffer tempSysBuffer(PAGE_SIZE, defaultGPUNode, false); SDMAQueue sdmaQueue; HsaSharedMemoryHandle sharedHandleLM; HsaMemMapFlags mapFlags = {0}; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, size, mflags, &toShareLocalBuffer)); /* Fill a Local Buffer with a pattern */ - ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(toShareLocalBuffer.As(), toShareLocalBuffer.Size(), &AlternateVAGPU, + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(toShareLocalBuffer, size, &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode))); tempSysBuffer.Fill(0xAAAAAAAA); /* Copy pattern in Local Memory before sharing it */ ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); - sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), toShareLocalBuffer.As(), + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), toShareLocalBuffer, tempSysBuffer.As(), size)); sdmaQueue.Wait4PacketConsumption(); /* Share it with the child process */ - ASSERT_SUCCESS(hsaKmtShareMemory(toShareLocalBuffer.As(), size, &sharedHandleLM)); + ASSERT_SUCCESS(hsaKmtShareMemory(toShareLocalBuffer, size, &sharedHandleLM)); ASSERT_GE(write(pipefd[1], reinterpret_cast(&sharedHandleLM), sizeof(sharedHandleLM)), 0); @@ -142,12 +154,12 @@ void KFDIPCTest::BasicTestParentProcess(int defaultGPUNode, pid_t cpid, int *pip /* Check for the new pattern filled in by child process */ sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), tempSysBuffer.As(), - toShareLocalBuffer.As(), size)); + toShareLocalBuffer, size)); sdmaQueue.Wait4PacketConsumption(); EXPECT_TRUE(WaitOnValue(tempSysBuffer.As(), 0xBBBBBBBB)); /* Clean up */ - EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(toShareLocalBuffer.As())); + EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(toShareLocalBuffer)); EXPECT_SUCCESS(sdmaQueue.Destroy()); } @@ -167,6 +179,7 @@ TEST_F(KFDIPCTest, BasicTest) { const std::vector& GpuNodes = m_NodeInfo.GetNodesWithGPU(); int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); int pipefd[2]; + HsaMemFlags mflags = {0}; ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; @@ -188,12 +201,15 @@ TEST_F(KFDIPCTest, BasicTest) { ASSERT_EQ(pipe(pipefd), 0); /* Create a child process and share the above Local Memory with it */ + mflags.ui32.NonPaged = 1; + mflags.ui32.CoarseGrain = 1; + mflags.ui32.HostAccess = 1; m_ChildPid = fork(); if (m_ChildPid == 0) - BasicTestChildProcess(defaultGPUNode, pipefd); /* Child Process */ + BasicTestChildProcess(defaultGPUNode, pipefd, mflags); /* Child Process */ else - BasicTestParentProcess(defaultGPUNode, m_ChildPid, pipefd); /* Parent proces */ + BasicTestParentProcess(defaultGPUNode, m_ChildPid, pipefd, mflags); /* Parent proces */ /* Code path executed by both parent and child with respective fds */ close(pipefd[1]); diff --git a/tests/kfdtest/src/KFDIPCTest.hpp b/tests/kfdtest/src/KFDIPCTest.hpp index 6a72f4a805..961ecbd9e9 100644 --- a/tests/kfdtest/src/KFDIPCTest.hpp +++ b/tests/kfdtest/src/KFDIPCTest.hpp @@ -120,8 +120,8 @@ class KFDIPCTest : public KFDBaseComponentTest { virtual void TearDown(); /* For IPC testing */ - void BasicTestChildProcess(int defaultGPUNode, int *pipefd); - void BasicTestParentProcess(int defaultGPUNode, pid_t childPid, int *pipefd); + void BasicTestChildProcess(int defaultGPUNode, int *pipefd, HsaMemFlags mflags); + void BasicTestParentProcess(int defaultGPUNode, pid_t childPid, int *pipefd, HsaMemFlags mflags); /* For CMA testing */ CMA_TEST_STATUS CrossMemoryAttachChildProcess(int defaultGPUNode, int writePipe, From 1fe2225f5270a1f23af1b2d905e521f873301937 Mon Sep 17 00:00:00 2001 From: changzhu Date: Wed, 28 Jul 2021 15:44:02 +0800 Subject: [PATCH 0883/1247] kfdtest: take back SVM test cases for gfx90c Change-Id: I28627f6eeaf9d213d7740849c6fadcccd91733c0 Signed-off-by: changzhu --- tests/kfdtest/scripts/kfdtest.exclude | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 1e64dc6af4..319c0fd983 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -202,18 +202,7 @@ FILTER[renoir]=\ "KFDEvictTest.*:"\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.SignalHandling:"\ -"KFDSVMRangeTest.BasicSystemMemTest:"\ -"KFDSVMRangeTest.BasicVramTest:"\ -"KFDSVMRangeTest.EvictSystemRangeTest:"\ -"KFDSVMRangeTest.PartialUnmapSysMemTest:"\ -"KFDSVMRangeTest.MigrateTest:"\ -"KFDSVMRangeTest.MigratePolicyTest:"\ -"KFDSVMRangeTest.MigrateGranularityTest:"\ -"KFDSVMRangeTest.MigrateLargeBufTest:"\ -"KFDSVMRangeTest.MultiThreadMigrationTest:"\ -"KFDSVMRangeTest.MigrateAccessInPlaceTest:"\ -"KFDSVMEvictTest.QueueTest" +"KFDMemoryTest.SignalHandling" # KFDExceptionTest.* (KFD-435) # KFDEvictTest.BurstyTest (KFD-425) From 3600cdfa0136d23abe7ea083799b0a66f2fe986d Mon Sep 17 00:00:00 2001 From: changzhu Date: Wed, 28 Jul 2021 15:33:51 +0800 Subject: [PATCH 0884/1247] kfdtest: blacklist some SVM test cases for gfx902 iommu path Change-Id: I77f734df7ca65f2aa176548fd02ddac6036fe138 Signed-off-by: changzhu --- tests/kfdtest/scripts/kfdtest.exclude | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 319c0fd983..025ac3e541 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -195,6 +195,10 @@ FILTER[raven_dgpuFallback]=\ FILTER[raven]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.SdmaConcurrentCopies:"\ +"KFDSVMRangeTest.EvictSystemRangeTest:"\ +"KFDSVMRangeTest.PartialUnmapSysMemTest:"\ +"KFDSVMRangeTest.PrefetchTest:"\ +"KFDSVMRangeTest.MultiThreadMigrationTest:"\ "KFDQMTest.MultipleCpQueuesStressDispatch" FILTER[renoir]=\ From 860931557c531185cb13debfd64f9070aa123034 Mon Sep 17 00:00:00 2001 From: chen gong Date: Mon, 7 Jun 2021 17:23:47 +0800 Subject: [PATCH 0885/1247] libhsakmt: add cyan skillfish support Signed-off-by: chen gong Signed-off-by: Oak Zeng Change-Id: Ie26c2e156c9c4e68a6b36fe699adccf3f808b5e7 --- src/libhsakmt.h | 19 ++++++++++--------- src/pmc_table.c | 1 + src/queues.c | 7 +++++++ src/topology.c | 6 ++++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index a616c822b9..9c08f78230 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -134,15 +134,16 @@ enum asic_family_type { CHIP_RENOIR, /* 13 */ CHIP_ARCTURUS, /* 14 */ CHIP_NAVI10, /* 15 */ - CHIP_NAVI12, /* 16 */ - CHIP_NAVI14, /* 17 */ - CHIP_SIENNA_CICHLID, /* 18 */ - CHIP_NAVY_FLOUNDER, /* 19 */ - CHIP_DIMGREY_CAVEFISH, /* 20 */ - CHIP_VANGOGH, /* 21 */ - CHIP_ALDEBARAN, /* 22 */ - CHIP_BEIGE_GOBY, /* 23 */ - CHIP_YELLOW_CARP, /* 24 */ + CHIP_CYAN_SKILLFISH, /* 16 */ + CHIP_NAVI12, /* 17 */ + CHIP_NAVI14, /* 18 */ + CHIP_SIENNA_CICHLID, /* 19 */ + CHIP_NAVY_FLOUNDER, /* 20 */ + CHIP_DIMGREY_CAVEFISH, /* 21 */ + CHIP_VANGOGH, /* 22 */ + CHIP_ALDEBARAN, /* 23 */ + CHIP_BEIGE_GOBY, /* 24 */ + CHIP_YELLOW_CARP, /* 25 */ CHIP_LAST }; diff --git a/src/pmc_table.c b/src/pmc_table.c index 8697661f3e..3000b95d32 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -2135,6 +2135,7 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, case CHIP_NAVI14: case CHIP_VANGOGH: case CHIP_YELLOW_CARP: + case CHIP_CYAN_SKILLFISH: *block = navi_blocks[block_id]; break; default: diff --git a/src/queues.c b/src/queues.c index e667f1a1ce..3de137bf40 100644 --- a/src/queues.c +++ b/src/queues.c @@ -157,6 +157,12 @@ const struct device_info navi10_device_info = { .doorbell_size = DOORBELL_SIZE_GFX9, }; +const struct device_info cyan_skillfish_device_info = { + .asic_family = CHIP_CYAN_SKILLFISH, + .eop_buffer_size = 4096, + .doorbell_size = DOORBELL_SIZE_GFX9, +}; + const struct device_info navi12_device_info = { .asic_family = CHIP_NAVI12, .eop_buffer_size = 4096, @@ -223,6 +229,7 @@ static const struct device_info *dev_lookup_table[] = { [CHIP_ARCTURUS] = &arcturus_device_info, [CHIP_ALDEBARAN] = &aldebaran_device_info, [CHIP_NAVI10] = &navi10_device_info, + [CHIP_CYAN_SKILLFISH] = &cyan_skillfish_device_info, [CHIP_NAVI12] = &navi12_device_info, [CHIP_NAVI14] = &navi14_device_info, [CHIP_SIENNA_CICHLID] = &sienna_cichlid_device_info, diff --git a/src/topology.c b/src/topology.c index 3761c4f076..55d567f74c 100644 --- a/src/topology.c +++ b/src/topology.c @@ -236,6 +236,12 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x731A, 10, 1, 0, "Navi10", CHIP_NAVI10 }, { 0x731E, 10, 1, 0, "Navi10", CHIP_NAVI10 }, { 0x731F, 10, 1, 0, "Navi10", CHIP_NAVI10 }, + /* cyan_skillfish */ + { 0x13F9, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, + { 0x13FA, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, + { 0x13FB, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, + { 0x13FC, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, + { 0x13FE, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, /* Navi14 */ { 0x7340, 10, 1, 2, "Navi14", CHIP_NAVI14 }, { 0x7341, 10, 1, 2, "Navi14", CHIP_NAVI14 }, From 413979c3195deba17ba768c23759416d97f3febb Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 29 Jul 2021 14:29:02 -0400 Subject: [PATCH 0886/1247] kfdtest: fix KFDIPCTest on gfx1xxx hsaKmtAllocMemory with mflags.HostAccess=1 will fail on desktop gfx1xxx GPU because it's small bar PCIe. Remove this flag. Child process exit status should use current_test_info, not unit test info. Change-Id: I060f23aa98a42251921d003878e70a409c92a4de Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDIPCTest.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index c4a758f6f1..c18b471307 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -57,7 +57,7 @@ KFDIPCTest::~KFDIPCTest(void) { * starts the next test while the parent is still active. */ if (m_ChildPid == 0) - exit(::testing::UnitTest::GetInstance()->failed_test_count()); + exit(::testing::UnitTest::GetInstance()->current_test_info()->result()->Failed()); } /* Import shared Local Memory from parent process. Check for the pattern @@ -203,7 +203,6 @@ TEST_F(KFDIPCTest, BasicTest) { /* Create a child process and share the above Local Memory with it */ mflags.ui32.NonPaged = 1; mflags.ui32.CoarseGrain = 1; - mflags.ui32.HostAccess = 1; m_ChildPid = fork(); if (m_ChildPid == 0) From 991cde0656281ce460322637c942f136c8f87258 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 29 Jul 2021 16:59:16 -0400 Subject: [PATCH 0887/1247] libhsakmt: update query pointer info document Query pointer info return HsaPointerInfo with MemFlags for all pointer type now. Change-Id: I3c02b7b71ba0af953035e3ed9cd6bb6435bb9b65 Signed-off-by: Philip Yang --- include/hsakmttypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index a8a396946a..7cfca1082a 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1292,7 +1292,7 @@ typedef enum _HSA_POINTER_TYPE { typedef struct _HsaPointerInfo { HSA_POINTER_TYPE Type; // Pointer type HSAuint32 Node; // Node where the memory is located - HsaMemFlags MemFlags; // Only valid for HSA_POINTER_ALLOCATED + HsaMemFlags MemFlags; // HsaMemFlags used to alloc memory void *CPUAddress; // Start address for CPU access HSAuint64 GPUAddress; // Start address for GPU access HSAuint64 SizeInBytes; // Size in bytes From 80e4cb62616e8c357593bf28c4e78ef8f0b2954c Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Thu, 29 Jul 2021 09:27:53 +0800 Subject: [PATCH 0888/1247] libhsakmt: reorder the list of GPUs in enum asic_family_type Put CHIP_gfx90a next to CHIP_gfx908 to keep all the GFXv9 chips together. Signed-off-by: Aaron Liu Change-Id: I32b4b7f7746ae4bdaff7e178f13307c1bcfa93e1 --- src/libhsakmt.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 9c08f78230..4af19763f2 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -133,15 +133,15 @@ enum asic_family_type { CHIP_RAVEN, /* 12 */ CHIP_RENOIR, /* 13 */ CHIP_ARCTURUS, /* 14 */ - CHIP_NAVI10, /* 15 */ - CHIP_CYAN_SKILLFISH, /* 16 */ - CHIP_NAVI12, /* 17 */ - CHIP_NAVI14, /* 18 */ - CHIP_SIENNA_CICHLID, /* 19 */ - CHIP_NAVY_FLOUNDER, /* 20 */ - CHIP_DIMGREY_CAVEFISH, /* 21 */ - CHIP_VANGOGH, /* 22 */ - CHIP_ALDEBARAN, /* 23 */ + CHIP_ALDEBARAN, /* 15 */ + CHIP_NAVI10, /* 16 */ + CHIP_CYAN_SKILLFISH, /* 17 */ + CHIP_NAVI12, /* 18 */ + CHIP_NAVI14, /* 19 */ + CHIP_SIENNA_CICHLID, /* 20 */ + CHIP_NAVY_FLOUNDER, /* 21 */ + CHIP_DIMGREY_CAVEFISH, /* 22 */ + CHIP_VANGOGH, /* 23 */ CHIP_BEIGE_GOBY, /* 24 */ CHIP_YELLOW_CARP, /* 25 */ CHIP_LAST From fc216a36684188618fb683028d328788f0d51094 Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Fri, 23 Jul 2021 06:27:37 +0800 Subject: [PATCH 0889/1247] libhsakmt: expand control stack size limit for all gfx103x Expand check control stack size for Beige Goby, Van Gogh and Yellow Carp. Signed-off-by: Aaron Liu Change-Id: I6c8345bbb751eb9179fc740fe1674504b4295a5f --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 3de137bf40..a4a7df6070 100644 --- a/src/queues.c +++ b/src/queues.c @@ -475,7 +475,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + ctl_stack_size); if (q->dev_info->asic_family >= CHIP_NAVI10 && - q->dev_info->asic_family <= CHIP_DIMGREY_CAVEFISH) { + q->dev_info->asic_family <= CHIP_YELLOW_CARP) { /* HW design limits control stack size to 0x7000. * This is insufficient for theoretical PM4 cases * but sufficient for AQL, limited by SPI events. From 6778da44609a5f6b7db6a128222660abbba6280d Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 30 Jul 2021 12:50:08 -0400 Subject: [PATCH 0890/1247] CMakeLists: Re-enable pkgconfig, require libdrm This was an implicit requirement, but the change to needing libdrm/drm.h means we should be explicitly looking for this package during the build Change-Id: I75064bf6f5477cb1ca99318681afca76c0060f72 --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3bf5a46436..2d98ddba84 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,18 +148,20 @@ target_include_directories( ${HSAKMT_TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) -target_compile_options(${HSAKMT_TARGET} PRIVATE ${HSAKMT_C_FLAGS}) set_property(TARGET ${HSAKMT_TARGET} PROPERTY LINK_FLAGS ${HSAKMT_LINK_FLAGS}) ## Set the VERSION and SOVERSION values set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" ) set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" ) -#find_package(PkgConfig) +find_package(PkgConfig) +pkg_check_modules(LIBDRM REQUIRED libdrm) target_link_libraries ( ${HSAKMT_TARGET} PRIVATE pthread rt numa ) +target_compile_options(${HSAKMT_TARGET} PRIVATE ${LIBDRM_CFLAGS} ${HSAKMT_C_FLAGS}) + ## Define default paths and packages. if( CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT ) set ( CMAKE_INSTALL_PREFIX "/opt/rocm" ) From 888061026548fb72c4d7c38db5cb7ef3b86a88bf Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 16 Jul 2021 15:45:31 -0400 Subject: [PATCH 0891/1247] libhsakmt: Add gfx version defines, helpers Add defines for converting between full gfx version (in decimal) to major/minor/stepping values, and major/minor/stepping values to full gfx version (in hex). The first is to be used when grabbing the gfx version from sysfs, the second when returning the gfx version in get_gfxv_by_node_id() for comparisions in Thunk. Add enum for defines of full gfx versions for various cards. These are to be used in conditionals to replace CHIP_*/asic_family comparisons. On NPI bringup, addition to this enum will not be required for functionality unless explicitly needed for a conditional. Most ASICs in this enum simply for completeness. Add get_gfxv_by_node_id(): returns full gfx version in hex. Signed-off-by: Graham Sider Change-Id: I9d1dba7fdd4a587a000eea2f4141a1d301bf4776 --- src/libhsakmt.h | 39 +++++++++++++++++++++++++++++++++++++++ src/topology.c | 9 +++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 4af19763f2..c44df5c37f 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -117,6 +117,44 @@ extern int hsakmt_debug_level; } \ }) +/* Expects gfxv (full) in decimal */ +#define HSA_GET_GFX_VERSION_MAJOR(gfxv) (((gfxv) / 10000) % 100) +#define HSA_GET_GFX_VERSION_MINOR(gfxv) (((gfxv) / 100) % 100) +#define HSA_GET_GFX_VERSION_STEP(gfxv) ((gfxv) % 100) + +/* Expects HSA_ENGINE_ID.ui32, returns gfxv (full) in hex */ +#define HSA_GET_GFX_VERSION_FULL(ui32) \ + (((ui32.Major) << 16) | ((ui32.Minor) << 8) | (ui32.Stepping)) + +enum full_gfx_versions { + GFX_VERSION_KAVERI = 0x070000, + GFX_VERSION_HAWAII = 0x070001, + GFX_VERSION_CARRIZO = 0x080001, + GFX_VERSION_TONGA = 0x080002, + GFX_VERSION_FIJI = 0x080003, + GFX_VERSION_POLARIS10 = 0x080003, + GFX_VERSION_POLARIS11 = 0x080003, + GFX_VERSION_POLARIS12 = 0x080003, + GFX_VERSION_VEGAM = 0x080003, + GFX_VERSION_VEGA10 = 0x090000, + GFX_VERSION_RAVEN = 0x090002, + GFX_VERSION_RENOIR = 0x090002, + GFX_VERSION_VEGA12 = 0x090004, + GFX_VERSION_VEGA20 = 0x090006, + GFX_VERSION_ARCTURUS = 0x090008, + GFX_VERSION_ALDEBARAN = 0x090010, + GFX_VERSION_NAVI10 = 0x100100, + GFX_VERSION_NAVI12 = 0x100101, + GFX_VERSION_NAVI14 = 0x100102, + GFX_VERSION_CYAN_SKILLFISH = 0x100103, + GFX_VERSION_SIENNA_CICHLID = 0x100300, + GFX_VERSION_NAVY_FLOUNDER = 0x100301, + GFX_VERSION_DIMGREY_CAVEFISH = 0x100302, + GFX_VERSION_VANGOGH = 0x100303, + GFX_VERSION_BEIGE_GOBY = 0x100304, + GFX_VERSION_YELLOW_CARP = 0x100305, +}; + enum asic_family_type { CHIP_KAVERI = 0, CHIP_HAWAII, /* 1 */ @@ -162,6 +200,7 @@ HSAKMT_STATUS init_kfd_version(void); HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); +uint32_t get_gfxv_by_node_id(HSAuint32 node_id); bool prefer_ats(HSAuint32 node_id); uint16_t get_device_id_by_node_id(HSAuint32 node_id); bool is_kaveri(HSAuint32 node_id); diff --git a/src/topology.c b/src/topology.c index 55d567f74c..91f8055831 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1091,8 +1091,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->EngineId.ui32.Stepping = step & 0xff; } else { props->EngineId.ui32.Major = hsa_gfxip->major & 0x3f; - props->EngineId.ui32.Minor = hsa_gfxip->minor; - props->EngineId.ui32.Stepping = hsa_gfxip->stepping; + props->EngineId.ui32.Minor = hsa_gfxip->minor & 0xff; + props->EngineId.ui32.Stepping = hsa_gfxip->stepping & 0xff; } if (!hsa_gfxip->amd_name) { @@ -2271,6 +2271,11 @@ out: return err; } +uint32_t get_gfxv_by_node_id(HSAuint32 node_id) +{ + return HSA_GET_GFX_VERSION_FULL(g_props[node_id].node.EngineId.ui32); +} + uint16_t get_device_id_by_node_id(HSAuint32 node_id) { if (!g_props || !g_system || g_system->NumNodes <= node_id) From 2f261a8dab1ebc74bf09c627284084a8bc8fe9fb Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 16 Jul 2021 15:48:01 -0400 Subject: [PATCH 0892/1247] libhsakmt: Deprecate is_gfx700() Condition generalized by (get_gfxv_by_node_id(NodeId) == GFX_VERSION_KAVERI) Signed-off-by: Graham Sider Change-Id: Iff4f77043e3ce8e48a9e84fbf257ebf59abf5bbb --- src/libhsakmt.h | 1 - src/memory.c | 10 +++++----- src/topology.c | 9 ++------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index c44df5c37f..d847003392 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -203,7 +203,6 @@ HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); uint32_t get_gfxv_by_node_id(HSAuint32 node_id); bool prefer_ats(HSAuint32 node_id); uint16_t get_device_id_by_node_id(HSAuint32 node_id); -bool is_kaveri(HSAuint32 node_id); uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id); uint32_t get_direct_link_cpu(uint32_t gpu_node); int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); diff --git a/src/memory.c b/src/memory.c index 63b88e4d5d..145aa0472a 100644 --- a/src/memory.c +++ b/src/memory.c @@ -51,7 +51,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, pr_debug("[%s] node %d; default %d; alternate %d\n", __func__, Node, DefaultPolicy, AlternatePolicy); - if (!is_kaveri(Node)) + result = validate_nodeid(Node, &gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) + return result; + + if (get_gfxv_by_node_id(Node) != GFX_VERSION_KAVERI) /* This is a legacy API useful on Kaveri only. On dGPU * the alternate aperture is setup and used * automatically for coherent allocations. Don't let @@ -59,10 +63,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, */ return HSAKMT_STATUS_NOT_IMPLEMENTED; - result = validate_nodeid(Node, &gpu_id); - if (result != HSAKMT_STATUS_SUCCESS) - return result; - /* * We accept any legal policy and alternate address location. * You get CC everywhere anyway. diff --git a/src/topology.c b/src/topology.c index 91f8055831..38ef39742b 100644 --- a/src/topology.c +++ b/src/topology.c @@ -2154,7 +2154,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, * For dGPU the topology node contains Local Memory and it is added by * the for loop above */ - if (is_kaveri(NodeId) && i < NumBanks && g_props[NodeId].node.LocalMemSize > 0 && + if (get_gfxv_by_node_id(NodeId) == GFX_VERSION_KAVERI && i < NumBanks && + g_props[NodeId].node.LocalMemSize > 0 && fmm_get_aperture_base_and_limit(FMM_GPUVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { MemoryProperties[i].HeapType = HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE; @@ -2291,12 +2292,6 @@ bool prefer_ats(HSAuint32 node_id) && g_props[node_id].node.NumFComputeCores; } -bool is_kaveri(HSAuint32 node_id) -{ - return g_props[node_id].node.EngineId.ui32.Major == 7 - && g_props[node_id].node.EngineId.ui32.Minor == 0; -} - uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) { unsigned int i; From bc4aca039c61b983e102d711df22bcccc667d3d2 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 16 Jul 2021 16:19:02 -0400 Subject: [PATCH 0893/1247] libhsakmt: Use node_id in topology_is_svm_needed() Signed-off-by: Graham Sider Change-Id: I7773ccb959e60e43b816477c5659f1fe835ecbbd --- src/fmm.c | 9 +++++---- src/fmm.h | 3 ++- src/libhsakmt.h | 2 +- src/memory.c | 3 ++- src/topology.c | 8 ++------ 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 9bcf9f9115..02e044bb88 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1318,7 +1318,8 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo return mem; } -void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags mflags) +void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, + uint64_t MemorySizeInBytes, HsaMemFlags mflags) { manageable_aperture_t *aperture; int32_t gpu_mem_id; @@ -1339,7 +1340,7 @@ void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInB ioc_flags |= fmm_translate_hsa_to_ioc_flags(mflags); - if (topology_is_svm_needed(get_device_id_by_gpu_id(gpu_id))) { + if (topology_is_svm_needed(node_id)) { aperture = svm.dgpu_aperture; if (mflags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; @@ -2386,7 +2387,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) fmm_init_rbtree(); for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) { - if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) + if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].node_id)) continue; gpu_mem[gpu_mem_id].mmio_aperture.base = map_mmio( gpu_mem[gpu_mem_id].node_id, @@ -3170,7 +3171,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, gpu_mem_id = gpu_mem_find_by_gpu_id(infoArgs.gpu_id); if (gpu_mem_id < 0) goto error_free_metadata; - if (topology_is_svm_needed(gpu_mem[gpu_mem_id].device_id)) { + if (topology_is_svm_needed(gpu_mem[gpu_mem_id].node_id)) { aperture = svm.dgpu_aperture; aperture_base = NULL; } else { diff --git a/src/fmm.h b/src/fmm.h index e06acfde24..b49e73f7bf 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -50,7 +50,8 @@ void fmm_destroy_process_apertures(void); /* Memory interface */ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes); -void *fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags); +void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, + uint64_t MemorySizeInBytes, HsaMemFlags flags); void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset); void *fmm_allocate_host(uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags); diff --git a/src/libhsakmt.h b/src/libhsakmt.h index d847003392..3059eebf6a 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -214,7 +214,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); -bool topology_is_svm_needed(uint16_t device_id); +bool topology_is_svm_needed(uint32_t node_id); HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, enum asic_family_type *asic); diff --git a/src/memory.c b/src/memory.c index 145aa0472a..6d2a4f4450 100644 --- a/src/memory.c +++ b/src/memory.c @@ -171,7 +171,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, } /* GPU allocated VRAM */ - *MemoryAddress = fmm_allocate_device(gpu_id, *MemoryAddress, SizeInBytes, MemFlags); + *MemoryAddress = fmm_allocate_device(gpu_id, PreferredNode, *MemoryAddress, + SizeInBytes, MemFlags); if (!(*MemoryAddress)) { pr_err("[%s] failed to allocate %lu bytes from device\n", diff --git a/src/topology.c b/src/topology.c index 38ef39742b..12337fcff7 100644 --- a/src/topology.c +++ b/src/topology.c @@ -798,16 +798,12 @@ void topology_setup_is_dgpu_param(HsaNodeProperties *props) is_dgpu = true; } -bool topology_is_svm_needed(uint16_t device_id) +bool topology_is_svm_needed(uint32_t node_id) { - const struct hsa_gfxip_table *hsa_gfxip; - if (is_dgpu) return true; - hsa_gfxip = find_hsa_gfxip_device(device_id); - - if (hsa_gfxip && hsa_gfxip->asic_family >= CHIP_VEGA10) + if (get_gfxv_by_node_id(node_id) >= GFX_VERSION_VEGA10) return true; return false; From 0b46d418c38fe10b9ebc45d632b5a7328da88633 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 16 Jul 2021 16:58:27 -0400 Subject: [PATCH 0894/1247] libhsakmt: Use gfx version conditions in pmc_table.c get_block_properties() switch statement now based on major GFX version instead of asic_family. Addition for NPI bringup only needed on major GFX revisions unless differing perf_counter_block assignment necessary. Signed-off-by: Graham Sider Change-Id: Ia02676d2a3a8f968b67feeafbab534ef4c01c0dd --- src/pmc_table.c | 58 +++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/src/pmc_table.c b/src/pmc_table.c index 3000b95d32..7b1f9c5eba 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -2088,8 +2088,8 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, enum perf_block_id block_id, struct perf_counter_block *block) { + uint32_t gfxv = get_gfxv_by_node_id(node_id); uint16_t dev_id = get_device_id_by_node_id(node_id); - enum asic_family_type asic; if (block_id > PERFCOUNTER_BLOCKID__MAX || block_id < PERFCOUNTER_BLOCKID__FIRST) @@ -2100,42 +2100,34 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, return HSAKMT_STATUS_SUCCESS; } - if (topology_get_asic_family(dev_id, &asic) != HSAKMT_STATUS_SUCCESS) - return HSAKMT_STATUS_INVALID_PARAMETER; - - switch (asic) { - case CHIP_KAVERI: - *block = kaveri_blocks[block_id]; + /* Major GFX Version */ + switch (gfxv >> 16) { + case 7: + if (gfxv == GFX_VERSION_KAVERI) + *block = kaveri_blocks[block_id]; + else + *block = hawaii_blocks[block_id]; break; - case CHIP_HAWAII: - *block = hawaii_blocks[block_id]; + case 8: + if (gfxv == GFX_VERSION_TONGA) + return HSAKMT_STATUS_INVALID_PARAMETER; + else if (gfxv == GFX_VERSION_CARRIZO) + *block = carrizo_blocks[block_id]; + else { + /* + * Fiji/Polaris/VegaM cards are of the same GFXIP Engine Version (8.0.3). + * Only way to differentiate b/t Fiji and Polaris/VegaM is via DID. + */ + if (dev_id == 0x7300 || dev_id == 0x730F) + *block = fiji_blocks[block_id]; + else + *block = polaris_blocks[block_id]; + } break; - case CHIP_CARRIZO: - *block = carrizo_blocks[block_id]; - break; - case CHIP_FIJI: - *block = fiji_blocks[block_id]; - break; - case CHIP_POLARIS10: - case CHIP_POLARIS11: - case CHIP_POLARIS12: - case CHIP_VEGAM: - *block = polaris_blocks[block_id]; - break; - case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_VEGA20: - case CHIP_RAVEN: - case CHIP_RENOIR: - case CHIP_ARCTURUS: - case CHIP_ALDEBARAN: + case 9: *block = vega_blocks[block_id]; break; - case CHIP_NAVI10: - case CHIP_NAVI14: - case CHIP_VANGOGH: - case CHIP_YELLOW_CARP: - case CHIP_CYAN_SKILLFISH: + case 10: *block = navi_blocks[block_id]; break; default: From 8f883d083067bbc1a1ecf80e5d3cbcb56f278111 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 19 Jul 2021 16:48:56 -0400 Subject: [PATCH 0895/1247] libhsakmt: Replace asic_family with gfx version All conditional logic previously dependent on asic_family now dependent on gfx version. As such, HSA_FORCE_ASIC_TYPE is now deprecated and HSA_OVERRIDE_GFX_VERSION should be used instead. device_info structs containing asic_family, eop_buffer_size, and doorbell_size removed (alongside dev_lookup_table[]). eop_buffer_size and doorbell_size now determined via macro, with eop_buffer_size and full gfx version field being added to queue struct. Certain funcs previously dependent on DID or GPU ID now dependent on Node ID (due to use of get_gfxv_by_node_id()). Fixes sleeper bug where ctl_stack_size and debug_memory_size were being incorrectly programmed on gfx90a. Signed-off-by: Graham Sider Change-Id: Iaabdc5c73920ad83b4b379cd6086992357b8ff10 --- src/libhsakmt.h | 7 +- src/queues.c | 282 +++++++----------------------------------------- src/topology.c | 2 +- 3 files changed, 45 insertions(+), 246 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 3059eebf6a..ca0249ce9e 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -196,7 +196,7 @@ struct hsa_gfxip_table { HSAKMT_STATUS init_kfd_version(void); -#define IS_SOC15(chip) ((chip) >= CHIP_VEGA10) +#define IS_SOC15(gfxv) ((gfxv) >= GFX_VERSION_VEGA10) HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id); HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t* node_id); @@ -256,7 +256,8 @@ uint32_t get_num_sysfs_nodes(void); bool is_forked_child(void); /* Calculate VGPR and SGPR register file size per CU */ -#define VGPR_SIZE_PER_CU(asic_family) ((asic_family == CHIP_ARCTURUS || \ - asic_family == CHIP_ALDEBARAN) ? 0x80000 : 0x40000) +#define VGPR_SIZE_PER_CU(gfxv) \ + (((gfxv) == GFX_VERSION_ARCTURUS || \ + (gfxv) == GFX_VERSION_ALDEBARAN) ? 0x80000 : 0x40000) #define SGPR_SIZE_PER_CU 0x4000 #endif diff --git a/src/queues.c b/src/queues.c index a4a7df6070..1615aa9d93 100644 --- a/src/queues.c +++ b/src/queues.c @@ -37,208 +37,24 @@ #include /* 1024 doorbells, 4 or 8 bytes each doorbell depending on ASIC generation */ -#define DOORBELL_SIZE_GFX7 4 -#define DOORBELL_SIZE_GFX8 4 -#define DOORBELL_SIZE_GFX9 8 -#define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) +#define DOORBELL_SIZE(gfxv) (((gfxv) >= 0x90000) ? 8 : 4) +#define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) + +#define EOP_BUFFER_SIZE(gfxv) \ + (((gfxv) == GFX_VERSION_TONGA) ? TONGA_PAGE_SIZE : \ + (((gfxv) >= 0x80000) ? 4096 : 0)) + +#define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv) \ + (VGPR_SIZE_PER_CU(gfxv) + SGPR_SIZE_PER_CU + \ + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) + +#define CNTL_STACK_BYTES_PER_WAVE(gfxv) \ + ((gfxv) >= GFX_VERSION_NAVI10 ? 12 : 8) #define LDS_SIZE_PER_CU 0x10000 #define HWREG_SIZE_PER_CU 0x1000 -#define WG_CONTEXT_DATA_SIZE_PER_CU(asic_family) (VGPR_SIZE_PER_CU(asic_family) + SGPR_SIZE_PER_CU + LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) -#define CNTL_STACK_BYTES_PER_WAVE(asic_family) (asic_family >= CHIP_NAVI10 ? 12 : 8) #define DEBUGGER_BYTES_ALIGN 64 -#define DEBUGGER_BYTES_PER_WAVE(asic_family) 32 - -struct device_info { - enum asic_family_type asic_family; - uint32_t eop_buffer_size; - uint32_t doorbell_size; -}; - -const struct device_info kaveri_device_info = { - .asic_family = CHIP_KAVERI, - .eop_buffer_size = 0, - .doorbell_size = DOORBELL_SIZE_GFX7, -}; - -const struct device_info hawaii_device_info = { - .asic_family = CHIP_HAWAII, - .eop_buffer_size = 0, - .doorbell_size = DOORBELL_SIZE_GFX7, -}; - -const struct device_info carrizo_device_info = { - .asic_family = CHIP_CARRIZO, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX8, -}; - -const struct device_info tonga_device_info = { - .asic_family = CHIP_TONGA, - .eop_buffer_size = TONGA_PAGE_SIZE, - .doorbell_size = DOORBELL_SIZE_GFX8, -}; - -const struct device_info fiji_device_info = { - .asic_family = CHIP_FIJI, - .eop_buffer_size = TONGA_PAGE_SIZE, - .doorbell_size = DOORBELL_SIZE_GFX8, -}; - -const struct device_info polaris10_device_info = { - .asic_family = CHIP_POLARIS10, - .eop_buffer_size = TONGA_PAGE_SIZE, - .doorbell_size = DOORBELL_SIZE_GFX8, -}; - -const struct device_info polaris11_device_info = { - .asic_family = CHIP_POLARIS11, - .eop_buffer_size = TONGA_PAGE_SIZE, - .doorbell_size = DOORBELL_SIZE_GFX8, -}; - -const struct device_info polaris12_device_info = { - .asic_family = CHIP_POLARIS12, - .eop_buffer_size = TONGA_PAGE_SIZE, - .doorbell_size = DOORBELL_SIZE_GFX8, -}; - -const struct device_info vegam_device_info = { - .asic_family = CHIP_VEGAM, - .eop_buffer_size = TONGA_PAGE_SIZE, - .doorbell_size = DOORBELL_SIZE_GFX8, -}; - -const struct device_info vega10_device_info = { - .asic_family = CHIP_VEGA10, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info vega12_device_info = { - .asic_family = CHIP_VEGA12, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info raven_device_info = { - .asic_family = CHIP_RAVEN, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info renoir_device_info = { - .asic_family = CHIP_RENOIR, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info vega20_device_info = { - .asic_family = CHIP_VEGA20, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info arcturus_device_info = { - .asic_family = CHIP_ARCTURUS, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info aldebaran_device_info = { - .asic_family = CHIP_ALDEBARAN, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info navi10_device_info = { - .asic_family = CHIP_NAVI10, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info cyan_skillfish_device_info = { - .asic_family = CHIP_CYAN_SKILLFISH, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info navi12_device_info = { - .asic_family = CHIP_NAVI12, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info navi14_device_info = { - .asic_family = CHIP_NAVI14, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info sienna_cichlid_device_info = { - .asic_family = CHIP_SIENNA_CICHLID, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info navy_flounder_device_info = { - .asic_family = CHIP_NAVY_FLOUNDER, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info dimgrey_cavefish_device_info = { - .asic_family = CHIP_DIMGREY_CAVEFISH, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info vangogh_device_info = { - .asic_family = CHIP_VANGOGH, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info beige_goby_device_info = { - .asic_family = CHIP_BEIGE_GOBY, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -const struct device_info yellow_carp_device_info = { - .asic_family = CHIP_YELLOW_CARP, - .eop_buffer_size = 4096, - .doorbell_size = DOORBELL_SIZE_GFX9, -}; - -static const struct device_info *dev_lookup_table[] = { - [CHIP_KAVERI] = &kaveri_device_info, - [CHIP_HAWAII] = &hawaii_device_info, - [CHIP_CARRIZO] = &carrizo_device_info, - [CHIP_TONGA] = &tonga_device_info, - [CHIP_FIJI] = &fiji_device_info, - [CHIP_POLARIS10] = &polaris10_device_info, - [CHIP_POLARIS11] = &polaris11_device_info, - [CHIP_POLARIS12] = &polaris12_device_info, - [CHIP_VEGAM] = &vegam_device_info, - [CHIP_VEGA10] = &vega10_device_info, - [CHIP_VEGA12] = &vega12_device_info, - [CHIP_VEGA20] = &vega20_device_info, - [CHIP_RAVEN] = &raven_device_info, - [CHIP_RENOIR] = &renoir_device_info, - [CHIP_ARCTURUS] = &arcturus_device_info, - [CHIP_ALDEBARAN] = &aldebaran_device_info, - [CHIP_NAVI10] = &navi10_device_info, - [CHIP_CYAN_SKILLFISH] = &cyan_skillfish_device_info, - [CHIP_NAVI12] = &navi12_device_info, - [CHIP_NAVI14] = &navi14_device_info, - [CHIP_SIENNA_CICHLID] = &sienna_cichlid_device_info, - [CHIP_NAVY_FLOUNDER] = &navy_flounder_device_info, - [CHIP_DIMGREY_CAVEFISH] = &dimgrey_cavefish_device_info, - [CHIP_VANGOGH] = &vangogh_device_info, - [CHIP_BEIGE_GOBY] = &beige_goby_device_info, - [CHIP_YELLOW_CARP] = &yellow_carp_device_info, -}; +#define DEBUGGER_BYTES_PER_WAVE 32 struct queue { uint32_t queue_id; @@ -249,7 +65,8 @@ struct queue { uint32_t ctx_save_restore_size; uint32_t ctl_stack_size; uint32_t debug_memory_size; - const struct device_info *dev_info; + uint32_t eop_buffer_size; + uint32_t gfxv; bool use_ats; /* This queue structure is allocated from GPU with page aligned size * but only small bytes are used. We use the extra space in the end for @@ -293,30 +110,18 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) return ret; } -static const struct device_info *get_device_info_by_dev_id(uint16_t dev_id) -{ - enum asic_family_type asic; - - if (topology_get_asic_family(dev_id, &asic) != HSAKMT_STATUS_SUCCESS) - return NULL; - - return dev_lookup_table[asic]; -} - -static void get_doorbell_map_info(uint16_t dev_id, +static void get_doorbell_map_info(uint32_t node_id, struct process_doorbells *doorbell) { - const struct device_info *dev_info; - - dev_info = get_device_info_by_dev_id(dev_id); - /* * GPUVM doorbell on Tonga requires a workaround for VM TLB ACTIVE bit * lookup bug. Remove ASIC check when this is implemented in amdgpu. */ - doorbell->use_gpuvm = (is_dgpu && - dev_info->asic_family != CHIP_TONGA); - doorbell->size = DOORBELLS_PAGE_SIZE(dev_info->doorbell_size); + uint32_t gfxv = get_gfxv_by_node_id(node_id); + doorbell->use_gpuvm = (is_dgpu && gfxv != GFX_VERSION_TONGA); + doorbell->size = DOORBELLS_PAGE_SIZE(DOORBELL_SIZE(gfxv)); + + return; } void destroy_process_doorbells(void) @@ -414,8 +219,7 @@ static HSAKMT_STATUS map_doorbell(HSAuint32 NodeId, HSAuint32 gpu_id, return HSAKMT_STATUS_SUCCESS; } - get_doorbell_map_info(get_device_id_by_node_id(NodeId), - &doorbells[NodeId]); + get_doorbell_map_info(NodeId, &doorbells[NodeId]); if (doorbells[NodeId].use_gpuvm) { status = map_doorbell_dgpu(NodeId, gpu_id, doorbell_mmap_offset); @@ -459,23 +263,23 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) { HsaNodeProperties node; - if (q->dev_info->asic_family < CHIP_CARRIZO) + if (q->gfxv < GFX_VERSION_CARRIZO) return false; if (hsaKmtGetNodeProperties(nodeid, &node)) return false; if (node.NumFComputeCores && node.NumSIMDPerCU) { uint32_t ctl_stack_size, wg_data_size; uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; - uint32_t wave_num = (q->dev_info->asic_family < CHIP_NAVI10) + uint32_t wave_num = (q->gfxv < GFX_VERSION_NAVI10) ? MIN(cu_num * 40, node.NumShaderBanks / node.NumArrays * 512) : cu_num * 32; - ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(q->dev_info->asic_family) + 8; - wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->dev_info->asic_family); + ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(q->gfxv) + 8; + wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->gfxv); q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + ctl_stack_size); - if (q->dev_info->asic_family >= CHIP_NAVI10 && - q->dev_info->asic_family <= CHIP_YELLOW_CARP) { + if (q->gfxv >= GFX_VERSION_NAVI10 && + q->gfxv <= GFX_VERSION_YELLOW_CARP) { /* HW design limits control stack size to 0x7000. * This is insufficient for theoretical PM4 cases * but sufficient for AQL, limited by SPI events. @@ -484,7 +288,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) } q->debug_memory_size = - ALIGN_UP(wave_num * DEBUGGER_BYTES_PER_WAVE(q->dev_info->asic_family), DEBUGGER_BYTES_ALIGN); + ALIGN_UP(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN); q->ctx_save_restore_size = q->ctl_stack_size + PAGE_ALIGN_UP(wg_data_size + q->debug_memory_size); @@ -583,7 +387,7 @@ static void free_queue(struct queue *q) { if (q->eop_buffer) free_exec_aligned_memory(q->eop_buffer, - q->dev_info->eop_buffer_size, + q->eop_buffer_size, PAGE_SIZE, q->use_ats); if (q->ctx_save_restore) free_exec_aligned_memory(q->ctx_save_restore, @@ -599,23 +403,21 @@ static int handle_concrete_asic(struct queue *q, HsaEvent *Event, volatile HSAint64 *ErrPayload) { - const struct device_info *dev_info = q->dev_info; bool ret; - if (!dev_info || args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA || - args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI) + if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA || + args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI) return HSAKMT_STATUS_SUCCESS; - if (dev_info->eop_buffer_size > 0) { - q->eop_buffer = - allocate_exec_aligned_memory(q->dev_info->eop_buffer_size, + if (q->eop_buffer_size > 0) { + q->eop_buffer = allocate_exec_aligned_memory(q->eop_buffer_size, q->use_ats, NodeId, true, /* Unused for VRAM */false); if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; args->eop_buffer_address = (uintptr_t)q->eop_buffer; - args->eop_buffer_size = dev_info->eop_buffer_size; + args->eop_buffer_size = q->eop_buffer_size; } ret = update_ctx_save_restore_size(NodeId, q); @@ -663,10 +465,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, { HSAKMT_STATUS result; uint32_t gpu_id; - uint16_t dev_id; uint64_t doorbell_mmap_offset; unsigned int doorbell_offset; - const struct device_info *dev_info; int err; HsaNodeProperties props; uint32_t cu_num, i; @@ -684,9 +484,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, use_ats = prefer_ats(NodeId); - dev_id = get_device_id_by_node_id(NodeId); - dev_info = get_device_info_by_dev_id(dev_id); - struct queue *q = allocate_exec_aligned_memory(sizeof(*q), use_ats, NodeId, false, true); @@ -695,8 +492,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, memset(q, 0, sizeof(*q)); + q->gfxv = get_gfxv_by_node_id(NodeId); q->use_ats = use_ats; - q->dev_info = dev_info; + q->eop_buffer_size = EOP_BUFFER_SIZE(q->gfxv); /* By default, CUs are all turned on. Initialize cu_mask to '1 * for all CU bits. @@ -759,7 +557,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, q->queue_id = args.queue_id; - if (IS_SOC15(dev_info->asic_family)) { + if (IS_SOC15(q->gfxv)) { /* On SOC15 chips, the doorbell offset within the * doorbell page is included in the doorbell offset * returned by KFD. This allows CP queue doorbells to be @@ -775,7 +573,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, * doorbell page is based on the queue ID. */ doorbell_mmap_offset = args.doorbell_offset; - doorbell_offset = q->queue_id * dev_info->doorbell_size; + doorbell_offset = q->queue_id * DOORBELL_SIZE(q->gfxv); } err = map_doorbell(NodeId, gpu_id, doorbell_mmap_offset); diff --git a/src/topology.c b/src/topology.c index 12337fcff7..ded9711fb2 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1111,8 +1111,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, } /* Get VGPR/SGPR size in byte per CU */ - props->VGPRSizePerCU = VGPR_SIZE_PER_CU(hsa_gfxip->asic_family); props->SGPRSizePerCU = SGPR_SIZE_PER_CU; + props->VGPRSizePerCU = VGPR_SIZE_PER_CU(HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32)); } else if (props->DeviceId) /* still return success */ From 1ad1a2a817985f7d25ba50dd719f0d18f83f5803 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 16 Jul 2021 17:52:37 -0400 Subject: [PATCH 0896/1247] libhsakmt: Remove asic_family vars/helpers Signed-off-by: Graham Sider Change-Id: I1bd054d8f9ffc56e2c48a5df4899966918f26e1b --- src/libhsakmt.h | 37 ------ src/openclose.c | 25 ---- src/topology.c | 338 ++++++++++++++++++++++-------------------------- 3 files changed, 157 insertions(+), 243 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index ca0249ce9e..314f28b6ca 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -41,10 +41,6 @@ extern bool is_dgpu; extern HsaVersionInfo kfd_version_info; -extern int force_asic; -extern char force_asic_name[HSA_PUBLIC_NAME_SIZE]; -extern struct hsa_gfxip_table force_asic_entry; - #undef HSAKMTAPI #define HSAKMTAPI __attribute__((visibility ("default"))) @@ -155,43 +151,12 @@ enum full_gfx_versions { GFX_VERSION_YELLOW_CARP = 0x100305, }; -enum asic_family_type { - CHIP_KAVERI = 0, - CHIP_HAWAII, /* 1 */ - CHIP_CARRIZO, /* 2 */ - CHIP_TONGA, /* 3 */ - CHIP_FIJI, /* 4 */ - CHIP_POLARIS10, /* 5 */ - CHIP_POLARIS11, /* 6 */ - CHIP_POLARIS12, /* 7 */ - CHIP_VEGAM, /* 8 */ - CHIP_VEGA10, /* 9 */ - CHIP_VEGA12, /* 10 */ - CHIP_VEGA20, /* 11 */ - CHIP_RAVEN, /* 12 */ - CHIP_RENOIR, /* 13 */ - CHIP_ARCTURUS, /* 14 */ - CHIP_ALDEBARAN, /* 15 */ - CHIP_NAVI10, /* 16 */ - CHIP_CYAN_SKILLFISH, /* 17 */ - CHIP_NAVI12, /* 18 */ - CHIP_NAVI14, /* 19 */ - CHIP_SIENNA_CICHLID, /* 20 */ - CHIP_NAVY_FLOUNDER, /* 21 */ - CHIP_DIMGREY_CAVEFISH, /* 22 */ - CHIP_VANGOGH, /* 23 */ - CHIP_BEIGE_GOBY, /* 24 */ - CHIP_YELLOW_CARP, /* 25 */ - CHIP_LAST -}; - struct hsa_gfxip_table { uint16_t device_id; // Device ID unsigned char major; // GFXIP Major engine version unsigned char minor; // GFXIP Minor engine version unsigned char stepping; // GFXIP Stepping info const char *amd_name; // CALName of the device - enum asic_family_type asic_family; // Device family id }; HSAKMT_STATUS init_kfd_version(void); @@ -215,8 +180,6 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); bool topology_is_svm_needed(uint32_t node_id); -HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, - enum asic_family_type *asic); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); diff --git a/src/openclose.c b/src/openclose.c index 55ba5f23f2..19dbf65894 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -135,31 +135,6 @@ static HSAKMT_STATUS init_vars_from_env(void) if (envvar) zfb_support = atoi(envvar); - /* Force all the GPUs to a certain type, use the below command: - * export HSA_FORCE_ASIC_TYPE="10.1.0 Navi10 14" - * meaning major.minor.step asic_name asic_id - */ - envvar = getenv("HSA_FORCE_ASIC_TYPE"); - if (envvar) { - uint32_t major, minor, step, asic_family; - - if ((sscanf(envvar, "%u.%u.%u %63s %u", &major, &minor, &step, - force_asic_name, &asic_family) != 5) - || (major > 63 || minor > 255 || step > 255) - || asic_family >= CHIP_LAST) { - pr_err("HSA_FORCE_ASIC_TYPE %s is invalid\n", envvar); - return HSAKMT_STATUS_ERROR; - } - - force_asic_entry.major = major; - force_asic_entry.minor = minor; - force_asic_entry.stepping = step; - - force_asic_entry.asic_family = asic_family; - - force_asic = 1; - } - return HSAKMT_STATUS_SUCCESS; } diff --git a/src/topology.c b/src/topology.c index ded9711fb2..b562d7690a 100644 --- a/src/topology.c +++ b/src/topology.c @@ -84,195 +84,188 @@ static const char *supported_processor_vendor_name[] = { static HSAKMT_STATUS topology_take_snapshot(void); static HSAKMT_STATUS topology_drop_snapshot(void); -int force_asic; -char force_asic_name[HSA_PUBLIC_NAME_SIZE]; -struct hsa_gfxip_table force_asic_entry = { - .amd_name = force_asic_name, -}; - static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Kaveri Family */ - { 0x1304, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1305, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1306, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1307, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1309, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130A, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130B, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130C, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130D, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130E, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x130F, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1310, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1311, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1312, 7, 0, 0, "Spooky", CHIP_KAVERI }, - { 0x1313, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1315, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x1316, 7, 0, 0, "Spooky", CHIP_KAVERI }, - { 0x1317, 7, 0, 0, "Spooky", CHIP_KAVERI }, - { 0x1318, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x131B, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x131C, 7, 0, 0, "Spectre", CHIP_KAVERI }, - { 0x131D, 7, 0, 0, "Spectre", CHIP_KAVERI }, + { 0x1304, 7, 0, 0, "Spectre" }, + { 0x1305, 7, 0, 0, "Spectre" }, + { 0x1306, 7, 0, 0, "Spectre" }, + { 0x1307, 7, 0, 0, "Spectre" }, + { 0x1309, 7, 0, 0, "Spectre" }, + { 0x130A, 7, 0, 0, "Spectre" }, + { 0x130B, 7, 0, 0, "Spectre" }, + { 0x130C, 7, 0, 0, "Spectre" }, + { 0x130D, 7, 0, 0, "Spectre" }, + { 0x130E, 7, 0, 0, "Spectre" }, + { 0x130F, 7, 0, 0, "Spectre" }, + { 0x1310, 7, 0, 0, "Spectre" }, + { 0x1311, 7, 0, 0, "Spectre" }, + { 0x1312, 7, 0, 0, "Spooky" }, + { 0x1313, 7, 0, 0, "Spectre" }, + { 0x1315, 7, 0, 0, "Spectre" }, + { 0x1316, 7, 0, 0, "Spooky" }, + { 0x1317, 7, 0, 0, "Spooky" }, + { 0x1318, 7, 0, 0, "Spectre" }, + { 0x131B, 7, 0, 0, "Spectre" }, + { 0x131C, 7, 0, 0, "Spectre" }, + { 0x131D, 7, 0, 0, "Spectre" }, /* Hawaii Family */ - { 0x67A0, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A1, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A2, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A8, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67A9, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67AA, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B0, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B1, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B8, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67B9, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67BA, 7, 0, 1, "Hawaii", CHIP_HAWAII }, - { 0x67BE, 7, 0, 1, "Hawaii", CHIP_HAWAII }, + { 0x67A0, 7, 0, 1, "Hawaii" }, + { 0x67A1, 7, 0, 1, "Hawaii" }, + { 0x67A2, 7, 0, 1, "Hawaii" }, + { 0x67A8, 7, 0, 1, "Hawaii" }, + { 0x67A9, 7, 0, 1, "Hawaii" }, + { 0x67AA, 7, 0, 1, "Hawaii" }, + { 0x67B0, 7, 0, 1, "Hawaii" }, + { 0x67B1, 7, 0, 1, "Hawaii" }, + { 0x67B8, 7, 0, 1, "Hawaii" }, + { 0x67B9, 7, 0, 1, "Hawaii" }, + { 0x67BA, 7, 0, 1, "Hawaii" }, + { 0x67BE, 7, 0, 1, "Hawaii" }, /* Carrizo Family */ - { 0x9870, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, - { 0x9874, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, - { 0x9875, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, - { 0x9876, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, - { 0x9877, 8, 0, 1, "Carrizo", CHIP_CARRIZO }, + { 0x9870, 8, 0, 1, "Carrizo" }, + { 0x9874, 8, 0, 1, "Carrizo" }, + { 0x9875, 8, 0, 1, "Carrizo" }, + { 0x9876, 8, 0, 1, "Carrizo" }, + { 0x9877, 8, 0, 1, "Carrizo" }, /* Tonga Family */ - { 0x6920, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x6921, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x6928, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x6929, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x692B, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x692F, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x6930, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x6938, 8, 0, 2, "Tonga", CHIP_TONGA }, - { 0x6939, 8, 0, 2, "Tonga", CHIP_TONGA }, + { 0x6920, 8, 0, 2, "Tonga" }, + { 0x6921, 8, 0, 2, "Tonga" }, + { 0x6928, 8, 0, 2, "Tonga" }, + { 0x6929, 8, 0, 2, "Tonga" }, + { 0x692B, 8, 0, 2, "Tonga" }, + { 0x692F, 8, 0, 2, "Tonga" }, + { 0x6930, 8, 0, 2, "Tonga" }, + { 0x6938, 8, 0, 2, "Tonga" }, + { 0x6939, 8, 0, 2, "Tonga" }, /* Fiji */ - { 0x7300, 8, 0, 3, "Fiji", CHIP_FIJI }, - { 0x730F, 8, 0, 3, "Fiji", CHIP_FIJI }, + { 0x7300, 8, 0, 3, "Fiji" }, + { 0x730F, 8, 0, 3, "Fiji" }, /* Polaris10 */ - { 0x67C0, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67C1, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67C2, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67C4, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67C7, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67C8, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67C9, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67CA, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67CC, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67CF, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67D0, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x67DF, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, - { 0x6FDF, 8, 0, 3, "Polaris10", CHIP_POLARIS10 }, + { 0x67C0, 8, 0, 3, "Polaris10" }, + { 0x67C1, 8, 0, 3, "Polaris10" }, + { 0x67C2, 8, 0, 3, "Polaris10" }, + { 0x67C4, 8, 0, 3, "Polaris10" }, + { 0x67C7, 8, 0, 3, "Polaris10" }, + { 0x67C8, 8, 0, 3, "Polaris10" }, + { 0x67C9, 8, 0, 3, "Polaris10" }, + { 0x67CA, 8, 0, 3, "Polaris10" }, + { 0x67CC, 8, 0, 3, "Polaris10" }, + { 0x67CF, 8, 0, 3, "Polaris10" }, + { 0x67D0, 8, 0, 3, "Polaris10" }, + { 0x67DF, 8, 0, 3, "Polaris10" }, + { 0x6FDF, 8, 0, 3, "Polaris10" }, /* Polaris11 */ - { 0x67E0, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67E1, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67E3, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67E7, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67E8, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67E9, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67EB, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67EF, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, - { 0x67FF, 8, 0, 3, "Polaris11", CHIP_POLARIS11 }, + { 0x67E0, 8, 0, 3, "Polaris11" }, + { 0x67E1, 8, 0, 3, "Polaris11" }, + { 0x67E3, 8, 0, 3, "Polaris11" }, + { 0x67E7, 8, 0, 3, "Polaris11" }, + { 0x67E8, 8, 0, 3, "Polaris11" }, + { 0x67E9, 8, 0, 3, "Polaris11" }, + { 0x67EB, 8, 0, 3, "Polaris11" }, + { 0x67EF, 8, 0, 3, "Polaris11" }, + { 0x67FF, 8, 0, 3, "Polaris11" }, /* Polaris12 */ - { 0x6980, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, - { 0x6981, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, - { 0x6985, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, - { 0x6986, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, - { 0x6987, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, - { 0x6995, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, - { 0x6997, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, - { 0x699F, 8, 0, 3, "Polaris12", CHIP_POLARIS12 }, + { 0x6980, 8, 0, 3, "Polaris12" }, + { 0x6981, 8, 0, 3, "Polaris12" }, + { 0x6985, 8, 0, 3, "Polaris12" }, + { 0x6986, 8, 0, 3, "Polaris12" }, + { 0x6987, 8, 0, 3, "Polaris12" }, + { 0x6995, 8, 0, 3, "Polaris12" }, + { 0x6997, 8, 0, 3, "Polaris12" }, + { 0x699F, 8, 0, 3, "Polaris12" }, /* VegaM */ - { 0x694C, 8, 0, 3, "VegaM", CHIP_VEGAM }, - { 0x694E, 8, 0, 3, "VegaM", CHIP_VEGAM }, - { 0x694F, 8, 0, 3, "VegaM", CHIP_VEGAM }, + { 0x694C, 8, 0, 3, "VegaM" }, + { 0x694E, 8, 0, 3, "VegaM" }, + { 0x694F, 8, 0, 3, "VegaM" }, /* Vega10 */ - { 0x6860, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x6861, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x6862, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x6863, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x6864, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x6867, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x6868, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x6869, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x686A, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x686B, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x686C, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x686D, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x686E, 9, 0, 0, "Vega10", CHIP_VEGA10 }, - { 0x687F, 9, 0, 0, "Vega10", CHIP_VEGA10 }, + { 0x6860, 9, 0, 0, "Vega10" }, + { 0x6861, 9, 0, 0, "Vega10" }, + { 0x6862, 9, 0, 0, "Vega10" }, + { 0x6863, 9, 0, 0, "Vega10" }, + { 0x6864, 9, 0, 0, "Vega10" }, + { 0x6867, 9, 0, 0, "Vega10" }, + { 0x6868, 9, 0, 0, "Vega10" }, + { 0x6869, 9, 0, 0, "Vega10" }, + { 0x686A, 9, 0, 0, "Vega10" }, + { 0x686B, 9, 0, 0, "Vega10" }, + { 0x686C, 9, 0, 0, "Vega10" }, + { 0x686D, 9, 0, 0, "Vega10" }, + { 0x686E, 9, 0, 0, "Vega10" }, + { 0x687F, 9, 0, 0, "Vega10" }, /* Vega12 */ - { 0x69A0, 9, 0, 4, "Vega12", CHIP_VEGA12 }, - { 0x69A1, 9, 0, 4, "Vega12", CHIP_VEGA12 }, - { 0x69A2, 9, 0, 4, "Vega12", CHIP_VEGA12 }, - { 0x69A3, 9, 0, 4, "Vega12", CHIP_VEGA12 }, - { 0x69Af, 9, 0, 4, "Vega12", CHIP_VEGA12 }, + { 0x69A0, 9, 0, 4, "Vega12" }, + { 0x69A1, 9, 0, 4, "Vega12" }, + { 0x69A2, 9, 0, 4, "Vega12" }, + { 0x69A3, 9, 0, 4, "Vega12" }, + { 0x69Af, 9, 0, 4, "Vega12" }, /* Raven */ - { 0x15DD, 9, 0, 2, "Raven", CHIP_RAVEN }, - { 0x15D8, 9, 0, 2, "Raven", CHIP_RAVEN }, + { 0x15DD, 9, 0, 2, "Raven" }, + { 0x15D8, 9, 0, 2, "Raven" }, /* Renoir */ - { 0x1636, 9, 0, 2, "Renoir", CHIP_RENOIR }, - { 0x1638, 9, 0, 2, "Renoir", CHIP_RENOIR }, - { 0x164C, 9, 0, 2, "Renoir", CHIP_RENOIR }, + { 0x1636, 9, 0, 2, "Renoir" }, + { 0x1638, 9, 0, 2, "Renoir" }, + { 0x164C, 9, 0, 2, "Renoir" }, /* Vega20 */ - { 0x66A0, 9, 0, 6, "Vega20", CHIP_VEGA20 }, - { 0x66A1, 9, 0, 6, "Vega20", CHIP_VEGA20 }, - { 0x66A2, 9, 0, 6, "Vega20", CHIP_VEGA20 }, - { 0x66A3, 9, 0, 6, "Vega20", CHIP_VEGA20 }, - { 0x66A4, 9, 0, 6, "Vega20", CHIP_VEGA20 }, - { 0x66A7, 9, 0, 6, "Vega20", CHIP_VEGA20 }, - { 0x66AF, 9, 0, 6, "Vega20", CHIP_VEGA20 }, + { 0x66A0, 9, 0, 6, "Vega20" }, + { 0x66A1, 9, 0, 6, "Vega20" }, + { 0x66A2, 9, 0, 6, "Vega20" }, + { 0x66A3, 9, 0, 6, "Vega20" }, + { 0x66A4, 9, 0, 6, "Vega20" }, + { 0x66A7, 9, 0, 6, "Vega20" }, + { 0x66AF, 9, 0, 6, "Vega20" }, /* Arcturus */ - { 0x7388, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, - { 0x738C, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, - { 0x738E, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, - { 0x7390, 9, 0, 8, "Arcturus", CHIP_ARCTURUS }, + { 0x7388, 9, 0, 8, "Arcturus" }, + { 0x738C, 9, 0, 8, "Arcturus" }, + { 0x738E, 9, 0, 8, "Arcturus" }, + { 0x7390, 9, 0, 8, "Arcturus" }, /* Aldebaran */ - { 0x7408, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, - { 0x740C, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, - { 0x740F, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, - { 0x7410, 9, 0, 10, "Aldebaran", CHIP_ALDEBARAN }, + { 0x7408, 9, 0, 10, "Aldebaran" }, + { 0x740C, 9, 0, 10, "Aldebaran" }, + { 0x740F, 9, 0, 10, "Aldebaran" }, + { 0x7410, 9, 0, 10, "Aldebaran" }, /* Navi10 */ - { 0x7310, 10, 1, 0, "Navi10", CHIP_NAVI10 }, - { 0x7312, 10, 1, 0, "Navi10", CHIP_NAVI10 }, - { 0x7318, 10, 1, 0, "Navi10", CHIP_NAVI10 }, - { 0x731A, 10, 1, 0, "Navi10", CHIP_NAVI10 }, - { 0x731E, 10, 1, 0, "Navi10", CHIP_NAVI10 }, - { 0x731F, 10, 1, 0, "Navi10", CHIP_NAVI10 }, + { 0x7310, 10, 1, 0, "Navi10" }, + { 0x7312, 10, 1, 0, "Navi10" }, + { 0x7318, 10, 1, 0, "Navi10" }, + { 0x731A, 10, 1, 0, "Navi10" }, + { 0x731E, 10, 1, 0, "Navi10" }, + { 0x731F, 10, 1, 0, "Navi10" }, /* cyan_skillfish */ - { 0x13F9, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, - { 0x13FA, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, - { 0x13FB, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, - { 0x13FC, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, - { 0x13FE, 10, 1, 3, "cyan_skillfish", CHIP_CYAN_SKILLFISH }, + { 0x13F9, 10, 1, 3, "cyan_skillfish" }, + { 0x13FA, 10, 1, 3, "cyan_skillfish" }, + { 0x13FB, 10, 1, 3, "cyan_skillfish" }, + { 0x13FC, 10, 1, 3, "cyan_skillfish" }, + { 0x13FE, 10, 1, 3, "cyan_skillfish" }, /* Navi14 */ - { 0x7340, 10, 1, 2, "Navi14", CHIP_NAVI14 }, - { 0x7341, 10, 1, 2, "Navi14", CHIP_NAVI14 }, - { 0x7347, 10, 1, 2, "Navi14", CHIP_NAVI14 }, + { 0x7340, 10, 1, 2, "Navi14" }, + { 0x7341, 10, 1, 2, "Navi14" }, + { 0x7347, 10, 1, 2, "Navi14" }, /* Navi12 */ - { 0x7360, 10, 1, 1, "Navi12", CHIP_NAVI12 }, - { 0x7362, 10, 1, 1, "Navi12", CHIP_NAVI12 }, + { 0x7360, 10, 1, 1, "Navi12" }, + { 0x7362, 10, 1, 1, "Navi12" }, /* SIENNA_CICHLID */ - { 0x73A0, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, - { 0x73A1, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, - { 0x73A2, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, - { 0x73A3, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, - { 0x73AB, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, - { 0x73AE, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, - { 0x73BF, 10, 3, 0, "SIENNA_CICHLID", CHIP_SIENNA_CICHLID }, + { 0x73A0, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73A1, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73A2, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73A3, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73AB, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73AE, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73BF, 10, 3, 0, "SIENNA_CICHLID" }, /* NAVY_FLOUNDER */ - { 0x73C0, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, - { 0x73C1, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, - { 0x73C3, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, - { 0x73DF, 10, 3, 1, "NAVY_FLOUNDER", CHIP_NAVY_FLOUNDER }, + { 0x73C0, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73C1, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73C3, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73DF, 10, 3, 1, "NAVY_FLOUNDER" }, /* DIMGREY_CAVEFISH */ - { 0x73E0, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, - { 0x73E1, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, - { 0x73E2, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, - { 0x73E3, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, - { 0x73FF, 10, 3, 2, "DIMGREY_CAVEFISH", CHIP_DIMGREY_CAVEFISH }, + { 0x73E0, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73E1, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73E2, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73FF, 10, 3, 2, "DIMGREY_CAVEFISH" }, /* VanGogh */ - { 0x163F, 10, 3, 3, "VanGogh", CHIP_VANGOGH }, + { 0x163F, 10, 3, 3, "VanGogh" }, /* Yellow_Carp */ - { 0x164D, 10, 3, 5, "YELLOW_CARP", CHIP_YELLOW_CARP }, - { 0x1681, 10, 3, 5, "YELLOW_CARP", CHIP_YELLOW_CARP }, + { 0x164D, 10, 3, 5, "YELLOW_CARP" }, + { 0x1681, 10, 3, 5, "YELLOW_CARP" }, }; /* information from /proc/cpuinfo */ @@ -766,9 +759,6 @@ static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id) { uint32_t i, table_size; - if (force_asic) - return &force_asic_entry; - table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); for (i = 0; i < table_size; i++) { if (gfxip_lookup_table[i].device_id == device_id) @@ -777,20 +767,6 @@ static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id) return NULL; } -HSAKMT_STATUS topology_get_asic_family(uint16_t device_id, - enum asic_family_type *asic) -{ - const struct hsa_gfxip_table *hsa_gfxip = - find_hsa_gfxip_device(device_id); - - if (!hsa_gfxip) - return HSAKMT_STATUS_INVALID_PARAMETER; - - *asic = hsa_gfxip->asic_family; - return HSAKMT_STATUS_SUCCESS; -} - - void topology_setup_is_dgpu_param(HsaNodeProperties *props) { /* if we found a dGPU node, then treat the whole system as dGPU */ @@ -1120,7 +1096,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->DeviceId); if (props->NumFComputeCores) - assert(props->EngineId.ui32.Major && "HSA_FORCE_ASIC_TYPE may be needed"); + assert(props->EngineId.ui32.Major && "HSA_OVERRIDE_GFX_VERSION may be needed"); err: free(read_buf); From 29369753f5d2dea1df58214ff326c736f6e83ca6 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 19 Jul 2021 17:11:59 -0400 Subject: [PATCH 0897/1247] libhsakmt: Remove redundant checks, small bug fixes Some g_system and Node ID checks already performed by validate_nodeid(). Fixes bug where hsaKmtGetNodeProperties() would return on validate_nodeid() error without releasing lock. Switches some conditionals to return INVALID_NODE_UNIT instead of INVALID_PARAMETER if NodeId >= NumNodes. Removes some g_system assertions to return error code rather than trigger a hard fault. Signed-off-by: Graham Sider Change-Id: Id1604b20c2cef8808b98cdad61bd47aa7ea3d229 --- src/topology.c | 37 ++++----------------------- tests/kfdtest/src/KFDTopologyTest.cpp | 2 +- 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/src/topology.c b/src/topology.c index b562d7690a..8475e41af8 100644 --- a/src/topology.c +++ b/src/topology.c @@ -2036,21 +2036,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, CHECK_KFD_OPEN(); pthread_mutex_lock(&hsakmt_mutex); - /* KFD ADD page 18, snapshot protocol violation */ - if (!g_system) { - err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(g_system); - goto out; - } - - if (NodeId >= g_system->NumNodes) { - err = HSAKMT_STATUS_INVALID_PARAMETER; - goto out; - } - err = validate_nodeid(NodeId, &gpu_id); if (err != HSAKMT_STATUS_SUCCESS) - return err; + goto out; *NodeProperties = g_props[NodeId].node; /* For CPU only node don't add any additional GPU memory banks. */ @@ -2085,19 +2073,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, CHECK_KFD_OPEN(); pthread_mutex_lock(&hsakmt_mutex); - /* KFD ADD page 18, snapshot protocol violation */ - if (!g_system) { - err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(g_system); - goto out; - } - - /* Check still necessary */ - if (NodeId >= g_system->NumNodes) { - err = HSAKMT_STATUS_INVALID_PARAMETER; - goto out; - } - err = validate_nodeid(NodeId, &gpu_id); if (err != HSAKMT_STATUS_SUCCESS) goto out; @@ -2183,13 +2158,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeCacheProperties(HSAuint32 NodeId, pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (!g_system) { + if (!g_system || NodeId >= g_system->NumNodes) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(g_system); goto out; } - if (NodeId >= g_system->NumNodes || NumCaches > g_props[NodeId].node.NumCaches) { + if (NumCaches > g_props[NodeId].node.NumCaches) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } @@ -2221,13 +2195,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, pthread_mutex_lock(&hsakmt_mutex); /* KFD ADD page 18, snapshot protocol violation */ - if (!g_system) { + if (!g_system || NodeId >= g_system->NumNodes ) { err = HSAKMT_STATUS_INVALID_NODE_UNIT; - assert(g_system); goto out; } - if (NodeId >= g_system->NumNodes || NumIoLinks > g_props[NodeId].node.NumIOLinks) { + if (NumIoLinks > g_props[NodeId].node.NumIOLinks) { err = HSAKMT_STATUS_INVALID_PARAMETER; goto out; } diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index bad2533d49..b4b406a24a 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -90,7 +90,7 @@ TEST_F(KFDTopologyTest, GetNodePropertiesInvalidNodeNum) { HsaNodeProperties nodeProperties; memset(&nodeProperties, 0, sizeof(nodeProperties)); - EXPECT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtGetNodeProperties(m_SystemProperties.NumNodes, &nodeProperties)); + EXPECT_EQ(HSAKMT_STATUS_INVALID_NODE_UNIT, hsaKmtGetNodeProperties(m_SystemProperties.NumNodes, &nodeProperties)); TEST_END } From bb8deeae11e274c75fcbddd09accf3d1af65dcbd Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 19 Jul 2021 17:26:22 -0400 Subject: [PATCH 0898/1247] libhsakmt: Retrieve gfx version from sysfs If lookup is successful, device info from find_hsa_gfxip_device() is used to populate props->EngineId.ui32 gfx version fields and node CAL name (props->AMDName). If unsuccessful, uses gfx_target_version sysfs field if present (otherwise results in error), and sets the node CAL name to "GFX" (e.g. 9.0.8 -> GFX090008). Signed-off-by: Graham Sider Change-Id: Id39c521b769aebbe40c934f19e03150a66884cce --- src/topology.c | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/topology.c b/src/topology.c index 8475e41af8..3ba7074343 100644 --- a/src/topology.c +++ b/src/topology.c @@ -928,6 +928,8 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, char namebuf[HSA_PUBLIC_NAME_SIZE]; const char *name; uint32_t sys_node_id; + uint32_t gfxv = 0; + uint8_t gfxv_major, gfxv_minor, gfxv_stepping; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -1043,10 +1045,16 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->NumSdmaQueuesPerEngine = prop_val; else if (strcmp(prop_name, "num_cp_queues") == 0) props->NumCpQueues = prop_val; + else if (strcmp(prop_name, "gfx_target_version") == 0) + gfxv = (uint32_t)prop_val; } + gfxv_major = HSA_GET_GFX_VERSION_MAJOR(gfxv); + gfxv_minor = HSA_GET_GFX_VERSION_MINOR(gfxv); + gfxv_stepping = HSA_GET_GFX_VERSION_STEP(gfxv); + hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); - if (hsa_gfxip) { + if (hsa_gfxip || gfxv) { envvar = getenv("HSA_OVERRIDE_GFX_VERSION"); if (envvar) { /* HSA_OVERRIDE_GFX_VERSION=major.minor.stepping */ @@ -1061,19 +1069,26 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->EngineId.ui32.Major = major & 0x3f; props->EngineId.ui32.Minor = minor & 0xff; props->EngineId.ui32.Stepping = step & 0xff; - } else { + } else if (hsa_gfxip) { props->EngineId.ui32.Major = hsa_gfxip->major & 0x3f; props->EngineId.ui32.Minor = hsa_gfxip->minor & 0xff; props->EngineId.ui32.Stepping = hsa_gfxip->stepping & 0xff; + } else { + props->EngineId.ui32.Major = gfxv_major & 0x3f; + props->EngineId.ui32.Minor = gfxv_minor & 0xff; + props->EngineId.ui32.Stepping = gfxv_stepping & 0xff; } - if (!hsa_gfxip->amd_name) { - ret = HSAKMT_STATUS_ERROR; - goto err; - } + /* Set the CAL name of the node. If DID-based hsa_gfxip lookup was + * successful, use that name. Otherwise, set to GFX. + */ + if (hsa_gfxip && hsa_gfxip->amd_name) + strncpy((char *)props->AMDName, hsa_gfxip->amd_name, + sizeof(props->AMDName)-1); + else + snprintf((char *)props->AMDName, sizeof(props->AMDName)-1, "GFX%06x", + HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32)); - /* Retrieve the CAL name of the node */ - strncpy((char *)props->AMDName, hsa_gfxip->amd_name, sizeof(props->AMDName)-1); if (!props->NumCPUCores) { /* Is dGPU Node, not APU * Retrieve the marketing name of the node using pcilib, From cff8e4992c0e8b516d8fa79301783e88e14d6c26 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Wed, 4 Aug 2021 10:56:33 +0800 Subject: [PATCH 0899/1247] libhsakmt: Add DIDs for Beige_Goby Signed-off-by: Chengming Gui Change-Id: Iff82aa0ab70df49ffc6d8004e40b5cccd637da44 --- src/topology.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/topology.c b/src/topology.c index 3ba7074343..72e3a8f313 100644 --- a/src/topology.c +++ b/src/topology.c @@ -263,6 +263,12 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73FF, 10, 3, 2, "DIMGREY_CAVEFISH" }, /* VanGogh */ { 0x163F, 10, 3, 3, "VanGogh" }, + /* BEIGE_GOBY */ + { 0x7420, 10, 3, 4, "BEIGE_GOBY" }, + { 0x7421, 10, 3, 4, "BEIGE_GOBY" }, + { 0x7422, 10, 3, 4, "BEIGE_GOBY" }, + { 0x7423, 10, 3, 4, "BEIGE_GOBY" }, + { 0x743F, 10, 3, 4, "BEIGE_GOBY" }, /* Yellow_Carp */ { 0x164D, 10, 3, 5, "YELLOW_CARP" }, { 0x1681, 10, 3, 5, "YELLOW_CARP" }, From 9745c58d4038e83a6313fad0ebe555daa7ced7f8 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 11 Aug 2021 16:59:44 -0400 Subject: [PATCH 0900/1247] libhsakmt: Hotfix for incorrect hex gfx versions Signed-off-by: Graham Sider Change-Id: Ic67b7a3e1649288c8fdef85fef875327ffc50948 --- src/libhsakmt.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 314f28b6ca..1232f80374 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -138,17 +138,17 @@ enum full_gfx_versions { GFX_VERSION_VEGA12 = 0x090004, GFX_VERSION_VEGA20 = 0x090006, GFX_VERSION_ARCTURUS = 0x090008, - GFX_VERSION_ALDEBARAN = 0x090010, - GFX_VERSION_NAVI10 = 0x100100, - GFX_VERSION_NAVI12 = 0x100101, - GFX_VERSION_NAVI14 = 0x100102, - GFX_VERSION_CYAN_SKILLFISH = 0x100103, - GFX_VERSION_SIENNA_CICHLID = 0x100300, - GFX_VERSION_NAVY_FLOUNDER = 0x100301, - GFX_VERSION_DIMGREY_CAVEFISH = 0x100302, - GFX_VERSION_VANGOGH = 0x100303, - GFX_VERSION_BEIGE_GOBY = 0x100304, - GFX_VERSION_YELLOW_CARP = 0x100305, + GFX_VERSION_ALDEBARAN = 0x09000A, + GFX_VERSION_NAVI10 = 0x0A0100, + GFX_VERSION_NAVI12 = 0x0A0101, + GFX_VERSION_NAVI14 = 0x0A0102, + GFX_VERSION_CYAN_SKILLFISH = 0x0A0103, + GFX_VERSION_SIENNA_CICHLID = 0x0A0300, + GFX_VERSION_NAVY_FLOUNDER = 0x0A0301, + GFX_VERSION_DIMGREY_CAVEFISH = 0x0A0302, + GFX_VERSION_VANGOGH = 0x0A0303, + GFX_VERSION_BEIGE_GOBY = 0x0A0304, + GFX_VERSION_YELLOW_CARP = 0x0A0305, }; struct hsa_gfxip_table { From 52f5b666d6cccc229ebfcb4f0326e86d033d62bd Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 17 Aug 2021 08:27:13 -0400 Subject: [PATCH 0901/1247] kfdtest.exclude: Remove KFDEvictTest.BurstyTest on all ASICs This is reported as failing on pretty much every ASIC, so move it to the temporary exclude list for all ASICs until the test can be fixed. Signed-off-by: Kent Russell Change-Id: I2973feb5705c178db32009e4603376d1f5e94a2f --- tests/kfdtest/scripts/kfdtest.exclude | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 025ac3e541..917df5d15d 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -61,12 +61,16 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # KFDQMTest.GPUDoorbellWrite fails intermittently (KFD-318) # KFDQMTest.mGPUShareBO (KFD-334) # KFDHWSTest.* (SWDEV-193035) +# KFDEvictTest.BurstyTest (ROCMOPS-464) +# KFDEvictTest.BurstyTest (SWDEV-291256) +# KFDEvictTest.BurstyTest (KFD-425) TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ "KFDQMTest.SdmaEventInterrupt:"\ "KFDMemoryTest.CacheInvalidateOnRemoteWrite:"\ "KFDDBGTest.BasicDebuggerSuspendResume:"\ +"KFDEvictTest.BurstyTest:"\ "KFDHWSTest.*" BLACKLIST_ALL_ASICS=\ @@ -153,12 +157,10 @@ FILTER[polaris12]=\ "$SDMA_BLACKLIST" # KFDIPCTest.BasicTest (ROCMOPS-459) .CMABasicTest (ROCMOPS-460) .CrossMemoryAttachTest (ROCMOPS-461) -# KFDEvictTest.BurstyTest (ROCMOPS-464) # KFDQMTest.AllSdmaQueues (ROCMOPS-463) FILTER[vega10]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDIPCTest.BasicTest:"\ -"KFDEvictTest.BurstyTest:"\ "KFDIPCTest.CMABasicTest:"\ "KFDIPCTest.CrossMemoryAttachTest:"\ "KFDQMTest.AllSdmaQueues" @@ -167,12 +169,9 @@ FILTER[vega12]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST"\ -# KFDEvictTest.BurstyTest (ROCMOPS-464) -# KFDEvictTest.BurstyTest (SWDEV-291256) FILTER[vega20]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ -"KFDEvictTest.BurstyTest:"\ "KFDEvictTest.BasicTest:"\ "KFDQMTest.GPUDoorbellWrite" @@ -209,18 +208,15 @@ FILTER[renoir]=\ "KFDMemoryTest.SignalHandling" # KFDExceptionTest.* (KFD-435) -# KFDEvictTest.BurstyTest (KFD-425) # KFQMTest.BasicCuMaskingEven (Issues with baseline measurement) FILTER[arcturus]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDEvictTest.BurstyTest" +"KFDQMTest.BasicCuMaskingEven"\ FILTER[aldebaran]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDExceptionTest.FaultStorm:"\ -"KFDEvictTest.BurstyTest:"\ "KFDMemoryTest.PtraceAccess:"\ "KFDMemoryTest.DeviceHdpFlush" From acd5eda54f38b98fadf592eb677dc7d652bebb32 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Thu, 12 Aug 2021 14:13:55 -0400 Subject: [PATCH 0902/1247] kfdtest: Update KFDCWSRTest for software scheduler and MES Update KFDCWSRTest.BasicTest to run CWSR test with legacy HWS, MES, and software scheduler. Signed-off-by: Mukul Joshi Change-Id: I09af904507109c2bbbace6ff616a57fa2b1b8aa9 --- tests/kfdtest/src/KFDCWSRTest.cpp | 77 ++++++++++++++----------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index daa92c9823..e12e697566 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -132,6 +132,14 @@ bool isOnEmulator() { return isEmuMode; } +static inline uint32_t checkCWSREnabled() { + uint32_t cwsr_enable = 0; + + fscanf_dec("/sys/module/amdgpu/parameters/cwsr_enable", &cwsr_enable); + + return cwsr_enable; +} + /** * KFDCWSRTest.BasicTest * @@ -139,83 +147,68 @@ bool isOnEmulator() { * It then triggers CWSR and ensures the shader stops running. * It then resumes the shader, ensures that it's running again and terminates it. */ - TEST_F(KFDCWSRTest, BasicTest) { TEST_START(TESTPROFILE_RUNALL); int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - if (m_FamilyId >= FAMILY_VI) { + if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { + const char *pIterateIsa; HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer resultBuf1(PAGE_SIZE, defaultGPUNode, true, false, false); - HsaMemoryBuffer resultBuf2(PAGE_SIZE, defaultGPUNode, true, false, false); - - int count1 = 40000000; - int count2 = 20000000; - - if (isOnEmulator()) { - // Divide the iterator times by 1000 so that the test can - // finish in a reasonable time. - count1 /= 1000; - count2 /= 1000; - LOG() << "On Emulators" << std::endl; - } - - unsigned int* result1 = resultBuf1.As(); - unsigned int* result2 = resultBuf2.As(); - const char *pIterateIsa; + uint64_t count1 = 400000000; if (m_FamilyId < FAMILY_AI) pIterateIsa = iterate_isa_gfx8; else pIterateIsa = iterate_isa_gfx9; + if (isOnEmulator()) { + // Divide the iterator times by 10000 so that the test can + // finish in a reasonable time. + count1 /= 10000; + LOG() << "On Emulators" << std::endl; + } + + unsigned int* result1 = resultBuf1.As(); + m_pIsaGen->CompileShader(pIterateIsa, "iterate_isa", isaBuffer); - PM4Queue queue1, queue2; + PM4Queue queue1; ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); - Dispatch *dispatch1, *dispatch2; + Dispatch *dispatch1; dispatch1 = new Dispatch(isaBuffer); - dispatch2 = new Dispatch(isaBuffer); dispatch1->SetArgs(reinterpret_cast(count1), result1); dispatch1->SetDim(wave_number, 1, 1); - dispatch2->SetArgs(reinterpret_cast(count2), result2); - dispatch2->SetDim(wave_number, 1, 1); // Submit the shader, queue1 dispatch1->Submit(queue1); - // Create queue2 during queue1 still running will trigger the CWSR - ASSERT_SUCCESS(queue2.Create(defaultGPUNode)); - // Submit the shader - dispatch2->Submit(queue2); + + //Give time for waves to launch before disabling queue. + Delay(1); + EXPECT_SUCCESS(queue1.Update(0/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false)); + Delay(5); + EXPECT_SUCCESS(queue1.Update(100/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false)); + dispatch1->Sync(); - dispatch2->Sync(); // Ensure all the waves complete as expected int i; for (i = 0 ; i < wave_number; ++i) { - if (result1[i] != count1) { - LOG() << "Dispatch 1, work item [" << std::dec << i << "] " - << result1[i] << " != " << count1 << std::endl; - break; - } - if (result2[i] != count2) { - LOG() << "Dispatch 2, work item [" << std::dec << i << "] " - << result2[i] << " != " << count2 << std::endl; - break; - } + if (result1[i] != count1) { + LOG() << "Dispatch 1, work item [" << std::dec << i << "] " + << result1[i] << " != " << count1 << std::endl; + break; + } } EXPECT_EQ(i, wave_number); EXPECT_SUCCESS(queue1.Destroy()); - EXPECT_SUCCESS(queue2.Destroy()); delete dispatch1; - delete dispatch2; - } else { LOG() << "Skipping test: No CWSR present for family ID 0x" << m_FamilyId << "." << std::endl; } @@ -240,7 +233,7 @@ TEST_F(KFDCWSRTest, InterruptRestore) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - if (m_FamilyId >= FAMILY_VI) { + if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); m_pIsaGen->CompileShader(infinite_isa, "infinite_isa", isaBuffer); From d0ae4b1f3f48e892b152165e06d16cd3eeaf3ba8 Mon Sep 17 00:00:00 2001 From: Tony Tye Date: Sat, 21 Aug 2021 21:47:09 +0000 Subject: [PATCH 0903/1247] Update HSA_RUNTIME_ENABLE_MIN_MAJOR to 10 The major version of the debug ioctl functions has been updated to 10. Change-Id: Iddc8656ad07c92aa30c6a7a99add28ab3a3f5a80 --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index e9e6b1406f..c2f0546b90 100644 --- a/src/debug.c +++ b/src/debug.c @@ -296,7 +296,7 @@ hsaKmtGetKernelDebugTrapVersionInfo( return HSAKMT_STATUS_SUCCESS; } -#define HSA_RUNTIME_ENABLE_MIN_MAJOR 9 +#define HSA_RUNTIME_ENABLE_MIN_MAJOR 10 #define HSA_RUNTIME_ENABLE_MIN_MINOR 0 HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, bool setupTtmp) From 79d00a35f93d4c432be7a550a93dd41452e04382 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 17 May 2021 10:37:41 -0400 Subject: [PATCH 0904/1247] CMakeLists: Combine binary and devel packages Consolidate all thunk components into a single development package Also declare build dependencies via find_package/library and report the location of said dependencies during build time Change-Id: I8524ad1a2d9a664accd93471abacc4bac82b9a7c Signed-off-by: Kent Russell --- CMakeLists.txt | 71 +++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d98ddba84..dc915c8c2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,11 +156,31 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" find_package(PkgConfig) pkg_check_modules(LIBDRM REQUIRED libdrm) +# get OS-info for OS-specific build dependencies +get_os_info() + +find_package(PkgConfig) +# Check for libraries required for building +find_library(LIBC NAMES libc.so.6 REQUIRED) +find_library(NUMA NAMES libnuma.so REQUIRED) +message(STATUS "LIBC:" ${LIBC}) +message(STATUS "NUMA:" ${NUMA}) + target_link_libraries ( ${HSAKMT_TARGET} - PRIVATE pthread rt numa + PRIVATE pthread rt ${LIBC} ${NUMA} ) target_compile_options(${HSAKMT_TARGET} PRIVATE ${LIBDRM_CFLAGS} ${HSAKMT_C_FLAGS}) +if(NOT DISTRO_ID MATCHES "ubuntu") + find_library(LIBGCC NAMES libgcc_s.so.1 REQUIRED) + message(STATUS "LIBGCC:" ${LIBGCC}) + target_link_libraries( ${HSAKMT_TARGET} PRIVATE ${LIBGCC} ) +else() + find_library(UDEV NAMES libudev.so libudev.a REQUIRED) + message(STATUS "UDEV:" ${UDEV}) + find_package(ZLIB REQUIRED) + target_link_libraries( ${HSAKMT_TARGET} PRIVATE ${ZLIB} ${UDEV} ) +endif() ## Define default paths and packages. if( CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT ) @@ -172,10 +192,10 @@ set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) # Installs binaries and exports the library usage data to ${HSAKMT_TARGET}Targets install ( TARGETS ${HSAKMT_TARGET} EXPORT ${HSAKMT_TARGET}Targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT binary - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT binary ) + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT binary ) +install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel ) # Install public headers install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -243,20 +263,15 @@ install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.pc DESTINATION ${CMAKE_INS set(CPACK_COMPONENTS_GROUPING IGNORE) set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_RPM_COMPONENT_INSTALL ON) -set(CPACK_DEBIAN_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE}) -set(CPACK_RPM_BINARY_PACKAGE_NAME ${HSAKMT_PACKAGE}) set(CPACK_PACKAGE_VENDOR "Advanced Micro Devices, Inc.") set(CPACK_PACKAGE_VERSION_MAJOR ${VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${VERSION_MINOR}) set(CPACK_PACKAGE_VERSION_PATCH ${VERSION_PATCH}) set(CPACK_PACKAGE_CONTACT "AMD GFX mailing list ") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md") -set(CPACK_COMPONENT_BINARY_DESCRIPTION "HSAKMT library for AMD KFD support\n This repository includes the user-mode API interfaces\n used to interact with the ROCk driver.") +set(CPACK_COMPONENT_DESCRIPTION "HSAKMT development package.\n This package includes the user-mode API interfaces\nused to interact with the ROCk driver.\n This package contains the headers, pkgonfig and\n cmake files for the ROCT package.") set ( ENABLE_LDCONFIG ON CACHE BOOL "Set library links and caches using ldconfig.") -## Component packaging, DEVEL -set(CPACK_COMPONENT_DEVEL_DESCRIPTION "HSAKMT development package.\n This package contains the headers, pkgonfig and\n cmake files for the ROCT package") - # Make proper version for appending # Default Value is 99999, setting it first set(ROCM_VERSION_FOR_PACKAGE "99999") @@ -266,8 +281,7 @@ endif() # Debian package specific variables set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface") -set(CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS "libc6, libnuma1, zlib1g, libudev1, rocm-core") -set(CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev") +set(CPACK_DEBIAN_PACKAGE_NAME "hsakmt-roct-dev") if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) set(CPACK_DEBIAN_PACKAGE_RELEASE $ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) else() @@ -276,18 +290,10 @@ endif() ## Process the Debian install/remove scripts to update the CPACK variables configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst.in DEBIAN/postinst @ONLY ) configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm.in DEBIAN/prerm @ONLY ) -set(CPACK_DEBIAN_BINARY_PACKAGE_CONTROL_EXTRA "DEBIAN/postinst;DEBIAN/prerm") - -# RPM package specific variables -# get OS-info -get_os_info() -if(DISTRO_ID MATCHES "sles" ) - set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, libnuma-devel, libgcc_s1, libdrm-devel, rocm-core") -elseif(DISTRO_ID MATCHES "centos" AND DISTRO_RELEASE MATCHES "6") - set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl, libgcc, rocm-core") -else() - set(CPACK_RPM_BINARY_PACKAGE_REQUIRES "glibc, numactl-libs, libgcc, rocm-core") -endif() +set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "DEBIAN/postinst;DEBIAN/prerm") +# Needed since some packages still say they need hsakmt-roct +set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") +set(CPACK_DEBIAN_PACKAGE_PROVIDES "hsakmt-roct") ## Process the Rpm install/remove scripts to update the CPACK variables configure_file ( "${CMAKE_CURRENT_SOURCE_DIR}/RPM/post.in" RPM/post @ONLY ) @@ -299,9 +305,10 @@ set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/RPM/post if(DEFINED CPACK_PACKAGING_INSTALL_PREFIX) set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX} ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}") endif() -set(CPACK_RPM_DEVEL_PACKAGE_NAME "hsakmt-roct-devel") +set(CPACK_RPM_PACKAGE_NAME "hsakmt-roct-devel") # Since we changed the package name to match RPM specs, take care of older builds that had -dev installed -set(CPACK_RPM_DEVEL_PACKAGE_OBSOLETES "hsakmt-roct-dev") +# Also cover the fact that this now replaces the old binary package hsakmt-roct +set(CPACK_RPM_PACKAGE_OBSOLETES "hsakmt-roct-dev hsakmt-roct") if(DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE}) set(CPACK_RPM_PACKAGE_RELEASE $ENV{CPACK_RPM_PACKAGE_RELEASE}) else() @@ -324,8 +331,10 @@ set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version -set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "${HSAKMT_PACKAGE} (=${PACKAGE_VERSION_STR}-${CPACK_DEBIAN_PACKAGE_RELEASE}), rocm-core") -set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "${HSAKMT_PACKAGE} = ${PACKAGE_VERSION_STR}-${CPACK_RPM_PACKAGE_RELEASE}, rocm-core") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core") +set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") + +set(CPACK_RPM_PACKAGE_REQUIRES "rocm-core") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") @@ -333,10 +342,8 @@ set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") # Remove dependency on rocm-core if -DROCM_DEP_ROCMCORE=ON not given to cmake if(NOT ROCM_DEP_ROCMCORE) - string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_BINARY_PACKAGE_REQUIRES ${CPACK_RPM_BINARY_PACKAGE_REQUIRES}) - string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_DEVEL_PACKAGE_REQUIRES ${CPACK_RPM_DEVEL_PACKAGE_REQUIRES}) - string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS ${CPACK_DEBIAN_BINARY_PACKAGE_DEPENDS}) - string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS ${CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_RPM_PACKAGE_REQUIRES}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS}) endif() include(CPack) From cb1c84d99b1dd9b136d778a0e4cf68087ec84581 Mon Sep 17 00:00:00 2001 From: Mike Li Date: Thu, 27 May 2021 17:03:02 -0400 Subject: [PATCH 0905/1247] Get Marketing name from libdrm-amdgpu Use the database provided by libdrm-amdgpu,as this is maintained by AMD Signed-off-by: Mike Li Change-Id: I319a46d833cb23173a77fdff1deae69ea58588b8 --- CMakeLists.txt | 28 ++++- src/fmm.c | 7 +- src/libhsakmt.h | 3 +- src/pci_ids.c | 304 ------------------------------------------------ src/pci_ids.h | 46 -------- src/topology.c | 55 ++++++--- 6 files changed, 64 insertions(+), 379 deletions(-) delete mode 100644 src/pci_ids.c delete mode 100644 src/pci_ids.h diff --git a/CMakeLists.txt b/CMakeLists.txt index dc915c8c2d..f6c85cb052 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,7 +122,6 @@ set ( HSAKMT_SRC "src/debug.c" "src/libhsakmt.c" "src/memory.c" "src/openclose.c" - "src/pci_ids.c" "src/perfctr.c" "src/pmc_table.c" "src/queues.c" @@ -155,7 +154,6 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" ) find_package(PkgConfig) -pkg_check_modules(LIBDRM REQUIRED libdrm) # get OS-info for OS-specific build dependencies get_os_info() @@ -166,11 +164,29 @@ find_library(NUMA NAMES libnuma.so REQUIRED) message(STATUS "LIBC:" ${LIBC}) message(STATUS "NUMA:" ${NUMA}) +## If environment variable DRM_DIR is set, the script +## will pick up the corresponding libraries from that path. +if( DEFINED ENV{DRM_DIR} ) +#assume header files and libraries are under the same path + set ( DRM_DIR $ENV{DRM_DIR} ) + set ( DRM_INCLUDE_DIRS ${DRM_DIR}/include ) + link_directories(${DRM_DIR}/lib64) + set ( DRM_LIBRARIES drm ) + set ( DRM_AMDGPU_LIBRARIES drm_amdgpu ) +else() +# The module name passed to pkg_check_modules() is determined by the +# name of file *.pc + pkg_check_modules(DRM REQUIRED libdrm) + pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) + include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) +endif() +include_directories(${DRM_INCLUDE_DIRS}) + target_link_libraries ( ${HSAKMT_TARGET} - PRIVATE pthread rt ${LIBC} ${NUMA} + PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt ${LIBC} ${NUMA} ) -target_compile_options(${HSAKMT_TARGET} PRIVATE ${LIBDRM_CFLAGS} ${HSAKMT_C_FLAGS}) +target_compile_options(${HSAKMT_TARGET} PRIVATE ${DRM_CFLAGS} ${HSAKMT_C_FLAGS}) if(NOT DISTRO_ID MATCHES "ubuntu") find_library(LIBGCC NAMES libgcc_s.so.1 REQUIRED) message(STATUS "LIBGCC:" ${LIBGCC}) @@ -331,10 +347,10 @@ set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version -set(CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") -set(CPACK_RPM_PACKAGE_REQUIRES "rocm-core") +set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") diff --git a/src/fmm.c b/src/fmm.c index 02e044bb88..2657415246 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2150,7 +2150,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; unsigned int guardPages = 1; - struct pci_ids pacc; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2197,13 +2196,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * gets called before hsaKmtAcquireSystemProperties() is called. */ - pacc = pci_ids_create(); - is_dgpu = false; for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); - ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc, NULL, NULL); + ret = topology_sysfs_get_node_props(i, &props, &gpu_id, NULL, NULL); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; @@ -2240,8 +2237,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) } } - pci_ids_destroy(pacc); - /* The ioctl will also return Number of Nodes if * args.kfd_process_device_apertures_ptr is set to NULL. This is not * required since Number of nodes is already known. Kernel will fill in diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 1232f80374..9d4be20409 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -28,7 +28,6 @@ #include "linux/kfd_ioctl.h" #include "hsakmt.h" -#include "pci_ids.h" #include #include #include @@ -175,7 +174,7 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - uint32_t *gpu_id, struct pci_ids pacc, + uint32_t *gpu_id, bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); diff --git a/src/pci_ids.c b/src/pci_ids.c deleted file mode 100644 index 0c8df69d28..0000000000 --- a/src/pci_ids.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright © 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including - * the next paragraph) shall be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -/* - * Parse a pci.ids text file to extract 'device_name' - * # Vendors, devices and subsystems. Please keep sorted. - * # Syntax: - * # vendor vendor_name - * # device device_name <-- single tab - * # subvendor subdevice subsystem_name <-- two tabs - */ - -/* - * Example section of file. Searching for 1002/130c - * - * 1002 Advanced Micro Devices, Inc. [AMD/ATI] - * # fields elided - * 130a Kaveri [Radeon R6 Graphics] - * 130b Kaveri [Radeon R4 Graphics] - * - * 130c Kaveri [Radeon R7 Graphics] <- result - * # ^-------------------------^ - * - * 130d Kaveri [Radeon R6 Graphics] - * 130e Kaveri [Radeon R5 Graphics] - * # fields elided - * # next vendor region starts - * 1003 ULSI Systems - * 0201 US201 - */ - -#define _GNU_SOURCE -#include - -#include "pci_ids.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -static const char *const pci_ids_paths[] = { - "/usr/share/hwdata/pci.ids", // update-pciids - "/usr/share/misc/pci.ids", // debian - "/usr/share/pci.ids", // redhat - "/var/lib/pciutils/pci.ids", // also debian - "pci.ids", -}; - -static struct pci_ids pci_ids_create_from_file(const char *path) -{ - struct pci_ids failure = { - .fd = -1, - }; - - int fd = open(path, O_RDONLY, 0); - - if (fd == -1) - return failure; - - struct stat sb; - - fstat(fd, &sb); - size_t sz = sb.st_size; - - if (sz == 0) { - close(fd); - return failure; - } - - sz = (sz < UINT32_MAX) ? sz : UINT32_MAX; - void *addr = mmap(0, sz, PROT_READ, MAP_PRIVATE, fd, 0); - - if (addr == MAP_FAILED) { - close(fd); - return failure; - } - - return (struct pci_ids){ - .fd = fd, - .addr = addr, - .size = sz, - }; -} - -struct pci_ids pci_ids_create(void) -{ - size_t sz = sizeof(pci_ids_paths) / sizeof(pci_ids_paths[0]); - - for (size_t i = 0; i < sz; i++) { - struct pci_ids res = pci_ids_create_from_file(pci_ids_paths[i]); - - if (res.fd != -1) - return res; - } - - return (struct pci_ids){ .fd = -1 }; -} - -void pci_ids_destroy(struct pci_ids f) -{ - if (f.fd != -1) { - munmap(f.addr, f.size); - close(f.fd); - } -} - -struct range { - // Iterator pair, start <= end. Can dereference [start end) - unsigned char *start; - unsigned char *end; -}; -static bool empty_range(struct range r) -{ - return r.start == r.end; -} - -static void write_as_hex(uint16_t x, char *b) -{ - static const char digits[] = "0123456789abcdef"; - - for (unsigned int i = 0; i < 4; i++) { - unsigned int index = 0xf & (x >> 4 * (3 - i)); - - b[i] = digits[index]; - } -} - -static struct range find_vendor(struct range r, uint16_t VendorId) - -{ - if (empty_range(r)) - return r; - - char needle[5] = { '\n' }; - - write_as_hex(VendorId, &needle[1]); - unsigned char *s = - memmem(r.start, r.end - r.start, needle, sizeof(needle)); - - if (s) { - r.start = s; - } else { - r.start = r.end; - assert(empty_range(r)); - } - return r; -} - -static struct range trim_whitespace(struct range r) -{ - while (!empty_range(r) && isspace(r.start[0])) - r.start++; - - while (!empty_range(r) && isspace(r.end[-1])) - r.end--; - - return r; -} - -static struct range skip_vendor_id(struct range r) -{ - const struct range failure = { 0, 0 }; - - assert(!empty_range(r)); - // Expect an initial newline to skip over - if (r.start[0] != '\n') - return failure; - - r.start++; - if (empty_range(r)) - return failure; - - // Skip rest of line - r.start = memchr(r.start, '\n', r.end - r.start); - if (!r.start) - return failure; - - return r; -} - -static struct range find_device(struct range r, uint16_t DeviceId) -{ - struct range failure = { 0, 0 }; - - if (empty_range(r)) - return failure; - - r = skip_vendor_id(r); - if (empty_range(r)) - return failure; - - assert(r.start[0] == '\n'); - - char needle[6] = { '\n', '\t' }; - - write_as_hex(DeviceId, &needle[2]); - - for (;;) { - size_t width = r.end - r.start; - - if (width < sizeof(needle)) - return failure; - - unsigned char *line_end = memchr(r.start + 1, '\n', width - 1); - - if (!line_end) { - // File may not end with a newline - line_end = r.end; - } - - if (memcmp(r.start, needle, sizeof(needle)) == 0) { - // Success - r.start += sizeof(needle); - r.end = line_end; - return trim_whitespace(r); - } - - if (isxdigit(r.start[1])) { - // Reached the end of this region - return failure; - } - - // Otherwise ignore whatever is on the line, e.g. '#' - r.start = line_end; - } -} - -static void copy_range_to_buffer(struct range r, char *buf, size_t size) -{ - assert(!empty_range(r)); - size_t to_copy = (r.end - r.start); - - to_copy = to_copy < (size - 1) ? to_copy : size - 1; - - memcpy(buf, r.start, to_copy); - buf[to_copy] = '\0'; -} - -static void write_fallback_to_buffer(char *buf, size_t size, uint16_t DeviceId) -{ - char tmp[] = "Device xxxx"; - - _Static_assert(sizeof(tmp) == 12, ""); - write_as_hex(DeviceId, &tmp[7]); - - size_t to_copy = (sizeof(tmp) <= size) ? sizeof(tmp) : size; - - memcpy(buf, tmp, to_copy); - buf[size - 1] = '\0'; -} - -char *pci_ids_lookup(struct pci_ids f, char *buf, size_t size, - uint16_t VendorId, uint16_t DeviceId) -{ - if (f.fd == -1) { - write_fallback_to_buffer(buf, size, DeviceId); - return buf; - } - - struct range whole_file = { - .start = f.addr, - .end = (unsigned char *)f.addr + f.size, - }; - - struct range vendor = find_vendor(whole_file, VendorId); - - struct range device = find_device(vendor, DeviceId); - - if (!empty_range(device)) - copy_range_to_buffer(device, buf, size); - else - write_fallback_to_buffer(buf, size, DeviceId); - - return buf; -} diff --git a/src/pci_ids.h b/src/pci_ids.h deleted file mode 100644 index 4c0d44c57e..0000000000 --- a/src/pci_ids.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright © 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including - * the next paragraph) shall be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef SRC_PCI_IDS_H_ -#define SRC_PCI_IDS_H_ - -#include -#include - -struct pci_ids { - int32_t fd; // -1 if file access failed - uint32_t size; - void *addr; -}; - -// Sixteen byte struct is passed in registers. Avoids calling malloc/free. -struct pci_ids pci_ids_create(void); -void pci_ids_destroy(struct pci_ids pacc); - -// Writes to buf. Returns buf. Does not fail. -char *pci_ids_lookup(struct pci_ids pacc, char *buf, size_t buf_size, - uint16_t VendorId, uint16_t DeviceId); - -#endif // SRC_PCI_IDS_H_ diff --git a/src/topology.c b/src/topology.c index 72e3a8f313..858483d232 100644 --- a/src/topology.c +++ b/src/topology.c @@ -34,6 +34,9 @@ #include #include +#include +#include +#include #include "libhsakmt.h" #include "fmm.h" @@ -916,10 +919,39 @@ exit: return ret; } +static int topology_get_marketing_name(int minor, uint16_t *marketing_name) +{ + int drm_fd; + uint32_t major_version; + uint32_t minor_version; + amdgpu_device_handle device_handle; + const char *name; + int i; + + if (marketing_name == NULL) + return -1; + drm_fd = drmOpenRender(minor); + if (drm_fd < 0) + return -1; + if (amdgpu_device_initialize(drm_fd, + &major_version, &minor_version, &device_handle) < 0) { + drmClose(drm_fd); + return -1; + } + name = amdgpu_get_marketing_name(device_handle); + if (name != NULL) { + for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + marketing_name[i] = name[i]; + marketing_name[i] = '\0'; + } + amdgpu_device_deinitialize(device_handle); + drmClose(drm_fd); + return 0; +} + HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, - struct pci_ids pacc, bool *p2p_links, uint32_t *num_p2pLinks) { @@ -928,11 +960,9 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, char prop_name[256]; char path[256]; unsigned long long prop_val; - uint32_t i, prog, major, minor, step; + uint32_t prog, major, minor, step; int read_size; const struct hsa_gfxip_table *hsa_gfxip; - char namebuf[HSA_PUBLIC_NAME_SIZE]; - const char *name; uint32_t sys_node_id; uint32_t gfxv = 0; uint8_t gfxv_major, gfxv_minor, gfxv_stepping; @@ -1097,14 +1127,12 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if (!props->NumCPUCores) { /* Is dGPU Node, not APU - * Retrieve the marketing name of the node using pcilib, - * convert UTF8 to UTF16 + * Retrieve the marketing name of the node. */ - name = pci_ids_lookup(pacc, namebuf, sizeof(namebuf), - props->VendorId, props->DeviceId); - for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) - props->MarketingName[i] = name[i]; - props->MarketingName[i] = '\0'; + if (topology_get_marketing_name(props->DrmRenderMinor, + props->MarketingName) != 0) + pr_info("failed to get marketing name for device ID 0x%x\n", + props->DeviceId); } /* Get VGPR/SGPR size in byte per CU */ @@ -1773,7 +1801,6 @@ HSAKMT_STATUS topology_take_snapshot(void) HsaSystemProperties sys_props; node_props_t *temp_props = 0; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - struct pci_ids pacc; struct proc_cpuinfo *cpuinfo; const uint32_t num_procs = get_nprocs(); uint32_t num_ioLinks; @@ -1800,12 +1827,11 @@ retry: ret = HSAKMT_STATUS_NO_MEMORY; goto err; } - pacc = pci_ids_create(); for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_props[i].node, &temp_props[i].gpu_id, - pacc, &p2p_links, &num_p2pLinks); + &p2p_links, &num_p2pLinks); if (ret != HSAKMT_STATUS_SUCCESS) { free_properties(temp_props, i); goto err; @@ -1910,7 +1936,6 @@ retry: temp_props[i].node.NumIOLinks = link_id; } } - pci_ids_destroy(pacc); } if (!p2p_links) { From af31e9364af17b79dca1235b07ec92b112a40fde Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 27 Aug 2021 07:47:20 -0400 Subject: [PATCH 0906/1247] CMakeLists: Make sure package provides hsakmt-roct for RPM RPMs were missed initially Change-Id: Id9d3fdcb41a25db90c9de71e2035de61928e9a32 --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f6c85cb052..077ab51c36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -310,6 +310,8 @@ set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "DEBIAN/postinst;DEBIAN/prerm") # Needed since some packages still say they need hsakmt-roct set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") set(CPACK_DEBIAN_PACKAGE_PROVIDES "hsakmt-roct") +set(CPACK_RPM_PACKAGE_OBSOLETES "hsakmt-roct") +set(CPACK_RPM_PACKAGE_PROVIDES "hsakmt-roct") ## Process the Rpm install/remove scripts to update the CPACK variables configure_file ( "${CMAKE_CURRENT_SOURCE_DIR}/RPM/post.in" RPM/post @ONLY ) From 294086096a27c6c4cbdc5281b38e3eafa7161c3a Mon Sep 17 00:00:00 2001 From: Jinzhou Su Date: Tue, 13 Jul 2021 11:16:07 +0800 Subject: [PATCH 0907/1247] libhsakmt: Add another gfx90c DID Add pci device ID. Signed-off-by: Jinzhou Su Signed-off-by: Huang Rui Change-Id: Idcf8e99583fc9991a0d71d1e0d7e741a132ed730 --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index 858483d232..aec250f905 100644 --- a/src/topology.c +++ b/src/topology.c @@ -205,6 +205,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x15DD, 9, 0, 2, "Raven" }, { 0x15D8, 9, 0, 2, "Raven" }, /* Renoir */ + { 0x15E7, 9, 0, 2, "Renoir" }, { 0x1636, 9, 0, 2, "Renoir" }, { 0x1638, 9, 0, 2, "Renoir" }, { 0x164C, 9, 0, 2, "Renoir" }, From bcd49af8bb38e187c3e4d7adf880a64a3a658568 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Tue, 31 Aug 2021 11:07:57 +0800 Subject: [PATCH 0908/1247] libhsakmt: Add new DIDs for gfx1030/gfx1031/gfx1032 Signed-off-by: Chengming Gui Change-Id: Iddb27ce2bedda9a57944eca1e2bb93001e61f733 --- src/topology.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/topology.c b/src/topology.c index aec250f905..9ff33ba0f1 100644 --- a/src/topology.c +++ b/src/topology.c @@ -252,6 +252,11 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73A1, 10, 3, 0, "SIENNA_CICHLID" }, { 0x73A2, 10, 3, 0, "SIENNA_CICHLID" }, { 0x73A3, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73A5, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73A8, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73A9, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73AC, 10, 3, 0, "SIENNA_CICHLID" }, + { 0x73AD, 10, 3, 0, "SIENNA_CICHLID" }, { 0x73AB, 10, 3, 0, "SIENNA_CICHLID" }, { 0x73AE, 10, 3, 0, "SIENNA_CICHLID" }, { 0x73BF, 10, 3, 0, "SIENNA_CICHLID" }, @@ -259,11 +264,23 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x73C0, 10, 3, 1, "NAVY_FLOUNDER" }, { 0x73C1, 10, 3, 1, "NAVY_FLOUNDER" }, { 0x73C3, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73DA, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73DB, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73DC, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73DD, 10, 3, 1, "NAVY_FLOUNDER" }, + { 0x73DE, 10, 3, 1, "NAVY_FLOUNDER" }, { 0x73DF, 10, 3, 1, "NAVY_FLOUNDER" }, /* DIMGREY_CAVEFISH */ { 0x73E0, 10, 3, 2, "DIMGREY_CAVEFISH" }, { 0x73E1, 10, 3, 2, "DIMGREY_CAVEFISH" }, { 0x73E2, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73E8, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73E9, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73EA, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73EB, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73EC, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73ED, 10, 3, 2, "DIMGREY_CAVEFISH" }, + { 0x73EF, 10, 3, 2, "DIMGREY_CAVEFISH" }, { 0x73FF, 10, 3, 2, "DIMGREY_CAVEFISH" }, /* VanGogh */ { 0x163F, 10, 3, 3, "VanGogh" }, From 89c9d6a3b9a1652ba72487c76a8a4c34ea6dbcea Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 30 Aug 2021 13:29:45 -0400 Subject: [PATCH 0909/1247] Revert "Get Marketing name from libdrm-amdgpu" This reverts commit cb1c84d99b1dd9b136d778a0e4cf68087ec84581. Reason for revert: aqlprofiler hasn't promoted yet Change-Id: I9e6a4169f014a6181f43ee8ef4810be2667d98bd --- CMakeLists.txt | 28 +---- src/fmm.c | 7 +- src/libhsakmt.h | 3 +- src/pci_ids.c | 304 ++++++++++++++++++++++++++++++++++++++++++++++++ src/pci_ids.h | 46 ++++++++ src/topology.c | 55 +++------ 6 files changed, 379 insertions(+), 64 deletions(-) create mode 100644 src/pci_ids.c create mode 100644 src/pci_ids.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 077ab51c36..2e117ddf87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,6 +122,7 @@ set ( HSAKMT_SRC "src/debug.c" "src/libhsakmt.c" "src/memory.c" "src/openclose.c" + "src/pci_ids.c" "src/perfctr.c" "src/pmc_table.c" "src/queues.c" @@ -154,6 +155,7 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" ) find_package(PkgConfig) +pkg_check_modules(LIBDRM REQUIRED libdrm) # get OS-info for OS-specific build dependencies get_os_info() @@ -164,29 +166,11 @@ find_library(NUMA NAMES libnuma.so REQUIRED) message(STATUS "LIBC:" ${LIBC}) message(STATUS "NUMA:" ${NUMA}) -## If environment variable DRM_DIR is set, the script -## will pick up the corresponding libraries from that path. -if( DEFINED ENV{DRM_DIR} ) -#assume header files and libraries are under the same path - set ( DRM_DIR $ENV{DRM_DIR} ) - set ( DRM_INCLUDE_DIRS ${DRM_DIR}/include ) - link_directories(${DRM_DIR}/lib64) - set ( DRM_LIBRARIES drm ) - set ( DRM_AMDGPU_LIBRARIES drm_amdgpu ) -else() -# The module name passed to pkg_check_modules() is determined by the -# name of file *.pc - pkg_check_modules(DRM REQUIRED libdrm) - pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) - include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) -endif() -include_directories(${DRM_INCLUDE_DIRS}) - target_link_libraries ( ${HSAKMT_TARGET} - PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt ${LIBC} ${NUMA} + PRIVATE pthread rt ${LIBC} ${NUMA} ) -target_compile_options(${HSAKMT_TARGET} PRIVATE ${DRM_CFLAGS} ${HSAKMT_C_FLAGS}) +target_compile_options(${HSAKMT_TARGET} PRIVATE ${LIBDRM_CFLAGS} ${HSAKMT_C_FLAGS}) if(NOT DISTRO_ID MATCHES "ubuntu") find_library(LIBGCC NAMES libgcc_s.so.1 REQUIRED) message(STATUS "LIBGCC:" ${LIBGCC}) @@ -349,10 +333,10 @@ set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version -set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") -set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") +set(CPACK_RPM_PACKAGE_REQUIRES "rocm-core") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") diff --git a/src/fmm.c b/src/fmm.c index 2657415246..02e044bb88 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2150,6 +2150,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; unsigned int guardPages = 1; + struct pci_ids pacc; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2196,11 +2197,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * gets called before hsaKmtAcquireSystemProperties() is called. */ + pacc = pci_ids_create(); + is_dgpu = false; for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); - ret = topology_sysfs_get_node_props(i, &props, &gpu_id, NULL, NULL); + ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc, NULL, NULL); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; @@ -2237,6 +2240,8 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) } } + pci_ids_destroy(pacc); + /* The ioctl will also return Number of Nodes if * args.kfd_process_device_apertures_ptr is set to NULL. This is not * required since Number of nodes is already known. Kernel will fill in diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 9d4be20409..1232f80374 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -28,6 +28,7 @@ #include "linux/kfd_ioctl.h" #include "hsakmt.h" +#include "pci_ids.h" #include #include #include @@ -174,7 +175,7 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - uint32_t *gpu_id, + uint32_t *gpu_id, struct pci_ids pacc, bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); diff --git a/src/pci_ids.c b/src/pci_ids.c new file mode 100644 index 0000000000..0c8df69d28 --- /dev/null +++ b/src/pci_ids.c @@ -0,0 +1,304 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * Parse a pci.ids text file to extract 'device_name' + * # Vendors, devices and subsystems. Please keep sorted. + * # Syntax: + * # vendor vendor_name + * # device device_name <-- single tab + * # subvendor subdevice subsystem_name <-- two tabs + */ + +/* + * Example section of file. Searching for 1002/130c + * + * 1002 Advanced Micro Devices, Inc. [AMD/ATI] + * # fields elided + * 130a Kaveri [Radeon R6 Graphics] + * 130b Kaveri [Radeon R4 Graphics] + * + * 130c Kaveri [Radeon R7 Graphics] <- result + * # ^-------------------------^ + * + * 130d Kaveri [Radeon R6 Graphics] + * 130e Kaveri [Radeon R5 Graphics] + * # fields elided + * # next vendor region starts + * 1003 ULSI Systems + * 0201 US201 + */ + +#define _GNU_SOURCE +#include + +#include "pci_ids.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static const char *const pci_ids_paths[] = { + "/usr/share/hwdata/pci.ids", // update-pciids + "/usr/share/misc/pci.ids", // debian + "/usr/share/pci.ids", // redhat + "/var/lib/pciutils/pci.ids", // also debian + "pci.ids", +}; + +static struct pci_ids pci_ids_create_from_file(const char *path) +{ + struct pci_ids failure = { + .fd = -1, + }; + + int fd = open(path, O_RDONLY, 0); + + if (fd == -1) + return failure; + + struct stat sb; + + fstat(fd, &sb); + size_t sz = sb.st_size; + + if (sz == 0) { + close(fd); + return failure; + } + + sz = (sz < UINT32_MAX) ? sz : UINT32_MAX; + void *addr = mmap(0, sz, PROT_READ, MAP_PRIVATE, fd, 0); + + if (addr == MAP_FAILED) { + close(fd); + return failure; + } + + return (struct pci_ids){ + .fd = fd, + .addr = addr, + .size = sz, + }; +} + +struct pci_ids pci_ids_create(void) +{ + size_t sz = sizeof(pci_ids_paths) / sizeof(pci_ids_paths[0]); + + for (size_t i = 0; i < sz; i++) { + struct pci_ids res = pci_ids_create_from_file(pci_ids_paths[i]); + + if (res.fd != -1) + return res; + } + + return (struct pci_ids){ .fd = -1 }; +} + +void pci_ids_destroy(struct pci_ids f) +{ + if (f.fd != -1) { + munmap(f.addr, f.size); + close(f.fd); + } +} + +struct range { + // Iterator pair, start <= end. Can dereference [start end) + unsigned char *start; + unsigned char *end; +}; +static bool empty_range(struct range r) +{ + return r.start == r.end; +} + +static void write_as_hex(uint16_t x, char *b) +{ + static const char digits[] = "0123456789abcdef"; + + for (unsigned int i = 0; i < 4; i++) { + unsigned int index = 0xf & (x >> 4 * (3 - i)); + + b[i] = digits[index]; + } +} + +static struct range find_vendor(struct range r, uint16_t VendorId) + +{ + if (empty_range(r)) + return r; + + char needle[5] = { '\n' }; + + write_as_hex(VendorId, &needle[1]); + unsigned char *s = + memmem(r.start, r.end - r.start, needle, sizeof(needle)); + + if (s) { + r.start = s; + } else { + r.start = r.end; + assert(empty_range(r)); + } + return r; +} + +static struct range trim_whitespace(struct range r) +{ + while (!empty_range(r) && isspace(r.start[0])) + r.start++; + + while (!empty_range(r) && isspace(r.end[-1])) + r.end--; + + return r; +} + +static struct range skip_vendor_id(struct range r) +{ + const struct range failure = { 0, 0 }; + + assert(!empty_range(r)); + // Expect an initial newline to skip over + if (r.start[0] != '\n') + return failure; + + r.start++; + if (empty_range(r)) + return failure; + + // Skip rest of line + r.start = memchr(r.start, '\n', r.end - r.start); + if (!r.start) + return failure; + + return r; +} + +static struct range find_device(struct range r, uint16_t DeviceId) +{ + struct range failure = { 0, 0 }; + + if (empty_range(r)) + return failure; + + r = skip_vendor_id(r); + if (empty_range(r)) + return failure; + + assert(r.start[0] == '\n'); + + char needle[6] = { '\n', '\t' }; + + write_as_hex(DeviceId, &needle[2]); + + for (;;) { + size_t width = r.end - r.start; + + if (width < sizeof(needle)) + return failure; + + unsigned char *line_end = memchr(r.start + 1, '\n', width - 1); + + if (!line_end) { + // File may not end with a newline + line_end = r.end; + } + + if (memcmp(r.start, needle, sizeof(needle)) == 0) { + // Success + r.start += sizeof(needle); + r.end = line_end; + return trim_whitespace(r); + } + + if (isxdigit(r.start[1])) { + // Reached the end of this region + return failure; + } + + // Otherwise ignore whatever is on the line, e.g. '#' + r.start = line_end; + } +} + +static void copy_range_to_buffer(struct range r, char *buf, size_t size) +{ + assert(!empty_range(r)); + size_t to_copy = (r.end - r.start); + + to_copy = to_copy < (size - 1) ? to_copy : size - 1; + + memcpy(buf, r.start, to_copy); + buf[to_copy] = '\0'; +} + +static void write_fallback_to_buffer(char *buf, size_t size, uint16_t DeviceId) +{ + char tmp[] = "Device xxxx"; + + _Static_assert(sizeof(tmp) == 12, ""); + write_as_hex(DeviceId, &tmp[7]); + + size_t to_copy = (sizeof(tmp) <= size) ? sizeof(tmp) : size; + + memcpy(buf, tmp, to_copy); + buf[size - 1] = '\0'; +} + +char *pci_ids_lookup(struct pci_ids f, char *buf, size_t size, + uint16_t VendorId, uint16_t DeviceId) +{ + if (f.fd == -1) { + write_fallback_to_buffer(buf, size, DeviceId); + return buf; + } + + struct range whole_file = { + .start = f.addr, + .end = (unsigned char *)f.addr + f.size, + }; + + struct range vendor = find_vendor(whole_file, VendorId); + + struct range device = find_device(vendor, DeviceId); + + if (!empty_range(device)) + copy_range_to_buffer(device, buf, size); + else + write_fallback_to_buffer(buf, size, DeviceId); + + return buf; +} diff --git a/src/pci_ids.h b/src/pci_ids.h new file mode 100644 index 0000000000..4c0d44c57e --- /dev/null +++ b/src/pci_ids.h @@ -0,0 +1,46 @@ +/* + * Copyright © 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef SRC_PCI_IDS_H_ +#define SRC_PCI_IDS_H_ + +#include +#include + +struct pci_ids { + int32_t fd; // -1 if file access failed + uint32_t size; + void *addr; +}; + +// Sixteen byte struct is passed in registers. Avoids calling malloc/free. +struct pci_ids pci_ids_create(void); +void pci_ids_destroy(struct pci_ids pacc); + +// Writes to buf. Returns buf. Does not fail. +char *pci_ids_lookup(struct pci_ids pacc, char *buf, size_t buf_size, + uint16_t VendorId, uint16_t DeviceId); + +#endif // SRC_PCI_IDS_H_ diff --git a/src/topology.c b/src/topology.c index 9ff33ba0f1..06f744153b 100644 --- a/src/topology.c +++ b/src/topology.c @@ -34,9 +34,6 @@ #include #include -#include -#include -#include #include "libhsakmt.h" #include "fmm.h" @@ -937,39 +934,10 @@ exit: return ret; } -static int topology_get_marketing_name(int minor, uint16_t *marketing_name) -{ - int drm_fd; - uint32_t major_version; - uint32_t minor_version; - amdgpu_device_handle device_handle; - const char *name; - int i; - - if (marketing_name == NULL) - return -1; - drm_fd = drmOpenRender(minor); - if (drm_fd < 0) - return -1; - if (amdgpu_device_initialize(drm_fd, - &major_version, &minor_version, &device_handle) < 0) { - drmClose(drm_fd); - return -1; - } - name = amdgpu_get_marketing_name(device_handle); - if (name != NULL) { - for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) - marketing_name[i] = name[i]; - marketing_name[i] = '\0'; - } - amdgpu_device_deinitialize(device_handle); - drmClose(drm_fd); - return 0; -} - HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, + struct pci_ids pacc, bool *p2p_links, uint32_t *num_p2pLinks) { @@ -978,9 +946,11 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, char prop_name[256]; char path[256]; unsigned long long prop_val; - uint32_t prog, major, minor, step; + uint32_t i, prog, major, minor, step; int read_size; const struct hsa_gfxip_table *hsa_gfxip; + char namebuf[HSA_PUBLIC_NAME_SIZE]; + const char *name; uint32_t sys_node_id; uint32_t gfxv = 0; uint8_t gfxv_major, gfxv_minor, gfxv_stepping; @@ -1145,12 +1115,14 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if (!props->NumCPUCores) { /* Is dGPU Node, not APU - * Retrieve the marketing name of the node. + * Retrieve the marketing name of the node using pcilib, + * convert UTF8 to UTF16 */ - if (topology_get_marketing_name(props->DrmRenderMinor, - props->MarketingName) != 0) - pr_info("failed to get marketing name for device ID 0x%x\n", - props->DeviceId); + name = pci_ids_lookup(pacc, namebuf, sizeof(namebuf), + props->VendorId, props->DeviceId); + for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + props->MarketingName[i] = name[i]; + props->MarketingName[i] = '\0'; } /* Get VGPR/SGPR size in byte per CU */ @@ -1819,6 +1791,7 @@ HSAKMT_STATUS topology_take_snapshot(void) HsaSystemProperties sys_props; node_props_t *temp_props = 0; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + struct pci_ids pacc; struct proc_cpuinfo *cpuinfo; const uint32_t num_procs = get_nprocs(); uint32_t num_ioLinks; @@ -1845,11 +1818,12 @@ retry: ret = HSAKMT_STATUS_NO_MEMORY; goto err; } + pacc = pci_ids_create(); for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_props[i].node, &temp_props[i].gpu_id, - &p2p_links, &num_p2pLinks); + pacc, &p2p_links, &num_p2pLinks); if (ret != HSAKMT_STATUS_SUCCESS) { free_properties(temp_props, i); goto err; @@ -1954,6 +1928,7 @@ retry: temp_props[i].node.NumIOLinks = link_id; } } + pci_ids_destroy(pacc); } if (!p2p_links) { From 978028d3f69c746653e4348a9172919315efd5cd Mon Sep 17 00:00:00 2001 From: changzhu Date: Wed, 18 Aug 2021 16:38:21 +0800 Subject: [PATCH 0910/1247] libhsakmt: fix KFDOpenCloseKFDTest.OpenAlreadyOpenedKFD Segmentation fault on gfx902 g_props is intialized in topology_take_snapshot which needs to call hsaKmtAcquireSystemProperties. In hsaKmtOpenKFD, it doesn't call hsaKmtAcquireSystemProperties. So it needs to change parameter of topology_is_svm_needed from node_id to EngineId to avoid Segmentation fault on gfx902. Signed-off-by: changzhu Change-Id: Iba8d20a9142510c70927454d26bdcaf579ad5574 --- src/fmm.c | 11 ++++++++--- src/libhsakmt.h | 2 +- src/topology.c | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 02e044bb88..950989e9bf 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -174,6 +174,7 @@ typedef struct { uint32_t device_id; uint32_t node_id; uint64_t local_mem_size; + HSA_ENGINE_ID EngineId; aperture_t lds_aperture; aperture_t scratch_aperture; aperture_t mmio_aperture; @@ -1340,7 +1341,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, ioc_flags |= fmm_translate_hsa_to_ioc_flags(mflags); - if (topology_is_svm_needed(node_id)) { + if (topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) { aperture = svm.dgpu_aperture; if (mflags.ui32.AQLQueueMemory) size = MemorySizeInBytes * 2; @@ -2217,6 +2218,10 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) goto sysfs_parse_failed; } + gpu_mem[gpu_mem_count].EngineId.ui32.Major = props.EngineId.ui32.Major; + gpu_mem[gpu_mem_count].EngineId.ui32.Minor = props.EngineId.ui32.Minor; + gpu_mem[gpu_mem_count].EngineId.ui32.Stepping = props.EngineId.ui32.Stepping; + gpu_mem[gpu_mem_count].drm_render_fd = fd; gpu_mem[gpu_mem_count].gpu_id = gpu_id; gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; @@ -2387,7 +2392,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) fmm_init_rbtree(); for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) { - if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].node_id)) + if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) continue; gpu_mem[gpu_mem_id].mmio_aperture.base = map_mmio( gpu_mem[gpu_mem_id].node_id, @@ -3171,7 +3176,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, gpu_mem_id = gpu_mem_find_by_gpu_id(infoArgs.gpu_id); if (gpu_mem_id < 0) goto error_free_metadata; - if (topology_is_svm_needed(gpu_mem[gpu_mem_id].node_id)) { + if (topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) { aperture = svm.dgpu_aperture; aperture_base = NULL; } else { diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 1232f80374..4c013c48c6 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -179,7 +179,7 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); -bool topology_is_svm_needed(uint32_t node_id); +bool topology_is_svm_needed(HSA_ENGINE_ID EngineId); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); diff --git a/src/topology.c b/src/topology.c index 06f744153b..8481d7e6c7 100644 --- a/src/topology.c +++ b/src/topology.c @@ -798,12 +798,12 @@ void topology_setup_is_dgpu_param(HsaNodeProperties *props) is_dgpu = true; } -bool topology_is_svm_needed(uint32_t node_id) +bool topology_is_svm_needed(HSA_ENGINE_ID EngineId) { if (is_dgpu) return true; - if (get_gfxv_by_node_id(node_id) >= GFX_VERSION_VEGA10) + if (HSA_GET_GFX_VERSION_FULL(EngineId.ui32) >= GFX_VERSION_VEGA10) return true; return false; @@ -2159,7 +2159,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeMemoryProperties(HSAuint32 NodeId, } /* Add SVM aperture */ - if (topology_is_svm_needed(get_device_id_by_gpu_id(gpu_id)) && i < NumBanks && + if (topology_is_svm_needed(g_props[NodeId].node.EngineId) && i < NumBanks && fmm_get_aperture_base_and_limit( FMM_SVM, gpu_id, &MemoryProperties[i].VirtualBaseAddress, &aperture_limit) == HSAKMT_STATUS_SUCCESS) { From 268bb1ae8498d52b237427720ebda2c5d28bfa8f Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 1 Sep 2021 07:44:17 -0400 Subject: [PATCH 0911/1247] Revert "Revert "Get Marketing name from libdrm-amdgpu"" This reverts commit 89c9d6a3b9a1652ba72487c76a8a4c34ea6dbcea. Reason for revert: aqlprofile has merged Change-Id: Iee0fbe8ac1e475123273b328949f68f10490d3ca --- CMakeLists.txt | 28 ++++- src/fmm.c | 7 +- src/libhsakmt.h | 3 +- src/pci_ids.c | 304 ------------------------------------------------ src/pci_ids.h | 46 -------- src/topology.c | 55 ++++++--- 6 files changed, 64 insertions(+), 379 deletions(-) delete mode 100644 src/pci_ids.c delete mode 100644 src/pci_ids.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e117ddf87..077ab51c36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,7 +122,6 @@ set ( HSAKMT_SRC "src/debug.c" "src/libhsakmt.c" "src/memory.c" "src/openclose.c" - "src/pci_ids.c" "src/perfctr.c" "src/pmc_table.c" "src/queues.c" @@ -155,7 +154,6 @@ set_property ( TARGET ${HSAKMT_TARGET} PROPERTY VERSION "${LIB_VERSION_STRING}" set_property ( TARGET ${HSAKMT_TARGET} PROPERTY SOVERSION "${LIB_VERSION_MAJOR}" ) find_package(PkgConfig) -pkg_check_modules(LIBDRM REQUIRED libdrm) # get OS-info for OS-specific build dependencies get_os_info() @@ -166,11 +164,29 @@ find_library(NUMA NAMES libnuma.so REQUIRED) message(STATUS "LIBC:" ${LIBC}) message(STATUS "NUMA:" ${NUMA}) +## If environment variable DRM_DIR is set, the script +## will pick up the corresponding libraries from that path. +if( DEFINED ENV{DRM_DIR} ) +#assume header files and libraries are under the same path + set ( DRM_DIR $ENV{DRM_DIR} ) + set ( DRM_INCLUDE_DIRS ${DRM_DIR}/include ) + link_directories(${DRM_DIR}/lib64) + set ( DRM_LIBRARIES drm ) + set ( DRM_AMDGPU_LIBRARIES drm_amdgpu ) +else() +# The module name passed to pkg_check_modules() is determined by the +# name of file *.pc + pkg_check_modules(DRM REQUIRED libdrm) + pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) + include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) +endif() +include_directories(${DRM_INCLUDE_DIRS}) + target_link_libraries ( ${HSAKMT_TARGET} - PRIVATE pthread rt ${LIBC} ${NUMA} + PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt ${LIBC} ${NUMA} ) -target_compile_options(${HSAKMT_TARGET} PRIVATE ${LIBDRM_CFLAGS} ${HSAKMT_C_FLAGS}) +target_compile_options(${HSAKMT_TARGET} PRIVATE ${DRM_CFLAGS} ${HSAKMT_C_FLAGS}) if(NOT DISTRO_ID MATCHES "ubuntu") find_library(LIBGCC NAMES libgcc_s.so.1 REQUIRED) message(STATUS "LIBGCC:" ${LIBGCC}) @@ -333,10 +349,10 @@ set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version -set(CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") -set(CPACK_RPM_PACKAGE_REQUIRES "rocm-core") +set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") diff --git a/src/fmm.c b/src/fmm.c index 950989e9bf..618cea9bdf 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2151,7 +2151,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; unsigned int guardPages = 1; - struct pci_ids pacc; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2198,13 +2197,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) * gets called before hsaKmtAcquireSystemProperties() is called. */ - pacc = pci_ids_create(); - is_dgpu = false; for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); - ret = topology_sysfs_get_node_props(i, &props, &gpu_id, pacc, NULL, NULL); + ret = topology_sysfs_get_node_props(i, &props, &gpu_id, NULL, NULL); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; @@ -2245,8 +2242,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) } } - pci_ids_destroy(pacc); - /* The ioctl will also return Number of Nodes if * args.kfd_process_device_apertures_ptr is set to NULL. This is not * required since Number of nodes is already known. Kernel will fill in diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 4c013c48c6..d531e87c90 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -28,7 +28,6 @@ #include "linux/kfd_ioctl.h" #include "hsakmt.h" -#include "pci_ids.h" #include #include #include @@ -175,7 +174,7 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - uint32_t *gpu_id, struct pci_ids pacc, + uint32_t *gpu_id, bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); diff --git a/src/pci_ids.c b/src/pci_ids.c deleted file mode 100644 index 0c8df69d28..0000000000 --- a/src/pci_ids.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright © 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including - * the next paragraph) shall be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -/* - * Parse a pci.ids text file to extract 'device_name' - * # Vendors, devices and subsystems. Please keep sorted. - * # Syntax: - * # vendor vendor_name - * # device device_name <-- single tab - * # subvendor subdevice subsystem_name <-- two tabs - */ - -/* - * Example section of file. Searching for 1002/130c - * - * 1002 Advanced Micro Devices, Inc. [AMD/ATI] - * # fields elided - * 130a Kaveri [Radeon R6 Graphics] - * 130b Kaveri [Radeon R4 Graphics] - * - * 130c Kaveri [Radeon R7 Graphics] <- result - * # ^-------------------------^ - * - * 130d Kaveri [Radeon R6 Graphics] - * 130e Kaveri [Radeon R5 Graphics] - * # fields elided - * # next vendor region starts - * 1003 ULSI Systems - * 0201 US201 - */ - -#define _GNU_SOURCE -#include - -#include "pci_ids.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -static const char *const pci_ids_paths[] = { - "/usr/share/hwdata/pci.ids", // update-pciids - "/usr/share/misc/pci.ids", // debian - "/usr/share/pci.ids", // redhat - "/var/lib/pciutils/pci.ids", // also debian - "pci.ids", -}; - -static struct pci_ids pci_ids_create_from_file(const char *path) -{ - struct pci_ids failure = { - .fd = -1, - }; - - int fd = open(path, O_RDONLY, 0); - - if (fd == -1) - return failure; - - struct stat sb; - - fstat(fd, &sb); - size_t sz = sb.st_size; - - if (sz == 0) { - close(fd); - return failure; - } - - sz = (sz < UINT32_MAX) ? sz : UINT32_MAX; - void *addr = mmap(0, sz, PROT_READ, MAP_PRIVATE, fd, 0); - - if (addr == MAP_FAILED) { - close(fd); - return failure; - } - - return (struct pci_ids){ - .fd = fd, - .addr = addr, - .size = sz, - }; -} - -struct pci_ids pci_ids_create(void) -{ - size_t sz = sizeof(pci_ids_paths) / sizeof(pci_ids_paths[0]); - - for (size_t i = 0; i < sz; i++) { - struct pci_ids res = pci_ids_create_from_file(pci_ids_paths[i]); - - if (res.fd != -1) - return res; - } - - return (struct pci_ids){ .fd = -1 }; -} - -void pci_ids_destroy(struct pci_ids f) -{ - if (f.fd != -1) { - munmap(f.addr, f.size); - close(f.fd); - } -} - -struct range { - // Iterator pair, start <= end. Can dereference [start end) - unsigned char *start; - unsigned char *end; -}; -static bool empty_range(struct range r) -{ - return r.start == r.end; -} - -static void write_as_hex(uint16_t x, char *b) -{ - static const char digits[] = "0123456789abcdef"; - - for (unsigned int i = 0; i < 4; i++) { - unsigned int index = 0xf & (x >> 4 * (3 - i)); - - b[i] = digits[index]; - } -} - -static struct range find_vendor(struct range r, uint16_t VendorId) - -{ - if (empty_range(r)) - return r; - - char needle[5] = { '\n' }; - - write_as_hex(VendorId, &needle[1]); - unsigned char *s = - memmem(r.start, r.end - r.start, needle, sizeof(needle)); - - if (s) { - r.start = s; - } else { - r.start = r.end; - assert(empty_range(r)); - } - return r; -} - -static struct range trim_whitespace(struct range r) -{ - while (!empty_range(r) && isspace(r.start[0])) - r.start++; - - while (!empty_range(r) && isspace(r.end[-1])) - r.end--; - - return r; -} - -static struct range skip_vendor_id(struct range r) -{ - const struct range failure = { 0, 0 }; - - assert(!empty_range(r)); - // Expect an initial newline to skip over - if (r.start[0] != '\n') - return failure; - - r.start++; - if (empty_range(r)) - return failure; - - // Skip rest of line - r.start = memchr(r.start, '\n', r.end - r.start); - if (!r.start) - return failure; - - return r; -} - -static struct range find_device(struct range r, uint16_t DeviceId) -{ - struct range failure = { 0, 0 }; - - if (empty_range(r)) - return failure; - - r = skip_vendor_id(r); - if (empty_range(r)) - return failure; - - assert(r.start[0] == '\n'); - - char needle[6] = { '\n', '\t' }; - - write_as_hex(DeviceId, &needle[2]); - - for (;;) { - size_t width = r.end - r.start; - - if (width < sizeof(needle)) - return failure; - - unsigned char *line_end = memchr(r.start + 1, '\n', width - 1); - - if (!line_end) { - // File may not end with a newline - line_end = r.end; - } - - if (memcmp(r.start, needle, sizeof(needle)) == 0) { - // Success - r.start += sizeof(needle); - r.end = line_end; - return trim_whitespace(r); - } - - if (isxdigit(r.start[1])) { - // Reached the end of this region - return failure; - } - - // Otherwise ignore whatever is on the line, e.g. '#' - r.start = line_end; - } -} - -static void copy_range_to_buffer(struct range r, char *buf, size_t size) -{ - assert(!empty_range(r)); - size_t to_copy = (r.end - r.start); - - to_copy = to_copy < (size - 1) ? to_copy : size - 1; - - memcpy(buf, r.start, to_copy); - buf[to_copy] = '\0'; -} - -static void write_fallback_to_buffer(char *buf, size_t size, uint16_t DeviceId) -{ - char tmp[] = "Device xxxx"; - - _Static_assert(sizeof(tmp) == 12, ""); - write_as_hex(DeviceId, &tmp[7]); - - size_t to_copy = (sizeof(tmp) <= size) ? sizeof(tmp) : size; - - memcpy(buf, tmp, to_copy); - buf[size - 1] = '\0'; -} - -char *pci_ids_lookup(struct pci_ids f, char *buf, size_t size, - uint16_t VendorId, uint16_t DeviceId) -{ - if (f.fd == -1) { - write_fallback_to_buffer(buf, size, DeviceId); - return buf; - } - - struct range whole_file = { - .start = f.addr, - .end = (unsigned char *)f.addr + f.size, - }; - - struct range vendor = find_vendor(whole_file, VendorId); - - struct range device = find_device(vendor, DeviceId); - - if (!empty_range(device)) - copy_range_to_buffer(device, buf, size); - else - write_fallback_to_buffer(buf, size, DeviceId); - - return buf; -} diff --git a/src/pci_ids.h b/src/pci_ids.h deleted file mode 100644 index 4c0d44c57e..0000000000 --- a/src/pci_ids.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright © 2020 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including - * the next paragraph) shall be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ - -#ifndef SRC_PCI_IDS_H_ -#define SRC_PCI_IDS_H_ - -#include -#include - -struct pci_ids { - int32_t fd; // -1 if file access failed - uint32_t size; - void *addr; -}; - -// Sixteen byte struct is passed in registers. Avoids calling malloc/free. -struct pci_ids pci_ids_create(void); -void pci_ids_destroy(struct pci_ids pacc); - -// Writes to buf. Returns buf. Does not fail. -char *pci_ids_lookup(struct pci_ids pacc, char *buf, size_t buf_size, - uint16_t VendorId, uint16_t DeviceId); - -#endif // SRC_PCI_IDS_H_ diff --git a/src/topology.c b/src/topology.c index 8481d7e6c7..be40aaff5c 100644 --- a/src/topology.c +++ b/src/topology.c @@ -34,6 +34,9 @@ #include #include +#include +#include +#include #include "libhsakmt.h" #include "fmm.h" @@ -934,10 +937,39 @@ exit: return ret; } +static int topology_get_marketing_name(int minor, uint16_t *marketing_name) +{ + int drm_fd; + uint32_t major_version; + uint32_t minor_version; + amdgpu_device_handle device_handle; + const char *name; + int i; + + if (marketing_name == NULL) + return -1; + drm_fd = drmOpenRender(minor); + if (drm_fd < 0) + return -1; + if (amdgpu_device_initialize(drm_fd, + &major_version, &minor_version, &device_handle) < 0) { + drmClose(drm_fd); + return -1; + } + name = amdgpu_get_marketing_name(device_handle); + if (name != NULL) { + for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) + marketing_name[i] = name[i]; + marketing_name[i] = '\0'; + } + amdgpu_device_deinitialize(device_handle); + drmClose(drm_fd); + return 0; +} + HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, uint32_t *gpu_id, - struct pci_ids pacc, bool *p2p_links, uint32_t *num_p2pLinks) { @@ -946,11 +978,9 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, char prop_name[256]; char path[256]; unsigned long long prop_val; - uint32_t i, prog, major, minor, step; + uint32_t prog, major, minor, step; int read_size; const struct hsa_gfxip_table *hsa_gfxip; - char namebuf[HSA_PUBLIC_NAME_SIZE]; - const char *name; uint32_t sys_node_id; uint32_t gfxv = 0; uint8_t gfxv_major, gfxv_minor, gfxv_stepping; @@ -1115,14 +1145,12 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if (!props->NumCPUCores) { /* Is dGPU Node, not APU - * Retrieve the marketing name of the node using pcilib, - * convert UTF8 to UTF16 + * Retrieve the marketing name of the node. */ - name = pci_ids_lookup(pacc, namebuf, sizeof(namebuf), - props->VendorId, props->DeviceId); - for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) - props->MarketingName[i] = name[i]; - props->MarketingName[i] = '\0'; + if (topology_get_marketing_name(props->DrmRenderMinor, + props->MarketingName) != 0) + pr_info("failed to get marketing name for device ID 0x%x\n", + props->DeviceId); } /* Get VGPR/SGPR size in byte per CU */ @@ -1791,7 +1819,6 @@ HSAKMT_STATUS topology_take_snapshot(void) HsaSystemProperties sys_props; node_props_t *temp_props = 0; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - struct pci_ids pacc; struct proc_cpuinfo *cpuinfo; const uint32_t num_procs = get_nprocs(); uint32_t num_ioLinks; @@ -1818,12 +1845,11 @@ retry: ret = HSAKMT_STATUS_NO_MEMORY; goto err; } - pacc = pci_ids_create(); for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_props[i].node, &temp_props[i].gpu_id, - pacc, &p2p_links, &num_p2pLinks); + &p2p_links, &num_p2pLinks); if (ret != HSAKMT_STATUS_SUCCESS) { free_properties(temp_props, i); goto err; @@ -1928,7 +1954,6 @@ retry: temp_props[i].node.NumIOLinks = link_id; } } - pci_ids_destroy(pacc); } if (!p2p_links) { From d91f92669b7a4c6f9cf0dc84d30800424ce6604a Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 29 Jun 2021 16:57:16 -0400 Subject: [PATCH 0912/1247] kfdtest: add KFDSVMRangeTest MigrateFileBackedRangeTest To test if SVM support address mmap on file backed memory. Change-Id: I227cc0b44d167c4fa2e63257e13a481aefa4f750 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index ecec01d4ac..6c7e3fd2af 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1197,3 +1197,63 @@ TEST_F(KFDSVMRangeTest, MultiThreadMigrationTest) { TEST_END } + +/* + * Test SVM support file backed range + * + * Create temp file, mmap to alloc memory backed on file. + * Create file backed svm range, to map to GPU for xnack on or off + * Use sdma to write data to memory, should write to file + * Close file, and then check if file data is updated correctly + */ +TEST_F(KFDSVMRangeTest, MigrateFileBackedRangeTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + char tmpfname[] = "/tmp/kfdtest-XXXXXX"; + int fd = mkostemp(tmpfname, 0600); + ASSERT_NE(-1, fd); + + size_t size = PAGE_SIZE; + char buf[size] = {0x30}; + + ASSERT_EQ(size, write(fd, buf, size)); + + void *MmapedFile = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + ASSERT_NE(nullptr, MmapedFile); + + HsaSVMRange filebackedRange(MmapedFile, size, defaultGPUNode, defaultGPUNode); + + SDMAQueue sdmaQueue; + EXPECT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(sdmaQueue.GetFamilyId(), + MmapedFile, 0x33333333, size)); + sdmaQueue.Wait4PacketConsumption(); + + EXPECT_SUCCESS(sdmaQueue.Destroy()); + munmap(MmapedFile, size); + EXPECT_SUCCESS(close(fd)); + + fd = open(tmpfname, O_RDONLY); + ASSERT_NE(-1, fd); + + ASSERT_EQ(size, read(fd, buf, size)); + EXPECT_EQ(0x33, buf[0]); + + EXPECT_SUCCESS(close(fd)); + EXPECT_SUCCESS(remove(tmpfname)); + + TEST_END +} From f9648c501464d32c7824441f4599d042fbced53a Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 17 Aug 2021 17:51:44 -0400 Subject: [PATCH 0913/1247] kfdtest: add KFDSVMRangeTest ReadOnlyRangeTest Map readonly range to GPU, test GPU can read range but write to range trigger GPU vm fault. Change-Id: Id2ce106055a353f21f4c34fa7f562d546523bc49 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 124 ++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 6c7e3fd2af..937849d6b3 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1257,3 +1257,127 @@ TEST_F(KFDSVMRangeTest, MigrateFileBackedRangeTest) { TEST_END } + +/* + * Test SVM support read only range + * + * Map read only range to GPU, test sdma can read the range + * write to range should trigger GPU vm fault for both xnack on and off + */ +TEST_F(KFDSVMRangeTest, ReadOnlyRangeTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + /* + * Use child process to run test because the test trigger GPU vm fault, KFD evict all user queues + * of the process and no more test can run after vm fault on the process. + */ + int pid = fork(); + if (pid == 0) { + if (hsaKmtOpenKFD() != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in child process" << std::endl; + exit(1); + } + } else { + int childStatus; + + waitpid(pid, &childStatus, 0); + if (is_dgpu()) { + EXPECT_EQ(true, WIFEXITED(childStatus)); + EXPECT_EQ(0, WEXITSTATUS(childStatus)); + } else { + EXPECT_EQ(true, WIFSIGNALED(childStatus)); + EXPECT_EQ(SIGSEGV, WTERMSIG(childStatus)); + } + + return; + } + + /* Use child process to run test */ + int ret = 0; + HsaSVMRange inBuffer(PAGE_SIZE * 2, defaultGPUNode); + HSAuint8 *pinBuf = inBuffer.As(); + + memset(pinBuf, 0x55, PAGE_SIZE); + + /* Map readonly pinBuf to GPU, sDMA should be able to read it */ + mprotect(pinBuf, PAGE_SIZE, PROT_READ); + + HsaSVMRange outputBuffer(PAGE_SIZE, defaultGPUNode); + HSAuint8 *pBuf = outputBuffer.As(); + + HsaEvent *vmFaultEvent; + HSAuint64 faultAddress; + HsaEventDescriptor eventDesc; + eventDesc.EventType = HSA_EVENTTYPE_MEMORY; + eventDesc.NodeId = defaultGPUNode; + eventDesc.SyncVar.SyncVar.UserData = NULL; + eventDesc.SyncVar.SyncVarSize = 0; + + ret = hsaKmtCreateEvent(&eventDesc, true, false, &vmFaultEvent); + if (ret != HSAKMT_STATUS_SUCCESS) { + WARN() << "Event create failed" << std::endl; + exit(ret); + } + + SDMAQueue sdmaQueue; + + ret = sdmaQueue.Create(defaultGPUNode); + if (ret != HSAKMT_STATUS_SUCCESS) { + WARN() << "Queue create failed" << std::endl; + goto queue_fail; + } + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pBuf, reinterpret_cast(pinBuf), PAGE_SIZE)); + sdmaQueue.Wait4PacketConsumption(); + EXPECT_EQ(0x55, pBuf[0]); + if (pBuf[0] != 0x55) + goto event_fail; + + /* sDMA write to readonly pinBuf should fail with GPU vm fault, check if pinBuf content is + * not changed, and KFD send HSA_EVENTTYPE_MEMORY event back with fault address pinBuf. + * + * This must be the last step of test because all queues are evicted after vm fault. + */ + + memset(pBuf, 0xAA, PAGE_SIZE); + sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), + pinBuf, reinterpret_cast(pBuf), PAGE_SIZE)); + + ret = hsaKmtWaitOnEvent(vmFaultEvent, g_TestTimeOut); + if (ret != HSAKMT_STATUS_SUCCESS) { + WARN() << "Wait failed. No Exception triggered" << std::endl; + goto event_fail; + } + if (vmFaultEvent->EventData.EventType != HSA_EVENTTYPE_MEMORY) { + WARN() << "Unexpected Event Received " << vmFaultEvent->EventData.EventType << std::endl; + ret = HSAKMT_STATUS_ERROR; + + goto event_fail; + } + faultAddress = vmFaultEvent->EventData.EventData.MemoryAccessFault.VirtualAddress; + if (faultAddress != (HSAuint64)pinBuf) { + WARN() << "Unexpected Fault Address " << faultAddress << std::endl; + ret = HSAKMT_STATUS_ERROR; + } + +event_fail: + EXPECT_SUCCESS(sdmaQueue.Destroy()); +queue_fail: + hsaKmtDestroyEvent(vmFaultEvent); + /* Child process exit, otherwise it will continue to run remaining tests */ + exit(ret); + + TEST_END +} From 445fcd803b37bed8c3ca7d97f81dccd49a1c088c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 10 Sep 2021 10:45:25 -0400 Subject: [PATCH 0914/1247] Revert "libhsakmt: Fix deadlocks in __fmm_release" This reverts commit 5fac7dcc3b7112f933ad07e90a57917b7ae6c1a5. Change-Id: Idf636655ac01f569d8e92e9c0e100e5db7d1cb61 Signed-off-by: Philip Yang --- src/fmm.c | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 618cea9bdf..420b6e0483 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -288,9 +288,6 @@ static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture); -static int _fmm_map_to_gpu(manageable_aperture_t *aperture, - void *address, uint64_t size, vm_object_t *obj, - uint32_t *nodes_to_map, uint32_t nodes_array_size); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address); @@ -1366,6 +1363,8 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, gpuid_to_nodeid(gpu_id, &vm_obj->node_id); } + pthread_mutex_unlock(&aperture->fmm_mutex); + if (mem) { int map_fd = gpu_mem[gpu_mem_id].drm_render_fd; int prot = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : @@ -1377,8 +1376,7 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - mem = NULL; - goto out_unlock; + return NULL; } /* * This madvise() call is needed to avoid additional references @@ -1389,9 +1387,6 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, madvise(mem, MemorySizeInBytes, MADV_DONTFORK); } -out_unlock: - pthread_mutex_unlock(&aperture->fmm_mutex); - return mem; } @@ -1433,6 +1428,8 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, gpuid_to_nodeid(gpu_id, &vm_obj->node_id); } + pthread_mutex_unlock(&aperture->fmm_mutex); + if (mem) { void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, @@ -1440,12 +1437,10 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, doorbell_mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - mem = NULL; + return NULL; } } - pthread_mutex_unlock(&aperture->fmm_mutex); - return mem; } @@ -1675,18 +1670,23 @@ static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) if (!object) return -EINVAL; + pthread_mutex_lock(&aperture->fmm_mutex); + /* If memory is user memory and it's still GPU mapped, munmap * would cause an eviction. If the restore happens quickly * enough, restore would also fail with an error message. So * free the BO before unmapping the pages. */ args.handle = object->handle; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) { + pthread_mutex_unlock(&aperture->fmm_mutex); return -errno; + } aperture_release_area(aperture, object->start, object->size); vm_remove_object(aperture, object); + pthread_mutex_unlock(&aperture->fmm_mutex); return 0; } @@ -1719,10 +1719,9 @@ HSAKMT_STATUS fmm_release(void *address) pthread_mutex_unlock(&aperture->fmm_mutex); munmap(address, size); } else { - int r = __fmm_release(object, aperture); - pthread_mutex_unlock(&aperture->fmm_mutex); - if (r) + + if (__fmm_release(object, aperture)) return HSAKMT_STATUS_ERROR; if (!aperture->is_cpu_accessible) @@ -2107,6 +2106,7 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) mflags.ui32.Reserved = 0; vm_obj->mflags = mflags; vm_obj->node_id = node_id; + pthread_mutex_unlock(&aperture->fmm_mutex); /* Map for CPU access*/ ret = mmap(mem, PAGE_SIZE, @@ -2115,14 +2115,14 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - mem = NULL; - /* Map for GPU access*/ - } else if (_fmm_map_to_gpu(aperture, mem, PAGE_SIZE, vm_obj, NULL, 0)) { - __fmm_release(vm_obj, aperture); - mem = NULL; + return NULL; } - pthread_mutex_unlock(&aperture->fmm_mutex); + /* Map for GPU access*/ + if (fmm_map_to_gpu(mem, PAGE_SIZE, NULL)) { + __fmm_release(vm_obj, aperture); + return NULL; + } return mem; } @@ -2905,8 +2905,13 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, free(object->mapped_node_id_array); object->mapped_node_id_array = NULL; - if (!ret) - ret = __fmm_release(object, aperture); + if (ret) + goto err; + + pthread_mutex_unlock(&aperture->fmm_mutex); + + /* free object in scratch backing aperture */ + return __fmm_release(object, aperture); err: pthread_mutex_unlock(&aperture->fmm_mutex); @@ -3414,8 +3419,8 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) * buffer. Deregistering imported graphics buffers or * userptrs means releasing the BO. */ - __fmm_release(object, aperture); pthread_mutex_unlock(&aperture->fmm_mutex); + __fmm_release(object, aperture); return HSAKMT_STATUS_SUCCESS; } From 91b0fce53563c52c8e0d217765c39c5bf0cb685e Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 10 Sep 2021 10:45:42 -0400 Subject: [PATCH 0915/1247] Revert "libhsakmt: Fix deadlock in map_mmio" This reverts commit 19536080a87aeb4c903ce766971461302d2ac677. Change-Id: Ibd32ac36bcb9ee418957d25b240ef9dd9172fbd3 Signed-off-by: Philip Yang --- src/fmm.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 420b6e0483..3b7f18af31 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2095,10 +2095,8 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) mem = __fmm_allocate_device(gpu_id, NULL, PAGE_SIZE, aperture, &mmap_offset, ioc_flags, &vm_obj); - if (!mem || !vm_obj) { - pthread_mutex_unlock(&aperture->fmm_mutex); + if (!mem || !vm_obj) return NULL; - } mflags.Value = 0; mflags.ui32.NonPaged = 1; From a809a5bf3287786a09b5b5ca939c4af008a6d533 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 10 Sep 2021 12:08:21 -0400 Subject: [PATCH 0916/1247] Revert "libhsakmt: fix multiple threads register userptr race" This reverts commit c4d5ee28f09c0471346cedc5f26d2c401f8d32f3. Change-Id: I954326d9a291280de7c9b7ef49a8cce0cd9dd9ad Signed-off-by: Philip Yang --- src/fmm.c | 88 +++++++++++++++++++++++-------------------------------- 1 file changed, 37 insertions(+), 51 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 3b7f18af31..ce71318647 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -958,10 +958,12 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, mflags = fmm_translate_ioc_to_hsa_flags(ioc_flags); /* Allocate object */ + pthread_mutex_lock(&aperture->fmm_mutex); vm_obj = aperture_allocate_object(aperture, mem, args.handle, MemorySizeInBytes, mflags); if (!vm_obj) goto err_object_allocation_failed; + pthread_mutex_unlock(&aperture->fmm_mutex); if (mmap_offset) *mmap_offset = args.mmap_offset; @@ -969,6 +971,7 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, return vm_obj; err_object_allocation_failed: + pthread_mutex_unlock(&aperture->fmm_mutex); free_args.handle = args.handle; kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &free_args); @@ -1292,7 +1295,9 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo return NULL; /* Allocate address space */ + pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, address, MemorySizeInBytes); + pthread_mutex_unlock(&aperture->fmm_mutex); /* * Now that we have the area reserved, allocate memory in the device @@ -1305,7 +1310,9 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo * allocation of memory in device failed. * Release region in aperture */ + pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, MemorySizeInBytes); + pthread_mutex_unlock(&aperture->fmm_mutex); /* Assign NULL to mem to indicate failure to calling function */ mem = NULL; @@ -1352,19 +1359,17 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, if (mflags.ui32.Uncached || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; - pthread_mutex_lock(&aperture->fmm_mutex); - mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); if (mem && vm_obj) { + pthread_mutex_lock(&aperture->fmm_mutex); /* Store memory allocation flags, not ioc flags */ vm_obj->mflags = mflags; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); + pthread_mutex_unlock(&aperture->fmm_mutex); } - pthread_mutex_unlock(&aperture->fmm_mutex); - if (mem) { int map_fd = gpu_mem[gpu_mem_id].drm_render_fd; int prot = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : @@ -1410,8 +1415,6 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - pthread_mutex_lock(&aperture->fmm_mutex); - mem = __fmm_allocate_device(gpu_id, NULL, MemorySizeInBytes, aperture, NULL, ioc_flags, &vm_obj); @@ -1424,12 +1427,12 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, mflags.ui32.HostAccess = 1; mflags.ui32.Reserved = 0xBe1; + pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->mflags = mflags; gpuid_to_nodeid(gpu_id, &vm_obj->node_id); + pthread_mutex_unlock(&aperture->fmm_mutex); } - pthread_mutex_unlock(&aperture->fmm_mutex); - if (mem) { void *ret = mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, @@ -1579,13 +1582,13 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, /* Paged memory is allocated as a userptr mapping, non-paged * memory is allocated from KFD */ - pthread_mutex_lock(&aperture->fmm_mutex); - if (!mflags.ui32.NonPaged && svm.userptr_for_paged_mem) { /* Allocate address space */ + pthread_mutex_lock(&aperture->fmm_mutex); mem = aperture_allocate_area(aperture, address, size); + pthread_mutex_unlock(&aperture->fmm_mutex); if (!mem) - goto out_unlock; + return NULL; /* Map anonymous pages */ if (mmap(mem, MemorySizeInBytes, PROT_READ | PROT_WRITE, @@ -1623,7 +1626,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); - goto out_unlock; + return NULL; } if (mflags.ui32.AQLQueueMemory) { @@ -1639,17 +1642,18 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, if (mem && vm_obj) { /* Store memory allocation flags, not ioc flags */ + pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->mflags = mflags; vm_obj->node_id = node_id; + pthread_mutex_unlock(&aperture->fmm_mutex); } - pthread_mutex_unlock(&aperture->fmm_mutex); return mem; out_release_area: /* Release address space */ + pthread_mutex_lock(&aperture->fmm_mutex); aperture_release_area(aperture, mem, size); -out_unlock: pthread_mutex_unlock(&aperture->fmm_mutex); return NULL; @@ -2089,9 +2093,6 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) ioc_flags = KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - - pthread_mutex_lock(&aperture->fmm_mutex); - mem = __fmm_allocate_device(gpu_id, NULL, PAGE_SIZE, aperture, &mmap_offset, ioc_flags, &vm_obj); @@ -2102,6 +2103,7 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) mflags.ui32.NonPaged = 1; mflags.ui32.HostAccess = 1; mflags.ui32.Reserved = 0; + pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->mflags = mflags; vm_obj->node_id = node_id; pthread_mutex_unlock(&aperture->fmm_mutex); @@ -2565,6 +2567,9 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, vm_object_t *object; int ret = 0; + if (!obj) + pthread_mutex_lock(&aperture->fmm_mutex); + object = obj; if (!object) { /* Find the object to retrieve the handle */ @@ -2627,6 +2632,9 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, exit_ok: err_object_not_found: err_map_failed: + if (!obj) + pthread_mutex_unlock(&aperture->fmm_mutex); + return ret; } @@ -2655,23 +2663,22 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert return -1; ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); - /* allocate object within the scratch backing aperture */ - pthread_mutex_lock(&aperture->fmm_mutex); - if (!ret && !is_debugger) { obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); if (!obj) - goto out_unlock; + return -1; /* Create a CPU mapping for the debugger */ map_fd = gpu_mem[gpu_mem_id].drm_render_fd; mmap_ret = mmap(address, size, PROT_NONE, MAP_PRIVATE | MAP_FIXED, map_fd, mmap_offset); - if (mmap_ret == MAP_FAILED) - goto out_release; + if (mmap_ret == MAP_FAILED) { + __fmm_release(obj, aperture); + return -1; + } } else { obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, &mmap_offset, @@ -2681,8 +2688,10 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert mmap_ret = mmap(address, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); - if (mmap_ret == MAP_FAILED) - goto out_release; + if (mmap_ret == MAP_FAILED) { + __fmm_release(obj, aperture); + return -1; + } } @@ -2691,14 +2700,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert if (ret != 0) __fmm_release(obj, aperture); - pthread_mutex_unlock(&aperture->fmm_mutex); return ret; - -out_release: - __fmm_release(obj, aperture); -out_unlock: - pthread_mutex_unlock(&aperture->fmm_mutex); - return -1; } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, @@ -3018,45 +3020,29 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, if (svm.check_userptr) fmm_check_user_memory(addr, size); - pthread_mutex_lock(&aperture->fmm_mutex); - - /* catch the race condition where some other thread added the userptr - * object already after the vm_find_object. - */ - obj = vm_find_object_by_userptr(aperture, addr, size); - if (obj) { - ++obj->registration_count; - goto out_found; - } - /* Allocate BO, userptr address is passed in mmap_offset */ svm_addr = __fmm_allocate_device(gpu_id, NULL, aligned_size, aperture, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), &obj); - if (!svm_addr) { - pthread_mutex_unlock(&aperture->fmm_mutex); + if (!svm_addr) return HSAKMT_STATUS_ERROR; - } if (obj) { + pthread_mutex_lock(&aperture->fmm_mutex); obj->userptr = addr; gpuid_to_nodeid(gpu_id, &obj->node_id); obj->userptr_size = size; obj->registration_count = 1; obj->user_node.key = rbtree_key((unsigned long)addr, size); rbtree_insert(&aperture->user_tree, &obj->user_node); - } else { pthread_mutex_unlock(&aperture->fmm_mutex); + } else return HSAKMT_STATUS_ERROR; - } -out_found: if (obj_ret) *obj_ret = obj; - - pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_SUCCESS; } From 8f7443494ad83119272745b228f854eb2bbc8993 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 10 Sep 2021 12:33:48 -0400 Subject: [PATCH 0917/1247] libhsakmt: fix multiple threads register userptr race v2 After calling ioctl to create userptr obj, take aperture lock, check if there is same userptr obj created after finding object, to catch the race that multiple threads register same userptr to multiple GPUs. If same userptr obj exist, then increase userptr registeration_count, and free the newly create obj. Change-Id: I63ae3a4f54da8aedd11c124d8d53ebe727b8203a Signed-off-by: Philip Yang --- src/fmm.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ce71318647..33a1075dc1 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3008,7 +3008,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); void *svm_addr; HSAuint32 gpu_id; - vm_object_t *obj; + vm_object_t *obj, *exist_obj; /* Find first GPU for creating the userptr BO */ if (!g_first_gpu_mem) @@ -3029,20 +3029,32 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, if (!svm_addr) return HSAKMT_STATUS_ERROR; - if (obj) { - pthread_mutex_lock(&aperture->fmm_mutex); + if (!obj) + return HSAKMT_STATUS_ERROR; + + pthread_mutex_lock(&aperture->fmm_mutex); + + /* catch the race condition where some other thread added the userptr + * object already after the vm_find_object. + */ + exist_obj = vm_find_object_by_userptr(aperture, addr, size); + if (exist_obj) { + ++exist_obj->registration_count; + } else { obj->userptr = addr; gpuid_to_nodeid(gpu_id, &obj->node_id); obj->userptr_size = size; obj->registration_count = 1; obj->user_node.key = rbtree_key((unsigned long)addr, size); rbtree_insert(&aperture->user_tree, &obj->user_node); - pthread_mutex_unlock(&aperture->fmm_mutex); - } else - return HSAKMT_STATUS_ERROR; + } + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (exist_obj) + __fmm_release(obj, aperture); if (obj_ret) - *obj_ret = obj; + *obj_ret = exist_obj ? exist_obj : obj; return HSAKMT_STATUS_SUCCESS; } From db1d6d92abec4f657ced2306f8d7d7982c1d1ec9 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Tue, 14 Sep 2021 16:19:46 +0800 Subject: [PATCH 0918/1247] libhsakmt: dump errno the queue destroy fails. to faciliate debugging, print errno when queue destroy fail current log give very little information when fail: [ RUN ] KFDQMTest.AllSdmaQueues [ ] Regular SDMA engines number: 1 SDMA queues per engine: 2 [ OK ] KFDQMTest.AllSdmaQueues (11 ms) [ RUN ] KFDQMTest.AllXgmiSdmaQueues [ ] XGMI SDMA engines number: 0 SDMA queues per engine: 2 [ OK ] KFDQMTest.AllXgmiSdmaQueues (6 ms) [ RUN ] KFDQMTest.AllQueues /home/foreman/build/hsakmt-roct-amdgpu-1.0.9.40500/sources/libhsakmt/tests/kfdtest/src/KFDQMTest.cpp:381: Failure Value of: (cpQueues[i].Destroy()) Actual: 1 Expected: HSAKMT_STATUS_SUCCESS Which is: 0 Signed-off-by: Yifan Zhang Change-Id: I5b1b5b616a5fd7ff198360c893a7aeed685022bd --- src/queues.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 1615aa9d93..51fc0bebd9 100644 --- a/src/queues.c +++ b/src/queues.c @@ -637,8 +637,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDestroyQueue(HSA_QUEUEID QueueId) int err = kmtIoctl(kfd_fd, AMDKFD_IOC_DESTROY_QUEUE, &args); - if (err == -1) + if (err == -1) { + pr_err("Failed to destroy queue: %s\n", strerror(errno)); return HSAKMT_STATUS_ERROR; + } free_queue(q); return HSAKMT_STATUS_SUCCESS; From f2f86591b66d5d5e71c663f5a63308df62b39b43 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 15 Sep 2021 11:34:05 -0400 Subject: [PATCH 0919/1247] kfdtest.exclude: Remove KFDSVMEvict from GFX10 This was never validated on GFX10, so remove it for now until it's either validated and confirmed, or move the check to the test itself Change-Id: Ie4d8b31885fbe6e5ed84b7b174c0bfed60879741 --- tests/kfdtest/scripts/kfdtest.exclude | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 917df5d15d..cdce90b9c6 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -222,6 +222,7 @@ FILTER[aldebaran]=\ # SP3 Compiler needs to be updated for GFX10. Temporarily disable all tests # that require shader compiler +# Adding KFDSVMEvictTest as SVM/HMM was never validated on GFX10 TEMP_GFX10_BLACKLIST=\ "KFDMemoryTest.FlatScratchAccess:"\ "KFDMemoryTest.PtraceAccessInvisibleVram:"\ @@ -232,7 +233,8 @@ TEMP_GFX10_BLACKLIST=\ "KFDEvictTest.QueueTest:"\ "KFDMemoryTest.MapUnmapToNodes:"\ "KFDMemoryTest.HostHdpFlush:"\ -"KFDMemoryTest.DeviceHdpFlush" +"KFDMemoryTest.DeviceHdpFlush:"\ +"KFDSVMEvictTest.*" FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From 88e6921d45b8bd472271fce59d614d7babec3696 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 16 Sep 2021 10:18:56 -0400 Subject: [PATCH 0920/1247] kfdtest: MigrateFileBackedRangeTest address sanitizer builds failing Use alloca to replace variable size array. Change-Id: I81e5a7d8f6ace203b81f5f5a651b2a6aa948821c Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 937849d6b3..faf0cd89fa 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1226,7 +1226,8 @@ TEST_F(KFDSVMRangeTest, MigrateFileBackedRangeTest) { ASSERT_NE(-1, fd); size_t size = PAGE_SIZE; - char buf[size] = {0x30}; + char *buf = reinterpret_cast(alloca(size)); + memset(buf, 0x30, size); ASSERT_EQ(size, write(fd, buf, size)); From 4144a0d2a49ef8c1d78d7e5ff36fd4eb0b6cf9e5 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 17 Sep 2021 09:50:32 -0400 Subject: [PATCH 0921/1247] CMakeLists: Make libdrm-amdgpu1 a suggestion This is required for Marketing Name, but Marketing Name isn't a hard requirement for ROCT, so make it a Suggested package Change-Id: Ibafcce2c59dc8bdba90c171e766122bebf548a48 Signed-off-by: Kent Russell --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 077ab51c36..5b2806b419 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,9 +350,15 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") +set(CPACK_DEBIAN_PACKAGE_SUGGESTS "libdrm-amdgpu-amdgpu1") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") +# NOTE: CentOS/RHEL 7 have no concept of "Suggests" so this will just +# fail to install libdrm-amdgpu on those OSes. +# If this package is required, either manually install the package +# or change the field below to REQUIRES +set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From 06b1f25172c5508deeb08c87cfa68eddb1419069 Mon Sep 17 00:00:00 2001 From: Aakash Sudhanwa Date: Fri, 24 Sep 2021 16:35:53 -0400 Subject: [PATCH 0922/1247] Revert "CMakeLists: Make libdrm-amdgpu1 a suggestion" This reverts commit 4144a0d2a49ef8c1d78d7e5ff36fd4eb0b6cf9e5. Reason for revert: Breaking on Centos7 Change-Id: Ibd5a7c5d4e8c07defec45821ae1faa525cfc916d --- CMakeLists.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b2806b419..077ab51c36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,15 +350,9 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") -set(CPACK_DEBIAN_PACKAGE_SUGGESTS "libdrm-amdgpu-amdgpu1") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") -# NOTE: CentOS/RHEL 7 have no concept of "Suggests" so this will just -# fail to install libdrm-amdgpu on those OSes. -# If this package is required, either manually install the package -# or change the field below to REQUIRES -set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From 04b149b7fde1c38be30da331807bd534bff6d986 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Wed, 22 Sep 2021 15:01:14 +0800 Subject: [PATCH 0923/1247] kfdtest.exclude: blacklist several tests case for gfx902 SDMA firmware of gfx902 has a regression which causes cp hang in kfdtest, blacklist related test cases. Signed-off-by: Yifan Zhang Change-Id: Ie75b72a952a29d6b1394c5fcb67ff9a5143b3b07 --- tests/kfdtest/scripts/kfdtest.exclude | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index cdce90b9c6..bf4cc8fff6 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -85,6 +85,7 @@ SDMA_BLACKLIST=\ "KFDMemoryTest.MemoryRegister:"\ "KFDMemoryTest.MMBench:"\ "KFDMemoryTest.SignalHandling:"\ +"KFDQMTest.AllQueues:"\ "KFDQMTest.*Sdma*:"\ "KFDQMTest.CreateQueueStressSingleThreaded:"\ "KFDQMTest.GPUDoorbellWrite:"\ @@ -177,8 +178,9 @@ FILTER[vega20]=\ FILTER[raven_dgpuFallback]=\ "$BLACKLIST_ALL_ASICS:"\ +"$SDMA_BLACKLIST:"\ +"KFDEvictTest.*:"\ "KFDMemoryTest.MemoryRegister:"\ -"KFDQMTest.SdmaConcurrentCopies:"\ "KFDSVMRangeTest.BasicSystemMemTest:"\ "KFDSVMRangeTest.BasicVramTest:"\ "KFDSVMRangeTest.EvictSystemRangeTest:"\ @@ -193,11 +195,13 @@ FILTER[raven_dgpuFallback]=\ FILTER[raven]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.SdmaConcurrentCopies:"\ +"$SDMA_BLACKLIST:"\ +"KFDEvictTest.*:"\ "KFDSVMRangeTest.EvictSystemRangeTest:"\ "KFDSVMRangeTest.PartialUnmapSysMemTest:"\ "KFDSVMRangeTest.PrefetchTest:"\ "KFDSVMRangeTest.MultiThreadMigrationTest:"\ +"KFDSVMEvictTest.QueueTest:"\ "KFDQMTest.MultipleCpQueuesStressDispatch" FILTER[renoir]=\ From d5d8c022f1878654ec7c856ef00e8478ccb0d84b Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 20 Jul 2021 14:36:15 -0400 Subject: [PATCH 0924/1247] kfdtest: Adjust KFDEvictTest memory size We were hitting memory map errors and segfaults when trying to use 7/8 VRAM on certain cards (dmesg showing "Failed to map to gpu 0/1"), as the original check didn't see if this would exceed the GTT size. Allocate the smaller of 1/3 of SRAM or 7/16 of VRAM (7/8 / 2) for the tests Signed-off-by: Kent Russell Change-Id: Ic11a5cee058535418eef903a28846e00e1839969 --- tests/kfdtest/src/KFDEvictTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 126541abcf..a9544eb47c 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -478,7 +478,7 @@ TEST_F(KFDEvictTest, BasicTest) { AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); /* Allocate gfx vram size of at most one third system memory */ - HSAuint64 size = sysMemSize / 3 < testSize ? sysMemSize / 3 : testSize; + HSAuint64 size = sysMemSize / 3 < testSize / 2 ? sysMemSize / 3 : testSize / 2; amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); @@ -576,7 +576,7 @@ TEST_F(KFDEvictTest, QueueTest) { AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); /* Allocate gfx vram size of at most one third system memory */ - HSAuint64 size = sysMemSize / 3 < testSize ? sysMemSize / 3 : testSize; + HSAuint64 size = sysMemSize / 3 < testSize / 2 ? sysMemSize / 3 : testSize / 2; amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); @@ -675,7 +675,7 @@ TEST_F(KFDEvictTest, BurstyTest) { AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); /* Allocate gfx vram size of at most one third system memory */ - HSAuint64 size = sysMemSize / 3 < testSize ? sysMemSize / 3 : testSize; + HSAuint64 size = sysMemSize / 3 < testSize / 2 ? sysMemSize / 3 : testSize / 2; amdgpu_bo_handle handle; AllocAmdgpuBo(rn, size, handle); From 392e0e2afcc037b0135b8df9c7aca9ddf8ff55c0 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 20 Sep 2021 09:23:21 -0400 Subject: [PATCH 0925/1247] kfdtest.exclude: Re-enable KFDEvictTests that were blacklisted After adjusting the memory usage, re-enable KFDEvictTest.BasicTest on gfx906 and KFDEvictTest.QueueTest on GFX10 Change-Id: I401e679e447f3150241078154635f0b30692513d Signed-off-by: Kent Russell --- tests/kfdtest/scripts/kfdtest.exclude | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index bf4cc8fff6..51850f69fa 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -173,7 +173,6 @@ FILTER[vega12]=\ FILTER[vega20]=\ "$BLACKLIST_ALL_ASICS:"\ "$SDMA_BLACKLIST:"\ -"KFDEvictTest.BasicTest:"\ "KFDQMTest.GPUDoorbellWrite" FILTER[raven_dgpuFallback]=\ @@ -234,7 +233,6 @@ TEMP_GFX10_BLACKLIST=\ "KFDQMTest.QueuePriorityOnSamePipe:"\ "KFDCWSRTest.BasicTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDEvictTest.QueueTest:"\ "KFDMemoryTest.MapUnmapToNodes:"\ "KFDMemoryTest.HostHdpFlush:"\ "KFDMemoryTest.DeviceHdpFlush:"\ From 5adb142dc7085b2f5a5b187e8d8ff41971c66379 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 29 Sep 2021 11:26:29 -0400 Subject: [PATCH 0926/1247] kfdtest.exclude: KFDSVMRangeTest.ReadOnlyRangeTest to blacklist This test requires KFD patch "drm/amdkfd: SVM map to gpus check vma boundary" to pass, the patch is on staging-dkms branch, not land on mainline branch. Temporary blacklist this to unblock QA, as QA reports kfdtest failure. Change-Id: I00515cd5d5d1c5612f4f8d48605d86f4a7e62ce2 Signed-off-by: Philip Yang --- tests/kfdtest/scripts/kfdtest.exclude | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 51850f69fa..2418998301 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -71,7 +71,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDMemoryTest.CacheInvalidateOnRemoteWrite:"\ "KFDDBGTest.BasicDebuggerSuspendResume:"\ "KFDEvictTest.BurstyTest:"\ -"KFDHWSTest.*" +"KFDHWSTest.*:"\ +"KFDSVMRangeTest.ReadOnlyRangeTest" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From f82927ad655c89d4f7f9942f2669ab3e1c3e8539 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 10 Sep 2021 12:58:13 -0400 Subject: [PATCH 0927/1247] libhsakmt: register and deregister userptr in parallel For userptr, after taking aperture lock, decrease registration_count and ensure object registration_count equal to 0 to release KFD and thunk object. Move decrementing of registration count from fmm_deregister_memory into __fmm_release to avoid a race condition when dropping the aperture lock in fmm_deregister_memory. Change-Id: I5381fa6b8a77a1516af2554e5174e91969c338c4 Signed-off-by: Philip Yang --- src/fmm.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 33a1075dc1..6154606786 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1676,6 +1676,14 @@ static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) pthread_mutex_lock(&aperture->fmm_mutex); + if (object->userptr) { + object->registration_count--; + if (object->registration_count > 0) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return 0; + } + } + /* If memory is user memory and it's still GPU mapped, munmap * would cause an eviction. If the restore happens quickly * enough, restore would also fail with an error message. So @@ -3404,12 +3412,6 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) return HSAKMT_STATUS_SUCCESS; } - if (object->registration_count > 1) { - --object->registration_count; - pthread_mutex_unlock(&aperture->fmm_mutex); - return HSAKMT_STATUS_SUCCESS; - } - if (object->metadata || object->userptr || object->is_imported_kfd_bo) { /* An object with metadata is an imported graphics * buffer. Deregistering imported graphics buffers or From b67bc6fcc901ab3a2e18d23c40ea97986e93dc71 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Thu, 30 Sep 2021 03:08:06 -0500 Subject: [PATCH 0928/1247] Check errno rather than return from kmtIoctl in hsaKmtRuntimeEnable. The return code is just -1 if any error occurs. To detect debugger unavailable we need to check the actual ioctl error code. Change-Id: I8a294c754196aec916809497ec8e810da2f072b8 Signed-off-by: Sean Keely --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index c2f0546b90..d2879e6a60 100644 --- a/src/debug.c +++ b/src/debug.c @@ -324,7 +324,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); if (err) { - if (err == EBUSY) + if (errno == EBUSY) return HSAKMT_STATUS_UNAVAILABLE; else return HSAKMT_STATUS_ERROR; From 5e3983cb938f7e44f4bdd24300f4d9b736997146 Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Sat, 2 Oct 2021 02:06:29 -0500 Subject: [PATCH 0929/1247] Check errno for error type in hsaKmtAllocQueueGWS. Check errno rather than err and drop negation. Change-Id: I1804d67292c3b054eab8fc077c8e9b22a8cdcc9f --- src/queues.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/queues.c b/src/queues.c index 51fc0bebd9..6a729b63ce 100644 --- a/src/queues.c +++ b/src/queues.c @@ -776,11 +776,11 @@ hsaKmtAllocQueueGWS( if (!err) return HSAKMT_STATUS_SUCCESS; - else if (err == -EINVAL) + else if (errno == EINVAL) return HSAKMT_STATUS_INVALID_PARAMETER; - else if (err == -EBUSY) + else if (errno == EBUSY) return HSAKMT_STATUS_OUT_OF_RESOURCES; - else if (err == -ENODEV) + else if (errno == ENODEV) return HSAKMT_STATUS_NOT_SUPPORTED; else return HSAKMT_STATUS_ERROR; From 5a8c9a54f167bb9b3150b5d5b9804487cc955540 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 8 Oct 2021 13:09:01 -0400 Subject: [PATCH 0930/1247] kfdtest.exclude: Add KFDEvictTest.Queuetest back to gfx1030's blacklist The shader is incompatible with gfx1030, so blacklist it there until we get to some more ASIC-agnostic shader generation Signed-off-by: Kent Russell Change-Id: I71f362a80e6e8f87ecece6afa6b34a5f02bc56bf --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 2418998301..2f7aac28c7 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -262,6 +262,7 @@ FILTER[navi14]=\ FILTER[sienna_cichlid]=\ "$BLACKLIST_ALL_ASICS:"\ "$TEMP_GFX10_BLACKLIST:"\ +"KFDEvictTest.QueueTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ From f7e90798a08f71c97fd1ede6a7d9298580a3c955 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 17 Sep 2021 09:50:32 -0400 Subject: [PATCH 0931/1247] CMakeLists: Refer to libdrm-amdgpu1 for DEB This is required for Marketing Name, but Marketing Name isn't a hard requirement for ROCT, so make it a Suggested package for DEBs. Adding a commented-out possible solution for RPM-based distros, as unfortunately SLE <10 and CentOS/RHEL <8 don't know what "SUGGESTS" means. Potentially we will add a check that the build environment that determines if we require the REQUIRES field to be used. Hopefully we can work out the RPM side quickly, but for now enable SUGGESTS on DEB as a start Signed-off-by: Kent Russell Change-Id: Icea689a14c185964e7f47fb23ee432f9b35ccb03 --- CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 077ab51c36..db28804314 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -350,10 +350,23 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") +set(CPACK_DEBIAN_PACKAGE_SUGGESTS "libdrm-amdgpu-amdgpu1") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") +# TODO: Implement this effectively +# NOTE: CentOS/RHEL <8 and SUSE <10 have no concept of "Suggests" +# so a "suggests" field will just cause rpm installation to fail +# If the HSAKMT_REQUIRES_LIBDRM variable is true in the build +# environment, we use REQUIRES. Otherwise we use SUGGESTS +# For the distros above, this variable should be set to true +#if(DEFINED HSAKMT_REQUIRES_LIBDRM) +# string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu" ) +#else() +# set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu" ) +#endif() + # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") From 28013420481d5bcd7e37fe45523b4a55b9e8b94f Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 19 Oct 2021 08:08:15 -0400 Subject: [PATCH 0932/1247] kfdtest.exclude: Remove KFDEvictTest.QueueTest on GFX10 Since it's shader-based, the GFX9 shader won't work for GFX10. Blacklist it for now until we move to a more ASIC-agnostic shader Signed-off-by: Kent Russell Change-Id: I353e9a3fe8c7ba6f33556e39c7e4f6b36e010480 --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 2f7aac28c7..392dba3cde 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -234,6 +234,7 @@ TEMP_GFX10_BLACKLIST=\ "KFDQMTest.QueuePriorityOnSamePipe:"\ "KFDCWSRTest.BasicTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ +"KFDEvictTest.QueueTest:"\ "KFDMemoryTest.MapUnmapToNodes:"\ "KFDMemoryTest.HostHdpFlush:"\ "KFDMemoryTest.DeviceHdpFlush:"\ @@ -262,7 +263,6 @@ FILTER[navi14]=\ FILTER[sienna_cichlid]=\ "$BLACKLIST_ALL_ASICS:"\ "$TEMP_GFX10_BLACKLIST:"\ -"KFDEvictTest.QueueTest:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ From 489db9fac6cb2a7c81af58c0062eb2742642e776 Mon Sep 17 00:00:00 2001 From: Philip Cox Date: Mon, 20 Sep 2021 13:14:21 -0400 Subject: [PATCH 0933/1247] libhsakmt: Add more debug information We need to add some more information about the debug features supported by the platform. We are adding the following: - debug supported - dispatch info always valid - precise memop supported - watchpoints shared Change-Id: I68deed98619396d17e28c6e18bad424b58297485 Signed-off-by: Philip Cox --- include/hsakmttypes.h | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 7cfca1082a..9063f851bd 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -216,7 +216,8 @@ typedef union unsigned int SRAM_EDCSupport: 1; // Indicates if GFX internal SRAM EDC/ECC functionality is active unsigned int SVMAPISupported : 1; // Whether or not the SVM API is supported unsigned int CoherentHostAccess: 1; // Whether or not device memory can be coherently accessed by the host CPU - unsigned int Reserved : 3; + unsigned int DebugSupportedFirmware : 1; + unsigned int Reserved : 2; } ui32; } HSA_CAPABILITY; @@ -232,9 +233,20 @@ typedef union // of the HSAuint64 WatchAddrMaskHiBit: 6; // watch address mask are used. // 0 is the least significant bit. - HSAuint64 TrapDataCount: 4; // Number of 32 bit TrapData - // registers supported. - HSAuint64 Reserved: 50; // + HSAuint64 DispatchInfoAlwaysValid: 1; // 0 if control of TTMP setup is + // controlled on a per process + // basis and is not always enabled + // 1 if TTMP setup is always + // enabled + HSAuint64 AddressWatchpointShareKind: 1; // whether the address watchpoint + // is per process or shared with + // all proccesses + // 0 if shared or unsuppoted + // (unsupported indicated by + // address_watchpoint_count == 0) + // All current devices have shared watchpoints + // 1 if unshared + HSAuint64 Reserved: 52; // }; } HSA_DEBUG_PROPERTIES; From 7e90eef254d38c7c9d0ab5bbc1b448563cff27f6 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 29 Oct 2021 08:09:21 -0400 Subject: [PATCH 0934/1247] CMakeLists: Refer to libdrm-amdgpu1 for RPM distros The infra is in place to make sure that we can get around the issue where older versions of rpm don't know what "Suggests" is. Using the HSAKMT_REQUIRES_LIBDRM variable will force the package to require libdrm-amdgpu1 instead of just suggesting it in newer distros. Change-Id: I5d595c4e4be5ad8e8c411dcec3a58365e1984f05 --- CMakeLists.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index db28804314..6f6bdfd68c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,17 +355,16 @@ set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") -# TODO: Implement this effectively # NOTE: CentOS/RHEL <8 and SUSE <10 have no concept of "Suggests" # so a "suggests" field will just cause rpm installation to fail # If the HSAKMT_REQUIRES_LIBDRM variable is true in the build # environment, we use REQUIRES. Otherwise we use SUGGESTS # For the distros above, this variable should be set to true -#if(DEFINED HSAKMT_REQUIRES_LIBDRM) -# string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu" ) -#else() -# set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu" ) -#endif() +if(DEFINED HSAKMT_REQUIRES_LIBDRM) + string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu" ) +else() + set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu" ) +endif() # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From fa2309fcdd5b5dacaddbc3407f1b7e3093e05b17 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Wed, 10 Nov 2021 13:49:05 -0500 Subject: [PATCH 0935/1247] Set License field for RPM package This really should be set to conform to distro standards. Change-Id: I63442b9ad58f70c4691c499b666f7c359dc9609d Signed-off-by: Jeremy Newton --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f6bdfd68c..b63ff21e18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -324,6 +324,7 @@ if(DEFINED CPACK_PACKAGING_INSTALL_PREFIX) set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "${CPACK_PACKAGING_INSTALL_PREFIX} ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}") endif() set(CPACK_RPM_PACKAGE_NAME "hsakmt-roct-devel") +set(CPACK_RPM_PACKAGE_LICENSE "MIT") # Since we changed the package name to match RPM specs, take care of older builds that had -dev installed # Also cover the fact that this now replaces the old binary package hsakmt-roct set(CPACK_RPM_PACKAGE_OBSOLETES "hsakmt-roct-dev hsakmt-roct") From 9529bf7063cb83ec81f60e7232242358deadeb34 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 10 Nov 2021 12:33:05 -0500 Subject: [PATCH 0936/1247] CMakeLists: Fix libdrm-amdgpu dependencies SUGGESTS doesn't get installed by default by dpkg, it requires --install-suggests. While the libdrm-amdgpu-amdgpu1 lib might not be critical to core thunk usage, it's necessary enough to increase the dependency from weak to strong, thus we use RECOMMENDS instead Since CPack doesn't support RECOMMENDS for RPMs, and SUGGESTS isn't strong enough, we just set it as REQUIRED. This should handle any standard installation situations instead of trying to be fancy to try to make things exactly the way we want them to be. Signed-off-by: Kent Russell Change-Id: I5a676fb4a3fbf5c7b35f503acc30cf01c056266d --- CMakeLists.txt | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b63ff21e18..764d9c00e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -351,21 +351,13 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") -set(CPACK_DEBIAN_PACKAGE_SUGGESTS "libdrm-amdgpu-amdgpu1") +set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-amdgpu1") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") -set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") - -# NOTE: CentOS/RHEL <8 and SUSE <10 have no concept of "Suggests" -# so a "suggests" field will just cause rpm installation to fail -# If the HSAKMT_REQUIRES_LIBDRM variable is true in the build -# environment, we use REQUIRES. Otherwise we use SUGGESTS -# For the distros above, this variable should be set to true -if(DEFINED HSAKMT_REQUIRES_LIBDRM) - string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu" ) -else() - set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu" ) -endif() +# Certain versions of rpm can't handle SUGGESTS or RECOMMENDS +# Also, CPACK_RPM_PACKAGE_RECOMMENDS isn't supported by +# CPack, so just set libdrm-amdgpu as a requirement for RPMs +set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core, libdrm-amdgpu") # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From e842d7f4806a1bb00744874dce130cd691ce44f8 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 12 Nov 2021 14:30:04 -0500 Subject: [PATCH 0937/1247] Revert "CMakeLists: Fix libdrm-amdgpu dependencies" This reverts commit 9529bf7063cb83ec81f60e7232242358deadeb34. Reason for revert: Infra still not ready for it yet Change-Id: I03e043c1ca7924264e3e70e3e82c73b4efc2ae75 --- CMakeLists.txt | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 764d9c00e9..b63ff21e18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -351,13 +351,21 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") -set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-amdgpu1") +set(CPACK_DEBIAN_PACKAGE_SUGGESTS "libdrm-amdgpu-amdgpu1") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") -# Certain versions of rpm can't handle SUGGESTS or RECOMMENDS -# Also, CPACK_RPM_PACKAGE_RECOMMENDS isn't supported by -# CPack, so just set libdrm-amdgpu as a requirement for RPMs -set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core, libdrm-amdgpu") +set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") + +# NOTE: CentOS/RHEL <8 and SUSE <10 have no concept of "Suggests" +# so a "suggests" field will just cause rpm installation to fail +# If the HSAKMT_REQUIRES_LIBDRM variable is true in the build +# environment, we use REQUIRES. Otherwise we use SUGGESTS +# For the distros above, this variable should be set to true +if(DEFINED HSAKMT_REQUIRES_LIBDRM) + string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu" ) +else() + set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu" ) +endif() # Set the names now using CPACK utility set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") From 86c27a7af819fa354df9183897ae306261721bf5 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Tue, 16 Nov 2021 11:01:48 -0500 Subject: [PATCH 0938/1247] Use recommends for libdrm-amdgpu-amdgpu1 For the use of libdrm-amdgpu-amdgpu1 and libdrm-amdgpu, we should use recommends, as we want these packages installed with a strong dependency but avoid a strict dependency, since this is enhancement feature. Using the newer libdrm, which is build for amdgpu-dkms, is ideal since it will produce more correct marketing names, but should not be mandated due to two reasons: - A user may not want to install both libdrms on their system - The system might not have the newer libdrm available This patch only fixes the Ubuntu/debian package since recommends is not properly implemented for the RPM generator for CPACK. For now, "suggests" will have to do, since it's the closest option we have. I will investigate if we can get around this issue. Change-Id: I33a90c3ead235bbbe265238c026933688ea63fe3 Signed-off-by: Jeremy Newton --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b63ff21e18..b656de7b57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -351,7 +351,7 @@ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Setting devel package dependendent version set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") -set(CPACK_DEBIAN_PACKAGE_SUGGESTS "libdrm-amdgpu-amdgpu1") +set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-amdgpu1") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") From 3c0e4fee0f87deed00b4add2e88e715b21629167 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Tue, 16 Nov 2021 12:48:58 -0500 Subject: [PATCH 0939/1247] Implement RPM recommends for libdrm-amdgpu CPack does not support recommends for RPM generation, so I've generated a template RPM SPEC files in order to make modifications to allow for support of recommends. The spec.in file was generated using the cpack option "CPACK_RPM_GENERATE_USER_BINARY_SPECFILE_TEMPLATE" and was modified very sparingly to avoid any maintanance burden, e.g. can be easily regenerated. The CPACK_RPM_USER_BINARY_SPECFILE is then used to specify the customized template file, instead of using the cmake's template. From what I understand, the point of these two options is to allow developers to tailor the specfile to their desire, since rpm spec files are much more advanced then the equilivent debian file. Change-Id: I80c69be58a3c57729ed997fd2ce01f5d16b9e9b9 Signed-off-by: Jeremy Newton --- CMakeLists.txt | 16 +++--- RPM/hsakmt-roct-devel.spec.in | 97 +++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 RPM/hsakmt-roct-devel.spec.in diff --git a/CMakeLists.txt b/CMakeLists.txt index b656de7b57..6d48d98b8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -333,6 +333,10 @@ if(DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE}) else() set(CPACK_RPM_PACKAGE_RELEASE "local") endif() +## Specify a RPM SPEC template +# This template was generated by using "CPACK_RPM_GENERATE_USER_BINARY_SPECFILE_TEMPLATE" +# but I've modified it to support CPACK_RPM_PACKAGE_RECOMMENDS for EL8+ and SLE +set ( CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_BINARY_DIR}/RPM/${CPACK_RPM_PACKAGE_NAME}.spec.in" ) # 'dist' breaks manual builds on debian systems due to empty Provides execute_process( COMMAND rpm --eval %{?dist} @@ -356,15 +360,13 @@ set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") -# NOTE: CentOS/RHEL <8 and SUSE <10 have no concept of "Suggests" -# so a "suggests" field will just cause rpm installation to fail -# If the HSAKMT_REQUIRES_LIBDRM variable is true in the build -# environment, we use REQUIRES. Otherwise we use SUGGESTS -# For the distros above, this variable should be set to true -if(DEFINED HSAKMT_REQUIRES_LIBDRM) +# NOTE: CentOS/RHEL <8 and SUSE <10 do not have support for rpm recommends, +# so HSAKMT_REQUIRES_LIBDRM should be defined when building EL7 to avoid +# producing binaries which do not install on CentOS7 or RHEL7 +if(HSAKMT_REQUIRES_LIBDRM) string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu" ) else() - set(CPACK_RPM_PACKAGE_SUGGESTS "libdrm-amdgpu" ) + set(CPACK_RPM_PACKAGE_RECOMMENDS "libdrm-amdgpu" ) endif() # Set the names now using CPACK utility diff --git a/RPM/hsakmt-roct-devel.spec.in b/RPM/hsakmt-roct-devel.spec.in new file mode 100644 index 0000000000..cb8b7e39b9 --- /dev/null +++ b/RPM/hsakmt-roct-devel.spec.in @@ -0,0 +1,97 @@ +# Restore old style debuginfo creation for rpm >= 4.14. +%undefine _debugsource_packages +%undefine _debuginfo_subpackages + +# -*- rpm-spec -*- +BuildRoot: %_topdir/@CPACK_PACKAGE_FILE_NAME@@CPACK_RPM_PACKAGE_COMPONENT_PART_PATH@ +Summary: @CPACK_RPM_PACKAGE_SUMMARY@ +Name: @CPACK_RPM_PACKAGE_NAME@ +Version: @CPACK_RPM_PACKAGE_VERSION@ +Release: @CPACK_RPM_PACKAGE_RELEASE@ +License: @CPACK_RPM_PACKAGE_LICENSE@ +Group: @CPACK_RPM_PACKAGE_GROUP@ +Vendor: @CPACK_RPM_PACKAGE_VENDOR@ + +@TMP_RPM_URL@ +@TMP_RPM_REQUIRES@ +@TMP_RPM_REQUIRES_PRE@ +@TMP_RPM_REQUIRES_POST@ +@TMP_RPM_REQUIRES_PREUN@ +@TMP_RPM_REQUIRES_POSTUN@ +@TMP_RPM_PROVIDES@ +@TMP_RPM_OBSOLETES@ +@TMP_RPM_CONFLICTS@ +@TMP_RPM_SUGGESTS@ +@TMP_RPM_AUTOPROV@ +@TMP_RPM_AUTOREQ@ +@TMP_RPM_AUTOREQPROV@ +@TMP_RPM_BUILDARCH@ +@TMP_RPM_PREFIXES@ +@TMP_RPM_EPOCH@ + +# Modifications to allow recommends to be used (not implemented in cpack): +%if "@CPACK_RPM_PACKAGE_RECOMMENDS@" != "" +Recommends: @CPACK_RPM_PACKAGE_RECOMMENDS@ +%endif +# End of modifications + +@TMP_RPM_DEBUGINFO@ + +%define _rpmdir %_topdir/RPMS +%define _srcrpmdir %_topdir/SRPMS +@FILE_NAME_DEFINE@ +%define _unpackaged_files_terminate_build 0 +@TMP_RPM_SPEC_INSTALL_POST@ +@CPACK_RPM_SPEC_MORE_DEFINE@ +@CPACK_RPM_COMPRESSION_TYPE_TMP@ + +%description +@CPACK_RPM_PACKAGE_DESCRIPTION@ + +# This is a shortcutted spec file generated by CMake RPM generator +# we skip _install step because CPack does that for us. +# We do only save CPack installed tree in _prepr +# and then restore it in build. +%prep +mv $RPM_BUILD_ROOT %_topdir/tmpBBroot + +%install +if [ -e $RPM_BUILD_ROOT ]; +then + rm -rf $RPM_BUILD_ROOT +fi +mv %_topdir/tmpBBroot $RPM_BUILD_ROOT + +@TMP_RPM_DEBUGINFO_INSTALL@ + +%clean + +%post +@RPM_SYMLINK_POSTINSTALL@ +@CPACK_RPM_SPEC_POSTINSTALL@ + +%posttrans +@CPACK_RPM_SPEC_POSTTRANS@ + +%postun +@CPACK_RPM_SPEC_POSTUNINSTALL@ + +%pre +@CPACK_RPM_SPEC_PREINSTALL@ + +%pretrans +@CPACK_RPM_SPEC_PRETRANS@ + +%preun +@CPACK_RPM_SPEC_PREUNINSTALL@ + +%files +%defattr(@TMP_DEFAULT_FILE_PERMISSIONS@,@TMP_DEFAULT_USER@,@TMP_DEFAULT_GROUP@,@TMP_DEFAULT_DIR_PERMISSIONS@) +@CPACK_RPM_INSTALL_FILES@ +@CPACK_RPM_ABSOLUTE_INSTALL_FILES@ +@CPACK_RPM_USER_INSTALL_FILES@ + +%changelog +@CPACK_RPM_SPEC_CHANGELOG@ + +@TMP_OTHER_COMPONENTS@ From 529c96c08b6c7ff459f41833e9788cc34a6ce8ac Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Wed, 17 Nov 2021 17:44:36 -0500 Subject: [PATCH 0940/1247] Fix to previous commit I used the binary directory instead of the source directory to specify the spec.in path, which passed local testing since these directories are in the same location. This is not guarenteed to be true. Change-Id: I1b49ca8453b9c074a947104c26fb39667d728a8e Signed-off-by: Jeremy Newton --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d48d98b8e..994b00fbf2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,7 +336,7 @@ endif() ## Specify a RPM SPEC template # This template was generated by using "CPACK_RPM_GENERATE_USER_BINARY_SPECFILE_TEMPLATE" # but I've modified it to support CPACK_RPM_PACKAGE_RECOMMENDS for EL8+ and SLE -set ( CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_BINARY_DIR}/RPM/${CPACK_RPM_PACKAGE_NAME}.spec.in" ) +set ( CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_SOURCE_DIR}/RPM/${CPACK_RPM_PACKAGE_NAME}.spec.in" ) # 'dist' breaks manual builds on debian systems due to empty Provides execute_process( COMMAND rpm --eval %{?dist} From 7649cd862e378a5727008fdcac7439ddcc7851e0 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Thu, 18 Nov 2021 15:51:09 -0500 Subject: [PATCH 0941/1247] Fix packaging of license file CPACK doesn't have proper logic for installing the license as described by CPACK_RESOURCE_FILE_LICENSE. For Debian packaging, the license is expected to be installed as: /usr/share/doc/PACKAGENAME/copyright To do this, I've added a bit of logic for CPACK to copy this into the package using CPACK_INSTALL_COMMANDS to prep the directory, and CPACK_INSTALLED_DIRECTORIES to add it to the package. This applies to both RPM and DEB, so I've added some logic to the spec file to exclude this file (note that CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION does not work for files installed with CPACK_INSTALLED_DIRECTORIES). For RPM install, I've just added a small bit of logic to the spec file to handle it. The file needs to be copied into the spec working directory, then a macro is used to handle the rest. Note the license macro does not work on EL6, but I don't think we want to support this. Change-Id: I06ce63d300419893cb8274bc504a15633e304d91 Signed-off-by: Jeremy Newton --- CMakeLists.txt | 7 ++++++- RPM/hsakmt-roct-devel.spec.in | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 994b00fbf2..e563552c5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -211,7 +211,7 @@ install ( TARGETS ${HSAKMT_TARGET} EXPORT ${HSAKMT_TARGET}Targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel ) -install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel ) +#install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel ) # Install public headers install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -303,6 +303,11 @@ if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) else() set(CPACK_DEBIAN_PACKAGE_RELEASE "local") endif() +# Logic to install debian copyright file (i.e. the license file) +set(CPACK_INSTALL_COMMANDS "install -m 644 -D ${CPACK_RESOURCE_FILE_LICENSE} \ + ${PROJECT_BINARY_DIR}/doc/${CPACK_DEBIAN_PACKAGE_NAME}/copyright") +set(CPACK_INSTALLED_DIRECTORIES "${PROJECT_BINARY_DIR}/doc/${CPACK_DEBIAN_PACKAGE_NAME}" + "/usr/share/doc/${CPACK_DEBIAN_PACKAGE_NAME}") ## Process the Debian install/remove scripts to update the CPACK variables configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst.in DEBIAN/postinst @ONLY ) configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm.in DEBIAN/prerm @ONLY ) diff --git a/RPM/hsakmt-roct-devel.spec.in b/RPM/hsakmt-roct-devel.spec.in index cb8b7e39b9..2cc9e9f8ad 100644 --- a/RPM/hsakmt-roct-devel.spec.in +++ b/RPM/hsakmt-roct-devel.spec.in @@ -55,6 +55,9 @@ Recommends: @CPACK_RPM_PACKAGE_RECOMMENDS@ %prep mv $RPM_BUILD_ROOT %_topdir/tmpBBroot +# The license macro expects the license to be copied into the working directory +cp -a @CPACK_RESOURCE_FILE_LICENSE@ ./LICENSE.md + %install if [ -e $RPM_BUILD_ROOT ]; then @@ -91,6 +94,10 @@ mv %_topdir/tmpBBroot $RPM_BUILD_ROOT @CPACK_RPM_ABSOLUTE_INSTALL_FILES@ @CPACK_RPM_USER_INSTALL_FILES@ +# RPM has a different way of packaging the license, ignore the debian one: +%license LICENSE.md +%exclude /usr/share/doc/@CPACK_DEBIAN_PACKAGE_NAME@ + %changelog @CPACK_RPM_SPEC_CHANGELOG@ From 348a3613d6cacdebd4833432a92c0c0924a83b2a Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Wed, 24 Nov 2021 17:04:56 -0500 Subject: [PATCH 0942/1247] Fix side-by-side copyright file The copyright file will conflict if multiple thunks are installed. This should resolve the issue by adding the version to the install path. Change-Id: Ieac5a3eba979b3e934fb9100f890b92fc7c35d71 Signed-off-by: Jeremy Newton --- CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e563552c5d..444477c106 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -295,6 +295,10 @@ if(DEFINED ENV{ROCM_LIBPATCH_VERSION}) set(ROCM_VERSION_FOR_PACKAGE $ENV{ROCM_LIBPATCH_VERSION}) endif() +# Prepare final version for the CPACK use +set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${ROCM_VERSION_FOR_PACKAGE}") +set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") + # Debian package specific variables set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface") set(CPACK_DEBIAN_PACKAGE_NAME "hsakmt-roct-dev") @@ -307,7 +311,7 @@ endif() set(CPACK_INSTALL_COMMANDS "install -m 644 -D ${CPACK_RESOURCE_FILE_LICENSE} \ ${PROJECT_BINARY_DIR}/doc/${CPACK_DEBIAN_PACKAGE_NAME}/copyright") set(CPACK_INSTALLED_DIRECTORIES "${PROJECT_BINARY_DIR}/doc/${CPACK_DEBIAN_PACKAGE_NAME}" - "/usr/share/doc/${CPACK_DEBIAN_PACKAGE_NAME}") + "/usr/share/doc/${CPACK_DEBIAN_PACKAGE_NAME}${CPACK_PACKAGE_VERSION}") ## Process the Debian install/remove scripts to update the CPACK variables configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst.in DEBIAN/postinst @ONLY ) configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm.in DEBIAN/prerm @ONLY ) @@ -354,10 +358,6 @@ if ( PROC_RESULT EQUAL "0" AND NOT EVAL_RESULT STREQUAL "" ) string ( APPEND CPACK_RPM_PACKAGE_RELEASE "%{?dist}" ) endif() -# Prepare final version for the CPACK use -set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${ROCM_VERSION_FOR_PACKAGE}") -set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") - # Setting devel package dependendent version set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-amdgpu1") From 3b6451778721f90d93f8fb8f1c91c302e498ccf4 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Tue, 30 Nov 2021 10:29:30 -0500 Subject: [PATCH 0943/1247] Add Makefile to gitignore Calling cmake replaces this file, so no need to commit it. Change-Id: Ic4747cc9eebd9cbfc61d524a31d2025c04eda12e Signed-off-by: Jeremy Newton --- .gitignore | 1 + Makefile | 18 ------------------ 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 57f4774867..1140a6dcaa 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ patches-* build/ outgoing/ +Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index c306e90756..0000000000 --- a/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -.PHONY: all clean lnx64a lnx - -all: lnx64a - -deb: - $(MAKE) -C src deb - -rpm: - $(MAKE) -C src rpm - -lnx64a: - $(MAKE) -C src lnx64a - -lnx: - $(MAKE) -C src lnx - -clean: - $(MAKE) -C src clean From 3f9075030471283ce9d143a1caa7ed63f18d957c Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Tue, 30 Nov 2021 09:17:59 -0500 Subject: [PATCH 0944/1247] Just install license into /opt/rocm*/share/doc This is causing issues with side by side, sorry for the noise. This license location isn't ideal but it's good enough for now. Change-Id: Iba2a84cedf22466fdaaf3c63b6ea49c9fc277967 Signed-off-by: Jeremy Newton --- CMakeLists.txt | 8 +++----- RPM/hsakmt-roct-devel.spec.in | 7 ------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 444477c106..273335d7cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -288,6 +288,9 @@ set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md") set(CPACK_COMPONENT_DESCRIPTION "HSAKMT development package.\n This package includes the user-mode API interfaces\nused to interact with the ROCk driver.\n This package contains the headers, pkgonfig and\n cmake files for the ROCT package.") set ( ENABLE_LDCONFIG ON CACHE BOOL "Set library links and caches using ldconfig.") +# Install License file +install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR}/${CPACK_PACKAGE_NAME} COMPONENT devel) + # Make proper version for appending # Default Value is 99999, setting it first set(ROCM_VERSION_FOR_PACKAGE "99999") @@ -307,11 +310,6 @@ if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) else() set(CPACK_DEBIAN_PACKAGE_RELEASE "local") endif() -# Logic to install debian copyright file (i.e. the license file) -set(CPACK_INSTALL_COMMANDS "install -m 644 -D ${CPACK_RESOURCE_FILE_LICENSE} \ - ${PROJECT_BINARY_DIR}/doc/${CPACK_DEBIAN_PACKAGE_NAME}/copyright") -set(CPACK_INSTALLED_DIRECTORIES "${PROJECT_BINARY_DIR}/doc/${CPACK_DEBIAN_PACKAGE_NAME}" - "/usr/share/doc/${CPACK_DEBIAN_PACKAGE_NAME}${CPACK_PACKAGE_VERSION}") ## Process the Debian install/remove scripts to update the CPACK variables configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/postinst.in DEBIAN/postinst @ONLY ) configure_file ( ${CMAKE_CURRENT_SOURCE_DIR}/DEBIAN/prerm.in DEBIAN/prerm @ONLY ) diff --git a/RPM/hsakmt-roct-devel.spec.in b/RPM/hsakmt-roct-devel.spec.in index 2cc9e9f8ad..cb8b7e39b9 100644 --- a/RPM/hsakmt-roct-devel.spec.in +++ b/RPM/hsakmt-roct-devel.spec.in @@ -55,9 +55,6 @@ Recommends: @CPACK_RPM_PACKAGE_RECOMMENDS@ %prep mv $RPM_BUILD_ROOT %_topdir/tmpBBroot -# The license macro expects the license to be copied into the working directory -cp -a @CPACK_RESOURCE_FILE_LICENSE@ ./LICENSE.md - %install if [ -e $RPM_BUILD_ROOT ]; then @@ -94,10 +91,6 @@ mv %_topdir/tmpBBroot $RPM_BUILD_ROOT @CPACK_RPM_ABSOLUTE_INSTALL_FILES@ @CPACK_RPM_USER_INSTALL_FILES@ -# RPM has a different way of packaging the license, ignore the debian one: -%license LICENSE.md -%exclude /usr/share/doc/@CPACK_DEBIAN_PACKAGE_NAME@ - %changelog @CPACK_RPM_SPEC_CHANGELOG@ From a06d1a38840448264f633e2c204bc13ba859cf76 Mon Sep 17 00:00:00 2001 From: Joseph Greathouse Date: Fri, 29 Oct 2021 18:55:56 -0500 Subject: [PATCH 0945/1247] Correct gfx90c gfx arch number in HSA topology The AMD compiler team has confirmed that they expect gfx90c to be gfx90c, with a major/minor/stepping of 9, 0, and 12 respectively. It appears that there is a typo in the libhsakmt topology information that lists this part as gfx902. This patch fixes the issue. Signed-off-by: Joseph Greathouse Change-Id: I6f907a7aa6f190b12aba8bb4210c7b341b3c720b --- src/topology.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/topology.c b/src/topology.c index be40aaff5c..7b5ed09bb4 100644 --- a/src/topology.c +++ b/src/topology.c @@ -204,11 +204,6 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Raven */ { 0x15DD, 9, 0, 2, "Raven" }, { 0x15D8, 9, 0, 2, "Raven" }, - /* Renoir */ - { 0x15E7, 9, 0, 2, "Renoir" }, - { 0x1636, 9, 0, 2, "Renoir" }, - { 0x1638, 9, 0, 2, "Renoir" }, - { 0x164C, 9, 0, 2, "Renoir" }, /* Vega20 */ { 0x66A0, 9, 0, 6, "Vega20" }, { 0x66A1, 9, 0, 6, "Vega20" }, @@ -227,6 +222,11 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x740C, 9, 0, 10, "Aldebaran" }, { 0x740F, 9, 0, 10, "Aldebaran" }, { 0x7410, 9, 0, 10, "Aldebaran" }, + /* Renoir */ + { 0x15E7, 9, 0, 12, "Renoir" }, + { 0x1636, 9, 0, 12, "Renoir" }, + { 0x1638, 9, 0, 12, "Renoir" }, + { 0x164C, 9, 0, 12, "Renoir" }, /* Navi10 */ { 0x7310, 10, 1, 0, "Navi10" }, { 0x7312, 10, 1, 0, "Navi10" }, From 2dbee30232e30fe3055a0af6fc24253b466d71ff Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 18 Nov 2021 22:27:32 -0600 Subject: [PATCH 0946/1247] kfdtest: free user ptr buffer at SetGetAttributesTest Explicitly free the user buffer ptr before test's tear down. Otherwise the svm_bo object will never be released, causing a BUG error. Due to a late callback to svm_migrate_page_free when prange not longer exist. Also did cosmetic adjustments. Signed-off-by: Alex Sierra Change-Id: I989c62de8a9634faa84e42def956cecb3f84e329 --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index faf0cd89fa..3d4e64b6dd 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -111,7 +111,7 @@ TEST_F(KFDSVMRangeTest, SetGetAttributesTest) { int i; unsigned int BufSize = PAGE_SIZE; - HsaSVMRange *sysBuffer; + HsaSVMRange *sysBuffer = new HsaSVMRange(BufSize); HSAuint32 nAttributes = 5; HSA_SVM_ATTRIBUTE outputAttributes[nAttributes]; HSA_SVM_ATTRIBUTE inputAttributes[] = { @@ -132,8 +132,8 @@ TEST_F(KFDSVMRangeTest, SetGetAttributesTest) { }; HSAint32 enable = -1; EXPECT_SUCCESS(hsaKmtGetXNACKMode(&enable)); - expectedDefaultResults[4] = (enable)?HSA_SVM_ATTR_ACCESS:HSA_SVM_ATTR_NO_ACCESS; - sysBuffer = new HsaSVMRange(BufSize); + expectedDefaultResults[4] = (enable) ? + HSA_SVM_ATTR_ACCESS : HSA_SVM_ATTR_NO_ACCESS; char *pBuf = sysBuffer->As(); LOG() << "Get default atrributes" << std::endl; @@ -155,14 +155,15 @@ TEST_F(KFDSVMRangeTest, SetGetAttributesTest) { nAttributes, inputAttributes)); EXPECT_SUCCESS(hsaKmtSVMGetAttr(pBuf, BufSize, nAttributes, outputAttributes)); - for (i = 0; i < nAttributes; i++) { + for (i = 0; i < nAttributes; i++) { if (outputAttributes[i].type == HSA_SVM_ATTR_ACCESS || outputAttributes[i].type == HSA_SVM_ATTR_ACCESS_IN_PLACE || outputAttributes[i].type == HSA_SVM_ATTR_NO_ACCESS) EXPECT_EQ(inputAttributes[i].type, outputAttributes[i].type); else EXPECT_EQ(inputAttributes[i].value, outputAttributes[i].value); - } + } + delete sysBuffer; TEST_END } From 033b52c4e424af100660a1a913e82e8e0e1be384 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Fri, 10 Dec 2021 08:21:37 +0800 Subject: [PATCH 0947/1247] kfdtest: skip hdp flush test in sriov mode skip HDP flush test when remap feature is not supported. Backgroud: the HDP register remap is skipped in sriov mode, it will cause mmio base is nullPtr. Signed-off-by: Yang Wang Reviewed-by: Felix Kuehling Change-Id: Ib9aea1900931e30571656397a485ee4db051ec0a --- tests/kfdtest/src/KFDMemoryTest.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 9f62727cff..f2f0b14046 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2084,7 +2084,11 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { break; } } - ASSERT_NE(mmioBase, nullPtr) << "mmio base is NULL"; + + if (mmioBase == nullPtr) { + LOG() << "Skipping test: bsecause mmioBase is nullPtr, the mmio remap feature is not supported." << std::endl; + return; + } memoryFlags.ui32.NonPaged = 1; memoryFlags.ui32.CoarseGrain = 0; @@ -2205,7 +2209,11 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { break; } } - ASSERT_NE(mmioBase, nullPtr) << "mmio base is NULL"; + + if (mmioBase == nullPtr) { + LOG() << "Skipping test: bsecause mmioBase is nullPtr, the mmio remap feature is not supported." << std::endl; + return; + } memoryFlags.ui32.NonPaged = 1; memoryFlags.ui32.CoarseGrain = 0; From 0340c68031d6dd6b1c82067a03d57469b4768029 Mon Sep 17 00:00:00 2001 From: Ruili Ji Date: Tue, 4 Jan 2022 18:05:41 +0800 Subject: [PATCH 0948/1247] kfdtest : adjust memory size for KFDMemoryTest. Total VRAM size on APU is 512M usually, Framebuffer also is allocated from VRAM. There is no enough memory for this case. /home/ruiliji2/p5/libhsakmt/tests/kfdtest/src/KFDMemoryTest.cpp:1285: Failure Value of: (hsaKmtMapMemoryToGPUNodes(bufs[i], bufSize, &altVa, mapFlags, 1, &defaultGPUNode)) [ FAILED ] KFDMemoryTest.MMBench (1034 ms) Change-Id: Ib4201291122d85f6512a85859aea9a4713fb4f5c (cherry picked from commit a9f924484e7022a2d53ee02811b080f0833eba55) --- tests/kfdtest/src/KFDMemoryTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index f2f0b14046..f7ac73f1c2 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1255,7 +1255,8 @@ TEST_F(KFDMemoryTest, MMBench) { memFlags.ui32.HostAccess = 0; memFlags.ui32.NonPaged = 1; /* Upper limit of buffer number to fit 90% vram size */ - bufLimit = ((vramSizeMB << 20) * 9 / 10) / bufSize ; + bufLimit = ((vramSizeMB << 20) * 8 / 10) / bufSize ; + if (bufLimit == 0) continue; // skip when bufSize > vram From 4986f4a5c27f670260e42aee44ce432207da273d Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Thu, 25 Nov 2021 15:01:10 -0500 Subject: [PATCH 0949/1247] libhsakmt:Add MADV_DONTFORK to device mappings Mapped memory areas become invalid after fork, and the child process is required to remap the memory areas after a fork. So we mark these device memory mappings with MADV_DONTFORK so that they are removed from the child process after fork. This was causing some issues when doing CRIU checkpoint/restore because CRIU and amdgpu_plugin were not able to handle these mappings. Change-Id: I50eb334aecea6dab7522d94da0273adcf4fb1ce0 Signed-off-by: David Yat Sin --- src/fmm.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/fmm.c b/src/fmm.c index 6154606786..cdcf1e48b9 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1629,6 +1629,8 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, return NULL; } + madvise(ret, MemorySizeInBytes, MADV_DONTFORK); + if (mflags.ui32.AQLQueueMemory) { uint64_t my_buf_size = size / 2; @@ -1636,6 +1638,10 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); + + madvise(VOID_PTR_ADD(mem, my_buf_size), + MemorySizeInBytes, + MADV_DONTFORK); } } } @@ -2701,7 +2707,7 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert return -1; } } - + madvise(mmap_ret, size, MADV_DONTFORK); /* map to GPU */ ret = _fmm_map_to_gpu(aperture, address, size, NULL, &gpu_id, sizeof(uint32_t)); @@ -3364,6 +3370,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo err = HSAKMT_STATUS_ERROR; goto err_free_obj; } + madvise(ret, (SizeInPages << PAGE_SHIFT), MADV_DONTFORK); } *MemoryAddress = reservedMem; From 7c62a1291827872bfee1f1e9991f887aa74bb4cf Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Mon, 6 Dec 2021 12:03:37 -0500 Subject: [PATCH 0950/1247] kfdtest: dynamically increase timeout for P2PBandWidthTest Incease more timeout according to peers number to pass the test on some PCIe link platforms. Change-Id: Ifcb8c7297d6960c96fc18d29bc0a48733ca50165 Signed-off-by: Eric Huang --- tests/kfdtest/src/KFDPerformanceTest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/tests/kfdtest/src/KFDPerformanceTest.cpp index 32f342bd4d..bdd548c8e9 100644 --- a/tests/kfdtest/src/KFDPerformanceTest.cpp +++ b/tests/kfdtest/src/KFDPerformanceTest.cpp @@ -167,6 +167,8 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { return; } + g_TestTimeOut *= numPeers; + std::vector sysNodes(nodes); // include sysMem node 0... sysNodes.insert(sysNodes.begin(),0); @@ -308,6 +310,8 @@ TEST_F(KFDPerformanceTest, P2PBandWidthTest) { (float)speed2 / 1024 << " GB/s" << std::endl; } } + + g_TestTimeOut /= numPeers; exit: /* New line.*/ LOG() << std::endl << msg.str() << std::endl; From 7cdf38f6c0242543a390eecceb344b0cb0042f09 Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Wed, 26 Jan 2022 14:23:22 +0800 Subject: [PATCH 0951/1247] libhsakmt: correct the gfx version for gfx90c The gfx version of gfx90c is 90C instead of 902. Signed-off-by: Aaron Liu Change-Id: Id009c9357f816b8ccab605090df47626f1a579ef --- src/libhsakmt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index d531e87c90..e0c42de22d 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -133,11 +133,11 @@ enum full_gfx_versions { GFX_VERSION_VEGAM = 0x080003, GFX_VERSION_VEGA10 = 0x090000, GFX_VERSION_RAVEN = 0x090002, - GFX_VERSION_RENOIR = 0x090002, GFX_VERSION_VEGA12 = 0x090004, GFX_VERSION_VEGA20 = 0x090006, GFX_VERSION_ARCTURUS = 0x090008, GFX_VERSION_ALDEBARAN = 0x09000A, + GFX_VERSION_RENOIR = 0x09000C, GFX_VERSION_NAVI10 = 0x0A0100, GFX_VERSION_NAVI12 = 0x0A0101, GFX_VERSION_NAVI14 = 0x0A0102, From 052b7957ea59fc905d327191e3293780cced8aeb Mon Sep 17 00:00:00 2001 From: Lang Yu Date: Wed, 19 Jan 2022 17:25:24 +0800 Subject: [PATCH 0952/1247] libhsakmt: Add another pci device id for cyan skillfish Add PCI DID for cyan skillfish. Signed-off-by: Lang Yu Change-Id: I1d06936cccdf99af76fe5ca3ff323538fac76c9c --- src/topology.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/topology.c b/src/topology.c index 7b5ed09bb4..e4da0f37bd 100644 --- a/src/topology.c +++ b/src/topology.c @@ -240,6 +240,7 @@ static const struct hsa_gfxip_table gfxip_lookup_table[] = { { 0x13FB, 10, 1, 3, "cyan_skillfish" }, { 0x13FC, 10, 1, 3, "cyan_skillfish" }, { 0x13FE, 10, 1, 3, "cyan_skillfish" }, + { 0x143F, 10, 1, 3, "cyan_skillfish" }, /* Navi14 */ { 0x7340, 10, 1, 2, "Navi14" }, { 0x7341, 10, 1, 2, "Navi14" }, From 046f2e91165d2d478d944ffa54b2530015541857 Mon Sep 17 00:00:00 2001 From: Saravanan Solaiyappan Date: Thu, 24 Feb 2022 12:01:39 -0500 Subject: [PATCH 0953/1247] Consider apt/yum upgrade operation check in package scripts. Include the upgrade operation check in the prerm and postun scripts in package. Signed-off-by: Saravanan Solaiyappan Change-Id: Ib95ea72f15bfbf4141b69b0a8ca4d3a71fe1c093 --- DEBIAN/prerm.in | 2 +- RPM/postun.in | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/DEBIAN/prerm.in b/DEBIAN/prerm.in index 5cc004f399..f16a1ab3b7 100755 --- a/DEBIAN/prerm.in +++ b/DEBIAN/prerm.in @@ -10,7 +10,7 @@ rm_ldconfig() { } case "$1" in - ( remove ) + ( remove | upgrade ) rm_ldconfig ;; ( purge ) diff --git a/RPM/postun.in b/RPM/postun.in index 8e8eb1fea4..0691ac8617 100644 --- a/RPM/postun.in +++ b/RPM/postun.in @@ -1,5 +1,6 @@ # second term originates from ENABLE_LDCONFIG = ON/OFF at package build -if [ $1 -eq 0 ] && [ "@ENABLE_LDCONFIG@" == "ON" ]; then +if [ $1 -le 1 ] && [ "@ENABLE_LDCONFIG@" == "ON" ]; then + # perform the below actions for rpm remove($1=0) or upgrade($1=1) operations rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf ldconfig fi From b8dc875b3c8424aeed4c52ec524a8262f8b49511 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Tue, 1 Mar 2022 12:03:28 -0500 Subject: [PATCH 0954/1247] libhsakmt: Update context save area size calculations Currently, context save area size passed to KFD includes the size of the debug area. Change this to report the actual size of the context save area to KFD. Signed-off-by: Mukul Joshi Change-Id: I5d440ae802255a97ade046775f6a000bae79d5d5 --- src/queues.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/queues.c b/src/queues.c index 6a729b63ce..c83dd93038 100644 --- a/src/queues.c +++ b/src/queues.c @@ -291,7 +291,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) ALIGN_UP(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN); q->ctx_save_restore_size = q->ctl_stack_size - + PAGE_ALIGN_UP(wg_data_size + q->debug_memory_size); + + PAGE_ALIGN_UP(wg_data_size); return true; } return false; @@ -423,14 +423,21 @@ static int handle_concrete_asic(struct queue *q, ret = update_ctx_save_restore_size(NodeId, q); if (ret) { + uint32_t total_mem_alloc_size = 0; HsaUserContextSaveAreaHeader *header; args->ctx_save_restore_size = q->ctx_save_restore_size; args->ctl_stack_size = q->ctl_stack_size; + + /* Total memory to be allocated is = + * (Control Stack size + WG size) + Debug memory area size + */ + total_mem_alloc_size = q->ctx_save_restore_size + + q->debug_memory_size; q->ctx_save_restore = - allocate_exec_aligned_memory(q->ctx_save_restore_size, - q->use_ats, - NodeId, false, false); + allocate_exec_aligned_memory(total_mem_alloc_size, + q->use_ats, NodeId, false, false); + if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; @@ -441,7 +448,7 @@ static int handle_concrete_asic(struct queue *q, if (Event) header->ErrorEventId = Event->EventId; header->ErrorReason = ErrPayload; - header->DebugOffset = q->ctx_save_restore_size - q->debug_memory_size; + header->DebugOffset = q->ctx_save_restore_size; header->DebugSize = q->debug_memory_size; } From dc33a092c0ebc4da736811d20d1718d04f90c04b Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Fri, 4 Mar 2022 15:12:37 -0600 Subject: [PATCH 0955/1247] kfdtest: remove log message at hsaKmtSVMSetAttr failure This error messages should be handled by the caller. Signed-off-by: Alex Sierra Change-Id: I68d879d6d41835f47b8ac138c2218eaa6b86a512 --- tests/kfdtest/src/KFDTestUtil.cpp | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index e52e7b496b..7473db456a 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -717,10 +717,8 @@ HSAKMT_STATUS RegisterSVMRange(HSAuint32 GPUNode, void *MemoryAddress, attrs[3].value = GPUNode; r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, nattr, attrs); - if (r) { - LOG() << "set range attrs failed" << std::endl; + if (r) return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } @@ -734,10 +732,8 @@ HSAKMT_STATUS SVMRangeGetPrefetchNode(void *MemoryAddress, HSAuint64 SizeInBytes attr.value = 0; r = hsaKmtSVMGetAttr(MemoryAddress, SizeInBytes, 1, &attr); - if (r) { - LOG() << "get prefetch node failed" << std::endl; + if (r) return HSAKMT_STATUS_ERROR; - } *PrefetchNode = attr.value; @@ -753,10 +749,8 @@ HSAKMT_STATUS SVMRangePrefetchToNode(void *MemoryAddress, HSAuint64 SizeInBytes, attr.value = PrefetchNode; r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); - if (r) { - LOG() << "set prefetch node failed" << std::endl; + if (r) return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } @@ -770,10 +764,8 @@ HSAKMT_STATUS SVMRangeMapToNode(void *MemoryAddress, HSAuint64 SizeInBytes, attr.value = NodeID; r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); - if (r) { - LOG() << "set map to node failed" << std::endl; + if (r) return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } @@ -787,10 +779,8 @@ HSAKMT_STATUS SVMRangeMapInPlaceToNode(void *MemoryAddress, HSAuint64 SizeInByte attr.value = NodeID; r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); - if (r) { - LOG() << "set map in place to node failed" << std::endl; + if (r) return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } @@ -804,10 +794,8 @@ HSAKMT_STATUS SVMRangSetGranularity(void *MemoryAddress, HSAuint64 SizeInBytes, attr.value = Granularity; r = hsaKmtSVMSetAttr(MemoryAddress, SizeInBytes, 1, &attr); - if (r) { - LOG() << "set granularity failed" << std::endl; + if (r) return HSAKMT_STATUS_ERROR; - } return HSAKMT_STATUS_SUCCESS; } From f88aaa933b0a84814c3d2e7816ccd00d36d60fa2 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 16 Mar 2022 13:19:41 -0400 Subject: [PATCH 0956/1247] libhsakmt: Update kfd_ioctl.h Import the latest version from the kernel tree. Signed-off-by: Felix Kuehling Change-Id: If5f998ad55085ebd5020adaa382181204d834e3e --- include/linux/kfd_ioctl.h | 111 +++++++++++++++++++++++++++++++++----- src/debug.c | 8 +-- 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 039b30b311..8a0ed49d70 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -32,9 +32,11 @@ * - 1.4 - Indicate new SRAM EDC bit in device properties * - 1.5 - Add SVM API * - 1.6 - Query clear flags in SVM get_attr API + * - 1.7 - Checkpoint Restore (CRIU) API + * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 6 +#define KFD_IOCTL_MINOR_VERSION 8 /* * Debug revision change log @@ -65,9 +67,13 @@ * 8.0 - Expand runtime information given to the debugger * 8.1 - Allow the debugger to set the exception mask * 9.0 - Handle multiple exceptions from single trap signal + * 10.0 - Query debug event returns both queue_id and gpu_id + * 10.1 - Add additional debug capability information + * 10.2 - Reserved + * 10.3 - Pass context_save_restore_area size to user-space */ -#define KFD_IOCTL_DBG_MAJOR_VERSION 9 -#define KFD_IOCTL_DBG_MINOR_VERSION 0 +#define KFD_IOCTL_DBG_MAJOR_VERSION 10 +#define KFD_IOCTL_DBG_MINOR_VERSION 3 struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ @@ -141,7 +147,8 @@ struct kfd_queue_snapshot_entry { __u32 gpu_id; __u32 ring_size; __u32 queue_type; - __u32 reserved[18]; + __u32 ctx_save_restore_area_size; + __u32 reserved[17]; }; struct kfd_dbg_device_info_entry { @@ -261,6 +268,7 @@ struct kfd_ioctl_dbg_wave_control_args { #define KFD_INVALID_GPUID 0xffffffff #define KFD_INVALID_QUEUEID 0xffffffff +#define KFD_INVALID_FD 0xffffffff enum kfd_dbg_trap_override_mode { KFD_DBG_TRAP_OVERRIDE_OR = 0, @@ -442,8 +450,8 @@ struct kfd_runtime_info { /* KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT: * exception_mask: exception to clear (IN) on query and report (OUT) * ptr: unused - * data1: source id (OUT) - * data2: unused + * data1: queue id (OUT) + * data2: gpu id (OUT) * data3: unused * data4: unused * @@ -838,6 +846,7 @@ enum kfd_smi_event { }; #define KFD_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1)) +#define KFD_SMI_EVENT_MSG_SIZE 96 struct kfd_ioctl_smi_events_args { __u32 gpuid; /* to KFD */ @@ -919,6 +928,81 @@ struct kfd_ioctl_spm_args { __u32 has_data_loss; }; +/************************************************************************************************** + * CRIU IOCTLs (Checkpoint Restore In Userspace) + * + * When checkpointing a process, the userspace application will perform: + * 1. PROCESS_INFO op to determine current process information. This pauses execution and evicts + * all the queues. + * 2. CHECKPOINT op to checkpoint process contents (BOs, queues, events, svm-ranges) + * 3. UNPAUSE op to un-evict all the queues + * + * When restoring a process, the CRIU userspace application will perform: + * + * 1. RESTORE op to restore process contents + * 2. RESUME op to start the process + * + * Note: Queues are forced into an evicted state after a successful PROCESS_INFO. User + * application needs to perform an UNPAUSE operation after calling PROCESS_INFO. + */ + +enum kfd_criu_op { + KFD_CRIU_OP_PROCESS_INFO, + KFD_CRIU_OP_CHECKPOINT, + KFD_CRIU_OP_UNPAUSE, + KFD_CRIU_OP_RESTORE, + KFD_CRIU_OP_RESUME, +}; + +/** + * kfd_ioctl_criu_args - Arguments perform CRIU operation + * @devices: [in/out] User pointer to memory location for devices information. + * This is an array of type kfd_criu_device_bucket. + * @bos: [in/out] User pointer to memory location for BOs information + * This is an array of type kfd_criu_bo_bucket. + * @priv_data: [in/out] User pointer to memory location for private data + * @priv_data_size: [in/out] Size of priv_data in bytes + * @num_devices: [in/out] Number of GPUs used by process. Size of @devices array. + * @num_bos [in/out] Number of BOs used by process. Size of @bos array. + * @num_objects: [in/out] Number of objects used by process. Objects are opaque to + * user application. + * @pid: [in/out] PID of the process being checkpointed + * @op [in] Type of operation (kfd_criu_op) + * + * Return: 0 on success, -errno on failure + */ +struct kfd_ioctl_criu_args { + __u64 devices; /* Used during ops: CHECKPOINT, RESTORE */ + __u64 bos; /* Used during ops: CHECKPOINT, RESTORE */ + __u64 priv_data; /* Used during ops: CHECKPOINT, RESTORE */ + __u64 priv_data_size; /* Used during ops: PROCESS_INFO, RESTORE */ + __u32 num_devices; /* Used during ops: PROCESS_INFO, RESTORE */ + __u32 num_bos; /* Used during ops: PROCESS_INFO, RESTORE */ + __u32 num_objects; /* Used during ops: PROCESS_INFO, RESTORE */ + __u32 pid; /* Used during ops: PROCESS_INFO, RESUME */ + __u32 op; +}; + +struct kfd_criu_device_bucket { + __u32 user_gpu_id; + __u32 actual_gpu_id; + __u32 drm_fd; + __u32 pad; +}; + +struct kfd_criu_bo_bucket { + __u64 addr; + __u64 size; + __u64 offset; + __u64 restored_offset; /* During restore, updated offset for BO */ + __u32 gpu_id; /* This is the user_gpu_id */ + __u32 alloc_flags; + __u32 dmabuf_fd; + __u32 pad; +}; + +/* CRIU IOCTLs - END */ +/**************************************************************************************************/ /* Register offset inside the remapped mmio page */ enum kfd_mmio_remap { @@ -1094,7 +1178,7 @@ struct kfd_ioctl_svm_args { __u32 op; __u32 nattr; /* Variable length array of attributes */ - struct kfd_ioctl_svm_attribute attrs[0]; + struct kfd_ioctl_svm_attribute attrs[]; }; /** @@ -1177,16 +1261,16 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_WAIT_EVENTS \ AMDKFD_IOWR(0x0C, struct kfd_ioctl_wait_events_args) -#define AMDKFD_IOC_DBG_REGISTER \ +#define AMDKFD_IOC_DBG_REGISTER_DEPRECATED \ AMDKFD_IOW(0x0D, struct kfd_ioctl_dbg_register_args) -#define AMDKFD_IOC_DBG_UNREGISTER \ +#define AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED \ AMDKFD_IOW(0x0E, struct kfd_ioctl_dbg_unregister_args) -#define AMDKFD_IOC_DBG_ADDRESS_WATCH \ +#define AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED \ AMDKFD_IOW(0x0F, struct kfd_ioctl_dbg_address_watch_args) -#define AMDKFD_IOC_DBG_WAVE_CONTROL \ +#define AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED \ AMDKFD_IOW(0x10, struct kfd_ioctl_dbg_wave_control_args) #define AMDKFD_IOC_SET_SCRATCH_BACKING_VA \ @@ -1240,8 +1324,11 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_SET_XNACK_MODE \ AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args) +#define AMDKFD_IOC_CRIU_OP \ + AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x22 +#define AMDKFD_COMMAND_END 0x23 /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ diff --git a/src/debug.c b/src/debug.c index d2879e6a60..2703722919 100644 --- a/src/debug.c +++ b/src/debug.c @@ -75,7 +75,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId) args.gpu_id = gpu_id; - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER, &args); + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_REGISTER_DEPRECATED, &args); if (err == 0) result = HSAKMT_STATUS_SUCCESS; @@ -102,7 +102,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgUnregister(HSAuint32 NodeId) struct kfd_ioctl_dbg_unregister_args args = {0}; args.gpu_id = gpu_id; - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER, &args); + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED, &args); if (err) return HSAKMT_STATUS_ERROR; @@ -165,7 +165,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgWavefrontControl(HSAuint32 NodeId, run_ptr += sizeof(DbgWaveMsgRing->MemoryVA); /* send to kernel */ - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_WAVE_CONTROL, args); + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED, args); free(args); @@ -253,7 +253,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(HSAuint32 NodeId, } /* send to kernel */ - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_ADDRESS_WATCH, args); + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED, args); free(args); From 8b54459e12c025c7ae3d84b3d099e1305097df12 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 28 Mar 2022 15:46:51 -0400 Subject: [PATCH 0957/1247] kfdtest: Add function to check for PCI Atomic Ops support While this is currently only used in one subtest, it's useful to have this separated into the test utilities. This will also allow us to check for PCI Atomics support before trying to run them. Signed-off-by: Kent Russell Change-Id: I9704d151bfaa627eceae8399cc46c15babde6ff1 --- tests/kfdtest/src/KFDTestUtil.cpp | 37 +++++++++++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 1 + 2 files changed, 38 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 7473db456a..476e0bb1ce 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -128,6 +128,43 @@ bool is_dgpu() { return is_dgpu_dev; } +bool hasPciAtomicsSupport(int node) { + /* If we can't get Node Properties, assume a lack of Atomics support */ + HsaNodeProperties *pNodeProperties = new HsaNodeProperties(); + if (hsaKmtGetNodeProperties(node, pNodeProperties)) { + LOG() << "Unable to get Node Properties for node " << node << std::endl; + return false; + } + + /* APUs don't have IO Links, but support Atomic Ops by default */ + if (pNodeProperties->NumCPUCores && pNodeProperties->NumFComputeCores) + return true; + + HsaIoLinkProperties *IolinkProperties = new HsaIoLinkProperties[pNodeProperties->NumIOLinks]; + if (hsaKmtGetNodeIoLinkProperties(node, pNodeProperties->NumIOLinks, IolinkProperties)) { + LOG() << "Unable to get Node IO Link Information for node " << node << std::endl; + return false; + } + + /* Make sure we're checking GPU-to-CPU connection here */ + for (int linkId = 0; linkId < pNodeProperties->NumIOLinks; linkId++) { + /* Make sure it's a CPU */ + HsaNodeProperties *linkProps = new HsaNodeProperties(); + if (hsaKmtGetNodeProperties(IolinkProperties[linkId].NodeTo, linkProps)) { + LOG() << "Unable to get connected device's IO Link information" << std::endl; + return false; + } + if (linkProps->NumCPUCores) { + /* IOLink flags are only valid if Override flag is set */ + return (IolinkProperties[linkId].Flags.ui32.Override && + !IolinkProperties[linkId].Flags.ui32.NoAtomics32bit && + !IolinkProperties[linkId].Flags.ui32.NoAtomics64bit); + } + } + + return false; +} + unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { unsigned int familyId = FAMILY_UNKNOWN; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index ee88ed3909..7c2f9c61ce 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -50,6 +50,7 @@ HSAKMT_STATUS CreateQueueTypeEvent(bool ManualReset, bool IsSignaled, unsigned i bool is_dgpu(); bool isTonga(const HsaNodeProperties *props); +bool hasPciAtomicsSupport(int node); unsigned int FamilyIdFromNode(const HsaNodeProperties *props); void GetHwQueueInfo(const HsaNodeProperties *props, From f62e9b9821d8ad402be1f1c1a59f9e7942cb4c1b Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Mon, 28 Mar 2022 15:48:09 -0400 Subject: [PATCH 0958/1247] kfdtest: Check for Atomic Ops support before running Atomics test If PCIe Atomics aren't supported, we shouldn't try to run a test that tests PCIe Atomics. Check for support, and bail early if it's not there Signed-off-by: Kent Russell Change-Id: Ie9aa0fed3ece07fb83a33e6cacef2961626afab4 --- tests/kfdtest/src/KFDQMTest.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 6d4cb7cbef..9b4003b68e 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1518,13 +1518,15 @@ unsigned int AtomicIncThread(void* pCtx) { TEST_F(KFDQMTest, Atomics) { TEST_START(TESTPROFILE_RUNALL); - /* CI doesn't support Atomics. KV does, but gets its own FAMILY_KV def */ - if (m_FamilyId == FAMILY_CI) { - LOG() << "Skipping test: CI doesn't support Atomics." << std::endl; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!hasPciAtomicsSupport(defaultGPUNode)) { + LOG() << "Skipping test: Node doesn't support Atomics." << std::endl; return; } - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer destBuf(PAGE_SIZE, defaultGPUNode); From 0d07b3477b3fe7db992f620011743eaa3c782ae5 Mon Sep 17 00:00:00 2001 From: Divya Shikre Date: Wed, 6 Apr 2022 16:32:23 -0400 Subject: [PATCH 0959/1247] kfdtest: Add log message in KFDEvictTest Map failures happen in AllocBuffers function when there isn't enough space to move BO to vram. In such cases, the function retries allocation/map until successful to continue testing eviction and restore. Print a message in KFDEvictTest when this happens to correlate to the message seen in the kernel log. amdgpu 0000:c1:00.0: amdgpu: Failed to map peer:0000:c1:00.0 mem_domain:4 Signed-off-by: Divya Shikre Change-Id: I0475d8d9521a07612182e54fc7cddb9bd44353e6 --- tests/kfdtest/src/KFDEvictTest.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index a9544eb47c..7ec86bc8bd 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -84,21 +84,23 @@ void KFDEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAui if (hsaKmtMapMemoryToGPUNodes(m_pBuf, vramBufSize, NULL, mapFlags, 1, reinterpret_cast(&defaultGPUNode)) == HSAKMT_STATUS_ERROR) { EXPECT_SUCCESS(hsaKmtFreeMemory(m_pBuf, vramBufSize)); - break; + LOG() << "Map failed for " << i << "/" << count << " buffer. Retrying allocation" << std::endl; + goto retry; } } pBuffers.push_back(m_pBuf); i++; retry = 0; - } else { - if (retry++ > ALLOCATE_RETRY_TIMES) { - break; - } - - /* Wait for 1 second to try allocate again */ - sleep(1); + continue; } +retry: + if (retry++ > ALLOCATE_RETRY_TIMES) { + break; + } + + /* Wait for 1 second to try allocate again */ + sleep(1); } } From 3ecd54f0984349d38a29771d1932721c15d06bbb Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 18 Mar 2022 16:08:59 -0400 Subject: [PATCH 0960/1247] kfdtest: Skip slow tests in MMBandWidth Some VRAM access tests in MMBandWidth can be very slow on systems with complicated PCIe topology. Skip tests that take a long time to avoid excessively long running tests with little benefit. Signed-off-by: Felix Kuehling Change-Id: I2950237347fc2f764f6aa3292ab819051472bf37 --- tests/kfdtest/src/KFDMemoryTest.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index f7ac73f1c2..31f199f79f 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2031,6 +2031,10 @@ TEST_F(KFDMemoryTest, MMBandWidth) { RECORD(mcpWTime) << MMBANDWIDTH_KEY_PREFIX << "mcpWTime"; RECORD(accessRTime) << MMBANDWIDTH_KEY_PREFIX << "accessRTime"; RECORD(accessWTime) << MMBANDWIDTH_KEY_PREFIX << "accessWTime"; + + // skip slow tests + if (mcpRTime + mcpWTime + accessRTime + accessWTime > 5000000) + break; } munmap(tmp, tmpBufferSize); From 5fd3c868b28fab59fa054b0c833c53540721921c Mon Sep 17 00:00:00 2001 From: Timothy Pearson Date: Sat, 23 Nov 2019 20:09:55 -0500 Subject: [PATCH 0961/1247] Initial support for POWER platforms Tested on Talos II with Vega 64 POWER systems allocate NUMA nodes on multiples of 8 to allow CPU onlining / offlining Set the correct NUMA mask bits when requesting node-bound memory allocations This is a cleanup/squash/rebase of: https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/pull/47 Change-Id: Id4af6dff7e66e9d464d6b17a1e99087eb3ac8e51 Signed-off-by: Jeremy Newton --- src/fmm.c | 4 ++ src/topology.c | 101 ++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 91 insertions(+), 14 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index cdcf1e48b9..2319513c29 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1514,7 +1514,11 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, if (!node_mask) return -ENOMEM; +#ifdef __PPC64__ + numa_bitmask_setbit(node_mask, node_id * 8); +#else numa_bitmask_setbit(node_mask, node_id); +#endif mode |= mflags.ui32.NoSubstitute ? MPOL_BIND : MPOL_PREFERRED; r = mbind(mem, SizeInBytes, mode, node_mask->maskp, num_node + 1, 0); numa_bitmask_free(node_mask); diff --git a/src/topology.c b/src/topology.c index e4da0f37bd..81ff62f572 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1,5 +1,6 @@ /* * Copyright © 2014 Advanced Micro Devices, Inc. + * Copyright 2016-2018 Raptor Engineering, LLC. All Rights Reserved. * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -76,12 +77,14 @@ static int processor_vendor = -1; /* Supported System Vendors */ enum SUPPORTED_PROCESSOR_VENDORS { GENUINE_INTEL = 0, - AUTHENTIC_AMD + AUTHENTIC_AMD, + IBM_POWER }; /* Adding newline to make the search easier */ static const char *supported_processor_vendor_name[] = { "GenuineIntel\n", - "AuthenticAMD\n" + "AuthenticAMD\n", + "\n" // POWER requires a different search method }; static HSAKMT_STATUS topology_take_snapshot(void); @@ -497,6 +500,13 @@ static int get_cpu_cache_info(const char *prefix, struct proc_cpuinfo *cpuinfo, int idx, num_idx, n; HsaCacheProperties *this_cache; char path[256], str[256]; + bool is_power9 = false; + + if (processor_vendor == IBM_POWER) { + if (strcmp(cpuinfo[0].model_name, "POWER9") == 0) { + is_power9 = true; + } + } this_cache = cpu_ci->cache_prop; num_idx = cpu_ci->num_caches; @@ -504,19 +514,27 @@ static int get_cpu_cache_info(const char *prefix, struct proc_cpuinfo *cpuinfo, /* If this cache is shared by multiple CPUs, we only need * to list it in the first CPU. */ - snprintf(path, 256, "%s/index%d/shared_cpu_list", prefix, idx); - /* shared_cpu_list is shown as n1,n2... or n1-n2,n3-n4... - * For both cases, this cache is listed to proc n1 only. - */ - fscanf_dec(path, (uint32_t *)&n); - if (cpu_ci->proc_num != n) { - /* proc is not n1. Skip and reduce the cache count. */ - --cpu_ci->num_caches; - continue; + if (is_power9) { + // POWER9 has SMT4 + if (cpu_ci->proc_num & 0x3) { + /* proc is not 0,4,8,etc. Skip and reduce the cache count. */ + --cpu_ci->num_caches; + continue; + } + } else { + snprintf(path, 256, "%s/index%d/shared_cpu_list", prefix, idx); + /* shared_cpu_list is shown as n1,n2... or n1-n2,n3-n4... + * For both cases, this cache is listed to proc n1 only. + */ + fscanf_dec(path, (uint32_t *)&n); + if (cpu_ci->proc_num != n) { + /* proc is not n1. Skip and reduce the cache count. */ + --cpu_ci->num_caches; + continue; + } + this_cache->ProcessorIdLow = cpuinfo[cpu_ci->proc_num].apicid; } - this_cache->ProcessorIdLow = cpuinfo[cpu_ci->proc_num].apicid; - /* CacheLevel */ snprintf(path, 256, "%s/index%d/level", prefix, idx); fscanf_dec(path, &this_cache->CacheLevel); @@ -845,6 +863,8 @@ static int topology_search_processor_vendor(const char *processor_name) for (i = 0; i < ARRAY_LEN(supported_processor_vendor_name); i++) { if (!strcmp(processor_name, supported_processor_vendor_name[i])) return i; + if (!strcmp(processor_name, "POWER9, altivec supported\n")) + return IBM_POWER; } return -1; } @@ -877,6 +897,52 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, return HSAKMT_STATUS_ERROR; } +#ifdef __PPC64__ + char *p2; + + /* Each line in /proc/cpuinfo that read_buf is constructed, the format + * is like this: + * "token : value\n" + * where token is our target like vendor_id, model name, apicid ... + * and value is the answer + */ + while (fgets(read_buf, sizeof(read_buf), fd)) { + /* processor number */ + if (!strncmp("processor ", read_buf, sizeof("processor ") - 1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + proc = atoi(p); + if (proc >= num_procs) { + pr_warn("cpuinfo contains processor %d larger than %u\n", + proc, num_procs); + ret = HSAKMT_STATUS_NO_MEMORY; + goto exit; + } + continue; + } + + /* vendor name / model name */ + if (!strncmp("cpu ", read_buf, sizeof("cpu ") - 1) && + (processor_vendor == -1)) { + p = strchr(read_buf, ':'); + p += 2; /* remove ": " */ + processor_vendor = topology_search_processor_vendor(p); + + p2 = strchr(p, ','); + if (p2 != NULL) { + p2++; + *p2 = 0; + } + if (strlen(p) < HSA_PUBLIC_NAME_SIZE) { + /* -1 to remove \n from p */ + strncpy(cpuinfo[proc].model_name, p, strlen(p) - 1); + cpuinfo[proc].model_name[strlen(p) - 1] = '\0'; + } else + strncpy(cpuinfo[proc].model_name, p, HSA_PUBLIC_NAME_SIZE); + continue; + } + } +#else /* Each line in /proc/cpuinfo that read_buf is constructed, the format * is like this: * "token : value\n" @@ -926,6 +992,7 @@ static HSAKMT_STATUS topology_parse_cpuinfo(struct proc_cpuinfo *cpuinfo, cpuinfo[proc].apicid = atoi(p); } } +#endif if (processor_vendor < 0) { pr_err("Failed to get Processor Vendor. Setting to %s", @@ -1289,7 +1356,13 @@ static int topology_create_temp_cpu_cache_list(int node, *temp_cpu_ci_list = NULL; /* Get info from /sys/devices/system/node/nodeX/cpuY/cache */ - snprintf(node_dir, MAXPATHSIZE, "/sys/devices/system/node/node%d", node); + int node_real = node; + if (processor_vendor == IBM_POWER) { + if (!strcmp(cpuinfo[0].model_name, "POWER9")) { + node_real = node * 8; + } + } + snprintf(node_dir, MAXPATHSIZE, "/sys/devices/system/node/node%d", node_real); /* Other than cpuY folders, this dir also has cpulist and cpumap */ max_cpus = num_subdirs(node_dir, "cpu"); if (max_cpus <= 0) { From 6c103877dd6d0d8dbe383ae7e58d18f2e0a04d54 Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Fri, 25 Mar 2022 14:33:14 +0800 Subject: [PATCH 0962/1247] kfdtest: decrease granularityMB for handling small vram system It's not possible to allocate the 3/4 vram size with granularityMB being 128 when vram size < 512MB and decrease granularityMB to 16 has no significant impact on ROCt test on other system. So let's decrease granularityMB on small vram system for handling LargestVramBufferTest(). Change-Id: Iea7c29abfd382a20761b653730fd09a220ad2fd0 Signed-off-by: Prike Liang --- tests/kfdtest/src/KFDMemoryTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 31f199f79f..8cf24ffb1b 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -965,7 +965,7 @@ void KFDMemoryTest::BinarySearchLargestBuffer(int allocNode, const HsaMemFlags & int ret; HsaMemMapFlags mapFlags = {0}; - HSAuint64 granularityMB = 128; + HSAuint64 granularityMB = highMB > 512 ? 128 : 16; /* Testing big buffers in VRAM */ unsigned int * pDb = NULL; From c86a0b8332cf9f680794ba7bf9d120c286194932 Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Fri, 25 Mar 2022 15:30:34 +0800 Subject: [PATCH 0963/1247] kfdtest: limit vram allocation size for MigrateAccessInPlaceTest On the some platform there's only 256MB vram and then will fail to allocate 256MB vram. So let's limit a small vram allocation for ensuring vram allocated successfully. Change-Id: Iba4c469de56925675e5624b300a6153e24ab19b3 Signed-off-by: Prike Liang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 3d4e64b6dd..655ad4ed66 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -690,7 +690,7 @@ TEST_F(KFDSVMRangeTest, MigrateAccessInPlaceTest) { return; } - unsigned int BufferSize = 256 << 20; + unsigned int BufferSize = MIN(256ULL << 20, GetVramSize(defaultGPUNode) / 2); SDMAQueue sdmaQueue; ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); From 347bf6a03c7c589f125432abbd8fb37521799e5d Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 7 Apr 2022 22:54:16 -0400 Subject: [PATCH 0964/1247] kfdtest: Reduce maximum number of events to 4095 The KFD patch "drm/amdkfd: Ignore bogus signals from MEC efficiently" will reserve one signal slot that user mode cannot use any more. Update the maximum event number in KFDEventTest to match that change. Signed-off-by: Felix Kuehling Change-Id: Ic789e16b6d73dfea66ab51c5bbc075c8e8e2d052 --- tests/kfdtest/src/KFDEventTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index ca16e871ff..35e24cb493 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -228,7 +228,7 @@ exit: TEST_F(KFDEventTest, MeasureInterruptConsumption) { TEST_START(TESTPROFILE_RUNALL); QueueAndSignalBenchmark latencyBench(128); - QueueAndSignalBenchmark sustainedBench(4096); + QueueAndSignalBenchmark sustainedBench(4095); printf("\033[2J"); while (true) { @@ -246,7 +246,7 @@ TEST_F(KFDEventTest, MeasureInterruptConsumption) { TEST_F(KFDEventTest, SignalMaxEvents) { TEST_START(TESTPROFILE_RUNALL); - static const unsigned int MAX_EVENT_NUMBER = 4096; + static const unsigned int MAX_EVENT_NUMBER = 4095; uint64_t time, latency; QueueAndSignalBenchmark maxEventTest(MAX_EVENT_NUMBER); From e738e57fc46f39223f43d9fd892eea6605f0b14c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 21 Apr 2022 16:26:36 -0400 Subject: [PATCH 0965/1247] kfdtest: Add test for invalid signal interrupts AQL firmware can sometime send invalid signal interrupts with 0 context ID. This test simulates this by submitting similar events using PM4 packets and measures the performance of signaling a normal event after that. Signed-off-by: Felix Kuehling Change-Id: I69028dc6dd98a5a93f18daad4efbe1b16b6098f9 --- tests/kfdtest/src/KFDEventTest.cpp | 82 ++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index 35e24cb493..a1ae0e6182 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -295,3 +295,85 @@ TEST_F(KFDEventTest, SignalMultipleEventsWaitForAll) { TEST_END; } + +/* Send an event interrupt with 0 context ID. Test that KFD handles it + * gracefully and with good performance. On current GPUs and firmware it + * should be handled on a fast path. + */ +TEST_F(KFDEventTest, SignalInvalidEvent) { + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + // Create some dummy events, to make the slow path a bit slower + static const unsigned int EVENT_NUMBER = 4094; + HsaEvent* pHsaEvent[EVENT_NUMBER]; + for (int i = 0; i < EVENT_NUMBER; i++) { + pHsaEvent[i] = NULL; + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &pHsaEvent[i])); + } + + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &m_pHsaEvent)); + ASSERT_NE(0, m_pHsaEvent->EventData.HWData2); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + static const unsigned int REPS = 2000; + HSAuint64 duration[REPS]; + HSAuint64 total = 0, min = 1000000, max = 0; + for (int i = 0; i < REPS; i++) { + // Invalid signal packet + queue.PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, false, 0, 0)); + // Submit valid signal packet + queue.PlacePacket(PM4ReleaseMemoryPacket(m_FamilyId, false, + m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); + + HSAuint64 startTime = GetSystemTickCountInMicroSec(); + queue.SubmitPacket(); + + EXPECT_SUCCESS(hsaKmtWaitOnEvent(m_pHsaEvent, g_TestTimeOut)); + + duration[i] = GetSystemTickCountInMicroSec() - startTime; + total += duration[i]; + if (duration[i] < min) + min = duration[i]; + if (duration[i] > max) + max = duration[i]; + } + + double mean = (double)(total - min - max) / (REPS - 2); + double variance = 0; + bool skippedMin = false, skippedMax = false; + HSAuint64 newMin = max, newMax = min; + for (int i = 0; i < REPS; i++) { + if (!skippedMin && duration[i] == min) { + skippedMin = true; + continue; + } + if (!skippedMax && duration[i] == max) { + skippedMax = true; + continue; + } + if (duration[i] < newMin) + newMin = duration[i]; + if (duration[i] > newMax) + newMax = duration[i]; + double diff = mean - duration[i]; + variance += diff*diff; + } + variance /= REPS - 2; + double stdDev = sqrt(variance); + + LOG() << "Time for event handling (min/avg/max [std.dev] in us) " << std::dec + << newMin << "/" << mean << "/" << newMax << " [" << stdDev << "]\n"; + + EXPECT_SUCCESS(queue.Destroy()); + + for (int i = 0; i < EVENT_NUMBER; i++) + EXPECT_SUCCESS(hsaKmtDestroyEvent(pHsaEvent[i])); + + TEST_END; +} From 23ec6e880e25bf1246d04839f828d39fcb3b0766 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 4 Apr 2022 20:16:59 -0400 Subject: [PATCH 0966/1247] kfdtest: Set XNACK mode according HSA_XNACK env setting According env setting HSA_XNACK=1 or 0, set XNACK mode ON or OFF to run KFDSVMRangeTest and KFDSVMEvictTest. If HSA_XNACK is not defined, use system boot-time XNACK mode setting. Restore to the original XNACK mode when test finished. Change-Id: Ia896a1b0a90854646c8a79acca38a7d46098efde Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDBaseComponentTest.hpp | 34 ++++++++++++++++++++++ tests/kfdtest/src/KFDSVMEvictTest.cpp | 20 +++++++++++++ tests/kfdtest/src/KFDSVMEvictTest.hpp | 4 +++ tests/kfdtest/src/KFDSVMRangeTest.cpp | 10 ++++--- 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 187d9fbd73..e27baf1cc1 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -73,6 +73,7 @@ class KFDBaseComponentTest : public testing::Test { unsigned int m_numSdmaQueuesPerEngine; HsaMemFlags m_MemoryFlags; HsaNodeInfo m_NodeInfo; + HSAint32 m_xnack; // @brief Executed before every test that uses KFDBaseComponentTest class and sets all common settings for the tests. virtual void SetUp(); @@ -86,6 +87,39 @@ class KFDBaseComponentTest : public testing::Test { LOG() << "SVM API not supported" << std::endl; return supported; } + + void SVMSetXNACKMode() { + if (!SVMAPISupported()) + return; + + m_xnack = -1; + + char *hsa_xnack = getenv("HSA_XNACK"); + if (!hsa_xnack) + return; + + HSAKMT_STATUS ret = hsaKmtGetXNACKMode(&m_xnack); + if (ret != HSAKMT_STATUS_SUCCESS) { + LOG() << "Failed " << ret << " to get XNACK mode" << std::endl; + return; + } + + // XNACK OFF if defined HSA_XNACK=0 + HSAint32 xnack_on = strncmp(hsa_xnack, "0", 1); + ret = hsaKmtSetXNACKMode(xnack_on); + if (ret != HSAKMT_STATUS_SUCCESS) + LOG() << "Failed " << ret << " to set XNACK mode " << xnack_on << std::endl; + } + + void SVMRestoreXNACKMode() { + if (!SVMAPISupported()) + return; + + if (m_xnack == -1) + return; + + hsaKmtSetXNACKMode(m_xnack); + } }; extern KFDBaseComponentTest* g_baseTest; diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index f326d32093..d41aedac74 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -35,6 +35,26 @@ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +void KFDSVMEvictTest::SetUp() { + ROUTINE_START + + KFDLocalMemoryTest::SetUp(); + + SVMSetXNACKMode(); + + ROUTINE_END +} + +void KFDSVMEvictTest::TearDown() { + ROUTINE_START + + SVMRestoreXNACKMode(); + + KFDLocalMemoryTest::TearDown(); + + ROUTINE_END +} + HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) { HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT; HSAuint64 sysMemSize = GetSysMemSize(); diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/tests/kfdtest/src/KFDSVMEvictTest.hpp index 7749ac2dee..2b8a1de957 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.hpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.hpp @@ -49,6 +49,10 @@ class KFDSVMEvictTest : public KFDLocalMemoryTest { } catch (...) {} } + protected: + virtual void SetUp(); + virtual void TearDown(); + protected: std::string CreateShader(); void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 655ad4ed66..6aad683f91 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -36,6 +36,8 @@ void KFDSVMRangeTest::SetUp() { m_pIsaGen = IsaGenerator::Create(m_FamilyId); + SVMSetXNACKMode(); + ROUTINE_END } @@ -46,6 +48,8 @@ void KFDSVMRangeTest::TearDown() { delete m_pIsaGen; m_pIsaGen = NULL; + SVMRestoreXNACKMode(); + KFDBaseComponentTest::TearDown(); ROUTINE_END @@ -1287,10 +1291,8 @@ TEST_F(KFDSVMRangeTest, ReadOnlyRangeTest) { */ int pid = fork(); if (pid == 0) { - if (hsaKmtOpenKFD() != HSAKMT_STATUS_SUCCESS) { - WARN() << "KFD open failed in child process" << std::endl; - exit(1); - } + TearDown(); + SetUp(); } else { int childStatus; From 65b1e0c058259b55b33b477860c92ce5a93b9149 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:44:29 -0400 Subject: [PATCH 0967/1247] kfdtest: Add LLVM AMDGPU assembler components Initial commit for transition from IsaGenerator/SP3 assembler model to the LLVM AMDGPU (AMDGCN) assembler backend: - Add Assembler class, may be instantiated for assembly similar to IsaGenerator. - Add Assembler and LLVM archive dependencies to build process. - CXX bumped to gnu++14 as required for LLVM compilation. - Compatible with LLVM 7.0 and greater (latest Lightning/llvm-git version should be used for up-to-date gfx support). Note that this is just a build dependency and *not* a runtime dependency. LLVM does not need to be installed on the host machine to run kfdtest. - CMake will first look for a Lightning build. Lightning itself does not need to be installed system-wide, just built. If this fails, it will attempt to find a system-wide LLVM install. General Assembler usage and notes: - Similar to IsaGenerator, applicable test classes will contain an Assembler object pointer which may be instantiated in the test constructor. - Instantiation requires the GFXIP version in order to find the appropriate LLVM AMDGPU Target ID. - The RunAssemble() member func takes in a standard const char* shader and fills the TextData member with the output binary; TextSize with the size of TextData. These may be accessed via GetInstrStream() and GetInstrStreamSize(), or the output binary may be copied into an IsaBuffer via CopyInstrStream(). RunAssembleBuf() combines RunAssemble() and CopyInstrStream() and additionally takes an optional BufSize parameter to specify the size of the output buffer (defaults to PAGE_SIZE). - Assembler object deletion is to be done in the base test destructor. Assembler-specific memory allocation is freed in the Assembler destructor. - For debug, one can call PrintTextHex() to print out a formatted hex representation of the output binary, or PrintELFHex() to print out the intermediate ELF object. Note that PrintTextHex() is public whereas PrintELFHex() is private. - Prints use the LLVM outs() call as that allows for use of the LLVM format_hex() func in the aforementioned debug prints. This is subject to change if the LOG() call would be preferred. RunAssemble control flow: - Ensure correct Assembler initialization and clear previous run TextData (if necessary). - Initialize LLVM AMDGPU target, required interfaces, and buffers. - Set parser to specified target/subtarget and assemble into ELF code object. - Extract .text section from ELF, allocate space for TextData and store. - On success, returns 0 (HSAKMT_STATUS_SUCCESS). On error, returns -1 (subject to change to be in line with HSAKMT_STATUS enum). Signed-off-by: Graham Sider Change-Id: I1d96230824db651d3ffbaa46eb68fc274e7066b5 --- tests/kfdtest/CMakeLists.txt | 38 +++- tests/kfdtest/src/Assemble.cpp | 379 +++++++++++++++++++++++++++++++++ tests/kfdtest/src/Assemble.hpp | 84 ++++++++ 3 files changed, 499 insertions(+), 2 deletions(-) create mode 100644 tests/kfdtest/src/Assemble.cpp create mode 100644 tests/kfdtest/src/Assemble.hpp diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 05557ec16d..39994dbff3 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -95,6 +95,39 @@ endif() message ( "Find libhsakmt at ${HSAKMT_LIBRARY_DIRS}" ) +if ( POLICY CMP0074 ) + cmake_policy( SET CMP0074 NEW ) +endif() + +find_path( LIGHTNING_CMAKE_DIR NAMES LLVMConfig.cmake + PATHS $ENV{OUT_DIR}/llvm/lib/cmake/llvm NO_CACHE NO_DEFAULT_PATH) + +if ( DEFINED LIGHTNING_CMAKE_DIR AND EXISTS ${LIGHTNING_CMAKE_DIR} ) + set ( LLVM_DIR ${LIGHTNING_CMAKE_DIR} ) +else() + message( WARNING "Couldn't find Lightning build. " + "Attempting to use system LLVM install..." ) +endif() + +find_package( LLVM REQUIRED CONFIG ) + +if( ${LLVM_PACKAGE_VERSION} VERSION_LESS "7.0" ) + message( FATAL_ERROR "Requires LLVM 7.0 or greater " + "(found ${LLVM_PACKAGE_VERSION})" ) +elseif( ${LLVM_PACKAGE_VERSION} VERSION_LESS "14.0" ) + message( WARNING "Not using latest LLVM version. " + "Some ASIC targets may not work!" ) +endif() + +message( STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}" ) +message( STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}" ) + +include_directories(${LLVM_INCLUDE_DIRS}) +separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS}) +add_definitions(${LLVM_DEFINITIONS_LIST}) + +llvm_map_components_to_libnames(llvm_libs AMDGPUAsmParser Core Support) + set ( SP3_DIR ${PROJECT_SOURCE_DIR}/sp3 ) include_directories(${PROJECT_SOURCE_DIR}/gtest-1.6.0) @@ -112,6 +145,7 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/Dispatch.cpp src/GoogleTestExtension.cpp src/IndirectBuffer.cpp + src/Assemble.cpp src/IsaGenerator.cpp src/IsaGenerator_Aldebaran.cpp src/IsaGenerator_Gfx10.cpp @@ -163,7 +197,7 @@ message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") ## Add --enable-new-dtags to generate DT_RUNPATH -set ( CMAKE_CXX_FLAGS "-std=gnu++11 -Wl,--enable-new-dtags" ) +set ( CMAKE_CXX_FLAGS "-std=gnu++14 -Wl,--enable-new-dtags" ) endif() if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2" ) @@ -185,7 +219,7 @@ link_directories(${SP3_DIR}) add_executable(kfdtest ${SRC_FILES}) -target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread m stdc++ rt amdsp3 numa) +target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} ${llvm_libs} pthread m stdc++ rt amdsp3 numa) configure_file ( scripts/kfdtest.exclude kfdtest.exclude COPYONLY ) configure_file ( scripts/run_kfdtest.sh run_kfdtest.sh COPYONLY ) diff --git a/tests/kfdtest/src/Assemble.cpp b/tests/kfdtest/src/Assemble.cpp new file mode 100644 index 0000000000..cf4b9e7de0 --- /dev/null +++ b/tests/kfdtest/src/Assemble.cpp @@ -0,0 +1,379 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2022, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +/** + * Self-contained assembler that uses the LLVM MC API to assemble AMDGCN + * instructions + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LLVM_VERSION_MAJOR > 13 +#include +#else +#include +#endif + +#include +#include "OSWrapper.hpp" +#include "Assemble.hpp" + +using namespace llvm; + +Assembler::Assembler(const uint32_t Gfxv) { + SetTargetAsic(Gfxv); + TextData = nullptr; + TextSize = 0; + LLVMInit(); +} + +Assembler::~Assembler() { + FlushText(); + llvm_shutdown(); +} + +const char* Assembler::GetInstrStream() { + return TextData; +} + +const size_t Assembler::GetInstrStreamSize() { + return TextSize; +} + +int Assembler::CopyInstrStream(char* OutBuf, const size_t BufSize) { + if (TextSize > BufSize) + return -2; + + std::copy(TextData, TextData + TextSize, OutBuf); + return 0; +} + +const char* Assembler::GetTargetAsic() { + return MCPU; +} + +/** + * Set MCPU via GFX Version from Thunk + * LLVM Target IDs use decimal for Maj/Min, hex for Step + */ +void Assembler::SetTargetAsic(const uint32_t Gfxv) { + const uint8_t Major = (Gfxv >> 16) & 0xff; + const uint8_t Minor = (Gfxv >> 8) & 0xff; + const uint8_t Step = Gfxv & 0xff; + + snprintf(MCPU, ASM_MCPU_LEN, "gfx%d%d%x", Major, Minor, Step); +} + +/** + * Initialize LLVM targets and assembly printers/parsers + */ +void Assembler::LLVMInit() { + LLVMInitializeAMDGPUTargetInfo(); + LLVMInitializeAMDGPUTargetMC(); + LLVMInitializeAMDGPUAsmParser(); +} + +/** + * Flush/reset TextData and TextSize to initial state + */ +void Assembler::FlushText() { + if (TextData) + delete[] TextData; + TextData = nullptr; + TextSize = 0; +} + +/** + * Print hex of ELF object to stdout (debug) + */ +void Assembler::PrintELFHex(const std::string Data) { + outs() << "ASM Info: assembled ELF hex data (length " << Data.length() << "):\n"; + outs() << "0x00:\t"; + for (size_t i = 0; i < Data.length(); ++i) { + char c = Data[i]; + outs() << format_hex(static_cast(c), 4); + if ((i+1) % 16 == 0) + outs() << "\n" << format_hex(i+1, 4) << ":\t"; + else + outs() << " "; + } + outs() << "\n"; +} + +/** + * Print hex of raw instruction stream to stdout (debug) + */ +void Assembler::PrintTextHex() { + outs() << "ASM Info: assembled .text hex data (length " << TextSize << "):\n"; + outs() << "0x00:\t"; + for (size_t i = 0; i < TextSize; i++) { + outs() << format_hex(static_cast(TextData[i]), 4); + if ((i+1) % 16 == 0) + outs() << "\n" << format_hex(i+1, 4) << ":\t"; + else + outs() << " "; + } + outs() << "\n"; +} + +/** + * Extract raw instruction stream from .text section in ELF object + * + * @param RawData Raw C string of ELF object + * @return 0 on success + */ +int Assembler::ExtractELFText(const char* RawData) { + const Elf64_Ehdr* ElfHeader; + const Elf64_Shdr* SectHeader; + const Elf64_Shdr* SectStrTable; + const char* SectStrAddr; + unsigned NumSects, SectIdx; + + if (!(ElfHeader = reinterpret_cast(RawData))) { + outs() << "ASM Error: elf data is invalid or corrupted\n"; + return -1; + } + if (ElfHeader->e_ident[EI_CLASS] != ELFCLASS64) { + outs() << "ASM Error: elf object must be of 64-bit type\n"; + return -1; + } + + SectHeader = reinterpret_cast(RawData + ElfHeader->e_shoff); + SectStrTable = &SectHeader[ElfHeader->e_shstrndx]; + SectStrAddr = static_cast(RawData + SectStrTable->sh_offset); + + // Loop through sections, break on .text + NumSects = ElfHeader->e_shnum; + for (SectIdx = 0; SectIdx < NumSects; SectIdx++) { + std::string SectName = std::string(SectStrAddr + SectHeader[SectIdx].sh_name); + if (SectName == std::string(".text")) { + TextSize = SectHeader[SectIdx].sh_size; + TextData = new char[TextSize]; + memcpy(TextData, RawData + SectHeader[SectIdx].sh_offset, TextSize); + break; + } + } + + if (SectIdx >= NumSects) { + outs() << "ASM Error: couldn't locate .text section\n"; + return -1; + } + + return 0; +} + +/** + * Assemble shader, fill member vars, and copy to output buffer + * + * @param AssemblySource Shader source represented as a raw C string + * @param OutBuf Raw instruction stream output buffer + * @param BufSize Size of OutBuf (defaults to PAGE_SIZE) + * @return Value of RunAssemble() (0 on success) + */ +int Assembler::RunAssembleBuf(const char* const AssemblySource, char* OutBuf, + const size_t BufSize) { + int ret = RunAssemble(AssemblySource); + return ret ? ret : CopyInstrStream(OutBuf, BufSize); +} + +/** + * Assemble shader and fill member vars + * + * @param AssemblySource Shader source represented as a raw C string + * @return 0 on success + */ +int Assembler::RunAssemble(const char* const AssemblySource) { + // Ensure target ASIC has been set + if (!MCPU) { + outs() << "ASM Error: target asic is uninitialized\n"; + return -1; + } + + // Delete TextData for any previous runs + FlushText(); + +#if 0 + outs() << "ASM Info: running assembly for target: " << MCPU << "\n"; + outs() << "ASM Info: source:\n"; + outs() << AssemblySource << "\n"; +#endif + + // Initialize MCOptions and target triple + const MCTargetOptions MCOptions; + Triple TheTriple; + + const Target* TheTarget = + TargetRegistry::lookupTarget(ArchName, TheTriple, Error); + if (!TheTarget) { + outs() << Error; + return -1; + } + + TheTriple.setArchName(ArchName); + TheTriple.setVendorName(VendorName); + TheTriple.setOSName(OSName); + + TripleName = TheTriple.getTriple(); + TheTriple.setTriple(Triple::normalize(TripleName)); + + // Create MemoryBuffer for assembly source + StringRef AssemblyRef(AssemblySource); + std::unique_ptr BufferPtr = + MemoryBuffer::getMemBuffer(AssemblyRef, "", false); + if (!BufferPtr->getBufferSize()) { + outs() << "ASM Error: assembly source is empty\n"; + return -1; + } + + // Instantiate SrcMgr and transfer BufferPtr ownership + SourceMgr SrcMgr; + SrcMgr.AddNewSourceBuffer(std::move(BufferPtr), SMLoc()); + + // Initialize MC interfaces and base class objects + std::unique_ptr MRI( + TheTarget->createMCRegInfo(TripleName)); + if (!MRI) { + outs() << "ASM Error: no register info for target " << MCPU << "\n"; + return -1; + } +#if LLVM_VERSION_MAJOR > 9 + std::unique_ptr MAI( + TheTarget->createMCAsmInfo(*MRI, TripleName, MCOptions)); +#else + std::unique_ptr MAI( + TheTarget->createMCAsmInfo(*MRI, TripleName)); +#endif + if (!MAI) { + outs() << "ASM Error: no assembly info for target " << MCPU << "\n"; + return -1; + } + std::unique_ptr MCII( + TheTarget->createMCInstrInfo()); + if (!MCII) { + outs() << "ASM Error: no instruction info for target " << MCPU << "\n"; + return -1; + } + std::unique_ptr STI( + TheTarget->createMCSubtargetInfo(TripleName, MCPU, std::string())); + if (!STI || !STI->isCPUStringValid(MCPU)) { + outs() << "ASM Error: no subtarget info for target " << MCPU << "\n"; + return -1; + } + + // Set up the MCContext for creating symbols and MCExpr's +#if LLVM_VERSION_MAJOR > 12 + MCContext Ctx(TheTriple, MAI.get(), MRI.get(), STI.get(), &SrcMgr, &MCOptions); +#else + MCObjectFileInfo MOFI; + MCContext Ctx(MAI.get(), MRI.get(), &MOFI, &SrcMgr, &MCOptions); + MOFI.InitMCObjectFileInfo(TheTriple, true, Ctx); +#endif + + // Finalize setup for output object code stream + std::string Data; + std::unique_ptr DataStream(std::make_unique(Data)); + std::unique_ptr BOS(std::make_unique(*DataStream)); + raw_pwrite_stream* OS = BOS.get(); + +#if LLVM_VERSION_MAJOR > 14 + MCCodeEmitter* CE = TheTarget->createMCCodeEmitter(*MCII, Ctx); +#else + MCCodeEmitter* CE = TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx); +#endif + MCAsmBackend* MAB = TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions); + + std::unique_ptr Streamer(TheTarget->createMCObjectStreamer( + TheTriple, Ctx, + std::unique_ptr(MAB), MAB->createObjectWriter(*OS), + std::unique_ptr(CE), *STI, MCOptions.MCRelaxAll, + MCOptions.MCIncrementalLinkerCompatible, /*DWARFMustBeAtTheEnd*/ false)); + + std::unique_ptr Parser( + createMCAsmParser(SrcMgr, Ctx, *Streamer, *MAI)); + + // Set parser to target parser and run + std::unique_ptr TAP( + TheTarget->createMCAsmParser(*STI, *Parser, *MCII, MCOptions)); + if (!TAP) { + outs() << "ASM Error: no assembly parsing support for target " << MCPU << "\n"; + return -1; + } + Parser->setTargetParser(*TAP); + + if (Parser->Run(true)) { + outs() << "ASM Error: assembly parser failed\n"; + return -1; + } + + BOS.reset(); + DataStream->flush(); + + int ret = ExtractELFText(Data.data()); + if (ret < 0 || !TextData) { + outs() << "ASM Error: .text extraction failed\n"; + return ret; + } + +#if 0 + PrintELFHex(Data); + PrintTextHex(); +#endif + + return 0; +} diff --git a/tests/kfdtest/src/Assemble.hpp b/tests/kfdtest/src/Assemble.hpp new file mode 100644 index 0000000000..d61229a5a5 --- /dev/null +++ b/tests/kfdtest/src/Assemble.hpp @@ -0,0 +1,84 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2022, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef _ASSEMBLE_H_ +#define _ASSEMBLE_H_ + +#define ASM_MCPU_LEN 16 + +class Assembler { + private: + const char* ArchName = "amdgcn"; + const char* VendorName = "amd"; + const char* OSName = "amdhsa"; + char MCPU[ASM_MCPU_LEN]; + + std::string TripleName; + std::string Error; + + char* TextData; + size_t TextSize; + + void SetTargetAsic(const uint32_t Gfxv); + + void LLVMInit(); + void FlushText(); + void PrintELFHex(const std::string Data); + int ExtractELFText(const char* RawData); + + public: + Assembler(const uint32_t Gfxv); + ~Assembler(); + + void PrintTextHex(); + const char* GetTargetAsic(); + + const char* GetInstrStream(); + const size_t GetInstrStreamSize(); + int CopyInstrStream(char* OutBuf, const size_t BufSize = PAGE_SIZE); + + int RunAssemble(const char* const AssemblySource); + int RunAssembleBuf(const char* const AssemblySource, char* OutBuf, + const size_t BufSize = PAGE_SIZE); +}; + +#endif // _ASSEMBLE_H_ From 2f73db8fb05fe8f96c82964734496c056234f16d Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:49:02 -0400 Subject: [PATCH 0968/1247] kfdtest: Add GetGfxVersion to KFDTestUtil Required to derive LLVM AMDGPU target ASIC (MCPU). Signed-off-by: Graham Sider Change-Id: If8f139b3858c9bf42feba23ae9210e14625dc08b --- tests/kfdtest/src/KFDTestUtil.cpp | 6 ++++++ tests/kfdtest/src/KFDTestUtil.hpp | 1 + 2 files changed, 7 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 476e0bb1ce..2eddc8857b 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -231,6 +231,12 @@ bool isTonga(const HsaNodeProperties *props) { return false; } +const uint32_t GetGfxVersion(const HsaNodeProperties *props) { + return ((props->EngineId.ui32.Major << 16) | + (props->EngineId.ui32.Minor << 8) | + (props->EngineId.ui32.Stepping)); +} + HSAuint64 GetSystemTickCountInMicroSec() { struct timeval t; gettimeofday(&t, 0); diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 7c2f9c61ce..938ff8bf69 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -52,6 +52,7 @@ bool is_dgpu(); bool isTonga(const HsaNodeProperties *props); bool hasPciAtomicsSupport(int node); unsigned int FamilyIdFromNode(const HsaNodeProperties *props); +const uint32_t GetGfxVersion(const HsaNodeProperties *props); void GetHwQueueInfo(const HsaNodeProperties *props, unsigned int *p_num_cp_queues, From 235636d59856e08f9e49fbec0015bc4652760e9c Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 8 Oct 2021 13:35:51 -0400 Subject: [PATCH 0969/1247] kfdtest: Instantiate Assembler in KFDBaseComponentTest Instantiate in KFDBaseComponentTest::SetUp() and destroy in TearDown(). This ensures m_pAsm is available for all tests. Signed-off-by: Graham Sider Change-Id: I8b98a5350a9739d71455f14552c9879bdb1c475d --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 6 ++++++ tests/kfdtest/src/KFDBaseComponentTest.hpp | 2 ++ 2 files changed, 8 insertions(+) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index f950a7a1f1..5618945505 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -68,6 +68,8 @@ void KFDBaseComponentTest::SetUp() { g_baseTest = this; + m_pAsm = new Assembler(GetGfxVersion(nodeProperties)); + ROUTINE_END } @@ -86,6 +88,10 @@ void KFDBaseComponentTest::TearDown() { EXPECT_SUCCESS(hsaKmtCloseKFD()); g_baseTest = NULL; + if (m_pAsm) + delete m_pAsm; + m_pAsm = nullptr; + ROUTINE_END } diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index e27baf1cc1..ec3053c3d3 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -34,6 +34,7 @@ #include "hsakmt.h" #include "OSWrapper.hpp" #include "KFDTestUtil.hpp" +#include "Assemble.hpp" // @class KFDBaseComponentTest class KFDBaseComponentTest : public testing::Test { @@ -74,6 +75,7 @@ class KFDBaseComponentTest : public testing::Test { HsaMemFlags m_MemoryFlags; HsaNodeInfo m_NodeInfo; HSAint32 m_xnack; + Assembler* m_pAsm; // @brief Executed before every test that uses KFDBaseComponentTest class and sets all common settings for the tests. virtual void SetUp(); From 039bce94a6a7f3c4133c745c628749dca5013012 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:45:28 -0400 Subject: [PATCH 0970/1247] kfdtest: Update KFDMemoryTest to LLVM Asm - Reformat shaders for legibility - Move assembly processes to from IsaGen (CompileShader) to Assembler (RunAssembleBuf) - LLVM syntax change on ScratchCopyDwordIsa_gfx10: hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO/HI) -> hwreg(HW_REG_FLAT_SCR_LO/HI) - Fix bug in CopyOnSignalIsa_gfx10 and PollMemoryIsa_gfx10 whereby flat_store_dword used vector reg format v[n,n]. Changed to v[n:n] Signed-off-by: Graham Sider Change-Id: Id182cfb8aeb7372366c59affb5cbdd145909ee96 --- tests/kfdtest/src/KFDMemoryTest.cpp | 614 +++++++++++++--------------- 1 file changed, 284 insertions(+), 330 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 8cf24ffb1b..980c9c7bf8 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -39,166 +39,124 @@ #include "SDMAPacket.hpp" #include "linux/kfd_ioctl.h" -const char* gfx8_ScratchCopyDword = -"\ -shader ScratchCopyDword\n\ -asic(VI)\n\ -type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ -/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ - s_mov_b32 flat_scratch_lo, 8/*2 dwords of scratch per thread*/\n\ - s_mov_b32 flat_scratch_hi, 0/*offset in units of 256bytes*/\n\ -/*copy a dword between the passed addresses*/\n\ - flat_load_dword v4, v[0:1] slc\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - flat_store_dword v[2:3], v4 slc\n\ - \n\ - s_endpgm\n\ - \n\ -end\n\ -"; - -const char* gfx9_ScratchCopyDword = -"\ -shader ScratchCopyDword\n\ -asic(GFX9)\n\ -type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ -/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ - s_mov_b32 flat_scratch_lo, s4\n\ - s_mov_b32 flat_scratch_hi, s5\n\ -/*copy a dword between the passed addresses*/\n\ - flat_load_dword v4, v[0:1] slc\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - flat_store_dword v[2:3], v4 slc\n\ - \n\ - s_endpgm\n\ - \n\ -end\n\ -"; -const char* gfx10_ScratchCopyDword = -"\ -shader ScratchCopyDword\n\ -asic(GFX10)\n\ -type(CS)\n\ -wave_size(32)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ -/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ - s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_LO), s4\n\ - s_setreg_b32 hwreg(HW_REG_SHADER_FLAT_SCRATCH_HI), s5\n\ -/*copy a dword between the passed addresses*/\n\ - flat_load_dword v4, v[0:1] slc\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - flat_store_dword v[2:3], v4 slc\n\ - \n\ - s_endpgm\n\ - \n\ -end\n\ -"; - -const char* aldbrn_ScratchCopyDword = -"\ -shader ScratchCopyDword\n\ -asic(ALDEBARAN)\n\ -type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ -/*set up the scratch parameters. This assumes a single 16-reg block.*/\n\ - s_mov_b32 flat_scratch_lo, s4\n\ - s_mov_b32 flat_scratch_hi, s5\n\ -/*copy a dword between the passed addresses*/\n\ - flat_load_dword v4, v[0:1] slc\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - flat_store_dword v[2:3], v4 slc\n\ - \n\ - s_endpgm\n\ - \n\ -end\n\ -"; - - +static const char* ScratchCopyDwordIsa_gfx8 = R"( + .text + // Copy the parameters from scalar registers to vector registers + v_mov_b32_e32 v0, s0 + v_mov_b32_e32 v1, s1 + v_mov_b32_e32 v2, s2 + v_mov_b32_e32 v3, s3 + // Setup the scratch parameters. This assumes a single 16-reg block + s_mov_b32 flat_scratch_lo, 8 + s_mov_b32 flat_scratch_hi, 0 + // Copy a dword between the passed addresses + flat_load_dword v4, v[0:1] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + flat_store_dword v[2:3], v4 slc + s_endpgm +)"; +static const char* ScratchCopyDwordIsa_gfx9 = R"( + .text + // Copy the parameters from scalar registers to vector registers + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 + // Setup the scratch parameters. This assumes a single 16-reg block + s_mov_b32 flat_scratch_lo, s4 + s_mov_b32 flat_scratch_hi, s5 + // Copy a dword between the passed addresses + flat_load_dword v4, v[0:1] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + flat_store_dword v[2:3], v4 slc + s_endpgm +)"; +static const char* ScratchCopyDwordIsa_gfx10 = R"( + .text + // Copy the parameters from scalar registers to vector registers + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 + // Setup the scratch parameters. This assumes a single 16-reg block + s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4 + s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5 + // Copy a dword between the passed addresses + flat_load_dword v4, v[0:1] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + flat_store_dword v[2:3], v4 slc + s_endpgm +)"; +static const char* ScratchCopyDwordIsa_gfx9aldbrn = R"( + .text + // Copy the parameters from scalar registers to vector registers + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 + // Setup the scratch parameters. This assumes a single 16-reg block + s_mov_b32 flat_scratch_lo, s4 + s_mov_b32 flat_scratch_hi, s5 + // Copy a dword between the passed addresses + flat_load_dword v4, v[0:1] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + flat_store_dword v[2:3], v4 slc + s_endpgm +)"; /* Continuously poll src buffer and check buffer value * After src buffer is filled with specific value (0x5678, * by host program), fill dst buffer with specific * value(0x5678) and quit */ -const char* gfx9_PollMemory = -"\ -shader ReadMemory\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ - s_movk_i32 s18, 0x5678\n\ - LOOP:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 LOOP\n\ - s_store_dword s18, s[2:3], 0x0 glc\n\ - s_endpgm\n\ - end\n\ -"; +static const char* PollMemoryIsa_gfx9 = R"( + .text + // Assume src address in s0, s1, and dst address in s2, s3 + s_movk_i32 s18, 0x5678 + LOOP: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 LOOP + s_store_dword s18, s[2:3], 0x0 glc + s_endpgm +)"; -/* Similar to gfx9_PollMemory except that the buffer +/* Similar to PollMemoryIsa_gfx9 except that the buffer * polled can be Non-coherant memory. SCC system-level * cache coherence is not supported in scalar (smem) path. * Use vmem operations with scc */ -const char* gfx9_PollNCMemory = -"\ -shader ReadMemory\n\ -asic(ALDEBARAN)\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ - v_mov_b32 v6, 0x5678\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - LOOP:\n\ - flat_load_dword v4, v[0:1] scc\n\ - v_cmp_eq_u32 vcc, v4, v6\n\ - s_cbranch_vccz LOOP\n\ - v_mov_b32 v0, s2\n\ - v_mov_b32 v1, s3\n\ - flat_store_dword v[0:1], v6 scc\n\ - s_endpgm\n\ - end\n\ -"; +static const char* PollNCMemoryIsa_gfx9 = R"( + .text + // Assume src address in s0, s1, and dst address in s2, s3 + v_mov_b32 v6, 0x5678 + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + LOOP: + flat_load_dword v4, v[0:1] scc + v_cmp_eq_u32 vcc, v4, v6 + s_cbranch_vccz LOOP + v_mov_b32 v0, s2 + v_mov_b32 v1, s3 + flat_store_dword v[0:1], v6 scc + s_endpgm +)"; -const char* gfx10_PollMemory = -"\ -shader ReadMemory\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume src address in s0, s1 and dst address in s2, s3*/\n\ - s_movk_i32 s18, 0x5678\n\ - v_mov_b32 v0, s2\n\ - v_mov_b32 v1, s3\n\ - v_mov_b32 v2, 0x5678\n\ - LOOP:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 LOOP\n\ - flat_store_dword v[0,1], v2 slc\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; +static const char* PollMemoryIsa_gfx10 = R"( + .text + // Assume src address in s0, s1, and dst address in s2, s3 + s_movk_i32 s18, 0x5678 + v_mov_b32 v0, s2 + v_mov_b32 v1, s3 + v_mov_b32 v2, 0x5678 + LOOP: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 LOOP + flat_store_dword v[0:1], v2 slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; /* Input: A buffer of at least 3 dwords. * DW0: used as a signal. 0xcafe means it is signaled @@ -209,119 +167,99 @@ type(CS)\n\ * Once signal buffer is signaled, it copies input buffer * to output buffer */ -const char* gfx9_CopyOnSignal = -"\ -shader CopyOnSignal\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume input buffer in s0, s1 */\n\ - s_mov_b32 s18, 0xcafe\n\ -POLLSIGNAL:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 POLLSIGNAL\n\ - s_load_dword s17, s[0:1], 0x4 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_store_dword s17, s[0:1], 0x8 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; +static const char* CopyOnSignalIsa_gfx9 = R"( + .text + // Assume input buffer in s0, s1 + s_mov_b32 s18, 0xcafe + POLLSIGNAL: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 POLLSIGNAL + s_load_dword s17, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s17, s[0:1], 0x8 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; -const char* gfx10_CopyOnSignal = -"\ -shader CopyOnSignal\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume input buffer in s0, s1 */\n\ - s_add_u32 s2, s0, 0x8\n\ - s_addc_u32 s3, s1, 0x0\n\ - s_mov_b32 s18, 0xcafe\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v4, s2\n\ - v_mov_b32 v5, s3\n\ -POLLSIGNAL:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 POLLSIGNAL\n\ - s_load_dword s17, s[0:1], 0x4 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - v_mov_b32 v2, s17\n\ - flat_store_dword v[4,5], v2 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; +static const char* CopyOnSignalIsa_gfx10 = R"( + .text + // Assume input buffer in s0, s1 + s_add_u32 s2, s0, 0x8 + s_addc_u32 s3, s1, 0x0 + s_mov_b32 s18, 0xcafe + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v4, s2 + v_mov_b32 v5, s3 + POLLSIGNAL: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 POLLSIGNAL + s_load_dword s17, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + v_mov_b32 v2, s17 + flat_store_dword v[4:5], v2 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; /* Input0: A buffer of at least 2 dwords. * DW0: used as a signal. Write 0xcafe to signal * DW1: Write to this buffer for other device to read. * Input1: mmio base address */ -const char* gfx9_WriteAndSignal = -"\ -shader WriteAndSignal\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume input buffer in s0, s1 */\n\ - s_mov_b32 s18, 0xbeef\n\ - s_store_dword s18, s[0:1], 0x4 glc\n\ - s_mov_b32 s18, 0x1\n\ - s_store_dword s18, s[2:3], 0 glc\n\ - s_mov_b32 s18, 0xcafe\n\ - s_store_dword s18, s[0:1], 0x0 glc\n\ - s_endpgm\n\ - end\n\ -"; +static const char* WriteAndSignalIsa_gfx9 = R"( + .text + // Assume input buffer in s0, s1 + s_mov_b32 s18, 0xbeef + s_store_dword s18, s[0:1], 0x4 glc + s_mov_b32 s18, 0x1 + s_store_dword s18, s[2:3], 0 glc + s_mov_b32 s18, 0xcafe + s_store_dword s18, s[0:1], 0x0 glc + s_endpgm +)"; /* Continuously poll the flag at src buffer * After the flag of s[0:1] is 1 filled, * copy the value from s[0:1]+4 to dst buffer */ -const char* gfx9_PollAndCopy = -"\ -shader CopyMemory\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume src buffer in s[0:1] and dst buffer in s[2:3]*/\n\ - s_movk_i32 s18, 0x1\n\ - LOOP:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_cmp_eq_i32 s16, s18\n\ - s_cbranch_scc0 LOOP\n\ - s_load_dword s17, s[0:1], 0x4 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_store_dword s17, s[2:3], 0x0 glc:1\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; +static const char* PollAndCopyIsa_gfx9 = R"( + .text + // Assume src buffer in s[0:1] and dst buffer in s[2:3] + s_movk_i32 s18, 0x1 + LOOP: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 LOOP + s_load_dword s17, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s17, s[2:3], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; -const char* gfx9aldbrn_PollAndCopy = -"\ -shader CopyMemory\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume src buffer in s[0:1] and dst buffer in s[2:3]*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v18, 0x1\n\ - LOOP:\n\ - flat_load_dword v16, v[0:1] glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - v_cmp_eq_i32 vcc, v16, v18\n\ - s_cbranch_vccz LOOP\n\ - buffer_invl2\n\ - s_load_dword s17, s[0:1], 0x4 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_store_dword s17, s[2:3], 0x0 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - buffer_wbl2\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; +static const char* PollAndCopyIsa_gfx9aldbrn = R"( + .text + // Assume src buffer in s[0:1] and dst buffer in s[2:3] + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v18, 0x1 + LOOP: + flat_load_dword v16, v[0:1] glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + v_cmp_eq_i32 vcc, v16, v18 + s_cbranch_vccz LOOP + buffer_invl2 + s_load_dword s17, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s17, s[2:3], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + buffer_wbl2 + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; /* Input0: A buffer of at least 2 dwords. * DW0: used as a signal. Write 0x1 to signal @@ -330,51 +268,45 @@ type(CS)\n\ * Input1: A buffer of at least 2 dwords. * DW0: used as the value to be written. */ -const char* gfx9aldbrn_WriteFlagAndValue = -"\ -shader WriteMemory\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume two inputs buffer in s[0:1] and s[2:3]*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - s_load_dword s18, s[2:3], 0x0 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - s_store_dword s18, s[0:1], 0x4 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - buffer_wbl2\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0)\n\ - v_mov_b32 v16, 0x1\n\ - flat_store_dword v[0:1], v16 glc\n\ - s_endpgm\n\ - end\n\ -"; +static const char* WriteFlagAndValueIsa_gfx9aldbrn = R"( + .text + // Assume two inputs buffer in s[0:1] and s[2:3] + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + s_load_dword s18, s[2:3], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s18, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + buffer_wbl2 + s_waitcnt vmcnt(0) & lgkmcnt(0) + v_mov_b32 v16, 0x1 + flat_store_dword v[0:1], v16 glc + s_endpgm +)"; -const char* gfx10_WriteAndSignal = -"\ -shader WriteAndSignal\n\ -wave_size(32)\n\ -type(CS)\n\ -/* Assume input buffer in s0, s1 */\n\ - s_add_u32 s4, s0, 0x4\n\ - s_addc_u32 s5, s1, 0x0\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ - v_mov_b32 v4, s4\n\ - v_mov_b32 v5, s5\n\ - v_mov_b32 v18, 0xbeef\n\ - flat_store_dword v[4:5], v18 glc\n\ - v_mov_b32 v18, 0x1\n\ - flat_store_dword v[2:3], v18 glc\n\ - v_mov_b32 v18, 0xcafe\n\ - flat_store_dword v[0:1], v18 glc\n\ - s_endpgm\n\ - end\n\ -"; +static const char* WriteAndSignalIsa_gfx10 = R"( + .text + // Assume input buffer in s0, s1 + s_add_u32 s4, s0, 0x4 + s_addc_u32 s5, s1, 0x0 + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 + v_mov_b32 v4, s4 + v_mov_b32 v5, s5 + v_mov_b32 v18, 0xbeef + flat_store_dword v[4:5], v18 glc + v_mov_b32 v18, 0x1 + flat_store_dword v[2:3], v18 glc + v_mov_b32 v18, 0xcafe + flat_store_dword v[0:1], v18 glc + s_endpgm +)"; -//These gfx9_PullMemory, gfx9_CopyOnSignal, gfx9_WriteAndSignal shaders can be used by both gfx9 and gfx10 +/* These PollMemoryIsa_gfx9, CopyOnSignalIsa_gfx9, + * WriteAndSignalIsa_gfx9 shaders can be used by both gfx9 and gfx10 + */ void KFDMemoryTest::SetUp() { ROUTINE_START @@ -508,16 +440,15 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); const char *pReadMemory; - if (m_FamilyId < FAMILY_NV) - pReadMemory = gfx9_PollMemory; - else - pReadMemory = gfx10_PollMemory; - if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) /* On A+A system memory is mapped as NC */ - m_pIsaGen->CompileShader(gfx9_PollNCMemory, "ReadMemory", isaBuffer); + pReadMemory = PollNCMemoryIsa_gfx9; + else if (m_FamilyId < FAMILY_NV) + pReadMemory = PollMemoryIsa_gfx9; else - m_pIsaGen->CompileShader(pReadMemory, "ReadMemory", isaBuffer); + pReadMemory = PollMemoryIsa_gfx10; + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pReadMemory, isaBuffer.As())); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); @@ -855,16 +786,17 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { // Initialize the srcBuffer to some fixed value srcMemBuffer.Fill(0x01010101); - const char *pScratchCopyDword; + const char *pScratchCopyDwordIsa; if (m_FamilyId < FAMILY_AI) - pScratchCopyDword = gfx8_ScratchCopyDword; + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx8; else if (m_FamilyId < FAMILY_AL) - pScratchCopyDword = gfx9_ScratchCopyDword; + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9; else if (m_FamilyId == FAMILY_AL) - pScratchCopyDword = aldbrn_ScratchCopyDword; + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9aldbrn; else - pScratchCopyDword = gfx10_ScratchCopyDword; - m_pIsaGen->CompileShader(pScratchCopyDword, "ScratchCopyDword", isaBuffer); + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx10; + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pScratchCopyDwordIsa, isaBuffer.As())); const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); @@ -1728,17 +1660,18 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { // dstBuffer is cpu accessible gtt memory HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - const char *pScratchCopyDword; + const char *pScratchCopyDwordIsa; if (m_FamilyId < FAMILY_AI) - pScratchCopyDword = gfx8_ScratchCopyDword; + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx8; else if (m_FamilyId < FAMILY_AL) - pScratchCopyDword = gfx9_ScratchCopyDword; + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9; else if (m_FamilyId == FAMILY_AL) - pScratchCopyDword = aldbrn_ScratchCopyDword; + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9aldbrn; else - pScratchCopyDword = gfx10_ScratchCopyDword; + pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx10; + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pScratchCopyDwordIsa, isaBuffer.As())); - m_pIsaGen->CompileShader(pScratchCopyDword, "ScratchCopyDword", isaBuffer); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(mem0, dstBuffer.As()); dispatch0.Submit(queue); @@ -2109,12 +2042,14 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - const char *pCopyOnSignal; + const char *pCopyOnSignalIsa; if (m_FamilyId < FAMILY_NV) - pCopyOnSignal = gfx9_CopyOnSignal; + pCopyOnSignalIsa = CopyOnSignalIsa_gfx9; else - pCopyOnSignal = gfx10_CopyOnSignal; - m_pIsaGen->CompileShader(pCopyOnSignal, "CopyOnSignal", isaBuffer); + pCopyOnSignalIsa = CopyOnSignalIsa_gfx10; + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pCopyOnSignalIsa, isaBuffer.As())); + Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(buffer, NULL); dispatch0.Submit(queue); @@ -2234,12 +2169,14 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(nodes[0])); HsaMemoryBuffer isaBuffer(PAGE_SIZE, nodes[0], true/*zero*/, false/*local*/, true/*exec*/); - const char *pCopyOnSignal; + const char *pCopyOnSignalIsa; if (m_FamilyId < FAMILY_NV) - pCopyOnSignal = gfx9_CopyOnSignal; + pCopyOnSignalIsa = CopyOnSignalIsa_gfx9; else - pCopyOnSignal = gfx10_CopyOnSignal; - m_pIsaGen->CompileShader(pCopyOnSignal, "CopyOnSignal", isaBuffer); + pCopyOnSignalIsa = CopyOnSignalIsa_gfx10; + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pCopyOnSignalIsa, isaBuffer.As())); + Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, NULL); dispatch.Submit(queue); @@ -2247,12 +2184,14 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue0; ASSERT_SUCCESS(queue0.Create(nodes[1])); HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); - const char *pWriteAndSignal; + const char *pWriteAndSignalIsa; if (m_FamilyId < FAMILY_NV) - pWriteAndSignal = gfx9_WriteAndSignal; + pWriteAndSignalIsa = WriteAndSignalIsa_gfx9; else - pWriteAndSignal = gfx10_WriteAndSignal; - m_pIsaGen->CompileShader(pWriteAndSignal, "WriteAndSignal", isaBuffer0); + pWriteAndSignalIsa = WriteAndSignalIsa_gfx10; + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pWriteAndSignalIsa, isaBuffer.As())); + Dispatch dispatch0(isaBuffer0); dispatch0.SetArgs(buffer, mmioBase); dispatch0.Submit(queue0); @@ -2304,7 +2243,9 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnSdmaWrite) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As())); + Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); dispatch.Submit(queue); @@ -2357,7 +2298,9 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnCPUWrite) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As())); + Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, buffer+100); dispatch.Submit(queue); @@ -2419,7 +2362,9 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnRemoteWrite) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9_PollMemory, "ReadMemory", isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As())); + Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); dispatch.Submit(queue); @@ -2500,7 +2445,9 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa_gfx9aldbrn, isaBuffer.As())); + Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); dispatch.Submit(queue); @@ -2515,7 +2462,9 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) { PM4Queue queue1; ASSERT_SUCCESS(queue1.Create(nondefaultNode)); HsaMemoryBuffer isaBuffer1(PAGE_SIZE, nondefaultNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9aldbrn_WriteFlagAndValue, "WriteMemory", isaBuffer1); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteFlagAndValueIsa_gfx9aldbrn, isaBuffer.As())); + Dispatch dispatch1(isaBuffer1); dispatch1.SetArgs(buffer.As(), buffer.As()+dwSource); dispatch1.Submit(queue1); @@ -2569,7 +2518,9 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithCPU) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa_gfx9aldbrn, isaBuffer.As())); + Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, buffer+dwLocation); dispatch.Submit(queue); @@ -2627,10 +2578,13 @@ TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + const char* pPollAndCopyIsa; if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) - m_pIsaGen->CompileShader(gfx9aldbrn_PollAndCopy, "CopyMemory", isaBuffer); + pPollAndCopyIsa = PollAndCopyIsa_gfx9aldbrn; else - m_pIsaGen->CompileShader(gfx9_PollAndCopy, "CopyMemory", isaBuffer); + pPollAndCopyIsa = PollAndCopyIsa_gfx9; + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pPollAndCopyIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(fineBuffer, fineBuffer+dwLocation); From 08d38fb14010fd0bcf24d73ce626d8b5bffc49d8 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:46:46 -0400 Subject: [PATCH 0971/1247] kfdtest: Update KFDQMTest to LLVM Asm - Reformat shaders for legibility - Move assembly processes to from IsaGen (CompileShader) to Assembler (RunAssembleBuf) Signed-off-by: Graham Sider Change-Id: I669f076b5c34eb90349865eeca1b29e17c9e80d6 --- tests/kfdtest/src/KFDQMTest.cpp | 200 ++++++++++++++++---------------- 1 file changed, 97 insertions(+), 103 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 9b4003b68e..c28715639f 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -34,6 +34,100 @@ #include "Dispatch.hpp" +/* A simple isa loop program with dense mathematic operations + * s1 controls the number iterations of the loop + * This shader can be used by GFX8, GFX9 and GFX10 + */ +static const char* LoopIsa = R"( + .text + s_movk_i32 s0, 0x0008 + s_movk_i32 s1, 0x00ff + v_mov_b32 v0, 0 + v_mov_b32 v1, 0 + v_mov_b32 v2, 0 + v_mov_b32 v3, 0 + v_mov_b32 v4, 0 + v_mov_b32 v5, 0 + v_mov_b32 v6, 0 + v_mov_b32 v7, 0 + v_mov_b32 v8, 0 + v_mov_b32 v9, 0 + v_mov_b32 v10, 0 + v_mov_b32 v11, 0 + v_mov_b32 v12, 0 + v_mov_b32 v13, 0 + v_mov_b32 v14, 0 + v_mov_b32 v15, 0 + v_mov_b32 v16, 0 + LOOP: + s_mov_b32 s8, s4 + s_mov_b32 s9, s1 + s_mov_b32 s10, s6 + s_mov_b32 s11, s7 + s_cmp_le_i32 s1, s0 + s_cbranch_scc1 END_OF_PGM + s_buffer_load_dwordx8 s[8:15], s[8:11], 0x10 + v_add_f32 v0, 2.0, v0 + v_cvt_f32_i32 v17, s1 + s_waitcnt lgkmcnt(0) + v_add_f32 v18, s8, v17 + v_add_f32 v19, s9, v17 + v_add_f32 v20, s10, v17 + v_add_f32 v21, s11, v17 + v_add_f32 v22, s12, v17 + v_add_f32 v23, s13, v17 + v_add_f32 v24, s14, v17 + v_add_f32 v17, s15, v17 + v_log_f32 v25, v18 + v_mul_f32 v25, v22, v25 + v_exp_f32 v25, v25 + v_log_f32 v26, v19 + v_mul_f32 v26, v23, v26 + v_exp_f32 v26, v26 + v_log_f32 v27, v20 + v_mul_f32 v27, v24, v27 + v_exp_f32 v27, v27 + v_log_f32 v28, v21 + v_mul_f32 v28, v17, v28 + v_exp_f32 v28, v28 + v_add_f32 v5, v5, v25 + v_add_f32 v6, v6, v26 + v_add_f32 v7, v7, v27 + v_add_f32 v8, v8, v28 + v_mul_f32 v18, 0x3fb8aa3b, v18 + v_exp_f32 v18, v18 + v_mul_f32 v19, 0x3fb8aa3b, v19 + v_exp_f32 v19, v19 + v_mul_f32 v20, 0x3fb8aa3b, v20 + v_exp_f32 v20, v20 + v_mul_f32 v21, 0x3fb8aa3b, v21 + v_exp_f32 v21, v21 + v_add_f32 v9, v9, v18 + v_add_f32 v10, v10, v19 + v_add_f32 v11, v11, v20 + v_add_f32 v12, v12, v21 + v_sqrt_f32 v18, v22 + v_sqrt_f32 v19, v23 + v_sqrt_f32 v20, v24 + v_sqrt_f32 v21, v17 + v_add_f32 v13, v13, v18 + v_add_f32 v14, v14, v19 + v_add_f32 v15, v15, v20 + v_add_f32 v16, v16, v21 + v_rsq_f32 v18, v22 + v_rsq_f32 v19, v23 + v_rsq_f32 v20, v24 + v_rsq_f32 v17, v17 + v_add_f32 v1, v1, v18 + v_add_f32 v2, v2, v19 + v_add_f32 v3, v3, v20 + v_add_f32 v4, v4, v17 + s_add_u32 s0, s0, 1 + s_branch LOOP + END_OF_PGM: + s_endpgm +)"; + void KFDQMTest::SetUp() { ROUTINE_START @@ -677,111 +771,12 @@ TEST_F(KFDQMTest, OverSubscribeCpQueues) { TEST_END } -/* A simple isa loop program with dense mathematic operations - * s1 controls the number iterations of the loop - * This shader can be used by GFX8, GFX9 and GFX10 - */ -static const char *loop_isa = \ -"\ -shader loop_isa\n\ -wave_size(32)\n\ -type(CS)\n\ - s_movk_i32 s0, 0x0008\n\ - s_movk_i32 s1, 0x00ff\n\ - v_mov_b32 v0, 0\n\ - v_mov_b32 v1, 0\n\ - v_mov_b32 v2, 0\n\ - v_mov_b32 v3, 0\n\ - v_mov_b32 v4, 0\n\ - v_mov_b32 v5, 0\n\ - v_mov_b32 v6, 0\n\ - v_mov_b32 v7, 0\n\ - v_mov_b32 v8, 0\n\ - v_mov_b32 v9, 0\n\ - v_mov_b32 v10, 0\n\ - v_mov_b32 v11, 0\n\ - v_mov_b32 v12, 0\n\ - v_mov_b32 v13, 0\n\ - v_mov_b32 v14, 0\n\ - v_mov_b32 v15, 0\n\ - v_mov_b32 v16, 0\n\ - LOOP:\n\ - s_mov_b32 s8, s4\n\ - s_mov_b32 s9, s1\n\ - s_mov_b32 s10, s6\n\ - s_mov_b32 s11, s7\n\ - s_cmp_le_i32 s1, s0\n\ - s_cbranch_scc1 END_OF_PGM\n\ - s_buffer_load_dwordx8 s[8:15], s[8:11], 0x10\n\ - v_add_f32 v0, 2.0, v0\n\ - v_cvt_f32_i32 v17, s1\n\ -s_waitcnt lgkmcnt(0)\n\ - v_add_f32 v18, s8, v17\n\ - v_add_f32 v19, s9, v17\n\ - v_add_f32 v20, s10, v17\n\ - v_add_f32 v21, s11, v17\n\ - v_add_f32 v22, s12, v17\n\ - v_add_f32 v23, s13, v17\n\ - v_add_f32 v24, s14, v17\n\ - v_add_f32 v17, s15, v17\n\ - v_log_f32 v25, v18\n\ - v_mul_f32 v25, v22, v25\n\ - v_exp_f32 v25, v25\n\ - v_log_f32 v26, v19\n\ - v_mul_f32 v26, v23, v26\n\ - v_exp_f32 v26, v26\n\ - v_log_f32 v27, v20\n\ - v_mul_f32 v27, v24, v27\n\ - v_exp_f32 v27, v27\n\ - v_log_f32 v28, v21\n\ - v_mul_f32 v28, v17, v28\n\ - v_exp_f32 v28, v28\n\ - v_add_f32 v5, v5, v25\n\ - v_add_f32 v6, v6, v26\n\ - v_add_f32 v7, v7, v27\n\ - v_add_f32 v8, v8, v28\n\ - v_mul_f32 v18, 0x3fb8aa3b, v18\n\ - v_exp_f32 v18, v18\n\ - v_mul_f32 v19, 0x3fb8aa3b, v19\n\ - v_exp_f32 v19, v19\n\ - v_mul_f32 v20, 0x3fb8aa3b, v20\n\ - v_exp_f32 v20, v20\n\ - v_mul_f32 v21, 0x3fb8aa3b, v21\n\ - v_exp_f32 v21, v21\n\ - v_add_f32 v9, v9, v18\n\ - v_add_f32 v10, v10, v19\n\ - v_add_f32 v11, v11, v20\n\ - v_add_f32 v12, v12, v21\n\ - v_sqrt_f32 v18, v22\n\ - v_sqrt_f32 v19, v23\n\ - v_sqrt_f32 v20, v24\n\ - v_sqrt_f32 v21, v17\n\ - v_add_f32 v13, v13, v18\n\ - v_add_f32 v14, v14, v19\n\ - v_add_f32 v15, v15, v20\n\ - v_add_f32 v16, v16, v21\n\ - v_rsq_f32 v18, v22\n\ - v_rsq_f32 v19, v23\n\ - v_rsq_f32 v20, v24\n\ - v_rsq_f32 v17, v17\n\ - v_add_f32 v1, v1, v18\n\ - v_add_f32 v2, v2, v19\n\ - v_add_f32 v3, v3, v20\n\ - v_add_f32 v4, v4, v17\n\ - s_add_u32 s0, s0, 1\n\ - s_branch LOOP\n\ - END_OF_PGM:\n\ - s_endpgm\n\ - end\n\ -"; - HSAint64 KFDQMTest::TimeConsumedwithCUMask(int node, uint32_t* mask, uint32_t mask_count) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer dstBuffer(PAGE_SIZE, node, true, false, false); HsaMemoryBuffer ctlBuffer(PAGE_SIZE, node, true, false, false); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - m_pIsaGen->CompileShader(loop_isa, "loop_isa", isaBuffer); + EXPECT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetDim(1024, 16, 16); @@ -838,7 +833,6 @@ TEST_F(KFDQMTest, BasicCuMaskingLinear) { TEST_START(TESTPROFILE_RUNALL); int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - m_pIsaGen = IsaGenerator::Create(m_FamilyId); if (m_FamilyId >= FAMILY_VI) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); @@ -982,7 +976,7 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { HSAint32 *syncBuffer = syncBuf.As(); HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(loop_isa, "loop_isa", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As())); Dispatch dispatch[2] = { Dispatch(isaBuffer, true), @@ -1047,7 +1041,7 @@ TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { HSAint32 *syncBuffer = syncBuf.As(); HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(loop_isa, "loop_isa", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As())); Dispatch dispatch[2] = { Dispatch(isaBuffer, true), From c845b976d0e297c769f5ce8a5e31d9293464e5de Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:47:23 -0400 Subject: [PATCH 0972/1247] kfdtest: Update KFDCWSRTest to LLVM Asm - Reformat shaders for legibility - Move assembly processes to from IsaGen (CompileShader) to Assembler (RunAssembleBuf) Signed-off-by: Graham Sider Change-Id: I174f1ea5332c499440b30d9bcf06836274428a0f --- tests/kfdtest/src/KFDCWSRTest.cpp | 122 +++++++++++++----------------- tests/kfdtest/src/KFDCWSRTest.hpp | 4 +- 2 files changed, 54 insertions(+), 72 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index e12e697566..c7a2e4b312 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -38,76 +38,63 @@ * v6 - counter */ -static const char* iterate_isa_gfx8 = \ -"\ -shader iterate_isa\n\ -wave_size(32)\n\ -type(CS)\n\ - // copy the parameters from scalar registers to vector registers\n\ - v_mov_b32 v2, s0 // v[2:3] = s[0:1] \n\ - v_mov_b32 v3, s1 // v[2:3] = s[0:1] \n\ - v_mov_b32 v0, s4 // use workgroup id as index \n\ - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 \n\ - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 \n\ - v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 \n\ - v_add_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 \n\ - v_mov_b32 v6, 0 \n\ -LOOP: \n\ - v_add_u32 v6, vcc, 1, v6 \n\ - // compare the result value (v6) to iteration value (v2), and \n\ - // jump if equal (i.e. if VCC is not zero after the comparison) \n\ - v_cmp_lt_u32 vcc, v6, v2 \n\ - s_cbranch_vccnz LOOP \n\ - flat_store_dword v[4:5], v6 \n\ - s_waitcnt vmcnt(0)&lgkmcnt(0) \n\ - s_endpgm \n\ -end \n\ -"; +static const char* IterateIsa_gfx8 = R"( + .text + // Copy the parameters from scalar registers to vector registers + v_mov_b32 v2, s0 // v[2:3] = s[0:1] + v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 + v_add_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v6, 0 + LOOP: + v_add_u32 v6, vcc, 1, v6 + // Compare the result value (v6) to iteration value (v2), and + // jump if equal (i.e. if VCC is not zero after the comparison) + v_cmp_lt_u32 vcc, v6, v2 + s_cbranch_vccnz LOOP + flat_store_dword v[4:5], v6 + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; -//This shader can be used by gfx9 and gfx10 -static const char* iterate_isa_gfx9 = \ -"\ -shader iterate_isa\n\ -wave_size(32)\n\ -type(CS)\n\ - // copy the parameters from scalar registers to vector registers\n\ - v_mov_b32 v2, s0 // v[2:3] = s[0:1] \n\ - v_mov_b32 v3, s1 // v[2:3] = s[0:1] \n\ - v_mov_b32 v0, s4 // use workgroup id as index \n\ - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 \n\ - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 \n\ - v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 \n\ - v_add_co_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 \n\ - v_mov_b32 v6, 0 \n\ -LOOP: \n\ - v_add_co_u32 v6, vcc, 1, v6 \n\ - // compare the result value (v6) to iteration value (v2), and \n\ - // jump if equal (i.e. if VCC is not zero after the comparison) \n\ - v_cmp_lt_u32 vcc, v6, v2 \n\ - s_cbranch_vccnz LOOP \n\ - flat_store_dword v[4:5], v6 \n\ - s_waitcnt vmcnt(0)&lgkmcnt(0) \n\ - s_endpgm \n\ -end \n\ -"; +// This shader can be used by gfx9 and gfx10 +static const char* IterateIsa_gfx9 = R"( + .text + // Copy the parameters from scalar registers to vector registers + v_mov_b32 v2, s0 // v[2:3] = s[0:1] + v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 + v_add_co_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v6, 0 + LOOP: + v_add_co_u32 v6, vcc, 1, v6 + // Compare the result value (v6) to iteration value (v2), and + // jump if equal (i.e. if VCC is not zero after the comparison) + v_cmp_lt_u32 vcc, v6, v2 + s_cbranch_vccnz LOOP + flat_store_dword v[4:5], v6 + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; -static const char* infinite_isa = \ -"\ -shader infinite_isa \n\ -wave_size(32) \n\ -type(CS) \n\ -LOOP: \n\ - s_branch LOOP \n\ -end \n\ -"; +static const char* InfiniteIsa = R"( + .text + LOOP: + s_branch LOOP + s_endpgm +)"; void KFDCWSRTest::SetUp() { ROUTINE_START KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - wave_number = 1; ROUTINE_END @@ -115,9 +102,6 @@ void KFDCWSRTest::SetUp() { void KFDCWSRTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; KFDBaseComponentTest::TearDown(); @@ -159,9 +143,9 @@ TEST_F(KFDCWSRTest, BasicTest) { uint64_t count1 = 400000000; if (m_FamilyId < FAMILY_AI) - pIterateIsa = iterate_isa_gfx8; + pIterateIsa = IterateIsa_gfx8; else - pIterateIsa = iterate_isa_gfx9; + pIterateIsa = IterateIsa_gfx9; if (isOnEmulator()) { // Divide the iterator times by 10000 so that the test can @@ -172,7 +156,7 @@ TEST_F(KFDCWSRTest, BasicTest) { unsigned int* result1 = resultBuf1.As(); - m_pIsaGen->CompileShader(pIterateIsa, "iterate_isa", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pIterateIsa, isaBuffer.As())); PM4Queue queue1; @@ -236,7 +220,7 @@ TEST_F(KFDCWSRTest, InterruptRestore) { if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->CompileShader(infinite_isa, "infinite_isa", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(InfiniteIsa, isaBuffer.As())); PM4Queue queue1, queue2, queue3; diff --git a/tests/kfdtest/src/KFDCWSRTest.hpp b/tests/kfdtest/src/KFDCWSRTest.hpp index 779180ea3d..53c925b0aa 100644 --- a/tests/kfdtest/src/KFDCWSRTest.hpp +++ b/tests/kfdtest/src/KFDCWSRTest.hpp @@ -27,12 +27,11 @@ #include #include "PM4Queue.hpp" -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" class KFDCWSRTest : public KFDBaseComponentTest { public: - KFDCWSRTest() :m_pIsaGen(NULL) {} + KFDCWSRTest() {} ~KFDCWSRTest() {} protected: @@ -41,7 +40,6 @@ class KFDCWSRTest : public KFDBaseComponentTest { protected: // Members unsigned wave_number; - IsaGenerator* m_pIsaGen; }; #endif // __KFD_CWSR_TEST__H__ From b44d6762bd84afbf01420bc243ceb63a065556ca Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:47:40 -0400 Subject: [PATCH 0973/1247] kfdtest: Update KFDEvictTest to LLVM Asm - Reformat shaders for legibility - Move assembly processes to from IsaGen (CompileShader) to Assembler (RunAssembleBuf) Signed-off-by: Graham Sider Change-Id: I7333d0e45ccd3f43690a2a01227f89a6e04fcecb --- tests/kfdtest/src/KFDEvictTest.cpp | 253 +++++++++++++---------------- tests/kfdtest/src/KFDEvictTest.hpp | 5 +- 2 files changed, 117 insertions(+), 141 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 7ec86bc8bd..bf721238c8 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -36,23 +36,132 @@ #define SDMA_NOP 0x0 +/* Shader to read local buffers using multiple wavefronts in parallel + * until address buffer is filled with specific value 0x5678 by host program, + * then each wavefront fills value 0x5678 at corresponding result buffer and quit + * + * Initial state: + * s[0:1] - address buffer base address + * s[2:3] - result buffer base address + * s4 - workgroup id + * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 + * Registers: + * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X + * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 + * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 + * v[6:7] - local buf address used for read test + * + * This shader can be used by gfx9 and gfx10 + * + */ + +static const char* ReadMemoryIsa_gfx9 = R"( + .text + // Compute address of corresponding output buffer + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 + v_add_co_u32 v5, vcc, v5, vcc_lo + // Compute input buffer offset used to store corresponding local buffer address + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 + v_add_co_u32 v3, vcc, v3, vcc_lo + // load 64bit local buffer address stored at v[2:3] to v[6:7] + flat_load_dwordx2 v[6:7], v[2:3] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + v_mov_b32 v8, 0x5678 + s_movk_i32 s8, 0x5678 + L_REPEAT: + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s16, s8 + s_cbranch_scc1 L_QUIT // if notified to quit by host + // Loop read 64M local buffer starting at v[6:7] + // every 4k page only read once + v_mov_b32 v9, 0 + v_mov_b32 v10, 0x1000 // 4k page + v_mov_b32 v11, 0x4000000 // 64M size + v_mov_b32 v12, v6 + v_mov_b32 v13, v7 + L_LOOP_READ: + flat_load_dwordx2 v[14:15], v[12:13] slc + v_add_co_u32 v9, vcc, v9, v10 + v_add_co_u32 v12, vcc, v12, v10 + v_add_co_u32 v13, vcc, v13, vcc_lo + v_cmp_lt_u32 vcc, v9, v11 + s_cbranch_vccnz L_LOOP_READ + s_branch L_REPEAT + L_QUIT: + flat_store_dword v[4:5], v8 + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish + s_endpgm +)"; + +static const char* ReadMemoryIsa_gfx8 = R"( + .text + // Compute address of corresponding output buffer + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 + v_addc_u32 v5, vcc, v5, 0, vcc + // Compute input buffer offset used to store corresponding local buffer address + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 + v_addc_u32 v3, vcc, v3, 0, vcc + // Load 64bit local buffer address stored at v[2:3] to v[6:7] + flat_load_dwordx2 v[6:7], v[2:3] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + v_mov_b32 v8, 0x5678 + s_movk_i32 s8, 0x5678 + L_REPEAT: + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s16, s8 + s_cbranch_scc1 L_QUIT // if notified to quit by host + // Loop read 64M local buffer starting at v[6:7] + // every 4k page only read once + v_mov_b32 v9, 0 + v_mov_b32 v10, 0x1000 // 4k page + v_mov_b32 v11, 0x4000000 // 64M size + v_mov_b32 v12, v6 + v_mov_b32 v13, v7 + L_LOOP_READ: + flat_load_dwordx2 v[14:15], v[12:13] slc + v_add_u32 v9, vcc, v9, v10 + v_add_u32 v12, vcc, v12, v10 + v_addc_u32 v13, vcc, v13, 0, vcc + v_cmp_lt_u32 vcc, v9, v11 + s_cbranch_vccnz L_LOOP_READ + s_branch L_REPEAT + L_QUIT: + flat_store_dword v[4:5], v8 + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish + s_endpgm +)"; + +std::string KFDEvictTest::CreateShader() { + if (m_FamilyId < FAMILY_AI) + return ReadMemoryIsa_gfx8; + else + return ReadMemoryIsa_gfx9; +} + + void KFDEvictTest::SetUp() { ROUTINE_START KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDEvictTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - KFDBaseComponentTest::TearDown(); ROUTINE_END @@ -286,136 +395,6 @@ void KFDEvictTest::AmdgpuCommandSubmissionSdmaNop(int rn, amdgpu_bo_handle handl EXPECT_EQ(0, amdgpu_cs_ctx_free(contextHandle)); } -/* Shader to read local buffers using multiple wavefronts in parallel - * until address buffer is filled with specific value 0x5678 by host program, - * then each wavefront fills value 0x5678 at corresponding result buffer and quit - * - * Initial state: - * s[0:1] - address buffer base address - * s[2:3] - result buffer base address - * s4 - workgroup id - * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 - * Registers: - * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X - * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 - * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 - * v[6:7] - local buf address used for read test - * - * This shader can be used by gfx9 and gfx10 - * - */ - -static const char* gfx9_ReadMemory = -"\ - shader ReadMemory\n\ - wave_size(32)\n\ - type(CS)\n\ - \n\ - // compute address of corresponding output buffer\n\ - v_mov_b32 v0, s4 // use workgroup id as index\n\ - v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ - v_mov_b32 v5, s3\n\ - v_add_co_u32 v5, vcc, v5, vcc_lo\n\ - \n\ - // compute input buffer offset used to store corresponding local buffer address\n\ - v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ - v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ - v_mov_b32 v3, s1\n\ - v_add_co_u32 v3, vcc, v3, vcc_lo\n\ - \n\ - // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ - flat_load_dwordx2 v[6:7], v[2:3] slc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - \n\ - v_mov_b32 v8, 0x5678\n\ - s_movk_i32 s8, 0x5678\n\ -L_REPEAT:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - s_cmp_eq_i32 s16, s8\n\ - s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ - // loop read 64M local buffer starting at v[6:7]\n\ - // every 4k page only read once\n\ - v_mov_b32 v9, 0\n\ - v_mov_b32 v10, 0x1000 // 4k page\n\ - v_mov_b32 v11, 0x4000000 // 64M size\n\ - v_mov_b32 v12, v6\n\ - v_mov_b32 v13, v7\n\ -L_LOOP_READ:\n\ - flat_load_dwordx2 v[14:15], v[12:13] slc\n\ - v_add_co_u32 v9, vcc, v9, v10 \n\ - v_add_co_u32 v12, vcc, v12, v10\n\ - v_add_co_u32 v13, vcc, v13, vcc_lo\n\ - v_cmp_lt_u32 vcc, v9, v11\n\ - s_cbranch_vccnz L_LOOP_READ\n\ - s_branch L_REPEAT\n\ -L_QUIT:\n\ - flat_store_dword v[4:5], v8\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ - s_endpgm\n\ - end\n\ -"; - -static const char* gfx8_ReadMemory = -"\ - shader ReadMemory\n\ - asic(VI)\n\ - type(CS)\n\ - \n\ - // compute address of corresponding output buffer\n\ - v_mov_b32 v0, s4 // use workgroup id as index\n\ - v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ - v_mov_b32 v5, s3\n\ - v_addc_u32 v5, vcc, v5, 0, vcc\n\ - \n\ - // compute input buffer offset used to store corresponding local buffer address\n\ - v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ - v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ - v_mov_b32 v3, s1\n\ - v_addc_u32 v3, vcc, v3, 0, vcc\n\ - \n\ - // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ - flat_load_dwordx2 v[6:7], v[2:3] slc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - \n\ - v_mov_b32 v8, 0x5678\n\ - s_movk_i32 s8, 0x5678\n\ -L_REPEAT:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - s_cmp_eq_i32 s16, s8\n\ - s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ - // loop read 64M local buffer starting at v[6:7]\n\ - // every 4k page only read once\n\ - v_mov_b32 v9, 0\n\ - v_mov_b32 v10, 0x1000 // 4k page\n\ - v_mov_b32 v11, 0x4000000 // 64M size\n\ - v_mov_b32 v12, v6\n\ - v_mov_b32 v13, v7\n\ -L_LOOP_READ:\n\ - flat_load_dwordx2 v[14:15], v[12:13] slc\n\ - v_add_u32 v9, vcc, v9, v10 \n\ - v_add_u32 v12, vcc, v12, v10\n\ - v_addc_u32 v13, vcc, v13, 0, vcc\n\ - v_cmp_lt_u32 vcc, v9, v11\n\ - s_cbranch_vccnz L_LOOP_READ\n\ - s_branch L_REPEAT\n\ -L_QUIT:\n\ - flat_store_dword v[4:5], v8\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ - s_endpgm\n\ - end\n\ -"; - -std::string KFDEvictTest::CreateShader() { - if (m_FamilyId < FAMILY_AI) - return gfx8_ReadMemory; - else - return gfx9_ReadMemory; -} - /* Evict and restore procedure basic test * * Use N_PROCESSES processes to allocate vram buf size larger than total vram size @@ -567,7 +546,7 @@ TEST_F(KFDEvictTest, QueueTest) { HsaMemoryBuffer addrBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer resultBuffer(PAGE_SIZE, defaultGPUNode); - m_pIsaGen->CompileShader(CreateShader().c_str(), "ReadMemory", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CreateShader().c_str(), isaBuffer.As())); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index 2b838a5388..d70aada6b4 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -27,15 +27,13 @@ #include #include #include "KFDMultiProcessTest.hpp" -#include "IsaGenerator.hpp" #include "PM4Queue.hpp" // @class KFDEvictTest // Test eviction and restore procedure using two processes class KFDEvictTest : public KFDMultiProcessTest { public: - KFDEvictTest(void): m_pIsaGen(NULL) {} - + KFDEvictTest(void) {} ~KFDEvictTest(void) {} protected: @@ -52,7 +50,6 @@ class KFDEvictTest : public KFDMultiProcessTest { PM4Queue *computeQueue); protected: // Members - IsaGenerator* m_pIsaGen; HsaMemFlags m_Flags; void* m_pBuf; }; From ba9ccd32a191212b9adfe32018350d092e2f7656 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:47:58 -0400 Subject: [PATCH 0974/1247] kfdtest: Update KFDGWSTest to LLVM Asm - Reformat shaders for legibility - Move assembly processes to from IsaGen (CompileShader) to Assembler (RunAssembleBuf) - Change gds:1 modifier to gds - Change offset0:0 modifier to offset:0 Signed-off-by: Graham Sider Change-Id: I2a863695bcf7344cf184a809704948ba3a0d230f --- tests/kfdtest/src/KFDGWSTest.cpp | 124 +++++++++++++------------------ tests/kfdtest/src/KFDGWSTest.hpp | 6 +- 2 files changed, 54 insertions(+), 76 deletions(-) diff --git a/tests/kfdtest/src/KFDGWSTest.cpp b/tests/kfdtest/src/KFDGWSTest.cpp index 4c8aefc447..a6b3bec17b 100644 --- a/tests/kfdtest/src/KFDGWSTest.cpp +++ b/tests/kfdtest/src/KFDGWSTest.cpp @@ -26,23 +26,19 @@ #include "PM4Packet.hpp" #include "Dispatch.hpp" -/* Shader to initialize gws counter to 1*/ -const char* gfx9_10_GwsInit = -"\ -shader GwsInit\n\ -type(CS)\n\ -wave_size(32)\n\ - s_mov_b32 m0, 0\n\ - s_nop 0\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt 0\n\ - v_mov_b32 v0, s16\n\ - s_waitcnt 0\n\ - ds_gws_init v0 gds:1 offset0:0\n\ - s_waitcnt 0\n\ - s_endpgm\n\ - end\n\ -"; +/* Shader to initialize gws counter to 1 */ +static const char* GwsInitIsa_gfx9_10 = R"( + .text + s_mov_b32 m0, 0 + s_nop 0 + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt 0 + v_mov_b32 v0, s16 + s_waitcnt 0 + ds_gws_init v0 offset:0 gds + s_waitcnt 0 + s_endpgm +)"; /* Atomically increase a value in memory * This is expected to be executed from @@ -50,67 +46,53 @@ wave_size(32)\n\ * GWS semaphore is used to guarantee * the operation is atomic. */ -const char* gfx9_AtomicIncrease = -"\ -shader AtomicIncrease\n\ -type(CS)\n\ -/* Assume src address in s0, s1 */\n\ - s_mov_b32 m0, 0\n\ - s_nop 0\n\ - ds_gws_sema_p gds:1 offset0:0\n\ - s_waitcnt 0\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt 0\n\ - s_add_u32 s16, s16, 1\n\ - s_store_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt lgkmcnt(0)\n\ - ds_gws_sema_v gds:1 offset0:0\n\ - s_waitcnt 0\n\ - s_endpgm\n\ - end\n\ -"; +static const char* AtomicIncreaseIsa_gfx9 = R"( + .text + // Assume src address in s0, s1 + s_mov_b32 m0, 0 + s_nop 0 + ds_gws_sema_p offset:0 gds + s_waitcnt 0 + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt 0 + s_add_u32 s16, s16, 1 + s_store_dword s16, s[0:1], 0x0 glc + s_waitcnt lgkmcnt(0) + ds_gws_sema_v offset:0 gds + s_waitcnt 0 + s_endpgm +)"; -const char* gfx10_AtomicIncrease = -"\ -shader AtomicIncrease\n\ -asic(GFX10)\n\ -type(CS)\n\ -wave_size(32)\n\ -/* Assume src address in s0, s1 */\n\ - s_mov_b32 m0, 0\n\ - s_mov_b32 exec_lo, 0x1\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - ds_gws_sema_p gds:1 offset0:0\n\ - s_waitcnt 0\n\ - flat_load_dword v2, v[0:1] glc:1 dlc:1\n\ - s_waitcnt 0\n\ - v_add_nc_u32 v2, v2, 1\n\ - flat_store_dword v[0:1], v2\n\ - s_waitcnt_vscnt null, 0\n\ - ds_gws_sema_v gds:1 offset0:0\n\ - s_waitcnt 0\n\ - s_endpgm\n\ - end\n\ -"; +static const char* AtomicIncreaseIsa_gfx10 = R"( + .text + // Assume src address in s0, s1 + s_mov_b32 m0, 0 + s_mov_b32 exec_lo, 0x1 + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + ds_gws_sema_p offset:0 gds + s_waitcnt 0 + flat_load_dword v2, v[0:1] glc dlc + s_waitcnt 0 + v_add_nc_u32 v2, v2, 1 + flat_store_dword v[0:1], v2 + s_waitcnt_vscnt null, 0 + ds_gws_sema_v offset:0 gds + s_waitcnt 0 + s_endpgm +)"; void KFDGWSTest::SetUp() { ROUTINE_START KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDGWSTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - KFDBaseComponentTest::TearDown(); ROUTINE_END @@ -160,21 +142,21 @@ TEST_F(KFDGWSTest, Semaphore) { pNodeProperties->NumGws,&firstGWS)); EXPECT_EQ(0, firstGWS); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - m_pIsaGen->CompileShader(gfx9_10_GwsInit, "GwsInit", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(GwsInitIsa_gfx9_10, isaBuffer.As())); + Dispatch dispatch0(isaBuffer); buffer.Fill(numResources, 0, 4); dispatch0.SetArgs(buffer.As(), NULL); dispatch0.Submit(queue); dispatch0.Sync(); - const char *pAtomicIncrease; + const char *pAtomicIncreaseIsa; if (m_FamilyId <= FAMILY_AL) - pAtomicIncrease = gfx9_AtomicIncrease; + pAtomicIncreaseIsa = AtomicIncreaseIsa_gfx9; else - pAtomicIncrease = gfx10_AtomicIncrease; + pAtomicIncreaseIsa = AtomicIncreaseIsa_gfx10; - m_pIsaGen->CompileShader(pAtomicIncrease, "AtomicIncrease", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pAtomicIncreaseIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), NULL); diff --git a/tests/kfdtest/src/KFDGWSTest.hpp b/tests/kfdtest/src/KFDGWSTest.hpp index 15e61ee235..8413145982 100644 --- a/tests/kfdtest/src/KFDGWSTest.hpp +++ b/tests/kfdtest/src/KFDGWSTest.hpp @@ -26,20 +26,16 @@ #include -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" class KFDGWSTest : public KFDBaseComponentTest { public: - KFDGWSTest() :m_pIsaGen(NULL) {} + KFDGWSTest() {} ~KFDGWSTest() {} protected: virtual void SetUp(); virtual void TearDown(); - - protected: // Members - IsaGenerator* m_pIsaGen; }; #endif // __KFD_GWS_TEST__H__ From a7b85fdb08ba6c314259e7f265f8cdfe586e861a Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 24 Sep 2021 18:48:15 -0400 Subject: [PATCH 0975/1247] kfdtest: Update KFDSVMEvictTest to LLVM Asm - Reformat shaders for legibility - Move assembly processes to from IsaGen (CompileShader) to Assembler (RunAssembleBuf) Signed-off-by: Graham Sider Change-Id: Id1eb3856bc74bf0da46685c5dc08e91f5df66d4f --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 237 ++++++++++++-------------- tests/kfdtest/src/KFDSVMEvictTest.hpp | 1 - 2 files changed, 111 insertions(+), 127 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index d41aedac74..e712e7b44a 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -35,6 +35,109 @@ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +/* Shader to read local buffers using multiple wavefronts in parallel + * until address buffer is filled with specific value 0x5678 by host program, + * then each wavefront fills value 0x5678 at corresponding result buffer and quit + * + * initial state: + * s[0:1] - address buffer base address + * s[2:3] - result buffer base address + * s4 - workgroup id + * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 + * registers: + * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X + * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 + * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 + * v[6:7] - local buf address used for read test + */ +static const char* gfx9_ReadMemory = R"( + .text + // Compute address of corresponding output buffer + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 + v_add_u32 v5, vcc_lo, v5 + // Compute input buffer offset used to store corresponding local buffer address + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 + v_add_u32 v3, vcc_lo, v3 + // Load 64bit local buffer address stored at v[2:3] to v[6:7] + flat_load_dwordx2 v[6:7], v[2:3] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + v_mov_b32 v8, 0x5678 + s_movk_i32 s8, 0x5678 + L_REPEAT: + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s16, s8 + s_cbranch_scc1 L_QUIT // if notified to quit by host + // Loop read 64M local buffer starting at v[6:7] + // every 4k page only read once + v_mov_b32 v9, 0 + v_mov_b32 v10, 0x1000 // 4k page + v_mov_b32 v11, 0x4000000 // 64M size + v_mov_b32 v12, v6 + v_mov_b32 v13, v7 + L_LOOP_READ: + flat_load_dwordx2 v[14:15], v[12:13] slc + v_add_u32 v9, v9, v10 + v_add_co_u32 v12, vcc, v12, v10 + v_add_u32 v13, vcc_lo, v13 + v_cmp_lt_u32 vcc, v9, v11 + s_cbranch_vccnz L_LOOP_READ + s_branch L_REPEAT + L_QUIT: + flat_store_dword v[4:5], v8 + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish + s_endpgm +)"; + +static const char* gfx8_ReadMemory = R"( + .text + // Compute address of corresponding output buffer + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 + v_addc_u32 v5, vcc, v5, 0, vcc + // Compute input buffer offset used to store corresponding local buffer address + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 + v_addc_u32 v3, vcc, v3, 0, vcc + // Load 64bit local buffer address stored at v[2:3] to v[6:7] + flat_load_dwordx2 v[6:7], v[2:3] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + v_mov_b32 v8, 0x5678 + s_movk_i32 s8, 0x5678 + L_REPEAT: + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s16, s8 + s_cbranch_scc1 L_QUIT // if notified to quit by host + // Loop read 64M local buffer starting at v[6:7] + // every 4k page only read once + v_mov_b32 v9, 0 + v_mov_b32 v10, 0x1000 // 4k page + v_mov_b32 v11, 0x4000000 // 64M size + v_mov_b32 v12, v6 + v_mov_b32 v13, v7 + L_LOOP_READ: + flat_load_dwordx2 v[14:15], v[12:13] slc + v_add_u32 v9, vcc, v9, v10 + v_add_u32 v12, vcc, v12, v10 + v_addc_u32 v13, vcc, v13, 0, vcc + v_cmp_lt_u32 vcc, v9, v11 + s_cbranch_vccnz L_LOOP_READ + s_branch L_REPEAT + L_QUIT: + flat_store_dword v[4:5], v8 + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish + s_endpgm +)"; + void KFDSVMEvictTest::SetUp() { ROUTINE_START @@ -55,6 +158,13 @@ void KFDSVMEvictTest::TearDown() { ROUTINE_END } +std::string KFDSVMEvictTest::CreateShader() { + if (m_FamilyId >= FAMILY_AI) + return gfx9_ReadMemory; + else + return gfx8_ReadMemory; +} + HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) { HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT; HSAuint64 sysMemSize = GetSysMemSize(); @@ -234,131 +344,6 @@ TEST_F(KFDSVMEvictTest, BasicTest) { TEST_END } -/* Shader to read local buffers using multiple wavefronts in parallel - * until address buffer is filled with specific value 0x5678 by host program, - * then each wavefront fills value 0x5678 at corresponding result buffer and quit - * - * initial state: - * s[0:1] - address buffer base address - * s[2:3] - result buffer base address - * s4 - workgroup id - * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 - * registers: - * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X - * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 - * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 - * v[6:7] - local buf address used for read test - */ -static const char* gfx9_ReadMemory = -"\ - shader ReadMemory\n\ - type(CS)\n\ - \n\ - // compute address of corresponding output buffer\n\ - v_mov_b32 v0, s4 // use workgroup id as index\n\ - v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ - v_mov_b32 v5, s3\n\ - v_add_u32 v5, vcc_lo, v5\n\ - \n\ - // compute input buffer offset used to store corresponding local buffer address\n\ - v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ - v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ - v_mov_b32 v3, s1\n\ - v_add_u32 v3, vcc_lo, v3\n\ - \n\ - // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ - flat_load_dwordx2 v[6:7], v[2:3] slc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - \n\ - v_mov_b32 v8, 0x5678\n\ - s_movk_i32 s8, 0x5678\n\ -L_REPEAT:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - s_cmp_eq_i32 s16, s8\n\ - s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ - // loop read 64M local buffer starting at v[6:7]\n\ - // every 4k page only read once\n\ - v_mov_b32 v9, 0\n\ - v_mov_b32 v10, 0x1000 // 4k page\n\ - v_mov_b32 v11, 0x4000000 // 64M size\n\ - v_mov_b32 v12, v6\n\ - v_mov_b32 v13, v7\n\ -L_LOOP_READ:\n\ - flat_load_dwordx2 v[14:15], v[12:13] slc\n\ - v_add_u32 v9, v9, v10 \n\ - v_add_co_u32 v12, vcc, v12, v10\n\ - v_add_u32 v13, vcc_lo, v13\n\ - v_cmp_lt_u32 vcc, v9, v11\n\ - s_cbranch_vccnz L_LOOP_READ\n\ - s_branch L_REPEAT\n\ -L_QUIT:\n\ - flat_store_dword v[4:5], v8\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ - s_endpgm\n\ - end\n\ -"; - -static const char* gfx8_ReadMemory = -"\ - shader ReadMemory\n\ - asic(VI)\n\ - type(CS)\n\ - \n\ - // compute address of corresponding output buffer\n\ - v_mov_b32 v0, s4 // use workgroup id as index\n\ - v_lshlrev_b32 v0, 2, v0 // v0 *= 4\n\ - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4\n\ - v_mov_b32 v5, s3\n\ - v_addc_u32 v5, vcc, v5, 0, vcc\n\ - \n\ - // compute input buffer offset used to store corresponding local buffer address\n\ - v_lshlrev_b32 v0, 1, v0 // v0 *= 8\n\ - v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8\n\ - v_mov_b32 v3, s1\n\ - v_addc_u32 v3, vcc, v3, 0, vcc\n\ - \n\ - // load 64bit local buffer address stored at v[2:3] to v[6:7]\n\ - flat_load_dwordx2 v[6:7], v[2:3] slc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - \n\ - v_mov_b32 v8, 0x5678\n\ - s_movk_i32 s8, 0x5678\n\ -L_REPEAT:\n\ - s_load_dword s16, s[0:1], 0x0 glc\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish\n\ - s_cmp_eq_i32 s16, s8\n\ - s_cbranch_scc1 L_QUIT // if notified to quit by host\n\ - // loop read 64M local buffer starting at v[6:7]\n\ - // every 4k page only read once\n\ - v_mov_b32 v9, 0\n\ - v_mov_b32 v10, 0x1000 // 4k page\n\ - v_mov_b32 v11, 0x4000000 // 64M size\n\ - v_mov_b32 v12, v6\n\ - v_mov_b32 v13, v7\n\ -L_LOOP_READ:\n\ - flat_load_dwordx2 v[14:15], v[12:13] slc\n\ - v_add_u32 v9, vcc, v9, v10 \n\ - v_add_u32 v12, vcc, v12, v10\n\ - v_addc_u32 v13, vcc, v13, 0, vcc\n\ - v_cmp_lt_u32 vcc, v9, v11\n\ - s_cbranch_vccnz L_LOOP_READ\n\ - s_branch L_REPEAT\n\ -L_QUIT:\n\ - flat_store_dword v[4:5], v8\n\ - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish\n\ - s_endpgm\n\ - end\n\ -"; - -std::string KFDSVMEvictTest::CreateShader() { - if (m_FamilyId >= FAMILY_AI) - return gfx9_ReadMemory; - else - return gfx8_ReadMemory; -} - /* Evict and restore queue test * * N_PROCESSES processes read all local buffers in parallel while buffers are evicted and restored @@ -434,7 +419,7 @@ TEST_F(KFDSVMEvictTest, QueueTest) { for (i = 0; i < wavefront_num; i++) *(localBufAddr + i) = pBuffers[i]; - m_pIsaGen->CompileShader(CreateShader().c_str(), "ReadMemory", isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CreateShader().c_str(), isaBuffer.As())); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/tests/kfdtest/src/KFDSVMEvictTest.hpp index 2b8a1de957..3f26287cc0 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.hpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.hpp @@ -28,7 +28,6 @@ #include #include "KFDLocalMemoryTest.hpp" #include "KFDBaseComponentTest.hpp" -#include "IsaGenerator.hpp" // @class KFDEvictTest // Test eviction and restore procedure using two processes From 34ca37d9e8a959b48ee336808e2a415ef38eaf17 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 2 Nov 2021 13:45:08 -0400 Subject: [PATCH 0976/1247] kfdtest: Add ShaderStore.cpp/hpp Initial commit for ShaderStore.hpp. Will contain consts char*'s for all shaders used within KFDTest. The LLVM assembler now takes care of the correct instructions to be used for various GFX versions using directives embedded into the shader assembly. Signed-off-by: Graham Sider Change-Id: I2887a03b33d5c2cc382e4f96c2bc3e067715ab54 --- tests/kfdtest/CMakeLists.txt | 1 + tests/kfdtest/src/KFDBaseComponentTest.hpp | 1 + tests/kfdtest/src/ShaderStore.cpp | 65 ++++++++++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 33 +++++++++++ 4 files changed, 100 insertions(+) create mode 100644 tests/kfdtest/src/ShaderStore.cpp create mode 100644 tests/kfdtest/src/ShaderStore.hpp diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 39994dbff3..e175253dad 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -146,6 +146,7 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/GoogleTestExtension.cpp src/IndirectBuffer.cpp src/Assemble.cpp + src/ShaderStore.cpp src/IsaGenerator.cpp src/IsaGenerator_Aldebaran.cpp src/IsaGenerator_Gfx10.cpp diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index ec3053c3d3..cc87465ad5 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -35,6 +35,7 @@ #include "OSWrapper.hpp" #include "KFDTestUtil.hpp" #include "Assemble.hpp" +#include "ShaderStore.hpp" // @class KFDBaseComponentTest class KFDBaseComponentTest : public testing::Test { diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp new file mode 100644 index 0000000000..5f46bf844a --- /dev/null +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +/** + * Common + */ + +const char *NoopIsa = R"( + .text + s_endpgm +)"; + +const char *CopyDwordIsa = R"( + .text + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 + flat_load_dword v4, v[0:1] glc slc + s_waitcnt 0 + flat_store_dword v[2:3], v4 glc slc + s_endpgm +)"; + +const char *InfiniteLoopIsa = R"( + .text + LOOP: + s_branch LOOP + s_endpgm +)"; + +const char *AtomicIncIsa = R"( + .text + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + .if (.amdgcn.gfx_generation_number >= 8) + v_mov_b32 v2, 1 + flat_atomic_add v3, v[0:1], v2 glc slc + .else + v_mov_b32 v2, -1 + flat_atomic_inc v3, v[0:1], v2 glc slc + .endif + s_waitcnt 0 + s_endpgm +)"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp new file mode 100644 index 0000000000..2344c5bca2 --- /dev/null +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _SHADERSTORE_H_ +#define _SHADERSTORE_H_ + +/* Common */ +extern const char *NoopIsa; +extern const char *CopyDwordIsa; +extern const char *InfiniteLoopIsa; +extern const char *AtomicIncIsa; + +#endif // _SHADERSTORE_H_ From c926d83b5a5ece74324e7e422315e6c5fd6e4afe Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 2 Nov 2021 13:47:01 -0400 Subject: [PATCH 0977/1247] kfdtest: Move KFDMemoryTest shaders to ShaderStore Signed-off-by: Graham Sider Change-Id: I3335ca1f9dbe849233cf85253e0e92b56a20b8c9 --- tests/kfdtest/src/KFDGraphicsInterop.cpp | 3 +- tests/kfdtest/src/KFDMemoryTest.cpp | 364 ++--------------------- tests/kfdtest/src/KFDMemoryTest.hpp | 5 +- tests/kfdtest/src/ShaderStore.cpp | 225 ++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 9 + 5 files changed, 264 insertions(+), 342 deletions(-) diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/tests/kfdtest/src/KFDGraphicsInterop.cpp index bf2a928db1..48224bce0f 100644 --- a/tests/kfdtest/src/KFDGraphicsInterop.cpp +++ b/tests/kfdtest/src/KFDGraphicsInterop.cpp @@ -101,7 +101,8 @@ TEST_F(KFDGraphicsInterop, RegisterGraphicsHandle) { // Copy contents to a system memory buffer for comparison HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/); diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 980c9c7bf8..4e9cb5fd19 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -39,292 +39,17 @@ #include "SDMAPacket.hpp" #include "linux/kfd_ioctl.h" -static const char* ScratchCopyDwordIsa_gfx8 = R"( - .text - // Copy the parameters from scalar registers to vector registers - v_mov_b32_e32 v0, s0 - v_mov_b32_e32 v1, s1 - v_mov_b32_e32 v2, s2 - v_mov_b32_e32 v3, s3 - // Setup the scratch parameters. This assumes a single 16-reg block - s_mov_b32 flat_scratch_lo, 8 - s_mov_b32 flat_scratch_hi, 0 - // Copy a dword between the passed addresses - flat_load_dword v4, v[0:1] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) - flat_store_dword v[2:3], v4 slc - s_endpgm -)"; -static const char* ScratchCopyDwordIsa_gfx9 = R"( - .text - // Copy the parameters from scalar registers to vector registers - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, s2 - v_mov_b32 v3, s3 - // Setup the scratch parameters. This assumes a single 16-reg block - s_mov_b32 flat_scratch_lo, s4 - s_mov_b32 flat_scratch_hi, s5 - // Copy a dword between the passed addresses - flat_load_dword v4, v[0:1] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) - flat_store_dword v[2:3], v4 slc - s_endpgm -)"; -static const char* ScratchCopyDwordIsa_gfx10 = R"( - .text - // Copy the parameters from scalar registers to vector registers - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, s2 - v_mov_b32 v3, s3 - // Setup the scratch parameters. This assumes a single 16-reg block - s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4 - s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5 - // Copy a dword between the passed addresses - flat_load_dword v4, v[0:1] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) - flat_store_dword v[2:3], v4 slc - s_endpgm -)"; -static const char* ScratchCopyDwordIsa_gfx9aldbrn = R"( - .text - // Copy the parameters from scalar registers to vector registers - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, s2 - v_mov_b32 v3, s3 - // Setup the scratch parameters. This assumes a single 16-reg block - s_mov_b32 flat_scratch_lo, s4 - s_mov_b32 flat_scratch_hi, s5 - // Copy a dword between the passed addresses - flat_load_dword v4, v[0:1] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) - flat_store_dword v[2:3], v4 slc - s_endpgm -)"; - -/* Continuously poll src buffer and check buffer value - * After src buffer is filled with specific value (0x5678, - * by host program), fill dst buffer with specific - * value(0x5678) and quit - */ -static const char* PollMemoryIsa_gfx9 = R"( - .text - // Assume src address in s0, s1, and dst address in s2, s3 - s_movk_i32 s18, 0x5678 - LOOP: - s_load_dword s16, s[0:1], 0x0 glc - s_cmp_eq_i32 s16, s18 - s_cbranch_scc0 LOOP - s_store_dword s18, s[2:3], 0x0 glc - s_endpgm -)"; - -/* Similar to PollMemoryIsa_gfx9 except that the buffer - * polled can be Non-coherant memory. SCC system-level - * cache coherence is not supported in scalar (smem) path. - * Use vmem operations with scc - */ -static const char* PollNCMemoryIsa_gfx9 = R"( - .text - // Assume src address in s0, s1, and dst address in s2, s3 - v_mov_b32 v6, 0x5678 - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - LOOP: - flat_load_dword v4, v[0:1] scc - v_cmp_eq_u32 vcc, v4, v6 - s_cbranch_vccz LOOP - v_mov_b32 v0, s2 - v_mov_b32 v1, s3 - flat_store_dword v[0:1], v6 scc - s_endpgm -)"; - -static const char* PollMemoryIsa_gfx10 = R"( - .text - // Assume src address in s0, s1, and dst address in s2, s3 - s_movk_i32 s18, 0x5678 - v_mov_b32 v0, s2 - v_mov_b32 v1, s3 - v_mov_b32 v2, 0x5678 - LOOP: - s_load_dword s16, s[0:1], 0x0 glc - s_cmp_eq_i32 s16, s18 - s_cbranch_scc0 LOOP - flat_store_dword v[0:1], v2 slc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_endpgm -)"; - -/* Input: A buffer of at least 3 dwords. - * DW0: used as a signal. 0xcafe means it is signaled - * DW1: Input buffer for device to read. - * DW2: Output buffer for device to write. - * Once receive signal, device will copy DW1 to DW2 - * This shader continously poll the signal buffer, - * Once signal buffer is signaled, it copies input buffer - * to output buffer - */ -static const char* CopyOnSignalIsa_gfx9 = R"( - .text - // Assume input buffer in s0, s1 - s_mov_b32 s18, 0xcafe - POLLSIGNAL: - s_load_dword s16, s[0:1], 0x0 glc - s_cmp_eq_i32 s16, s18 - s_cbranch_scc0 POLLSIGNAL - s_load_dword s17, s[0:1], 0x4 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_store_dword s17, s[0:1], 0x8 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_endpgm -)"; - -static const char* CopyOnSignalIsa_gfx10 = R"( - .text - // Assume input buffer in s0, s1 - s_add_u32 s2, s0, 0x8 - s_addc_u32 s3, s1, 0x0 - s_mov_b32 s18, 0xcafe - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v4, s2 - v_mov_b32 v5, s3 - POLLSIGNAL: - s_load_dword s16, s[0:1], 0x0 glc - s_cmp_eq_i32 s16, s18 - s_cbranch_scc0 POLLSIGNAL - s_load_dword s17, s[0:1], 0x4 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - v_mov_b32 v2, s17 - flat_store_dword v[4:5], v2 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_endpgm -)"; - -/* Input0: A buffer of at least 2 dwords. - * DW0: used as a signal. Write 0xcafe to signal - * DW1: Write to this buffer for other device to read. - * Input1: mmio base address - */ -static const char* WriteAndSignalIsa_gfx9 = R"( - .text - // Assume input buffer in s0, s1 - s_mov_b32 s18, 0xbeef - s_store_dword s18, s[0:1], 0x4 glc - s_mov_b32 s18, 0x1 - s_store_dword s18, s[2:3], 0 glc - s_mov_b32 s18, 0xcafe - s_store_dword s18, s[0:1], 0x0 glc - s_endpgm -)"; - -/* Continuously poll the flag at src buffer - * After the flag of s[0:1] is 1 filled, - * copy the value from s[0:1]+4 to dst buffer - */ -static const char* PollAndCopyIsa_gfx9 = R"( - .text - // Assume src buffer in s[0:1] and dst buffer in s[2:3] - s_movk_i32 s18, 0x1 - LOOP: - s_load_dword s16, s[0:1], 0x0 glc - s_cmp_eq_i32 s16, s18 - s_cbranch_scc0 LOOP - s_load_dword s17, s[0:1], 0x4 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_store_dword s17, s[2:3], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_endpgm -)"; - -static const char* PollAndCopyIsa_gfx9aldbrn = R"( - .text - // Assume src buffer in s[0:1] and dst buffer in s[2:3] - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v18, 0x1 - LOOP: - flat_load_dword v16, v[0:1] glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - v_cmp_eq_i32 vcc, v16, v18 - s_cbranch_vccz LOOP - buffer_invl2 - s_load_dword s17, s[0:1], 0x4 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_store_dword s17, s[2:3], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - buffer_wbl2 - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_endpgm -)"; - -/* Input0: A buffer of at least 2 dwords. - * DW0: used as a signal. Write 0x1 to signal - * DW1: Write the value from 2nd input buffer - * for other device to read. - * Input1: A buffer of at least 2 dwords. - * DW0: used as the value to be written. - */ -static const char* WriteFlagAndValueIsa_gfx9aldbrn = R"( - .text - // Assume two inputs buffer in s[0:1] and s[2:3] - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - s_load_dword s18, s[2:3], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_store_dword s18, s[0:1], 0x4 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - buffer_wbl2 - s_waitcnt vmcnt(0) & lgkmcnt(0) - v_mov_b32 v16, 0x1 - flat_store_dword v[0:1], v16 glc - s_endpgm -)"; - -static const char* WriteAndSignalIsa_gfx10 = R"( - .text - // Assume input buffer in s0, s1 - s_add_u32 s4, s0, 0x4 - s_addc_u32 s5, s1, 0x0 - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, s2 - v_mov_b32 v3, s3 - v_mov_b32 v4, s4 - v_mov_b32 v5, s5 - v_mov_b32 v18, 0xbeef - flat_store_dword v[4:5], v18 glc - v_mov_b32 v18, 0x1 - flat_store_dword v[2:3], v18 glc - v_mov_b32 v18, 0xcafe - flat_store_dword v[0:1], v18 glc - s_endpgm -)"; - -/* These PollMemoryIsa_gfx9, CopyOnSignalIsa_gfx9, - * WriteAndSignalIsa_gfx9 shaders can be used by both gfx9 and gfx10 - */ - void KFDMemoryTest::SetUp() { ROUTINE_START KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDMemoryTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - KFDBaseComponentTest::TearDown(); ROUTINE_END @@ -442,11 +167,9 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { const char *pReadMemory; if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) /* On A+A system memory is mapped as NC */ - pReadMemory = PollNCMemoryIsa_gfx9; - else if (m_FamilyId < FAMILY_NV) - pReadMemory = PollMemoryIsa_gfx9; + pReadMemory = PollNCMemoryIsa; else - pReadMemory = PollMemoryIsa_gfx10; + pReadMemory = PollMemoryIsa; ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pReadMemory, isaBuffer.As())); @@ -605,7 +328,8 @@ TEST_F(KFDMemoryTest, MemoryRegister) { ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); /* First submit just so the queues are not empty, and to get the * TLB populated (in case we need to flush TLBs somewhere after @@ -786,17 +510,7 @@ TEST_F(KFDMemoryTest, FlatScratchAccess) { // Initialize the srcBuffer to some fixed value srcMemBuffer.Fill(0x01010101); - const char *pScratchCopyDwordIsa; - if (m_FamilyId < FAMILY_AI) - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx8; - else if (m_FamilyId < FAMILY_AL) - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9; - else if (m_FamilyId == FAMILY_AL) - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9aldbrn; - else - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx10; - - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pScratchCopyDwordIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ScratchCopyDwordIsa, isaBuffer.As())); const HsaNodeProperties *pNodeProperties = m_NodeInfo.GetNodeProperties(defaultGPUNode); @@ -1660,17 +1374,7 @@ TEST_F(KFDMemoryTest, PtraceAccessInvisibleVram) { // dstBuffer is cpu accessible gtt memory HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - const char *pScratchCopyDwordIsa; - if (m_FamilyId < FAMILY_AI) - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx8; - else if (m_FamilyId < FAMILY_AL) - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9; - else if (m_FamilyId == FAMILY_AL) - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx9aldbrn; - else - pScratchCopyDwordIsa = ScratchCopyDwordIsa_gfx10; - - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pScratchCopyDwordIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ScratchCopyDwordIsa, isaBuffer.As())); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(mem0, dstBuffer.As()); @@ -2042,13 +1746,8 @@ TEST_F(KFDMemoryTest, HostHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - const char *pCopyOnSignalIsa; - if (m_FamilyId < FAMILY_NV) - pCopyOnSignalIsa = CopyOnSignalIsa_gfx9; - else - pCopyOnSignalIsa = CopyOnSignalIsa_gfx10; - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pCopyOnSignalIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyOnSignalIsa, isaBuffer.As())); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(buffer, NULL); @@ -2169,13 +1868,8 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(nodes[0])); HsaMemoryBuffer isaBuffer(PAGE_SIZE, nodes[0], true/*zero*/, false/*local*/, true/*exec*/); - const char *pCopyOnSignalIsa; - if (m_FamilyId < FAMILY_NV) - pCopyOnSignalIsa = CopyOnSignalIsa_gfx9; - else - pCopyOnSignalIsa = CopyOnSignalIsa_gfx10; - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pCopyOnSignalIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyOnSignalIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, NULL); @@ -2184,13 +1878,8 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { PM4Queue queue0; ASSERT_SUCCESS(queue0.Create(nodes[1])); HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); - const char *pWriteAndSignalIsa; - if (m_FamilyId < FAMILY_NV) - pWriteAndSignalIsa = WriteAndSignalIsa_gfx9; - else - pWriteAndSignalIsa = WriteAndSignalIsa_gfx10; - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pWriteAndSignalIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteAndSignalIsa, isaBuffer.As())); Dispatch dispatch0(isaBuffer0); dispatch0.SetArgs(buffer, mmioBase); @@ -2244,7 +1933,7 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnSdmaWrite) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); @@ -2299,7 +1988,7 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnCPUWrite) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, buffer+100); @@ -2363,7 +2052,7 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnRemoteWrite) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa_gfx9, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); @@ -2379,7 +2068,9 @@ TEST_F(KFDMemoryTest, CacheInvalidateOnRemoteWrite) { ASSERT_SUCCESS(queue1.Create(nondefaultNode)); buffer.Fill(0x5678, sdmaQueue, dwLocation1*sizeof(int), 4); HsaMemoryBuffer isaBuffer1(PAGE_SIZE, nondefaultNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->GetCopyDwordIsa(isaBuffer1); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); + Dispatch dispatch1(isaBuffer1); dispatch1.SetArgs(buffer.As()+dwLocation1, buffer.As()); dispatch1.Submit(queue1); @@ -2446,7 +2137,7 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa_gfx9aldbrn, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), buffer.As()+dwLocation); @@ -2463,7 +2154,7 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) { ASSERT_SUCCESS(queue1.Create(nondefaultNode)); HsaMemoryBuffer isaBuffer1(PAGE_SIZE, nondefaultNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteFlagAndValueIsa_gfx9aldbrn, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteFlagAndValueIsa, isaBuffer.As())); Dispatch dispatch1(isaBuffer1); dispatch1.SetArgs(buffer.As(), buffer.As()+dwSource); @@ -2519,7 +2210,7 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithCPU) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa_gfx9aldbrn, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer, buffer+dwLocation); @@ -2559,12 +2250,17 @@ TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) { return; } - unsigned int *fineBuffer = NULL; - unsigned int tmp; - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); const int dwLocation = 0x80; + if (!m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) { + LOG() << "Skipping test: XGMI link to CPU is required." << std::endl; + return; + } + + unsigned int *fineBuffer = NULL; + unsigned int tmp; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode /* system */, PAGE_SIZE, m_MemoryFlags, reinterpret_cast(&fineBuffer))); ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(fineBuffer, PAGE_SIZE, NULL)); @@ -2578,13 +2274,7 @@ TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) { ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - const char* pPollAndCopyIsa; - if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) - pPollAndCopyIsa = PollAndCopyIsa_gfx9aldbrn; - else - pPollAndCopyIsa = PollAndCopyIsa_gfx9; - - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pPollAndCopyIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollAndCopyIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(fineBuffer, fineBuffer+dwLocation); diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp index ea93395f71..03149e5639 100644 --- a/tests/kfdtest/src/KFDMemoryTest.hpp +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -22,7 +22,6 @@ */ #include "KFDBaseComponentTest.hpp" -#include "IsaGenerator.hpp" #ifndef __KFD_MEMORY_TEST__H__ #define __KFD_MEMORY_TEST__H__ @@ -33,15 +32,13 @@ */ class KFDMemoryTest : public KFDBaseComponentTest { public: - KFDMemoryTest(void) :m_pIsaGen(NULL) {} + KFDMemoryTest(void) {} ~KFDMemoryTest(void) {} protected: virtual void SetUp(); virtual void TearDown(); protected: - IsaGenerator* m_pIsaGen; - void BinarySearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, HSAuint64 highMB, int nodeToMap, HSAuint64 *lastSizeMB); diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 5f46bf844a..e7cb07470b 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -63,3 +63,228 @@ const char *AtomicIncIsa = R"( s_waitcnt 0 s_endpgm )"; + +/** + * KFDMemoryTest + */ + +const char *ScratchCopyDwordIsa = R"( + .text + // Copy the parameters from scalar registers to vector registers + .if (.amdgcn.gfx_generation_number >= 9) + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 + .else + v_mov_b32_e32 v0, s0 + v_mov_b32_e32 v1, s1 + v_mov_b32_e32 v2, s2 + v_mov_b32_e32 v3, s3 + .endif + // Setup the scratch parameters. This assumes a single 16-reg block + .if (.amdgcn.gfx_generation_number >= 10) + s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s4 + s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s5 + .elseif (.amdgcn.gfx_generation_number == 9) + s_mov_b32 flat_scratch_lo, s4 + s_mov_b32 flat_scratch_hi, s5 + .else + s_mov_b32 flat_scratch_lo, 8 + s_mov_b32 flat_scratch_hi, 0 + .endif + // Copy a dword between the passed addresses + flat_load_dword v4, v[0:1] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + flat_store_dword v[2:3], v4 slc + s_endpgm +)"; + +/* Continuously poll src buffer and check buffer value + * After src buffer is filled with specific value (0x5678, + * by host program), fill dst buffer with specific + * value(0x5678) and quit + */ +const char *PollMemoryIsa = R"( + .text + // Assume src address in s0, s1, and dst address in s2, s3 + s_movk_i32 s18, 0x5678 + .if (.amdgcn.gfx_generation_number >= 10) + v_mov_b32 v0, s2 + v_mov_b32 v1, s3 + v_mov_b32 v2, 0x5678 + .endif + LOOP: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 LOOP + .if (.amdgcn.gfx_generation_number >= 10) + flat_store_dword v[0:1], v2 slc + .else + s_store_dword s18, s[2:3], 0x0 glc + .endif + s_endpgm +)"; + +/* Similar to PollMemoryIsa except that the buffer + * polled can be Non-coherant memory. SCC system-level + * cache coherence is not supported in scalar (smem) path. + * Use vmem operations with scc + * + * Note: Only works on Aldebaran, and even then the scc modifier + * has been defeatured. This shader is more or less + * deprecated. + */ +const char *PollNCMemoryIsa = R"( + .text + // Assume src address in s0, s1, and dst address in s2, s3 + v_mov_b32 v6, 0x5678 + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + LOOP: + flat_load_dword v4, v[0:1] scc + v_cmp_eq_u32 vcc, v4, v6 + s_cbranch_vccz LOOP + v_mov_b32 v0, s2 + v_mov_b32 v1, s3 + flat_store_dword v[0:1], v6 scc + s_endpgm +)"; + +/* Input: A buffer of at least 3 dwords. + * DW0: used as a signal. 0xcafe means it is signaled + * DW1: Input buffer for device to read. + * DW2: Output buffer for device to write. + * Once receive signal, device will copy DW1 to DW2 + * This shader continously poll the signal buffer, + * Once signal buffer is signaled, it copies input buffer + * to output buffer + */ +const char *CopyOnSignalIsa = R"( + .text + // Assume input buffer in s0, s1 + .if (.amdgcn.gfx_generation_number >= 10) + s_add_u32 s2, s0, 0x8 + s_addc_u32 s3, s1, 0x0 + s_mov_b32 s18, 0xcafe + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v4, s2 + v_mov_b32 v5, s3 + .else + s_mov_b32 s18, 0xcafe + .endif + POLLSIGNAL: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 POLLSIGNAL + s_load_dword s17, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + .if (.amdgcn.gfx_generation_number >= 10) + v_mov_b32 v2, s17 + flat_store_dword v[4:5], v2 glc + .else + s_store_dword s17, s[0:1], 0x8 glc + .endif + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; + +/* Continuously poll the flag at src buffer + * After the flag of s[0:1] is 1 filled, + * copy the value from s[0:1]+4 to dst buffer + * + * Note: Only works on GFX9 (only used in + * aldebaran tests) + */ +const char *PollAndCopyIsa = R"( + .text + // Assume src buffer in s[0:1] and dst buffer in s[2:3] + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_stepping == 10) + // Path for Aldebaran + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v18, 0x1 + LOOP_ALDBRN: + flat_load_dword v16, v[0:1] glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + v_cmp_eq_i32 vcc, v16, v18 + s_cbranch_vccz LOOP_ALDBRN + buffer_invl2 + s_load_dword s17, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s17, s[2:3], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + buffer_wbl2 + .else + s_movk_i32 s18, 0x1 + LOOP: + s_load_dword s16, s[0:1], 0x0 glc + s_cmp_eq_i32 s16, s18 + s_cbranch_scc0 LOOP + s_load_dword s17, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s17, s[2:3], 0x0 glc + .endif + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; + +/* Input0: A buffer of at least 2 dwords. + * DW0: used as a signal. Write 0x1 to signal + * DW1: Write the value from 2nd input buffer + * for other device to read. + * Input1: A buffer of at least 2 dwords. + * DW0: used as the value to be written. + * + * Note: Only works on Aldebaran + */ +const char *WriteFlagAndValueIsa = R"( + .text + // Assume two inputs buffer in s[0:1] and s[2:3] + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + s_load_dword s18, s[2:3], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s18, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + buffer_wbl2 + s_waitcnt vmcnt(0) & lgkmcnt(0) + v_mov_b32 v16, 0x1 + flat_store_dword v[0:1], v16 glc + s_endpgm +)"; + +/* Input0: A buffer of at least 2 dwords. + * DW0: used as a signal. Write 0xcafe to signal + * DW1: Write to this buffer for other device to read. + * Input1: mmio base address + */ +const char *WriteAndSignalIsa = R"( + .text + // Assume input buffer in s0, s1 + .if (.amdgcn.gfx_generation_number >= 10) + s_add_u32 s4, s0, 0x4 + s_addc_u32 s5, s1, 0x0 + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + v_mov_b32 v2, s2 + v_mov_b32 v3, s3 + v_mov_b32 v4, s4 + v_mov_b32 v5, s5 + v_mov_b32 v18, 0xbeef + flat_store_dword v[4:5], v18 glc + v_mov_b32 v18, 0x1 + flat_store_dword v[2:3], v18 glc + v_mov_b32 v18, 0xcafe + flat_store_dword v[0:1], v18 glc + .else + s_mov_b32 s18, 0xbeef + s_store_dword s18, s[0:1], 0x4 glc + s_mov_b32 s18, 0x1 + s_store_dword s18, s[2:3], 0 glc + s_mov_b32 s18, 0xcafe + s_store_dword s18, s[0:1], 0x0 glc + .endif + s_endpgm +)"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index 2344c5bca2..b98088b843 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -30,4 +30,13 @@ extern const char *CopyDwordIsa; extern const char *InfiniteLoopIsa; extern const char *AtomicIncIsa; +/* KFDMemoryTest */ +extern const char *ScratchCopyDwordIsa; +extern const char *PollMemoryIsa; +extern const char *PollNCMemoryIsa; +extern const char *CopyOnSignalIsa; +extern const char *PollAndCopyIsa; +extern const char *WriteFlagAndValueIsa; +extern const char *WriteAndSignalIsa; + #endif // _SHADERSTORE_H_ From aced779f1b8b7458ed4d92e3bd540ad57cb9c9ba Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 2 Nov 2021 13:51:21 -0400 Subject: [PATCH 0978/1247] kfdtest: Move KFDQMTest shaders to ShaderStore Signed-off-by: Graham Sider Change-Id: Id50aea16528c4bed4530f95644a02f59efddae3e --- tests/kfdtest/src/KFDQMTest.cpp | 114 +++--------------------------- tests/kfdtest/src/KFDQMTest.hpp | 4 +- tests/kfdtest/src/ShaderStore.cpp | 98 +++++++++++++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 3 + 4 files changed, 110 insertions(+), 109 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index c28715639f..ffc568ebdf 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -34,117 +34,17 @@ #include "Dispatch.hpp" -/* A simple isa loop program with dense mathematic operations - * s1 controls the number iterations of the loop - * This shader can be used by GFX8, GFX9 and GFX10 - */ -static const char* LoopIsa = R"( - .text - s_movk_i32 s0, 0x0008 - s_movk_i32 s1, 0x00ff - v_mov_b32 v0, 0 - v_mov_b32 v1, 0 - v_mov_b32 v2, 0 - v_mov_b32 v3, 0 - v_mov_b32 v4, 0 - v_mov_b32 v5, 0 - v_mov_b32 v6, 0 - v_mov_b32 v7, 0 - v_mov_b32 v8, 0 - v_mov_b32 v9, 0 - v_mov_b32 v10, 0 - v_mov_b32 v11, 0 - v_mov_b32 v12, 0 - v_mov_b32 v13, 0 - v_mov_b32 v14, 0 - v_mov_b32 v15, 0 - v_mov_b32 v16, 0 - LOOP: - s_mov_b32 s8, s4 - s_mov_b32 s9, s1 - s_mov_b32 s10, s6 - s_mov_b32 s11, s7 - s_cmp_le_i32 s1, s0 - s_cbranch_scc1 END_OF_PGM - s_buffer_load_dwordx8 s[8:15], s[8:11], 0x10 - v_add_f32 v0, 2.0, v0 - v_cvt_f32_i32 v17, s1 - s_waitcnt lgkmcnt(0) - v_add_f32 v18, s8, v17 - v_add_f32 v19, s9, v17 - v_add_f32 v20, s10, v17 - v_add_f32 v21, s11, v17 - v_add_f32 v22, s12, v17 - v_add_f32 v23, s13, v17 - v_add_f32 v24, s14, v17 - v_add_f32 v17, s15, v17 - v_log_f32 v25, v18 - v_mul_f32 v25, v22, v25 - v_exp_f32 v25, v25 - v_log_f32 v26, v19 - v_mul_f32 v26, v23, v26 - v_exp_f32 v26, v26 - v_log_f32 v27, v20 - v_mul_f32 v27, v24, v27 - v_exp_f32 v27, v27 - v_log_f32 v28, v21 - v_mul_f32 v28, v17, v28 - v_exp_f32 v28, v28 - v_add_f32 v5, v5, v25 - v_add_f32 v6, v6, v26 - v_add_f32 v7, v7, v27 - v_add_f32 v8, v8, v28 - v_mul_f32 v18, 0x3fb8aa3b, v18 - v_exp_f32 v18, v18 - v_mul_f32 v19, 0x3fb8aa3b, v19 - v_exp_f32 v19, v19 - v_mul_f32 v20, 0x3fb8aa3b, v20 - v_exp_f32 v20, v20 - v_mul_f32 v21, 0x3fb8aa3b, v21 - v_exp_f32 v21, v21 - v_add_f32 v9, v9, v18 - v_add_f32 v10, v10, v19 - v_add_f32 v11, v11, v20 - v_add_f32 v12, v12, v21 - v_sqrt_f32 v18, v22 - v_sqrt_f32 v19, v23 - v_sqrt_f32 v20, v24 - v_sqrt_f32 v21, v17 - v_add_f32 v13, v13, v18 - v_add_f32 v14, v14, v19 - v_add_f32 v15, v15, v20 - v_add_f32 v16, v16, v21 - v_rsq_f32 v18, v22 - v_rsq_f32 v19, v23 - v_rsq_f32 v20, v24 - v_rsq_f32 v17, v17 - v_add_f32 v1, v1, v18 - v_add_f32 v2, v2, v19 - v_add_f32 v3, v3, v20 - v_add_f32 v4, v4, v17 - s_add_u32 s0, s0, 1 - s_branch LOOP - END_OF_PGM: - s_endpgm -)"; - void KFDQMTest::SetUp() { ROUTINE_START KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDQMTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - KFDBaseComponentTest::TearDown(); ROUTINE_END @@ -1134,7 +1034,7 @@ TEST_F(KFDQMTest, EmptyDispatch) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->GetNoopIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As())); SyncDispatch(isaBuffer, NULL, NULL); @@ -1153,7 +1053,7 @@ TEST_F(KFDQMTest, SimpleWriteDispatch) { srcBuffer.Fill(0x01010101); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); SyncDispatch(isaBuffer, srcBuffer.As(), destBuffer.As()); @@ -1188,7 +1088,7 @@ TEST_F(KFDQMTest, MultipleCpQueuesStressDispatch) { destBuffer.Fill(0xFF); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); for (i = 0; i < MAX_CP_QUEUES; ++i) ASSERT_SUCCESS(queues[i].Create(defaultGPUNode)) << " QueueId=" << i; @@ -1527,7 +1427,7 @@ TEST_F(KFDQMTest, Atomics) { PM4Queue queue; - m_pIsaGen->GetAtomicIncIsa(isaBuf); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(AtomicIncIsa, isaBuf.As())); Dispatch dispatch(isaBuf); dispatch.SetArgs(destBuf.As(), NULL); @@ -1592,10 +1492,12 @@ TEST_F(KFDQMTest, mGPUShareBO) { srcNodeMem.Fill(0x05050505); - m_pIsaGen->GetCopyDwordIsa(isaBufferSrc); + ASSERT_SUCCESS(m_pAsm->RunAssemble(CopyDwordIsa)); + + m_pAsm->CopyInstrStream(isaBufferSrc.As()); SyncDispatch(isaBufferSrc, srcNodeMem.As(), shared_addr.As(), src_node); - m_pIsaGen->GetCopyDwordIsa(isaBufferDst); + m_pAsm->CopyInstrStream(isaBufferDst.As()); SyncDispatch(isaBufferDst, shared_addr.As(), dstNodeMem.As(), dst_node); EXPECT_EQ(dstNodeMem.As()[0], 0x05050505); diff --git a/tests/kfdtest/src/KFDQMTest.hpp b/tests/kfdtest/src/KFDQMTest.hpp index b0d3f66073..dfc36d17d4 100644 --- a/tests/kfdtest/src/KFDQMTest.hpp +++ b/tests/kfdtest/src/KFDQMTest.hpp @@ -27,13 +27,12 @@ #include #include "PM4Queue.hpp" -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" #include "Dispatch.hpp" class KFDQMTest : public KFDBaseComponentTest { public: - KFDQMTest():m_pIsaGen(NULL) {} + KFDQMTest() {} ~KFDQMTest() {} @@ -49,7 +48,6 @@ class KFDQMTest : public KFDBaseComponentTest { const double CuVariance = 0.15; const double CuNegVariance = 1.0 - CuVariance; const double CuPosVariance = 1.0 + CuVariance; - IsaGenerator* m_pIsaGen; }; #endif // __KFD_QCM_TEST__H__ diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index e7cb07470b..28025f77d9 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -288,3 +288,101 @@ const char *WriteAndSignalIsa = R"( .endif s_endpgm )"; + +/** + * KFDQMTest + */ + +/* A simple isa loop program with dense mathematic operations + * s1 controls the number iterations of the loop + * This shader can be used by GFX8, GFX9 and GFX10 + */ +const char *LoopIsa = R"( + .text + s_movk_i32 s0, 0x0008 + s_movk_i32 s1, 0x00ff + v_mov_b32 v0, 0 + v_mov_b32 v1, 0 + v_mov_b32 v2, 0 + v_mov_b32 v3, 0 + v_mov_b32 v4, 0 + v_mov_b32 v5, 0 + v_mov_b32 v6, 0 + v_mov_b32 v7, 0 + v_mov_b32 v8, 0 + v_mov_b32 v9, 0 + v_mov_b32 v10, 0 + v_mov_b32 v11, 0 + v_mov_b32 v12, 0 + v_mov_b32 v13, 0 + v_mov_b32 v14, 0 + v_mov_b32 v15, 0 + v_mov_b32 v16, 0 + LOOP: + s_mov_b32 s8, s4 + s_mov_b32 s9, s1 + s_mov_b32 s10, s6 + s_mov_b32 s11, s7 + s_cmp_le_i32 s1, s0 + s_cbranch_scc1 END_OF_PGM + s_buffer_load_dwordx8 s[8:15], s[8:11], 0x10 + v_add_f32 v0, 2.0, v0 + v_cvt_f32_i32 v17, s1 + s_waitcnt lgkmcnt(0) + v_add_f32 v18, s8, v17 + v_add_f32 v19, s9, v17 + v_add_f32 v20, s10, v17 + v_add_f32 v21, s11, v17 + v_add_f32 v22, s12, v17 + v_add_f32 v23, s13, v17 + v_add_f32 v24, s14, v17 + v_add_f32 v17, s15, v17 + v_log_f32 v25, v18 + v_mul_f32 v25, v22, v25 + v_exp_f32 v25, v25 + v_log_f32 v26, v19 + v_mul_f32 v26, v23, v26 + v_exp_f32 v26, v26 + v_log_f32 v27, v20 + v_mul_f32 v27, v24, v27 + v_exp_f32 v27, v27 + v_log_f32 v28, v21 + v_mul_f32 v28, v17, v28 + v_exp_f32 v28, v28 + v_add_f32 v5, v5, v25 + v_add_f32 v6, v6, v26 + v_add_f32 v7, v7, v27 + v_add_f32 v8, v8, v28 + v_mul_f32 v18, 0x3fb8aa3b, v18 + v_exp_f32 v18, v18 + v_mul_f32 v19, 0x3fb8aa3b, v19 + v_exp_f32 v19, v19 + v_mul_f32 v20, 0x3fb8aa3b, v20 + v_exp_f32 v20, v20 + v_mul_f32 v21, 0x3fb8aa3b, v21 + v_exp_f32 v21, v21 + v_add_f32 v9, v9, v18 + v_add_f32 v10, v10, v19 + v_add_f32 v11, v11, v20 + v_add_f32 v12, v12, v21 + v_sqrt_f32 v18, v22 + v_sqrt_f32 v19, v23 + v_sqrt_f32 v20, v24 + v_sqrt_f32 v21, v17 + v_add_f32 v13, v13, v18 + v_add_f32 v14, v14, v19 + v_add_f32 v15, v15, v20 + v_add_f32 v16, v16, v21 + v_rsq_f32 v18, v22 + v_rsq_f32 v19, v23 + v_rsq_f32 v20, v24 + v_rsq_f32 v17, v17 + v_add_f32 v1, v1, v18 + v_add_f32 v2, v2, v19 + v_add_f32 v3, v3, v20 + v_add_f32 v4, v4, v17 + s_add_u32 s0, s0, 1 + s_branch LOOP + END_OF_PGM: + s_endpgm +)"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index b98088b843..3d4df30e94 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -39,4 +39,7 @@ extern const char *PollAndCopyIsa; extern const char *WriteFlagAndValueIsa; extern const char *WriteAndSignalIsa; +/* KFDQMTest */ +extern const char *LoopIsa; + #endif // _SHADERSTORE_H_ From ad5f98814fc3763b5d15dd2c6360ac3e275bef63 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 2 Nov 2021 13:54:54 -0400 Subject: [PATCH 0979/1247] kfdtest: Move KFDCWSRTest shaders to ShaderStore Signed-off-by: Graham Sider Change-Id: I7c89fca94e92145a4115d1089348380807a868ee --- tests/kfdtest/src/KFDCWSRTest.cpp | 76 +------------------------------ tests/kfdtest/src/ShaderStore.cpp | 52 +++++++++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 3 ++ 3 files changed, 57 insertions(+), 74 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index c7a2e4b312..5a4f853ee2 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -24,72 +24,6 @@ #include "KFDCWSRTest.hpp" #include "Dispatch.hpp" - -/* Initial state: - * s[0:1] - 64 bits iteration number; only the lower 32 bits are useful. - * s[2:3] - result buffer base address - * s4 - workgroup id - * v0 - workitem id, always 0 because - * NUM_THREADS_X(number of threads) in workgroup set to 1 - * Registers: - * v0 - calculated workitem = v0 + s4 * NUM_THREADS_X, which is s4 - * v2 - = s0, 32 bits iteration number - * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 - * v6 - counter - */ - -static const char* IterateIsa_gfx8 = R"( - .text - // Copy the parameters from scalar registers to vector registers - v_mov_b32 v2, s0 // v[2:3] = s[0:1] - v_mov_b32 v3, s1 // v[2:3] = s[0:1] - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 - v_add_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v6, 0 - LOOP: - v_add_u32 v6, vcc, 1, v6 - // Compare the result value (v6) to iteration value (v2), and - // jump if equal (i.e. if VCC is not zero after the comparison) - v_cmp_lt_u32 vcc, v6, v2 - s_cbranch_vccnz LOOP - flat_store_dword v[4:5], v6 - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_endpgm -)"; - -// This shader can be used by gfx9 and gfx10 -static const char* IterateIsa_gfx9 = R"( - .text - // Copy the parameters from scalar registers to vector registers - v_mov_b32 v2, s0 // v[2:3] = s[0:1] - v_mov_b32 v3, s1 // v[2:3] = s[0:1] - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 - v_add_co_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v6, 0 - LOOP: - v_add_co_u32 v6, vcc, 1, v6 - // Compare the result value (v6) to iteration value (v2), and - // jump if equal (i.e. if VCC is not zero after the comparison) - v_cmp_lt_u32 vcc, v6, v2 - s_cbranch_vccnz LOOP - flat_store_dword v[4:5], v6 - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_endpgm -)"; - -static const char* InfiniteIsa = R"( - .text - LOOP: - s_branch LOOP - s_endpgm -)"; - void KFDCWSRTest::SetUp() { ROUTINE_START @@ -137,16 +71,10 @@ TEST_F(KFDCWSRTest, BasicTest) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { - const char *pIterateIsa; HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); HsaMemoryBuffer resultBuf1(PAGE_SIZE, defaultGPUNode, true, false, false); uint64_t count1 = 400000000; - if (m_FamilyId < FAMILY_AI) - pIterateIsa = IterateIsa_gfx8; - else - pIterateIsa = IterateIsa_gfx9; - if (isOnEmulator()) { // Divide the iterator times by 10000 so that the test can // finish in a reasonable time. @@ -156,7 +84,7 @@ TEST_F(KFDCWSRTest, BasicTest) { unsigned int* result1 = resultBuf1.As(); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pIterateIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(IterateIsa, isaBuffer.As())); PM4Queue queue1; @@ -220,7 +148,7 @@ TEST_F(KFDCWSRTest, InterruptRestore) { if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(InfiniteIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(InfiniteLoopIsa, isaBuffer.As())); PM4Queue queue1, queue2, queue3; diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 28025f77d9..93a2d60184 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -386,3 +386,55 @@ const char *LoopIsa = R"( END_OF_PGM: s_endpgm )"; + + +/** + * KFDCWSRTest + */ + +/* Initial state: + * s[0:1] - 64 bits iteration number; only the lower 32 bits are useful. + * s[2:3] - result buffer base address + * s4 - workgroup id + * v0 - workitem id, always 0 because + * NUM_THREADS_X(number of threads) in workgroup set to 1 + * Registers: + * v0 - calculated workitem = v0 + s4 * NUM_THREADS_X, which is s4 + * v2 - = s0, 32 bits iteration number + * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 + * v6 - counter + */ +const char *IterateIsa = R"( + .text + // Copy the parameters from scalar registers to vector registers + v_mov_b32 v2, s0 // v[2:3] = s[0:1] + v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + .if (.amdgcn.gfx_generation_number >= 9) + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 + v_add_co_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v6, 0 + LOOP: + v_add_co_u32 v6, vcc, 1, v6 + // Compare the result value (v6) to iteration value (v2), and + // jump if equal (i.e. if VCC is not zero after the comparison) + v_cmp_lt_u32 vcc, v6, v2 + s_cbranch_vccnz LOOP + .else + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 + v_add_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v6, 0 + LOOP_GFX8: + v_add_u32 v6, vcc, 1, v6 + // Compare the result value (v6) to iteration value (v2), and + // jump if equal (i.e. if VCC is not zero after the comparison) + v_cmp_lt_u32 vcc, v6, v2 + s_cbranch_vccnz LOOP_GFX8 + .endif + flat_store_dword v[4:5], v6 + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_endpgm +)"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index 3d4df30e94..f3a9d92781 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -42,4 +42,7 @@ extern const char *WriteAndSignalIsa; /* KFDQMTest */ extern const char *LoopIsa; +/* KFDCWSRTest */ +extern const char *IterateIsa; + #endif // _SHADERSTORE_H_ From 91cf11967ec201e056b04ca77f65eede728a8736 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 2 Nov 2021 13:56:38 -0400 Subject: [PATCH 0980/1247] kfdtest: Move KFDEvictTest shaders to ShaderStore Signed-off-by: Graham Sider Change-Id: I4d7d349284ea213becdb4680b804dbd202196e1d --- tests/kfdtest/src/KFDEvictTest.cpp | 117 +---------------------------- tests/kfdtest/src/KFDEvictTest.hpp | 1 - tests/kfdtest/src/ShaderStore.cpp | 104 +++++++++++++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 3 + 4 files changed, 108 insertions(+), 117 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index bf721238c8..1effa639b0 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -36,121 +36,6 @@ #define SDMA_NOP 0x0 -/* Shader to read local buffers using multiple wavefronts in parallel - * until address buffer is filled with specific value 0x5678 by host program, - * then each wavefront fills value 0x5678 at corresponding result buffer and quit - * - * Initial state: - * s[0:1] - address buffer base address - * s[2:3] - result buffer base address - * s4 - workgroup id - * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 - * Registers: - * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X - * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 - * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 - * v[6:7] - local buf address used for read test - * - * This shader can be used by gfx9 and gfx10 - * - */ - -static const char* ReadMemoryIsa_gfx9 = R"( - .text - // Compute address of corresponding output buffer - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 - v_add_co_u32 v5, vcc, v5, vcc_lo - // Compute input buffer offset used to store corresponding local buffer address - v_lshlrev_b32 v0, 1, v0 // v0 *= 8 - v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 - v_mov_b32 v3, s1 - v_add_co_u32 v3, vcc, v3, vcc_lo - // load 64bit local buffer address stored at v[2:3] to v[6:7] - flat_load_dwordx2 v[6:7], v[2:3] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - v_mov_b32 v8, 0x5678 - s_movk_i32 s8, 0x5678 - L_REPEAT: - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - s_cmp_eq_i32 s16, s8 - s_cbranch_scc1 L_QUIT // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] - // every 4k page only read once - v_mov_b32 v9, 0 - v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size - v_mov_b32 v12, v6 - v_mov_b32 v13, v7 - L_LOOP_READ: - flat_load_dwordx2 v[14:15], v[12:13] slc - v_add_co_u32 v9, vcc, v9, v10 - v_add_co_u32 v12, vcc, v12, v10 - v_add_co_u32 v13, vcc, v13, vcc_lo - v_cmp_lt_u32 vcc, v9, v11 - s_cbranch_vccnz L_LOOP_READ - s_branch L_REPEAT - L_QUIT: - flat_store_dword v[4:5], v8 - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish - s_endpgm -)"; - -static const char* ReadMemoryIsa_gfx8 = R"( - .text - // Compute address of corresponding output buffer - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 - v_addc_u32 v5, vcc, v5, 0, vcc - // Compute input buffer offset used to store corresponding local buffer address - v_lshlrev_b32 v0, 1, v0 // v0 *= 8 - v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 - v_mov_b32 v3, s1 - v_addc_u32 v3, vcc, v3, 0, vcc - // Load 64bit local buffer address stored at v[2:3] to v[6:7] - flat_load_dwordx2 v[6:7], v[2:3] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - v_mov_b32 v8, 0x5678 - s_movk_i32 s8, 0x5678 - L_REPEAT: - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - s_cmp_eq_i32 s16, s8 - s_cbranch_scc1 L_QUIT // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] - // every 4k page only read once - v_mov_b32 v9, 0 - v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size - v_mov_b32 v12, v6 - v_mov_b32 v13, v7 - L_LOOP_READ: - flat_load_dwordx2 v[14:15], v[12:13] slc - v_add_u32 v9, vcc, v9, v10 - v_add_u32 v12, vcc, v12, v10 - v_addc_u32 v13, vcc, v13, 0, vcc - v_cmp_lt_u32 vcc, v9, v11 - s_cbranch_vccnz L_LOOP_READ - s_branch L_REPEAT - L_QUIT: - flat_store_dword v[4:5], v8 - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish - s_endpgm -)"; - -std::string KFDEvictTest::CreateShader() { - if (m_FamilyId < FAMILY_AI) - return ReadMemoryIsa_gfx8; - else - return ReadMemoryIsa_gfx9; -} - - void KFDEvictTest::SetUp() { ROUTINE_START @@ -546,7 +431,7 @@ TEST_F(KFDEvictTest, QueueTest) { HsaMemoryBuffer addrBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer resultBuffer(PAGE_SIZE, defaultGPUNode); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CreateShader().c_str(), isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadMemoryIsa, isaBuffer.As())); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/tests/kfdtest/src/KFDEvictTest.hpp index d70aada6b4..30f0a856be 100644 --- a/tests/kfdtest/src/KFDEvictTest.hpp +++ b/tests/kfdtest/src/KFDEvictTest.hpp @@ -40,7 +40,6 @@ class KFDEvictTest : public KFDMultiProcessTest { virtual void SetUp(); virtual void TearDown(); - std::string CreateShader(); void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, std::vector &pBuffers); void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 93a2d60184..fdf536b4db 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -438,3 +438,107 @@ const char *IterateIsa = R"( s_waitcnt vmcnt(0) & lgkmcnt(0) s_endpgm )"; + +/** + * KFDEvictTest + */ + +/* Shader to read local buffers using multiple wavefronts in parallel + * until address buffer is filled with specific value 0x5678 by host program, + * then each wavefront fills value 0x5678 at corresponding result buffer and quit + * + * Initial state: + * s[0:1] - address buffer base address + * s[2:3] - result buffer base address + * s4 - workgroup id + * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 + * Registers: + * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X + * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 + * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 + * v[6:7] - local buf address used for read test + */ +const char *ReadMemoryIsa = R"( + .text + .if (.amdgcn.gfx_generation_number >= 9) + // Compute address of corresponding output buffer + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 + v_add_co_u32 v5, vcc, v5, vcc_lo + // Compute input buffer offset used to store corresponding local buffer address + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 + v_add_co_u32 v3, vcc, v3, vcc_lo + // Load 64bit local buffer address stored at v[2:3] to v[6:7] + flat_load_dwordx2 v[6:7], v[2:3] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + v_mov_b32 v8, 0x5678 + s_movk_i32 s8, 0x5678 + L_REPEAT: + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s16, s8 + s_cbranch_scc1 L_QUIT // if notified to quit by host + // Loop read 64M local buffer starting at v[6:7] + // every 4k page only read once + v_mov_b32 v9, 0 + v_mov_b32 v10, 0x1000 // 4k page + v_mov_b32 v11, 0x4000000 // 64M size + v_mov_b32 v12, v6 + v_mov_b32 v13, v7 + L_LOOP_READ: + flat_load_dwordx2 v[14:15], v[12:13] slc + v_add_co_u32 v9, vcc, v9, v10 + v_add_co_u32 v12, vcc, v12, v10 + v_add_co_u32 v13, vcc, v13, vcc_lo + v_cmp_lt_u32 vcc, v9, v11 + s_cbranch_vccnz L_LOOP_READ + s_branch L_REPEAT + L_QUIT: + flat_store_dword v[4:5], v8 + .else + // Compute address of corresponding output buffer + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 + v_addc_u32 v5, vcc, v5, 0, vcc + // Compute input buffer offset used to store corresponding local buffer address + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 + v_addc_u32 v3, vcc, v3, 0, vcc + // Load 64bit local buffer address stored at v[2:3] to v[6:7] + flat_load_dwordx2 v[6:7], v[2:3] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + v_mov_b32 v8, 0x5678 + s_movk_i32 s8, 0x5678 + L_REPEAT_GFX8: + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s16, s8 + s_cbranch_scc1 L_QUIT_8 // if notified to quit by host + // Loop read 64M local buffer starting at v[6:7] + // every 4k page only read once + v_mov_b32 v9, 0 + v_mov_b32 v10, 0x1000 // 4k page + v_mov_b32 v11, 0x4000000 // 64M size + v_mov_b32 v12, v6 + v_mov_b32 v13, v7 + L_LOOP_READ_GFX8: + flat_load_dwordx2 v[14:15], v[12:13] slc + v_add_u32 v9, vcc, v9, v10 + v_add_u32 v12, vcc, v12, v10 + v_addc_u32 v13, vcc, v13, 0, vcc + v_cmp_lt_u32 vcc, v9, v11 + s_cbranch_vccnz L_LOOP_READ_GFX8 + s_branch L_REPEAT_GFX8 + L_QUIT_8: + flat_store_dword v[4:5], v8 + .endif + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish + s_endpgm +)"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index f3a9d92781..de73034ee4 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -45,4 +45,7 @@ extern const char *LoopIsa; /* KFDCWSRTest */ extern const char *IterateIsa; +/* KFDEvictTest */ +extern const char *ReadMemoryIsa; + #endif // _SHADERSTORE_H_ From b2b54dffe6886cef1c7126f92a7808f0c1a6aac4 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 2 Nov 2021 14:04:48 -0400 Subject: [PATCH 0981/1247] kfdtest: Move KFDGWSTest shaders to ShaderStore Signed-off-by: Graham Sider Change-Id: I9608b8bea32d64d4d0e1a329191f9a62e3a771e7 --- tests/kfdtest/src/KFDGWSTest.cpp | 66 +------------------------------ tests/kfdtest/src/ShaderStore.cpp | 56 ++++++++++++++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 4 ++ 3 files changed, 62 insertions(+), 64 deletions(-) diff --git a/tests/kfdtest/src/KFDGWSTest.cpp b/tests/kfdtest/src/KFDGWSTest.cpp index a6b3bec17b..99e9248d8f 100644 --- a/tests/kfdtest/src/KFDGWSTest.cpp +++ b/tests/kfdtest/src/KFDGWSTest.cpp @@ -26,62 +26,6 @@ #include "PM4Packet.hpp" #include "Dispatch.hpp" -/* Shader to initialize gws counter to 1 */ -static const char* GwsInitIsa_gfx9_10 = R"( - .text - s_mov_b32 m0, 0 - s_nop 0 - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt 0 - v_mov_b32 v0, s16 - s_waitcnt 0 - ds_gws_init v0 offset:0 gds - s_waitcnt 0 - s_endpgm -)"; - -/* Atomically increase a value in memory - * This is expected to be executed from - * multiple work groups simultaneously. - * GWS semaphore is used to guarantee - * the operation is atomic. - */ -static const char* AtomicIncreaseIsa_gfx9 = R"( - .text - // Assume src address in s0, s1 - s_mov_b32 m0, 0 - s_nop 0 - ds_gws_sema_p offset:0 gds - s_waitcnt 0 - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt 0 - s_add_u32 s16, s16, 1 - s_store_dword s16, s[0:1], 0x0 glc - s_waitcnt lgkmcnt(0) - ds_gws_sema_v offset:0 gds - s_waitcnt 0 - s_endpgm -)"; - -static const char* AtomicIncreaseIsa_gfx10 = R"( - .text - // Assume src address in s0, s1 - s_mov_b32 m0, 0 - s_mov_b32 exec_lo, 0x1 - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - ds_gws_sema_p offset:0 gds - s_waitcnt 0 - flat_load_dword v2, v[0:1] glc dlc - s_waitcnt 0 - v_add_nc_u32 v2, v2, 1 - flat_store_dword v[0:1], v2 - s_waitcnt_vscnt null, 0 - ds_gws_sema_v offset:0 gds - s_waitcnt 0 - s_endpgm -)"; - void KFDGWSTest::SetUp() { ROUTINE_START @@ -142,7 +86,7 @@ TEST_F(KFDGWSTest, Semaphore) { pNodeProperties->NumGws,&firstGWS)); EXPECT_EQ(0, firstGWS); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(GwsInitIsa_gfx9_10, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(GwsInitIsa, isaBuffer.As())); Dispatch dispatch0(isaBuffer); buffer.Fill(numResources, 0, 4); @@ -150,13 +94,7 @@ TEST_F(KFDGWSTest, Semaphore) { dispatch0.Submit(queue); dispatch0.Sync(); - const char *pAtomicIncreaseIsa; - if (m_FamilyId <= FAMILY_AL) - pAtomicIncreaseIsa = AtomicIncreaseIsa_gfx9; - else - pAtomicIncreaseIsa = AtomicIncreaseIsa_gfx10; - - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pAtomicIncreaseIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(GwsAtomicIncreaseIsa, isaBuffer.As())); Dispatch dispatch(isaBuffer); dispatch.SetArgs(buffer.As(), NULL); diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index fdf536b4db..e720d4b8ce 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -542,3 +542,59 @@ const char *ReadMemoryIsa = R"( s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish s_endpgm )"; + +/** + * KFDGWSTest + */ + +/* Shader to initialize gws counter to 1 */ +const char *GwsInitIsa = R"( + .text + s_mov_b32 m0, 0 + s_nop 0 + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt 0 + v_mov_b32 v0, s16 + s_waitcnt 0 + ds_gws_init v0 offset:0 gds + s_waitcnt 0 + s_endpgm +)"; + +/* Atomically increase a value in memory + * This is expected to be executed from + * multiple work groups simultaneously. + * GWS semaphore is used to guarantee + * the operation is atomic. + */ +const char *GwsAtomicIncreaseIsa = R"( + .text + // Assume src address in s0, s1 + .if (.amdgcn.gfx_generation_number >= 10) + s_mov_b32 m0, 0 + s_mov_b32 exec_lo, 0x1 + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + ds_gws_sema_p offset:0 gds + s_waitcnt 0 + flat_load_dword v2, v[0:1] glc dlc + s_waitcnt 0 + v_add_nc_u32 v2, v2, 1 + flat_store_dword v[0:1], v2 + s_waitcnt_vscnt null, 0 + ds_gws_sema_v offset:0 gds + .else + s_mov_b32 m0, 0 + s_nop 0 + ds_gws_sema_p offset:0 gds + s_waitcnt 0 + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt 0 + s_add_u32 s16, s16, 1 + s_store_dword s16, s[0:1], 0x0 glc + s_waitcnt lgkmcnt(0) + ds_gws_sema_v offset:0 gds + .endif + s_waitcnt 0 + s_endpgm +)"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index de73034ee4..231e7f73d6 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -48,4 +48,8 @@ extern const char *IterateIsa; /* KFDEvictTest */ extern const char *ReadMemoryIsa; +/* KFDGWSTest */ +extern const char *GwsInitIsa; +extern const char *GwsAtomicIncreaseIsa; + #endif // _SHADERSTORE_H_ From 5ceb35f4280e3ff3b2d0b2a0b626112276bcf7c2 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 4 Nov 2021 17:08:43 -0400 Subject: [PATCH 0982/1247] kfdtest: Add macros to simplify instr differences Makes use of macros to simplify shader code with instruction-level differences depending on GFX version. These macros are extensible and are prepended to every shader so that they are usable everywhere. This patch introduces three macros used within IterateIsa and ReadMemoryIsa shaders. Signed-off-by: Graham Sider Change-Id: If954e1b6d2027e9f55bf7e99bd9df2668d1da524 --- tests/kfdtest/src/ShaderStore.cpp | 207 ++++++++++++++---------------- 1 file changed, 95 insertions(+), 112 deletions(-) diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index e720d4b8ce..63cd68a063 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -21,6 +21,41 @@ * */ +/** + * Macros + */ + +/* Create macro for portable v_add_co_u32, v_add_co_ci_u32, + * and v_cmp_lt_u32 + */ +#define SHADER_MACROS \ + " .text\n"\ + " .macro V_ADD_CO_U32 vdst, src0, vsrc1\n"\ + " .if (.amdgcn.gfx_generation_number >= 10)\n"\ + " v_add_co_u32 \\vdst, vcc_lo, \\src0, \\vsrc1\n"\ + " .elseif (.amdgcn.gfx_generation_number >= 9)\n"\ + " v_add_co_u32 \\vdst, vcc, \\src0, \\vsrc1\n"\ + " .else\n"\ + " v_add_u32 \\vdst, vcc, \\src0, \\vsrc1\n"\ + " .endif\n"\ + " .endm\n"\ + " .macro V_ADD_CO_CI_U32 vdst, src0, vsrc1\n"\ + " .if (.amdgcn.gfx_generation_number >= 10)\n"\ + " v_add_co_ci_u32 \\vdst, vcc_lo, \\src0, \\vsrc1, vcc_lo\n"\ + " .elseif (.amdgcn.gfx_generation_number >= 9)\n"\ + " v_addc_co_u32 \\vdst, vcc, \\src0, \\vsrc1, vcc\n"\ + " .else\n"\ + " v_addc_u32 \\vdst, vcc, \\src0, \\vsrc1, vcc\n"\ + " .endif\n"\ + " .endm\n"\ + " .macro V_CMP_LT_U32 src0, vsrc1\n"\ + " .if (.amdgcn.gfx_generation_number >= 10)\n"\ + " v_cmp_lt_u32 vcc_lo, \\src0, \\vsrc1\n"\ + " .else\n"\ + " v_cmp_lt_u32 vcc, \\src0, \\vsrc1\n"\ + " .endif\n"\ + " .endm\n" + /** * Common */ @@ -404,37 +439,24 @@ const char *LoopIsa = R"( * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 * v6 - counter */ -const char *IterateIsa = R"( - .text +const char *IterateIsa = SHADER_MACROS R"( // Copy the parameters from scalar registers to vector registers - v_mov_b32 v2, s0 // v[2:3] = s[0:1] - v_mov_b32 v3, s1 // v[2:3] = s[0:1] - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - .if (.amdgcn.gfx_generation_number >= 9) - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 - v_add_co_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v6, 0 - LOOP: - v_add_co_u32 v6, vcc, 1, v6 - // Compare the result value (v6) to iteration value (v2), and - // jump if equal (i.e. if VCC is not zero after the comparison) - v_cmp_lt_u32 vcc, v6, v2 - s_cbranch_vccnz LOOP - .else - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 - v_add_u32 v5, vcc, v5, vcc_lo // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v6, 0 - LOOP_GFX8: - v_add_u32 v6, vcc, 1, v6 - // Compare the result value (v6) to iteration value (v2), and - // jump if equal (i.e. if VCC is not zero after the comparison) - v_cmp_lt_u32 vcc, v6, v2 - s_cbranch_vccnz LOOP_GFX8 - .endif - flat_store_dword v[4:5], v6 + v_mov_b32 v2, s0 // v[2:3] = s[0:1] + v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + V_ADD_CO_U32 v4, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 + V_ADD_CO_CI_U32 v5, v5, 0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v6, 0 + LOOP: + V_ADD_CO_U32 v6, 1, v6 + + // Compare the result value (v6) to iteration value (v2), and + // jump if equal (i.e. if VCC is not zero after the comparison) + V_CMP_LT_U32 v6, v2 + s_cbranch_vccnz LOOP + flat_store_dword v[4:5], v6 s_waitcnt vmcnt(0) & lgkmcnt(0) s_endpgm )"; @@ -458,88 +480,49 @@ const char *IterateIsa = R"( * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 * v[6:7] - local buf address used for read test */ -const char *ReadMemoryIsa = R"( - .text - .if (.amdgcn.gfx_generation_number >= 9) - // Compute address of corresponding output buffer - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 - v_add_co_u32 v5, vcc, v5, vcc_lo - // Compute input buffer offset used to store corresponding local buffer address - v_lshlrev_b32 v0, 1, v0 // v0 *= 8 - v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 - v_mov_b32 v3, s1 - v_add_co_u32 v3, vcc, v3, vcc_lo - // Load 64bit local buffer address stored at v[2:3] to v[6:7] - flat_load_dwordx2 v[6:7], v[2:3] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - v_mov_b32 v8, 0x5678 - s_movk_i32 s8, 0x5678 - L_REPEAT: - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - s_cmp_eq_i32 s16, s8 - s_cbranch_scc1 L_QUIT // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] - // every 4k page only read once - v_mov_b32 v9, 0 - v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size - v_mov_b32 v12, v6 - v_mov_b32 v13, v7 - L_LOOP_READ: - flat_load_dwordx2 v[14:15], v[12:13] slc - v_add_co_u32 v9, vcc, v9, v10 - v_add_co_u32 v12, vcc, v12, v10 - v_add_co_u32 v13, vcc, v13, vcc_lo - v_cmp_lt_u32 vcc, v9, v11 - s_cbranch_vccnz L_LOOP_READ - s_branch L_REPEAT - L_QUIT: - flat_store_dword v[4:5], v8 - .else - // Compute address of corresponding output buffer - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 - v_addc_u32 v5, vcc, v5, 0, vcc - // Compute input buffer offset used to store corresponding local buffer address - v_lshlrev_b32 v0, 1, v0 // v0 *= 8 - v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 - v_mov_b32 v3, s1 - v_addc_u32 v3, vcc, v3, 0, vcc - // Load 64bit local buffer address stored at v[2:3] to v[6:7] - flat_load_dwordx2 v[6:7], v[2:3] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - v_mov_b32 v8, 0x5678 - s_movk_i32 s8, 0x5678 - L_REPEAT_GFX8: - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - s_cmp_eq_i32 s16, s8 - s_cbranch_scc1 L_QUIT_8 // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] - // every 4k page only read once - v_mov_b32 v9, 0 - v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size - v_mov_b32 v12, v6 - v_mov_b32 v13, v7 - L_LOOP_READ_GFX8: - flat_load_dwordx2 v[14:15], v[12:13] slc - v_add_u32 v9, vcc, v9, v10 - v_add_u32 v12, vcc, v12, v10 - v_addc_u32 v13, vcc, v13, 0, vcc - v_cmp_lt_u32 vcc, v9, v11 - s_cbranch_vccnz L_LOOP_READ_GFX8 - s_branch L_REPEAT_GFX8 - L_QUIT_8: - flat_store_dword v[4:5], v8 - .endif - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish +const char *ReadMemoryIsa = SHADER_MACROS R"( + // Compute address of corresponding output buffer + v_mov_b32 v0, s4 // use workgroup id as index + v_lshlrev_b32 v0, 2, v0 // v0 *= 4 + V_ADD_CO_U32 v4, s2, v0 // v[4:5] = s[2:3] + v0 * 4 + v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 + V_ADD_CO_CI_U32 v5, v5, 0 // v[4:5] = s[2:3] + v0 * 4 + + // Compute input buffer offset used to store corresponding local buffer address + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + V_ADD_CO_U32 v2, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v0 * 8 + V_ADD_CO_CI_U32 v3, v3, 0 // v[2:3] = s[0:1] + v0 * 8 + + // Load 64bit local buffer address stored at v[2:3] to v[6:7] + flat_load_dwordx2 v[6:7], v[2:3] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + v_mov_b32 v8, 0x5678 + s_movk_i32 s8, 0x5678 + L_REPEAT: + s_load_dword s16, s[0:1], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s16, s8 + s_cbranch_scc1 L_QUIT // if notified to quit by host + + // Loop read 64M local buffer starting at v[6:7] + // every 4k page only read once + v_mov_b32 v9, 0 + v_mov_b32 v10, 0x1000 // 4k page + v_mov_b32 v11, 0x4000000 // 64M size + v_mov_b32 v12, v6 + v_mov_b32 v13, v7 + L_LOOP_READ: + flat_load_dwordx2 v[14:15], v[12:13] slc + V_ADD_CO_U32 v9, v9, v10 + V_ADD_CO_U32 v12, v12, v10 + V_ADD_CO_CI_U32 v13, v13, 0 + V_CMP_LT_U32 v9, v11 + s_cbranch_vccnz L_LOOP_READ + s_branch L_REPEAT + L_QUIT: + flat_store_dword v[4:5], v8 + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish s_endpgm )"; From 097b11abad5bc91a973fdc50e804c9542f53e7d7 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 2 Nov 2021 14:05:43 -0400 Subject: [PATCH 0983/1247] kfdtest: Remove KFDSVMEvictTest ReadMemory shader Use ReadMemoryIsa transferred and updated from KFDEvictTest. Signed-off-by: Graham Sider Change-Id: I566f9ec36398bc4d08ab90231688600356df4d6a --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 112 +------------------------- 1 file changed, 1 insertion(+), 111 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index e712e7b44a..319b054a64 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -35,109 +35,6 @@ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) -/* Shader to read local buffers using multiple wavefronts in parallel - * until address buffer is filled with specific value 0x5678 by host program, - * then each wavefront fills value 0x5678 at corresponding result buffer and quit - * - * initial state: - * s[0:1] - address buffer base address - * s[2:3] - result buffer base address - * s4 - workgroup id - * v0 - workitem id, always 0 because NUM_THREADS_X(number of threads) in workgroup set to 1 - * registers: - * v0 - calculated workitem id, v0 = v0 + s4 * NUM_THREADS_X - * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 - * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 - * v[6:7] - local buf address used for read test - */ -static const char* gfx9_ReadMemory = R"( - .text - // Compute address of corresponding output buffer - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_co_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 - v_add_u32 v5, vcc_lo, v5 - // Compute input buffer offset used to store corresponding local buffer address - v_lshlrev_b32 v0, 1, v0 // v0 *= 8 - v_add_co_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 - v_mov_b32 v3, s1 - v_add_u32 v3, vcc_lo, v3 - // Load 64bit local buffer address stored at v[2:3] to v[6:7] - flat_load_dwordx2 v[6:7], v[2:3] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - v_mov_b32 v8, 0x5678 - s_movk_i32 s8, 0x5678 - L_REPEAT: - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - s_cmp_eq_i32 s16, s8 - s_cbranch_scc1 L_QUIT // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] - // every 4k page only read once - v_mov_b32 v9, 0 - v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size - v_mov_b32 v12, v6 - v_mov_b32 v13, v7 - L_LOOP_READ: - flat_load_dwordx2 v[14:15], v[12:13] slc - v_add_u32 v9, v9, v10 - v_add_co_u32 v12, vcc, v12, v10 - v_add_u32 v13, vcc_lo, v13 - v_cmp_lt_u32 vcc, v9, v11 - s_cbranch_vccnz L_LOOP_READ - s_branch L_REPEAT - L_QUIT: - flat_store_dword v[4:5], v8 - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish - s_endpgm -)"; - -static const char* gfx8_ReadMemory = R"( - .text - // Compute address of corresponding output buffer - v_mov_b32 v0, s4 // use workgroup id as index - v_lshlrev_b32 v0, 2, v0 // v0 *= 4 - v_add_u32 v4, vcc, s2, v0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v5, s3 - v_addc_u32 v5, vcc, v5, 0, vcc - // Compute input buffer offset used to store corresponding local buffer address - v_lshlrev_b32 v0, 1, v0 // v0 *= 8 - v_add_u32 v2, vcc, s0, v0 // v[2:3] = s[0:1] + v0 * 8 - v_mov_b32 v3, s1 - v_addc_u32 v3, vcc, v3, 0, vcc - // Load 64bit local buffer address stored at v[2:3] to v[6:7] - flat_load_dwordx2 v[6:7], v[2:3] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - v_mov_b32 v8, 0x5678 - s_movk_i32 s8, 0x5678 - L_REPEAT: - s_load_dword s16, s[0:1], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish - s_cmp_eq_i32 s16, s8 - s_cbranch_scc1 L_QUIT // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] - // every 4k page only read once - v_mov_b32 v9, 0 - v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size - v_mov_b32 v12, v6 - v_mov_b32 v13, v7 - L_LOOP_READ: - flat_load_dwordx2 v[14:15], v[12:13] slc - v_add_u32 v9, vcc, v9, v10 - v_add_u32 v12, vcc, v12, v10 - v_addc_u32 v13, vcc, v13, 0, vcc - v_cmp_lt_u32 vcc, v9, v11 - s_cbranch_vccnz L_LOOP_READ - s_branch L_REPEAT - L_QUIT: - flat_store_dword v[4:5], v8 - s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory writes to finish - s_endpgm -)"; - void KFDSVMEvictTest::SetUp() { ROUTINE_START @@ -158,13 +55,6 @@ void KFDSVMEvictTest::TearDown() { ROUTINE_END } -std::string KFDSVMEvictTest::CreateShader() { - if (m_FamilyId >= FAMILY_AI) - return gfx9_ReadMemory; - else - return gfx8_ReadMemory; -} - HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize) { HSAuint64 vramBufSizeInPages = vramBufSize >> PAGE_SHIFT; HSAuint64 sysMemSize = GetSysMemSize(); @@ -419,7 +309,7 @@ TEST_F(KFDSVMEvictTest, QueueTest) { for (i = 0; i < wavefront_num; i++) *(localBufAddr + i) = pBuffers[i]; - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CreateShader().c_str(), isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadMemoryIsa, isaBuffer.As())); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); From 549f7cdce20156e4ef78ac5b44dae662e21a4a25 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 30 Sep 2021 18:04:51 -0400 Subject: [PATCH 0984/1247] kfdtest: Remove IsaGen from KFDLocalMemoryTest Replace with LLVM-based Assembler. Signed-off-by: Graham Sider Change-Id: Ie83d27b6a93ac1b5169a830a7f274e360a31023e --- tests/kfdtest/src/KFDLocalMemoryTest.cpp | 13 ++++--------- tests/kfdtest/src/KFDLocalMemoryTest.hpp | 6 +----- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/tests/kfdtest/src/KFDLocalMemoryTest.cpp index a27b502f97..b37528c651 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.cpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.cpp @@ -33,18 +33,12 @@ void KFDLocalMemoryTest::SetUp() { KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDLocalMemoryTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - KFDBaseComponentTest::TearDown(); ROUTINE_END @@ -107,7 +101,7 @@ TEST_F(KFDLocalMemoryTest, BasicTest) { srcSysBuffer.Fill(0x01010101); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(srcLocalBuffer.As(), srcLocalBuffer.Size(), &AlternateVAGPU, mapFlags, 1, reinterpret_cast(&defaultGPUNode))); @@ -164,7 +158,7 @@ TEST_F(KFDLocalMemoryTest, VerifyContentsAfterUnmapAndMap) { SysBufferA.Fill(0x01010101); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.SetSkipWaitConsump(0); @@ -303,7 +297,8 @@ TEST_F(KFDLocalMemoryTest, Fragmentation) { PM4Queue queue; ASSERT_SUCCESS(queue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); /* Allocate and test memory using the strategy explained at the top */ HSAKMT_STATUS status; diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.hpp b/tests/kfdtest/src/KFDLocalMemoryTest.hpp index 519081cfc0..b3a9b2add6 100644 --- a/tests/kfdtest/src/KFDLocalMemoryTest.hpp +++ b/tests/kfdtest/src/KFDLocalMemoryTest.hpp @@ -26,20 +26,16 @@ #include -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" class KFDLocalMemoryTest : public KFDBaseComponentTest { public: - KFDLocalMemoryTest() :m_pIsaGen(NULL) {} + KFDLocalMemoryTest() {} ~KFDLocalMemoryTest() {} protected: virtual void SetUp(); virtual void TearDown(); - - protected: // Members - IsaGenerator* m_pIsaGen; }; #endif // __KFD_LOCALMEMORY_TEST__H__ From 469d5e67d234bcfe98a76c9d2538ce7e1031549a Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 30 Sep 2021 18:16:57 -0400 Subject: [PATCH 0985/1247] kfdtest: Remove IsaGen from KFDSVMRangeTest Replace with LLVM-based Assembler. Signed-off-by: Graham Sider Change-Id: Id05f8a18a188d0ad354b711c1c196b71dffcc756 --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 20 +++++++++----------- tests/kfdtest/src/KFDSVMRangeTest.hpp | 6 +----- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 6aad683f91..283a567ff4 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -34,8 +34,6 @@ void KFDSVMRangeTest::SetUp() { KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - SVMSetXNACKMode(); ROUTINE_END @@ -44,10 +42,6 @@ void KFDSVMRangeTest::SetUp() { void KFDSVMRangeTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - SVMRestoreXNACKMode(); KFDBaseComponentTest::TearDown(); @@ -80,7 +74,7 @@ TEST_F(KFDSVMRangeTest, BasicSystemMemTest) { srcSysBuffer.Fill(0x01010101); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.SetSkipWaitConsump(0); @@ -364,7 +358,8 @@ TEST_F(KFDSVMRangeTest, EvictSystemRangeTest) { ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); Dispatch dispatch0(isaBuffer); dispatch0.SetArgs(srcBuffer.As(), dstBuffer.As()); @@ -458,7 +453,8 @@ TEST_F(KFDSVMRangeTest, PartialUnmapSysMemTest) { munmap(pBuf2, Buf2Size); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); Dispatch dispatch(isaBuffer); @@ -507,7 +503,7 @@ TEST_F(KFDSVMRangeTest, BasicVramTest) { srcSysBuffer.Fill(0x01010101); - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); ASSERT_SUCCESS(queue.Create(defaultGPUNode)); queue.SetSkipWaitConsump(0); @@ -943,7 +939,9 @@ TEST_F(KFDSVMRangeTest, MigratePolicyTest) { #ifdef USE_PM4_QUEUE_TRIGGER_VM_FAULT HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode); PM4Queue queue; - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); for (HSAuint64 i = 0; i < BufferSize / 8; i += 512) { diff --git a/tests/kfdtest/src/KFDSVMRangeTest.hpp b/tests/kfdtest/src/KFDSVMRangeTest.hpp index 88bddd94a8..03a245dba3 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.hpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.hpp @@ -26,21 +26,17 @@ #include -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" class KFDSVMRangeTest : public KFDBaseComponentTest { public: - KFDSVMRangeTest() :m_pIsaGen(NULL) {} + KFDSVMRangeTest() {} ~KFDSVMRangeTest() {} void SplitRangeTest(int defaultGPUNode, int prefetch_location); protected: virtual void SetUp(); virtual void TearDown(); - - protected: // Members - IsaGenerator* m_pIsaGen; }; #endif // __KFD_LOCALMEMORY_TEST__H__ From 8a6743aef4a5b4fe15826ce093811c23cd719787 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 30 Sep 2021 18:17:26 -0400 Subject: [PATCH 0986/1247] kfdtest: Remove IsaGen from RDMATest Replace with LLVM-based Assembler. Signed-off-by: Graham Sider Change-Id: I5dff1b9402e294af33cec78a24e2e2decfb5b6d3 --- tests/kfdtest/src/RDMATest.cpp | 8 ++------ tests/kfdtest/src/RDMATest.hpp | 6 +----- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/kfdtest/src/RDMATest.cpp b/tests/kfdtest/src/RDMATest.cpp index 973dbdcdc8..d44d69895e 100644 --- a/tests/kfdtest/src/RDMATest.cpp +++ b/tests/kfdtest/src/RDMATest.cpp @@ -34,16 +34,11 @@ void RDMATest::SetUp() { KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void RDMATest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; KFDBaseComponentTest::TearDown(); @@ -77,7 +72,8 @@ TEST_F(RDMATest, GPUDirect) { srcSysBuffer.Fill(0xfe); /* Put 'copy dword' command to ISA buffer */ - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); Dispatch dispatch(isaBuffer); diff --git a/tests/kfdtest/src/RDMATest.hpp b/tests/kfdtest/src/RDMATest.hpp index 3e4b2331aa..1d16853838 100644 --- a/tests/kfdtest/src/RDMATest.hpp +++ b/tests/kfdtest/src/RDMATest.hpp @@ -26,20 +26,16 @@ #include -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" class RDMATest : public KFDBaseComponentTest { public: - RDMATest():m_pIsaGen(NULL) {} + RDMATest() {} ~RDMATest() {} protected: virtual void SetUp(); virtual void TearDown(); - - protected: // Members - IsaGenerator* m_pIsaGen; }; #endif // __RDMA_TEST__H__ From 780f0b618c9751e21308723ac6860ded4d83a3c3 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 30 Sep 2021 18:32:48 -0400 Subject: [PATCH 0987/1247] kfdtest: Remove IsaGen from KFDHWSTest Replace with LLVM-based Assembler. Signed-off-by: Graham Sider Change-Id: Ibbc4103d9498321b87feadf14a523b0d44d1851c --- tests/kfdtest/src/KFDHWSTest.cpp | 10 +++------- tests/kfdtest/src/KFDHWSTest.hpp | 7 +------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/tests/kfdtest/src/KFDHWSTest.cpp b/tests/kfdtest/src/KFDHWSTest.cpp index 66c0b5d8ff..3040b16ac6 100644 --- a/tests/kfdtest/src/KFDHWSTest.cpp +++ b/tests/kfdtest/src/KFDHWSTest.cpp @@ -28,18 +28,12 @@ void KFDHWSTest::SetUp() { KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDHWSTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - KFDBaseComponentTest::TearDown(); ROUTINE_END @@ -70,7 +64,9 @@ void KFDHWSTest::RunTest(unsigned nProcesses, unsigned nQueues, unsigned nLoops) // Run work on all queues HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - m_pIsaGen->GetNoopIsa(isaBuffer); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(NoopIsa, isaBuffer.As())); + for (l = 0; l < nLoops; l++) { for (q = 0; q < nQueues; q++) { if (dispatch[q]) diff --git a/tests/kfdtest/src/KFDHWSTest.hpp b/tests/kfdtest/src/KFDHWSTest.hpp index e3ea5155c2..cbec52fbed 100644 --- a/tests/kfdtest/src/KFDHWSTest.hpp +++ b/tests/kfdtest/src/KFDHWSTest.hpp @@ -27,14 +27,12 @@ #include #include "PM4Queue.hpp" -#include "IsaGenerator.hpp" #include "KFDMultiProcessTest.hpp" #include "Dispatch.hpp" class KFDHWSTest : public KFDMultiProcessTest { public: - KFDHWSTest():m_pIsaGen(NULL) {} - + KFDHWSTest() {} ~KFDHWSTest() {} protected: @@ -42,9 +40,6 @@ class KFDHWSTest : public KFDMultiProcessTest { virtual void TearDown(); void RunTest(unsigned nProcesses, unsigned nQueues, unsigned nLoops); - - protected: // Members - IsaGenerator* m_pIsaGen; }; #endif // __KFD_QCM_TEST__H__ From 4c7cf6e7d22f4439022e44756c6d5c0029179211 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 30 Sep 2021 18:33:06 -0400 Subject: [PATCH 0988/1247] kfdtest: Remove IsaGen from KFDExceptionTest Replace with LLVM-based Assembler. Signed-off-by: Graham Sider Change-Id: Ia64b8cc77382773b48de576d350bbed3c1efdb74 --- tests/kfdtest/src/KFDExceptionTest.cpp | 9 ++------- tests/kfdtest/src/KFDExceptionTest.hpp | 5 +---- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 11df6279c9..9fa15e0969 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -33,18 +33,12 @@ void KFDExceptionTest::SetUp() { KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDExceptionTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; - KFDBaseComponentTest::TearDown(); // WORKAROUND: This needs to be fixed in the kernel @@ -75,7 +69,8 @@ void KFDExceptionTest::TestMemoryException(int defaultGPUNode, HSAuint64 pSrc, eventDesc.SyncVar.SyncVar.UserData = NULL; eventDesc.SyncVar.SyncVarSize = 0; - m_pIsaGen->GetCopyDwordIsa(isaBuffer); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); + m_ChildStatus = queue.Create(defaultGPUNode); if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { WARN() << "Queue create failed" << std::endl; diff --git a/tests/kfdtest/src/KFDExceptionTest.hpp b/tests/kfdtest/src/KFDExceptionTest.hpp index 00b45fe5db..df57649e2c 100644 --- a/tests/kfdtest/src/KFDExceptionTest.hpp +++ b/tests/kfdtest/src/KFDExceptionTest.hpp @@ -26,12 +26,11 @@ #include -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" class KFDExceptionTest : public KFDBaseComponentTest { public: - KFDExceptionTest() :m_pIsaGen(NULL), m_ChildPid(-1) { + KFDExceptionTest() : m_ChildPid(-1) { /* Because there could be early return before m_ChildPid is set * by fork(), we should initialize m_ChildPid to a non-zero value * to avoid possible exit of the main process. @@ -59,8 +58,6 @@ class KFDExceptionTest : public KFDBaseComponentTest { protected: // Members pid_t m_ChildPid; HSAKMT_STATUS m_ChildStatus; - - IsaGenerator* m_pIsaGen; }; #endif // __KFD_EXCEPTION_TEST__H__ From ac48163885744426ebd01372ca3dd6ebf23f7f6e Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 30 Sep 2021 18:33:23 -0400 Subject: [PATCH 0989/1247] kfdtest: Remove IsaGen from KFDDBGTest KFDDBGTest is deprecated, so just removing references to IsaGen. Signed-off-by: Graham Sider Change-Id: I9f094d847a8ae43cb3793253b34a7d7ed2179ac1 --- tests/kfdtest/src/KFDDBGTest.cpp | 5 ----- tests/kfdtest/src/KFDDBGTest.hpp | 6 +----- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index b7a38bbd46..f256d8a135 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -176,16 +176,11 @@ void KFDDBGTest::SetUp() { KFDBaseComponentTest::SetUp(); - m_pIsaGen = IsaGenerator::Create(m_FamilyId); - ROUTINE_END } void KFDDBGTest::TearDown() { ROUTINE_START - if (m_pIsaGen) - delete m_pIsaGen; - m_pIsaGen = NULL; /* Reset the user trap handler */ hsaKmtSetTrapHandler(m_NodeInfo.HsaDefaultGPUNode(), 0, 0, 0, 0); diff --git a/tests/kfdtest/src/KFDDBGTest.hpp b/tests/kfdtest/src/KFDDBGTest.hpp index c4b46b296b..dccedc3fd3 100644 --- a/tests/kfdtest/src/KFDDBGTest.hpp +++ b/tests/kfdtest/src/KFDDBGTest.hpp @@ -26,20 +26,16 @@ #include -#include "IsaGenerator.hpp" #include "KFDBaseComponentTest.hpp" class KFDDBGTest : public KFDBaseComponentTest { public: - KFDDBGTest() :m_pIsaGen(NULL) {} + KFDDBGTest() {} ~KFDDBGTest() {} protected: virtual void SetUp(); virtual void TearDown(); - - protected: // Members - IsaGenerator* m_pIsaGen; }; #endif // __KFD_DBG_TEST__H__ From ffaa3d924695a622847cb07b039b109f8dbb8155 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 8 Oct 2021 14:52:45 -0400 Subject: [PATCH 0990/1247] kfdtest: Remove IsaGen from KFDIPCTest Signed-off-by: Graham Sider Change-Id: I3194e6a6bdab846da9cf346f47a7d7580e2def4d --- tests/kfdtest/src/KFDIPCTest.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kfdtest/src/KFDIPCTest.hpp b/tests/kfdtest/src/KFDIPCTest.hpp index 961ecbd9e9..3ce0aa12d8 100644 --- a/tests/kfdtest/src/KFDIPCTest.hpp +++ b/tests/kfdtest/src/KFDIPCTest.hpp @@ -23,7 +23,6 @@ #include "KFDBaseComponentTest.hpp" #include "BaseQueue.hpp" -#include "IsaGenerator.hpp" #ifndef __KFD_MEMORY_TEST__H__ #define __KFD_MEMORY_TEST__H__ From e6e498abf3f19a03c344955bbf4b7f5c331c6606 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 8 Oct 2021 14:50:04 -0400 Subject: [PATCH 0991/1247] kfdtest: Remove SP3/IsaGen - Remove SP3 lib directory - Remove IsaGenerator* files - Update CMakeLists accordingly Signed-off-by: Graham Sider Change-Id: I280161b0b238839ab318c18d6038cdd64fa66208 --- tests/kfdtest/CMakeLists.txt | 12 +- tests/kfdtest/sp3/README.txt | 6 - .../sp3/lib_helper/AMD_opensource_license.txt | 23 - .../kfdtest/sp3/lib_helper/CMakeLists_sp3.txt | 79 --- tests/kfdtest/sp3/lib_helper/build_sp3.sh | 57 -- tests/kfdtest/sp3/sp3.h | 643 ------------------ tests/kfdtest/src/IsaGenerator.cpp | 126 ---- tests/kfdtest/src/IsaGenerator.hpp | 52 -- tests/kfdtest/src/IsaGenerator_Aldebaran.cpp | 113 --- tests/kfdtest/src/IsaGenerator_Aldebaran.hpp | 49 -- tests/kfdtest/src/IsaGenerator_Gfx10.cpp | 142 ---- tests/kfdtest/src/IsaGenerator_Gfx10.hpp | 49 -- tests/kfdtest/src/IsaGenerator_Gfx72.cpp | 123 ---- tests/kfdtest/src/IsaGenerator_Gfx72.hpp | 49 -- tests/kfdtest/src/IsaGenerator_Gfx8.cpp | 128 ---- tests/kfdtest/src/IsaGenerator_Gfx8.hpp | 49 -- tests/kfdtest/src/IsaGenerator_Gfx9.cpp | 113 --- tests/kfdtest/src/IsaGenerator_Gfx9.hpp | 49 -- 18 files changed, 1 insertion(+), 1861 deletions(-) delete mode 100644 tests/kfdtest/sp3/README.txt delete mode 100644 tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt delete mode 100644 tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt delete mode 100755 tests/kfdtest/sp3/lib_helper/build_sp3.sh delete mode 100644 tests/kfdtest/sp3/sp3.h delete mode 100644 tests/kfdtest/src/IsaGenerator.cpp delete mode 100644 tests/kfdtest/src/IsaGenerator.hpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Aldebaran.cpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Aldebaran.hpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx10.cpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx10.hpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx72.cpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx72.hpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx8.cpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx8.hpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx9.cpp delete mode 100644 tests/kfdtest/src/IsaGenerator_Gfx9.hpp diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index e175253dad..b1208f54c2 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -128,12 +128,9 @@ add_definitions(${LLVM_DEFINITIONS_LIST}) llvm_map_components_to_libnames(llvm_libs AMDGPUAsmParser Core Support) -set ( SP3_DIR ${PROJECT_SOURCE_DIR}/sp3 ) - include_directories(${PROJECT_SOURCE_DIR}/gtest-1.6.0) include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(${PROJECT_SOURCE_DIR}/../../include) -include_directories(${SP3_DIR}) include_directories(${DRM_INCLUDE_DIRS}) @@ -147,12 +144,6 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/IndirectBuffer.cpp src/Assemble.cpp src/ShaderStore.cpp - src/IsaGenerator.cpp - src/IsaGenerator_Aldebaran.cpp - src/IsaGenerator_Gfx10.cpp - src/IsaGenerator_Gfx72.cpp - src/IsaGenerator_Gfx8.cpp - src/IsaGenerator_Gfx9.cpp src/LinuxOSWrapper.cpp src/PM4Packet.cpp src/PM4Queue.cpp @@ -216,11 +207,10 @@ endif () # The modules found by pkg_check_modules() in the default pkg config # path do not need to use link_directories() here. link_directories(${HSAKMT_LIBRARY_DIRS}) -link_directories(${SP3_DIR}) add_executable(kfdtest ${SRC_FILES}) -target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} ${llvm_libs} pthread m stdc++ rt amdsp3 numa) +target_link_libraries(kfdtest ${HSAKMT_LIBRARIES} ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} ${llvm_libs} pthread m stdc++ rt numa) configure_file ( scripts/kfdtest.exclude kfdtest.exclude COPYONLY ) configure_file ( scripts/run_kfdtest.sh run_kfdtest.sh COPYONLY ) diff --git a/tests/kfdtest/sp3/README.txt b/tests/kfdtest/sp3/README.txt deleted file mode 100644 index 7cbe800f50..0000000000 --- a/tests/kfdtest/sp3/README.txt +++ /dev/null @@ -1,6 +0,0 @@ -Note: This folder is primarily intended for AMD internal developers. - -The folder lib_helper contains the script to generate SP3 library libamdsp3.a -and the associated header files in the current folder for kfdtest to use. -cmake is required for the script to run. Just run ./build_sp3.sh after setting -up the environment variables (source build/envsetup.sh). diff --git a/tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt b/tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt deleted file mode 100644 index 673285ddb4..0000000000 --- a/tests/kfdtest/sp3/lib_helper/AMD_opensource_license.txt +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - diff --git a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt b/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt deleted file mode 100644 index ce8a3cb33f..0000000000 --- a/tests/kfdtest/sp3/lib_helper/CMakeLists_sp3.txt +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# - -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) - -project(amdsp3) - -#set ( CMAKE_VERBOSE_MAKEFILE on ) - -find_package(PkgConfig) - -set ( P4_PATH $ENV{WORK_ROOT}/p4/driver/drivers ) - -set ( SCLIB_SRC ${PROJECT_SOURCE_DIR} ) -#if( DEFINED ENV{SCLIB_SRC} ) -# set ( SCLIB_SRC $ENV{SCLIB_SRC} ) -#else() -# set ( SCLIB_SRC ${P4_PATH}/sc/Chip ) -#endif() - -include_directories(${SCLIB_SRC}/sp3) -#include_directories(${SCLIB_SRC}/sp3/release_headers) -include_directories(${SCLIB_SRC}/sp3/gen) - -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-asic.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-dispatch.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-eval.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-gc.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-int.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-lib.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-native.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-cipher.c ) -set ( SRC_FILES ${SRC_FILES} ${SCLIB_SRC}/sp3/sp3-vm.c ) - -aux_source_directory(${SCLIB_SRC}/sp3/gen SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/si/lib SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/ci/lib SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx8/lib SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx81/lib SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx9/lib SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx10/lib SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/aldbrn/lib SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx81/arch SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx9/arch SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/gfx10/arch SRC_FILES) -aux_source_directory(${SCLIB_SRC}/sp3/backend/aldbrn/arch SRC_FILES) - - -message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) -#message( STATUS "SRC_FILES: ") -#foreach(file ${SRC_FILES}) -# message(STATUS "${file}") -#endforeach() - -set ( CMAKE_C_FLAGS "-DSP3_STATIC_LIB -Wno-error -DPUBLIC_RELEASE -DLITTLEENDIAN_CPU -fPIC -DGFX101_BUILD -DALDBRN_BUILD" ) - -add_library(amdsp3 ${SRC_FILES}) - - diff --git a/tests/kfdtest/sp3/lib_helper/build_sp3.sh b/tests/kfdtest/sp3/lib_helper/build_sp3.sh deleted file mode 100755 index f93f145da6..0000000000 --- a/tests/kfdtest/sp3/lib_helper/build_sp3.sh +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (C) 2018 Advanced Micro Devices, Inc. All Rights Reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# - -#!/bin/bash - -if [ "$KFDTEST_ROOT" == "" ] || [ "$P4_ROOT" == "" ]; then - echo "Environment variables should be set before running this script" - exit 1 -fi - -cd $KFDTEST_ROOT/sp3/lib_helper - -SP3_PROJECT=$P4_ROOT/driver/drivers/sc/Chip/ -LIB_OUTPUT=$KFDTEST_ROOT/sp3/ - -cp CMakeLists_sp3.txt $SP3_PROJECT/CMakeLists.txt - -mkdir -p build -echo "Building SP3 lib" -pushd build -cmake $SP3_PROJECT/ -make -popd - -rsync --progress -a build/libamdsp3.a $LIB_OUTPUT -# Put the intermediate header files in the current folder for further processing -rsync --progress -a $SP3_PROJECT/sp3/public/lib/sp3.h . - -# Remove the build folder and CMakeLists.txt put into SP source folder -rm -r build -rm $SP3_PROJECT/CMakeLists.txt - -# Replace the license statement in the header files -{ cat AMD_opensource_license.txt; sed -e '1,/#ifndef/ { /#ifndef/b; d }' sp3.h; } > $LIB_OUTPUT/sp3.h - -# Delete the intermediate header files -rm sp3.h diff --git a/tests/kfdtest/sp3/sp3.h b/tests/kfdtest/sp3/sp3.h deleted file mode 100644 index 513167d595..0000000000 --- a/tests/kfdtest/sp3/sp3.h +++ /dev/null @@ -1,643 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef __SP3_H__ -#define __SP3_H__ - -#ifdef __cplusplus -extern "C" { -#endif - - -/// @file sp3.h -/// @brief sp3 API -#include - -// Export tags -#define SP3_EXPORT - - -/// @defgroup sp3main SP3 Main API -/// -/// Main API to assemble and disassemble SP3 shaders. -/// -/// @{ - - -/// Valid shader stages. -enum sp3_shtype { - SP3_SHTYPE_NONE = -1, - SP3_SHTYPE_PS = 0, - SP3_SHTYPE_VS = 1, - SP3_SHTYPE_GS = 2, - SP3_SHTYPE_ES = 3, - SP3_SHTYPE_HS = 4, - SP3_SHTYPE_LS = 5, - SP3_SHTYPE_CS = 6, -#ifdef NAVI10LITE_BUILD - SP3_SHTYPE_ACV = 7, -#endif -}; - -/// Assorted constants used by sp3 API. -enum sp3_count { - SP3_NUM_MRT = 8, ///< Maximum number of render targets supported. - SP3_NUM_STRM = 4, ///< Maximum number of streams supported. -}; - -/// Disassembly flags. Bitwise-OR flags to set options. -enum sp3_flag { - SP3DIS_NO_STATE = 0x01, ///< Do not include state header at top of shader. - SP3DIS_NO_BINARY = 0x02, ///< Do not include comments with raw binary microcode. - SP3DIS_COMMENTS = 0x04, ///< Do not include comments. - SP3DIS_NO_GPR_COUNT = 0x08, ///< Do not include GPR allocation counts. - SP3DIS_FORCEVALID = 0x10, ///< Force all bytes of microcode to be disassembled. - SP3DIS_NO_ASIC = 0x20, ///< Do not emit the asic header at top of shader. -}; - -/// Shader context. Contains no user-visible fields. -struct sp3_context; - -/// Memory object. Contains no user-visible fields. -struct sp3_vma; - -/// VM addresses are 64-bit and the address unit is 32 bits -typedef uint64_t sp3_vmaddr; - -/// Storage entry for register streams. -struct sp3_reg { - uint32_t index; ///< One of the MM aperture register addresses. - uint32_t value; ///< 32-bit register data. -}; - -/// Bits for a single instruction. -struct sp3_inst_bits { - uint32_t val[5]; ///< Largest single instruction in any backend is 5 dwords. -}; - -/// Wrapped shader metadata. -/// -/// After generation, shaders are encapsulated in sp3_shader structures. -/// -/// Those structures contain the shader binary, its register stream, constants and constant -/// buffers and metadata needed for SC compatibility. -/// -struct sp3_shader { - enum sp3_shtype type; ///< One of the SHTYPE_* constants. - uint32_t asic_int; ///< Internal ASIC index. Do not use. - char asic[0x100]; ///< ASIC name as a string ("RV870" etc). - uint32_t size; ///< Size of the compiled shader, in 32-bit words. - uint32_t nsgprs; ///< Number of scalar GPRs used. - uint32_t nvgprs; ///< Number of vector GPRs used. - uint32_t nsvgprs; ///< Number of shared vector GPRs used (only available in certain projects). - uint32_t naccvgprs; ///< Number of accumulator vector GPRs used (only available in certain projects). - uint32_t nsgprs_manual_alloc; - uint32_t nvgprs_manual_alloc; - uint32_t nsvgprs_manual_alloc; - uint32_t naccvgprs_manual_alloc; - uint32_t trap_present; - uint32_t user_sgpr_count; - uint32_t scratch_en; - uint32_t dispatch_draw_en; - uint32_t so_en; - uint32_t so_base0_en; - uint32_t so_base1_en; - uint32_t so_base2_en; - uint32_t so_base3_en; - uint32_t oc_lds_en; - uint32_t tg_size_en; - uint32_t tidig_comp_cnt; ///< Number of components(-1) enabled for thread id in group - uint32_t tgid_x_en; - uint32_t tgid_y_en; - uint32_t tgid_z_en; - uint32_t wave_cnt_en; - uint32_t primgen_en; - uint32_t pc_base_en; - uint32_t sgpr_scratch; - uint32_t sgpr_psvs_state; - uint32_t sgpr_gs2vs_offset; - uint32_t sgpr_so_write_index; - uint32_t sgpr_so_base_offset0; - uint32_t sgpr_so_base_offset1; - uint32_t sgpr_so_base_offset2; - uint32_t sgpr_so_base_offset3; - uint32_t sgpr_offchip_lds; - uint32_t sgpr_is_offchip; - uint32_t sgpr_ring_offset; - uint32_t sgpr_gs_wave_id; - uint32_t sgpr_global_wave_id; - uint32_t sgpr_tg_size; - uint32_t sgpr_tgid_x; - uint32_t sgpr_tgid_y; - uint32_t sgpr_tgid_z; - uint32_t sgpr_tf_base; - uint32_t sgpr_pc_base; - uint32_t sgpr_wave_cnt; - uint32_t wave_size; ///< Number of threads in a wavefront (only certain ASICs; 0 = don't care). - uint32_t pc_exports; ///< Range of parameters exported (if VS). - uint32_t pos_export; ///< Shader executes a position export (if VS). - uint32_t cb_exports; ///< Range of MRTs exported (if PS). - uint32_t mrtz_export_format;///< Export format of the mrtz export. - uint32_t z_export; ///< Shader executes a Z export (if PS). - uint32_t pops_en; ///< Shader is POPS (PS) - uint32_t pops_num_samples; ///< (PS) - uint32_t load_collision_waveid; ///< Shader sets load collision waveid (if PS). - uint32_t load_intrawave_collision; ///< Shader is in intrawave mode (if PS). - uint32_t stencil_test_export; ///< Shader exports stencil (if PS). - uint32_t stencil_op_export; ///< Shader exports stencil (if PS). - uint32_t kill_used; ///< Shader executes ALU KILL operations. - uint32_t cb_masks[SP3_NUM_MRT]; ///< Component masks for each MRT exported (if PS). - uint32_t emit_used; ///< EMIT opcodes used (if GS). - uint32_t covmask_export; ///< Shader exports coverage mask (if PS). - uint32_t mask_export; ///< Shader exports mask (if PS). - uint32_t strm_used[SP3_NUM_STRM]; ///< Streamout operations used (map). - uint32_t scratch_used; ///< Scratch SMX exports used. - uint32_t scratch_itemsize; ///< Scratch ring item size. - uint32_t reduction_used; ///< Reduction SMX exports used. - uint32_t ring_used; ///< ESGS/GSVS ring SMX exports used. - uint32_t ring_itemsize; ///< ESGS/GSVS ring item size (for ES/GS respectively). - uint32_t vertex_size[4]; ///< GSVS ring vertex size (for GS). - uint32_t mem_used; ///< Raw memory SMX exports used. - uint32_t rats_used; ///< Mask of RATs (UAVs) used - uint32_t group_size[3]; ///< Wavefront group size (for ELF files). - uint32_t alloc_lds; ///< Number of LDS bytes allocated for wave group. (translates to lds_size in CS and LS) - uint32_t *data; ///< Shader binary data. - uint32_t nregs; ///< Number of register writes in the stream. - uint64_t crc64; ///< CRC64 of compiled shader, may be used for identification/fingerprinting. - uint32_t crc32; ///< 32-bit CRC of compiled shader (based on crc64), may be used for identification/fingerprinting. - struct sp3_reg *regs; ///< Register writes (index-value pairs). - struct sp3_shader *merged_2nd_shader; ///< Merged es/gs, ls/hs shader, this points to start of the second shader (only certain ASICs). -}; - -/// Comment callback. -typedef const char *(*sp3_comment_cb)(void *, int); - - -/// Get version of the sp3 library. -/// -/// @return String containing the version number. -/// -SP3_EXPORT const char *sp3_version(void); - -/// Create a new sp3 context. -/// -/// @return A new context for use in assembling and disassembling shaders. Free with sp3_close(). -/// -SP3_EXPORT struct sp3_context *sp3_new(void); - -/// Set option for sp3. -/// -/// @param state sp3 context. -/// @param option Option name. Unknown options will raise an error. -/// @param value Option value. NULL is used to represent value-less options. -/// -/// Currently supported options: -/// -/// stdlib (string) -- absolute path to standard library files. May be a colon-separated list -/// of paths that will be used to search for stdlib files. Used by sp3_parse_library(). -/// -/// The following options are deprecated because they take integer arguments; you should use -/// sp3_set_option_int() for these settings going forward. They will continue to be accepted by -/// this API to support legacy users. -/// -/// Werror (boolean) -- indicates whether warnings should be treated as errors. -/// -/// wave_size (integer) -- sets the wave size being used by the draw calls that will be using -/// this shader. Ignored in certain ASICs. You may set this to 32, 64 or the special value 0 -/// to indicate no preference on wave size. The shader will be checked to ensure it is -/// compatible with the size specified here. -/// -/// omit_version (boolean) -- omit generation of the S_VERSION opcode. -/// -/// omit_code_end (boolean) -- omit generation of the S_CODE_END footer. -/// -/// allow_raw_bits (boolean) -- allow use of the raw_bits() function in sp3 shaders. This is a -/// dangerous option to allow in general so you must explicitly enable this option, otherwise -/// the raw_bits() function will always error out. -/// -SP3_EXPORT void sp3_set_option( - struct sp3_context *state, - const char *option, - const char *value); - -/// Set option for sp3. -/// -/// @param state sp3 context. -/// @param option Option name. Unknown options will raise an error. -/// @param value Option value. -/// -/// Currently supported options: -/// -/// Werror (boolean) -- indicates whether warnings should be treated as errors. -/// -/// wave_size (integer) -- sets the wave size being used by the draw calls that will be using -/// this shader. Ignored in certain ASICs. You may set this to 32, 64 or the special value 0 -/// to indicate no preference on wave size. The shader will be checked to ensure it is -/// compatible with the size specified here. -/// -/// omit_version (boolean) -- omit generation of the S_VERSION opcode. -/// -/// omit_code_end (boolean) -- omit generation of the S_CODE_END footer. -/// -/// allow_raw_bits (boolean) -- allow use of the raw_bits() function in sp3 shaders. This is a -/// dangerous option to allow in general so you must explicitly enable this option, otherwise -/// the raw_bits() function will always error out. -/// -/// secure_mode (boolean) -- run in secure mode. Disables macro language features in assembly -/// path including calls to custom functions. Useful if sp3 is used as a backend to a web-based -/// assembly tool. -/// -/// debug_encoding (boolean) -- if true, debug encoding selection logic for assembly. Only -/// supported in 10.4+ backends. -/// -/// no_vs_export_check (boolean) -- if true, disable VS export sanity check. Only supported in -/// 10.4+ backends. -/// -SP3_EXPORT void sp3_set_option_int( - struct sp3_context *state, - const char *option, - int32_t value); - -/// Parse a file into a context. -/// -/// Use sp3_compile to generate binary microcode after the shader is parsed. -/// -/// @param state Context to use for parsing. -/// @param file File to read. If NULL, parse from stdin. -/// -SP3_EXPORT void sp3_parse_file(struct sp3_context *state, const char *file); - -/// Parse a string into a context. -/// -/// Use sp3_compile to generate binary microcode after the shader is parsed. -/// -/// @param state Context to use for parsing. -/// @param string String to parse. -/// -SP3_EXPORT void sp3_parse_string(struct sp3_context *state, const char *string); - -/// Parse a file from the standard library into a context. -/// -/// Use sp3_compile to generate binary microcode after the shader is parsed. -/// -/// @param state Context to use for parsing. -/// @param name Path to the standard library; files in this directory are parsed. -/// -SP3_EXPORT void sp3_parse_library(struct sp3_context *state, const char *name); - -/// Call a sp3 function. -/// -SP3_EXPORT void sp3_call(struct sp3_context *state, const char *func); - -/// Compile a shader program that has been parsed into the context. -/// -/// @param state sp3 context. -/// @param cffunc Name of clause to call. By convention, this is "main". -/// @return A compiled and linked shader. Free memory with sp3_free_shader(). -/// -SP3_EXPORT struct sp3_shader *sp3_compile( - struct sp3_context *state, - const char *cffunc); - -/// Free a sp3_shader. -/// -/// @param sh Shader object to delete. -/// -SP3_EXPORT void sp3_free_shader(struct sp3_shader *sh); - -/// Get current ASIC name set for a context. -/// -/// @param state Context to query. -/// @return Name of ASIC. -/// -SP3_EXPORT const char *sp3_getasic(struct sp3_context *state); - -/// Set current ASIC name for a context. -/// -/// @param state Context to modify. -/// @param chip Case-insensitive string representing the ASIC to compile or disassemble for. -/// -SP3_EXPORT void sp3_setasic(struct sp3_context *state, const char *chip); - -/// Set global variable in context to an integer. -/// -SP3_EXPORT void sp3_set_param_int( - struct sp3_context *state, - const char *name, - int32_t value); - -/// Set global variable in context to an integer vector. -/// -SP3_EXPORT void sp3_set_param_intvec( - struct sp3_context *state, - const char *name, - uint32_t size, - const int32_t *value); - -/// Set global variable in context to a float. -/// -SP3_EXPORT void sp3_set_param_float( - struct sp3_context *state, - const char *name, - float value); - -/// Set global variable in context to a float vector. -/// -SP3_EXPORT void sp3_set_param_floatvec( - struct sp3_context *state, - const char *name, - uint32_t size, - const float *value); - -/// Set error message header. -/// -/// @param state Context to modify. -/// @param str Text to include in error message header. -/// -SP3_EXPORT void sp3_set_error_header(struct sp3_context *state, const char *str); - -/// Get ASIC metrics for the ASIC in current state. -/// -/// Used by ELF tools to fill in some CAL fields. -/// -/// @param state Context to query. -/// @param name Name of ASIC metric. -/// @return Value of ASIC metric. -/// -SP3_EXPORT int sp3_asicinfo(struct sp3_context *state, const char *name); - -/// Free a context allocated by sp3_new/open/parse. -/// -/// @param state Context to delete. -/// -SP3_EXPORT void sp3_close(struct sp3_context *state); - -/// Disassemble a shader. -/// -/// This call is likely to change to something that will take a filled sp3_shader structure -/// later on. -/// -/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). -/// @param bin Memory map with the opcodes (see sp3-vm.h). -/// @param base Start of the shader in the memory map (in VM entries, i.e. 32-bit words). -/// @param name Same to give the disassembled shader. -/// @param shader_type One of the SHTYPE_* constants. -/// @param include Literal text to include in the CF clause (NULL includes nothing). -/// @param max_len Maximum length of CF clause. Matters if SP3DIS_FORCEVALID is set. -/// @param flags A bitmask of SP3DIS_* flags. -/// -/// @return Shader disassembly as a string. Free memory with sp3_free(). -/// -SP3_EXPORT char *sp3_disasm( - struct sp3_context *state, - struct sp3_vma *bin, - sp3_vmaddr base, - const char *name, - enum sp3_shtype shader_type, - const char *include, - uint32_t max_len, - uint32_t flags); - -/// Disassemble a single shader instruction. -/// -/// This call is likely to change to something that will take a filled sp3_shader structure -/// later on. -/// -/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). -/// @param inst Pointer to dwords containing instruction (exact number of dwords required depends on instruction). -/// @param base Start of the shader in the memory map (in VM entries, i.e. 32-bit words). -/// @param addr Address of the instruction being disassembled (in VM entries, i.e. 32-bit words). -/// @param shader_type One of the SHTYPE_* constants. -/// @param flags A mask of SP3DIS_* flags. -/// -/// @return Shader disassembly as a string. Free memory with sp3_free(). -/// -SP3_EXPORT char *sp3_disasm_inst( - struct sp3_context *state, - const struct sp3_inst_bits *inst, - sp3_vmaddr base, - sp3_vmaddr addr, - enum sp3_shtype shader_type, - uint32_t flags); - -/// Parse a register stream. -/// -/// Can be called before sp3_disasm to preset things like ALU, boolean and loop constants. -/// -/// This call is likely to merge with sp3_disasm later on. -/// -/// @param state sp3 context to fill with state. -/// @param nregs Number of register entries. -/// @param regs Register stream to parse. -/// @param shader_type One of the SHTYPE_* constants. -/// -SP3_EXPORT void sp3_setregs( - struct sp3_context *state, - uint32_t nregs, - const struct sp3_reg *regs, - enum sp3_shtype shader_type); - - -/// Set shader comments -/// -/// @param state sp3 context. -/// @param map Map of comments (0 for no comment, other values will be passed to the callback). -/// @param f_top Callback returning comment to place above the opcode. -/// @param f_right Callback returning comment to place to the right of the opcode. -/// @param ctx Void pointer to pass to comment callbacks. -/// -SP3_EXPORT void sp3_setcomments( - struct sp3_context *state, - struct sp3_vma *map, - sp3_comment_cb f_top, - sp3_comment_cb f_right, - void *ctx); - -/// Set alternate shader entry points -/// -/// Used for disassembly; this marks an additional location in memory -/// (besides the start address) where shader code may be found. Generally -/// required for jump tables and any case where the shader may perform -/// indirect jumps to ensure that disassembly locates all shader -/// instructions. -/// -/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). -/// @param addr Address of the instruction being disassembled (in VM entries, i.e. 32-bit words). -/// -SP3_EXPORT void sp3_setentrypoint( - struct sp3_context *state, - sp3_vmaddr addr); - -/// Clear alternate shader entry points. -/// -/// Clear all entry points previously set with sp3_setentrypoint. -/// -/// @param state sp3 context (use sp3_new to allocate and sp3_setasic to set ASIC). -/// -SP3_EXPORT void sp3_clearentrypoints(struct sp3_context *state); - -/// Free memory allocated by sp3. -/// -/// Windows DLLs that allocate memory have to free it. This function -/// should be used to free the result of sp3_disasm, sp3_compile etc. -/// -SP3_EXPORT void sp3_free(void *ptr); - -/// SP3 API to merge two shaders given file names as input. -/// -SP3_EXPORT struct sp3_shader* sp3_merge_shaders( - struct sp3_context *pointer, - const char *first_file, - const char *second_file); - -/// SP3 API to merge two shaders given shader strings as input. -/// -SP3_EXPORT struct sp3_shader* sp3_merge_shader_strings( - struct sp3_context *pointer, - const char *first_string, - const char *second_string); - - -/// @} - - -/// @defgroup sp3vm SP3 Memory Objects -/// -/// The VM API is used to manage virtual memory maps. Those maps are used for binary storage -/// for disassembly, as they can naturally mirror the GPU's memory map (so no register -/// translation is needed). -/// -/// @{ - -/// Callback function that will fill a VMA on demand -/// -/// The VMA to be filled will be specified through the request address. -/// The callback should fill the VMA using sp3_vm_write calls. -/// -typedef void (* sp3_vmfill)(struct sp3_vma *vm, sp3_vmaddr addr, void *ctx); - -/// Create a new VM that is empty. -/// -/// Free the object with sp3_vm_free(). -/// -/// @return New VM object. -/// -SP3_EXPORT -struct sp3_vma *sp3_vm_new(void); - -/// Create a new VM that has a sp3_vmfill callback. -/// -/// Free the object with sp3_vm_free(). -/// -/// @param fill Function used to populate data in VM. The function will be pass the new VM object, the address and a context. -/// @param ctx User-specified context. Passed to the fill function and not used by sp3 itself. -/// @return New VM object. -/// -SP3_EXPORT -struct sp3_vma *sp3_vm_new_fill(sp3_vmfill fill, void *ctx); - -/// Create a new VM from an array of words. -/// -/// Free the object with sp3_vm_free(). -/// -/// @param base VM address to load array at. -/// @param len Number of 32-bit words in the array. -/// @param data Pointer to the array. -/// @return New VM object. -/// -SP3_EXPORT -struct sp3_vma *sp3_vm_new_ptr(sp3_vmaddr base, sp3_vmaddr len, const uint32_t *data); - -/// Find a VMA, optionally adding it. -/// -/// @param vm VM to search in. -/// @param addr Address to search for. -/// @param add Flag indicating whether a failure should result in adding a new VMA. -/// @return VM object matching the specified address. -/// -SP3_EXPORT -struct sp3_vma *sp3_vm_find(struct sp3_vma *vm, sp3_vmaddr addr, uint32_t add); - -/// Write a word to a VM. -/// -/// @param vm VM to write. -/// @param addr Address to write. -/// @param val 32-bits of data to write. -/// -SP3_EXPORT -void sp3_vm_write(struct sp3_vma *vm, sp3_vmaddr addr, uint32_t val); - -/// Read a word from a VM. -/// -/// @param vm VM to read. -/// @param addr Address to read. -/// @return 32-bits of data at specified address. -/// -SP3_EXPORT -uint32_t sp3_vm_read(struct sp3_vma *vm, sp3_vmaddr addr); - -/// Probe VM for presence. -/// -/// @param vm VM to probe. -/// @param addr Address to search for. -/// @return 1 if the specified address is backed in the VM, 0 otherwise. -/// -SP3_EXPORT -int sp3_vm_present(struct sp3_vma *vm, sp3_vmaddr addr); - -/// Return base address of VM. -/// -/// @param vm VM to query. -/// @return Base address. -/// -SP3_EXPORT -sp3_vmaddr sp3_vm_base(struct sp3_vma *vm); - -/// Return next VM. -/// -/// @param vm VM to query. -/// @return Next VM in list. -/// -SP3_EXPORT -struct sp3_vma *sp3_vm_next(struct sp3_vma *vm); - -/// Free a VM and all its storage. -/// -/// Use this function to free memory allocated by sp3_vm_new, sp3_vm_new_fill and -/// sp3_vm_new_ptr. -/// -/// @param vm VM to free. -/// -SP3_EXPORT -void sp3_vm_free(struct sp3_vma *vm); - - -/// @} - - -#ifdef __cplusplus -} -#endif - - -#endif /* __SP3_H__ */ diff --git a/tests/kfdtest/src/IsaGenerator.cpp b/tests/kfdtest/src/IsaGenerator.cpp deleted file mode 100644 index 3e69b5f9df..0000000000 --- a/tests/kfdtest/src/IsaGenerator.cpp +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include "IsaGenerator.hpp" - -#include -#include - -#include "IsaGenerator_Gfx72.hpp" -#include "IsaGenerator_Gfx8.hpp" -#include "IsaGenerator_Gfx9.hpp" -#include "IsaGenerator_Gfx10.hpp" -#include "IsaGenerator_Aldebaran.hpp" - -#include "GoogleTestExtension.hpp" - -#include "sp3.h" - -const std::string IsaGenerator::ADDRESS_WATCH_SP3( - "var REG_TRAPSTS_EXCP_MASK = 0x000001ff\n" - "var WAVE_COUNT_OFFSET = 12\n" - "var TMA_CYCLE_OFFSET = 16\n" - "\n" - "/*\n" - " * ttmp[0:1] -- The ISA address that triggered this trap handler\n" - " * ttmp[10:11] -- The TMA user provided, used to store the debug info in this shader\n" - " * v[10:14] ttmp[7:8] -- temp use inside this shader\n" - " * s5 -- store the counts that this trap been triggered\n" - " * Each time when the trap is triggered , this shader will write\n" - " * ttmp[0] : ttmp[1] : Trap_Status : [reserved]\n" - " * to TMA + (trap count * TMA_CYCLE_OFFSET)\n" - " * The TMA + WAVE_COUNT_OFFSET(the first [reserved] address)\n" - " * used to store the total triggered trap count.\n" - " */\n" - "shader main\n" - "\n" - " asic(VI)\n" - "\n" - " type(CS)\n" - " v_mov_b32 v10, ttmp10\n" - " v_mov_b32 v11, ttmp11\n" - " s_mov_b32 ttmp7, s5\n" - " s_mulk_i32 ttmp7, TMA_CYCLE_OFFSET\n" - " s_addk_i32 s5, 1\n" - " v_mov_b32 v12, ttmp0\n" - " v_add_u32 v10, vcc, ttmp7, v10\n" - " flat_store_dword v[10,11], v12 slc glc\n" - " v_mov_b32 v12, ttmp1\n" - " v_add_u32 v10, vcc, 4, v10\n" - " flat_store_dword v[10,11], v12 slc glc\n" - " s_getreg_b32 ttmp8, hwreg(HW_REG_TRAPSTS)\n" - " s_and_b32 ttmp8, ttmp8, REG_TRAPSTS_EXCP_MASK\n" - " v_mov_b32 v12, ttmp8\n" - " v_add_u32 v10, vcc, 4, v10\n" - " flat_store_dword v[10,11], v12 glc\n" - " v_mov_b32 v10, ttmp10\n" - " v_add_u32 v10, vcc, WAVE_COUNT_OFFSET, v10\n" - " v_mov_b32 v13, 1\n" - " flat_atomic_add v14, v[10:11], v13 slc glc\n" - " s_and_b32 ttmp1, ttmp1, 0xffff\n" - " s_rfe_b64 [ttmp0,ttmp1]\n" - "end\n" -); - -IsaGenerator* IsaGenerator::Create(unsigned int familyId) { - switch (familyId) { - case FAMILY_CI: - case FAMILY_KV: - return new IsaGenerator_Gfx72; - case FAMILY_VI: - case FAMILY_CZ: - return new IsaGenerator_Gfx8; - case FAMILY_AI: - case FAMILY_RV: - case FAMILY_AR: - return new IsaGenerator_Gfx9; - case FAMILY_AL: - return new IsaGenerator_Aldbrn; - case FAMILY_NV: - return new IsaGenerator_Gfx10; - - default: - LOG() << "Error: Invalid ISA" << std::endl; - return NULL; - } -} - -void IsaGenerator::GetAwTrapHandler(HsaMemoryBuffer& rBuf) { - CompileShader(ADDRESS_WATCH_SP3.c_str(), "main", rBuf); -} - -void IsaGenerator::CompileShader(const char* shaderCode, const char* shaderName, HsaMemoryBuffer& rBuf) { - sp3_context* pSp3 = sp3_new(); - sp3_setasic(pSp3, GetAsicName().c_str()); - sp3_parse_string(pSp3, shaderCode); - sp3_shader* pShader = sp3_compile(pSp3, shaderName); - - std::copy(pShader->data, pShader->data + pShader->size, rBuf.As()); - sp3_free_shader(pShader); - - /** Inside this close function, there is an unknown reason of free memory not used by compiler. - * Comment out this as a workaround. System will do the garbage collection after this - * application is closed. - */ - // sp3_close(pSp3); -} diff --git a/tests/kfdtest/src/IsaGenerator.hpp b/tests/kfdtest/src/IsaGenerator.hpp deleted file mode 100644 index 4b9c49ad9e..0000000000 --- a/tests/kfdtest/src/IsaGenerator.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef _ISAGENERATOR_H_ -#define _ISAGENERATOR_H_ - -#include "KFDTestUtil.hpp" - -/* isa generation class - interface */ -class IsaGenerator { - public: - static IsaGenerator* Create(unsigned int familyId); - - virtual ~IsaGenerator() {} - - virtual void GetNoopIsa(HsaMemoryBuffer& rBuf) = 0; - virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf) = 0; - virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) = 0; - virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf) = 0; - virtual void GetCwsrTrapHandler(HsaMemoryBuffer& rBuf) {} - virtual void GetAwTrapHandler(HsaMemoryBuffer& rBuf); - - void CompileShader(const char* shaderCode, const char* shaderName, HsaMemoryBuffer& rBuf); - - protected: - virtual const std::string& GetAsicName() = 0; - - private: - static const std::string ADDRESS_WATCH_SP3; -}; - -#endif // _ISAGENERATOR_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp b/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp deleted file mode 100644 index 2c377f9111..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Aldebaran.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include "IsaGenerator_Aldebaran.hpp" - -#include -#include - -const std::string IsaGenerator_Aldbrn::ASIC_NAME = "ALDEBARAN"; - -/* The binaries are generated from following ISA */ -#if 0 -/* flat_atomic_inc will not support by some PCIE, use flat_atomic_add instead */ -shader atomic_add -asic(ALDEBARAN) -type(CS) - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, 1 - flat_atomic_add v3, v[0:1], v2 slc glc scc - s_waitcnt 0 - s_endpgm -end - -shader copy_dword -asic(ALDEBARAN) -type(CS) -/* copy the parameters from scalar registers to vector registers */ - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, s2 - v_mov_b32 v3, s3 -/* copy a dword between the passed addresses */ - flat_load_dword v4, v[0:1] slc glc - s_waitcnt 0 - flat_store_dword v[2:3], v4 slc glc - s_endpgm -end - -shader main -asic(ALDEBARAN) -type(CS) -loop: - s_branch loop - s_endpgm -end - - -#endif - -const uint32_t IsaGenerator_Aldbrn::NOOP_ISA[] = { - 0xbf810000 -}; - -const uint32_t IsaGenerator_Aldbrn::COPY_DWORD_ISA[] = { - 0x7e000200, 0x7e020201, - 0x7e040202, 0x7e060203, - 0xdc530000, 0x047f0000, - 0xbf8c0000, 0xdc730000, - 0x007f0402, 0xbf810000 -}; - -const uint32_t IsaGenerator_Aldbrn::INFINITE_LOOP_ISA[] = { - 0xbf82ffff, 0xbf810000 -}; - -const uint32_t IsaGenerator_Aldbrn::ATOMIC_ADD_ISA[] = { - 0x7e000200, 0x7e020201, - 0x7e040281, 0xdf0b0000, - 0x037f0200, 0xbf8c0000, - 0xbf810000, 0x00000000 -}; - -void IsaGenerator_Aldbrn::GetNoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Aldbrn::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { - std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); -} - -void IsaGenerator_Aldbrn::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Aldbrn::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { - std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); -} - -const std::string& IsaGenerator_Aldbrn::GetAsicName() { - return ASIC_NAME; -} - diff --git a/tests/kfdtest/src/IsaGenerator_Aldebaran.hpp b/tests/kfdtest/src/IsaGenerator_Aldebaran.hpp deleted file mode 100644 index 5571b91c26..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Aldebaran.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2020 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef _ISAGENERATOR_ALDEBARAN_H_ -#define _ISAGENERATOR_ALDEBARAN_H_ - -#include -#include "IsaGenerator.hpp" - -class IsaGenerator_Aldbrn : public IsaGenerator { - public: - virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); - virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); - - protected: - virtual const std::string& GetAsicName(); - - private: - static const std::string ASIC_NAME; - - static const uint32_t NOOP_ISA[]; - static const uint32_t COPY_DWORD_ISA[]; - static const uint32_t INFINITE_LOOP_ISA[]; - static const uint32_t ATOMIC_ADD_ISA[]; -}; - -#endif // _ISAGENERATOR_ALDEBARAN_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Gfx10.cpp b/tests/kfdtest/src/IsaGenerator_Gfx10.cpp deleted file mode 100644 index d8d33086e5..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx10.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include "IsaGenerator_Gfx10.hpp" - -#include -#include - -/* The binaries are generated from following ISA */ -const std::string IsaGenerator_Gfx10::ASIC_NAME = "GFX10"; -#if 0 -static const char * atomic_add = \ -"\ -shader atomic_add \n\ -asic(GFX10) \n\ -wave_size(32) \n\ -type(CS) \n\ - v_mov_b32 v0, s0 \n\ - v_mov_b32 v1, s1 \n\ - v_mov_b32 v2, 1 \n\ - flat_atomic_add v3, v[0:1], v2 slc glc \n\ - s_waitcnt 0 \n\ - s_endpgm \n\ -end \n\ -"; - -static const char * copy_dword = \ -"\ -shader copy_dword \n\ -asic(GFX10) \n\ -wave_size(32) \n\ -type(CS) \n\ - v_mov_b32 v0, s0 \n\ - v_mov_b32 v1, s1 \n\ - v_mov_b32 v2, s2 \n\ - v_mov_b32 v3, s3 \n\ - flat_load_dword v4, v[0:1] slc glc \n\ - s_waitcnt 0 \n\ - flat_store_dword v[2:3], v4 slc glc \n\ - s_endpgm \n\ -end \n\ -"; - -static const char * loop= \ -"\ -shader loop \n\ -asic(GFX10) \n\ -type(CS) \n\ -wave_size(32) \n\ -loop: \n\ - s_branch loop \n\ - s_endpgm \n\ -end \n\ -"; - -static const char * noop= \ -"\ -shader noop \n\ -asic(GFX10) \n\ -type(CS) \n\ -wave_size(32) \n\ - s_endpgm \n\ -end \n\ -"; -#endif - -const uint32_t IsaGenerator_Gfx10::NOOP_ISA[] = { -0xb0804004, 0xbf810000, -0xbf9f0000, 0xbf9f0000, -0xbf9f0000, 0xbf9f0000, -0xbf9f0000 -}; - -const uint32_t IsaGenerator_Gfx10::COPY_DWORD_ISA[] = { -0xb0804004, 0x7e000200, -0x7e020201, 0x7e040202, -0x7e060203, 0xdc330000, -0x47d0000, 0xbf8c0000, -0xdc730000, 0x7d0402, -0xbf810000, 0xbf9f0000, -0xbf9f0000, 0xbf9f0000, -0xbf9f0000, 0xbf9f0000 -}; - -const uint32_t IsaGenerator_Gfx10::INFINITE_LOOP_ISA[] = { -0xbf82ffff, 0xb0804004, -0xbf810000, 0xbf9f0000, -0xbf9f0000, 0xbf9f0000, -0xbf9f0000, 0xbf9f0000 -}; - -const uint32_t IsaGenerator_Gfx10::ATOMIC_ADD_ISA[] = { -0xb0804004, 0x7e000200, -0x7e020201, 0x7e040281, -0xdccb0000, 0x37d0200, -0xbf8c0000, 0xbf810000, -0xbf9f0000, 0xbf9f0000, -0xbf9f0000, 0xbf9f0000, -0xbf9f0000 -}; - - -void IsaGenerator_Gfx10::GetNoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx10::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { - std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx10::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx10::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { - std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); -} - -const std::string& IsaGenerator_Gfx10::GetAsicName() { - return ASIC_NAME; -} - diff --git a/tests/kfdtest/src/IsaGenerator_Gfx10.hpp b/tests/kfdtest/src/IsaGenerator_Gfx10.hpp deleted file mode 100644 index e4a57cda56..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx10.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2019 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef _ISAGENERATOR_GFX10_H_ -#define _ISAGENERATOR_GFX10_H_ - -#include -#include "IsaGenerator.hpp" - -class IsaGenerator_Gfx10 : public IsaGenerator { - public: - virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); - virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); - - protected: - virtual const std::string& GetAsicName(); - - private: - static const std::string ASIC_NAME; - - static const uint32_t NOOP_ISA[]; - static const uint32_t COPY_DWORD_ISA[]; - static const uint32_t INFINITE_LOOP_ISA[]; - static const uint32_t ATOMIC_ADD_ISA[]; -}; - -#endif // _ISAGENERATOR_GFX9_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Gfx72.cpp b/tests/kfdtest/src/IsaGenerator_Gfx72.cpp deleted file mode 100644 index e0d98fb5c8..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx72.cpp +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include "IsaGenerator_Gfx72.hpp" - -#include -#include - -const std::string IsaGenerator_Gfx72::ASIC_NAME = "CI"; - -const uint32_t IsaGenerator_Gfx72::NOOP_ISA[] = { - 0xbf810000 // S_ENDPGM -}; - -/* The below arrays are filled with hex values in order not to reference - * proprietary header files, but we still leave the code here for future - * reference. - */ -#if 0 -const uint32_t IsaGenerator_Gfx72::COPY_DWORD_ISA[] = { - (63u << SQ_VOP1__ENCODING__SHIFT) | (0 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v0, s0 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (1 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v1, s1 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (2 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (2 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v2, s2 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (3 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (3 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v3, s3 (VOP1) - - (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_LOAD_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT)/*(3 << 16)*/, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) - (4u << SQ_FLAT_1__VDST__SHIFT) | (0 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V0:V1, VDST = V4 (FLAT_1) - - (383u << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_WAITCNT << SQ_SOPP__OP__SHIFT) | (0 << SQ_SOPP__SIMM16__SHIFT), // s_waitcnt 0 (SOPP) - - (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_STORE_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT), // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) - (4u << SQ_FLAT_1__DATA__SHIFT) | (2 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V2:V3, DATA = V4 (FLAT_1) - - 0xBF810000u // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 -}; - -const uint32_t IsaGenerator_Gfx72::INFINITE_LOOP_ISA[] = { - (0x17F << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_BRANCH << SQ_SOPP__OP__SHIFT) | ( (const uint32_t)-1 << SQ_SOPP__SIMM16__SHIFT), // s_branch -1 (PC <- PC + SIMM*4)+4 - 0xBF810000u // S_ENDPGM -}; - -const uint32_t IsaGenerator_Gfx72::ATOMIC_INC_ISA[] = { - (63u << SQ_VOP1__ENCODING__SHIFT) | (0 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v0, s0 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (1 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v1, s1 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (2 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0xC1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 0xFFFFFFFF, s2 (VOP1) - - (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_ATOMIC_INC << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (0 << SQ_FLAT_0__GLC__SHIFT), // SQ_FLAT_0, flat_atomic_inc, slc = 1, glc = 0 (FLAT_0) - (3u << SQ_FLAT_1__VDST__SHIFT) | (2u << SQ_FLAT_1__DATA__SHIFT) | (0 << SQ_FLAT_1__ADDR__SHIFT), // ADDR/dst = V0:V1, VDST/ret = V3, DATA/src=V2 (FLAT_1) - 0xBF810000u // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 -}; -#endif - -const uint32_t IsaGenerator_Gfx72::COPY_DWORD_ISA[] = { - 0x7e000200, // v_mov_b32 v0, s0 (VOP1) - 0x7e020201, // v_mov_b32 v1, s1 (VOP1) - 0x7e040202, // v_mov_b32 v2, s2 (VOP1) - 0x7e060203, // v_mov_b32 v3, s3 (VOP1) - - 0xdc330000, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) - 0x04000000, // ADDR = V0:V1, VDST = V4 (FLAT_1) - - 0xbf8c0000, // s_waitcnt 0 (SOPP) - - 0xdc730000, // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) - 0x00000402, // ADDR = V2:V3, DATA = V4 (FLAT_1) - - 0xbf810000 // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 -}; - -const uint32_t IsaGenerator_Gfx72::INFINITE_LOOP_ISA[] = { - 0xbf82ffff, // s_branch -1 (PC <- PC + SIMM*4)+4 - 0xbf810000 // S_ENDPGM -}; - -const uint32_t IsaGenerator_Gfx72::ATOMIC_INC_ISA[] = { - 0x7e000200, // v_mov_b32 v0, s0 (VOP1) - 0x7e020201, // v_mov_b32 v1, s1 (VOP1) - 0x7e0402c1, // v_mov_b32 0xFFFFFFFF, s2 (VOP1) - - 0xdcf20000, // SQ_FLAT_0, flat_atomic_inc, slc = 1, glc = 0 (FLAT_0) - 0x03000200, // ADDR/dst = V0:V1, VDST/ret = V3, DATA/src=V2 (FLAT_1) - 0xbf810000 // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 -}; - -void IsaGenerator_Gfx72::GetNoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx72::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { - std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx72::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx72::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { - std::copy(ATOMIC_INC_ISA, ATOMIC_INC_ISA+ARRAY_SIZE(ATOMIC_INC_ISA), rBuf.As()); -} - -const std::string& IsaGenerator_Gfx72::GetAsicName() { - return ASIC_NAME; -} diff --git a/tests/kfdtest/src/IsaGenerator_Gfx72.hpp b/tests/kfdtest/src/IsaGenerator_Gfx72.hpp deleted file mode 100644 index 5c39ffa216..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx72.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef _ISAGENERATOR_GFX72_H_ -#define _ISAGENERATOR_GFX72_H_ - -#include -#include "IsaGenerator.hpp" - -class IsaGenerator_Gfx72 : public IsaGenerator { - public: - virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); - virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); - - protected: - virtual const std::string& GetAsicName(); - - private: - static const std::string ASIC_NAME; - - static const uint32_t NOOP_ISA[]; - static const uint32_t COPY_DWORD_ISA[]; - static const uint32_t INFINITE_LOOP_ISA[]; - static const uint32_t ATOMIC_INC_ISA[]; -}; - -#endif // _ISAGENERATOR_GFX72_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Gfx8.cpp b/tests/kfdtest/src/IsaGenerator_Gfx8.cpp deleted file mode 100644 index 65e0df6836..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx8.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include "IsaGenerator_Gfx8.hpp" - -#include -#include - -const std::string IsaGenerator_Gfx8::ASIC_NAME = "VI"; - -const uint32_t IsaGenerator_Gfx8::NOOP_ISA[] = { - 0xbf810000 // S_ENDPGM -}; - -/** The below arrays are filled with hex values in order not to reference - * proprietary header files, but we still leave the code here for future - * reference. - */ -#if 0 -const uint32_t IsaGenerator_Gfx8::COPY_DWORD_ISA[] = { - (63u << SQ_VOP1__ENCODING__SHIFT) | (0 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (0 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v0, s0 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (1 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (1 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v1, s1 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (2 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (2 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v2, s2 (VOP1) - (63u << SQ_VOP1__ENCODING__SHIFT) | (3 << SQ_VOP1__VDST__SHIFT) | (SQ_V_MOV_B32 << SQ_VOP1__OP__SHIFT) | (3 << SQ_VOP1__SRC0__SHIFT), // v_mov_b32 v3, s3 (VOP1) - - (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_LOAD_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT)/*(3 << 16)*/, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) - (4u << SQ_FLAT_1__VDST__SHIFT) | (0 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V0:V1, VDST = V4 (FLAT_1) - - (383u << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_WAITCNT << SQ_SOPP__OP__SHIFT) | (0 << SQ_SOPP__SIMM16__SHIFT), // s_waitcnt 0 (SOPP) - - (55u << SQ_FLAT_0__ENCODING__SHIFT) | (SQ_FLAT_STORE_DWORD << SQ_FLAT_0__OP__SHIFT) | (1 << SQ_FLAT_0__SLC__SHIFT) | (1 << SQ_FLAT_0__GLC__SHIFT), // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) - (4u << SQ_FLAT_1__DATA__SHIFT) | (2 << SQ_FLAT_1__ADDR__SHIFT), // ADDR = V2:V3, DATA = V4 (FLAT_1) - - 0xBF810000u // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 -}; - -const uint32_t IsaGenerator_Gfx8::INFINITE_LOOP_ISA[] = { - (0x17F << SQ_SOPP__ENCODING__SHIFT) | (SQ_S_BRANCH << SQ_SOPP__OP__SHIFT) | ( (const uint32_t)-1 << SQ_SOPP__SIMM16__SHIFT), // s_branch -1 (PC <- PC + SIMM*4)+4 - 0xBF810000u // S_ENDPGM -}; -#endif - -const uint32_t IsaGenerator_Gfx8::COPY_DWORD_ISA[] = { - 0x7e000200, // v_mov_b32 v0, s0 (VOP1) - 0x7e020201, // v_mov_b32 v1, s1 (VOP1) - 0x7e040202, // v_mov_b32 v2, s2 (VOP1) - 0x7e060203, // v_mov_b32 v3, s3 (VOP1) - - 0xdc530000, // SQ_FLAT_0, flat_load_dword, slc = 1, glc = 1 (FLAT_0) - 0x04000000, // ADDR = V0:V1, VDST = V4 (FLAT_1) - - 0xbf8c0000, // s_waitcnt 0 (SOPP) - - 0xdc730000, // SQ_FLAT_0, flat_store_dword, slc = 1, glc = 1 (FLAT_0) - 0x00000402, // ADDR = V2:V3, DATA = V4 (FLAT_1) - - 0xbf810000 // s_endpgm, note that we rely on the implicit s_waitcnt 0,0,0 -}; - -const uint32_t IsaGenerator_Gfx8::INFINITE_LOOP_ISA[] = { - 0xbf82ffff, // s_branch -1 (PC <- PC + SIMM*4)+4 - 0xbf810000 // S_ENDPGM -}; - -/** - * The atomic_add_isa binary is generated from following ISA - * The original atomic_inc is not support by some PCIE, so use atomic_add instead - * - */ -/* -shader atomic_add -asic(VI) -type(CS) - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, 1 - flat_atomic_add v3, v[0:1], v2 slc glc - s_waitcnt 0 - s_endpgm -end -*/ - -const uint32_t IsaGenerator_Gfx8::ATOMIC_ADD_ISA[] = { - 0x7e000200, 0x7e020201, - 0x7e040281, 0xdd0b0000, - 0x03000200, 0xbf8c0000, - 0xbf810000, 0x00000000 -}; - -void IsaGenerator_Gfx8::GetNoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx8::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { - std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx8::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx8::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { - std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); -} - -const std::string& IsaGenerator_Gfx8::GetAsicName() { - return ASIC_NAME; -} diff --git a/tests/kfdtest/src/IsaGenerator_Gfx8.hpp b/tests/kfdtest/src/IsaGenerator_Gfx8.hpp deleted file mode 100644 index 7e5b9e3c89..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx8.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef _ISAGENERATOR_GFX8_H_ -#define _ISAGENERATOR_GFX8_H_ - -#include -#include "IsaGenerator.hpp" - -class IsaGenerator_Gfx8 : public IsaGenerator { - public: - virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); - virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); - - protected: - virtual const std::string& GetAsicName(); - - private: - static const std::string ASIC_NAME; - - static const uint32_t NOOP_ISA[]; - static const uint32_t COPY_DWORD_ISA[]; - static const uint32_t INFINITE_LOOP_ISA[]; - static const uint32_t ATOMIC_ADD_ISA[]; -}; - -#endif // _ISAGENERATOR_GFX72_H_ diff --git a/tests/kfdtest/src/IsaGenerator_Gfx9.cpp b/tests/kfdtest/src/IsaGenerator_Gfx9.cpp deleted file mode 100644 index 8eaab32a5e..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx9.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#include "IsaGenerator_Gfx9.hpp" - -#include -#include - -const std::string IsaGenerator_Gfx9::ASIC_NAME = "GFX9"; - -/* The binaries are generated from following ISA */ -#if 0 -/* flat_atomic_inc will not support by some PCIE, use flat_atomic_add instead */ -shader atomic_add -asic(GFX9) -type(CS) - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, 1 - flat_atomic_add v3, v[0:1], v2 slc glc - s_waitcnt 0 - s_endpgm -end - -shader copy_dword -asic(GFX9) -type(CS) -/* copy the parameters from scalar registers to vector registers */ - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - v_mov_b32 v2, s2 - v_mov_b32 v3, s3 -/* copy a dword between the passed addresses */ - flat_load_dword v4, v[0:1] slc glc - s_waitcnt 0 - flat_store_dword v[2:3], v4 slc glc - s_endpgm -end - -shader main -asic(GFX9) -type(CS) -loop: - s_branch loop - s_endpgm -end - - -#endif - -const uint32_t IsaGenerator_Gfx9::NOOP_ISA[] = { - 0xbf810000 -}; - -const uint32_t IsaGenerator_Gfx9::COPY_DWORD_ISA[] = { - 0x7e000200, 0x7e020201, - 0x7e040202, 0x7e060203, - 0xdc530000, 0x047f0000, - 0xbf8c0000, 0xdc730000, - 0x007f0402, 0xbf810000 -}; - -const uint32_t IsaGenerator_Gfx9::INFINITE_LOOP_ISA[] = { - 0xbf82ffff, 0xbf810000 -}; - -const uint32_t IsaGenerator_Gfx9::ATOMIC_ADD_ISA[] = { - 0x7e000200, 0x7e020201, - 0x7e040281, 0xdd0b0000, - 0x037f0200, 0xbf8c0000, - 0xbf810000, 0x00000000 -}; - -void IsaGenerator_Gfx9::GetNoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(NOOP_ISA, NOOP_ISA+ARRAY_SIZE(NOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx9::GetCopyDwordIsa(HsaMemoryBuffer& rBuf) { - std::copy(COPY_DWORD_ISA, COPY_DWORD_ISA+ARRAY_SIZE(COPY_DWORD_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx9::GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf) { - std::copy(INFINITE_LOOP_ISA, INFINITE_LOOP_ISA+ARRAY_SIZE(INFINITE_LOOP_ISA), rBuf.As()); -} - -void IsaGenerator_Gfx9::GetAtomicIncIsa(HsaMemoryBuffer& rBuf) { - std::copy(ATOMIC_ADD_ISA, ATOMIC_ADD_ISA+ARRAY_SIZE(ATOMIC_ADD_ISA), rBuf.As()); -} - -const std::string& IsaGenerator_Gfx9::GetAsicName() { - return ASIC_NAME; -} - diff --git a/tests/kfdtest/src/IsaGenerator_Gfx9.hpp b/tests/kfdtest/src/IsaGenerator_Gfx9.hpp deleted file mode 100644 index 32103c0a15..0000000000 --- a/tests/kfdtest/src/IsaGenerator_Gfx9.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2014-2018 Advanced Micro Devices, Inc. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef _ISAGENERATOR_GFX9_H_ -#define _ISAGENERATOR_GFX9_H_ - -#include -#include "IsaGenerator.hpp" - -class IsaGenerator_Gfx9 : public IsaGenerator { - public: - virtual void GetNoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetCopyDwordIsa(HsaMemoryBuffer& rBuf); - virtual void GetInfiniteLoopIsa(HsaMemoryBuffer& rBuf); - virtual void GetAtomicIncIsa(HsaMemoryBuffer& rBuf); - - protected: - virtual const std::string& GetAsicName(); - - private: - static const std::string ASIC_NAME; - - static const uint32_t NOOP_ISA[]; - static const uint32_t COPY_DWORD_ISA[]; - static const uint32_t INFINITE_LOOP_ISA[]; - static const uint32_t ATOMIC_ADD_ISA[]; -}; - -#endif // _ISAGENERATOR_GFX9_H_ From 025c6146d98c0c1bd184440f778d105dd45e67f5 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 9 Nov 2021 10:59:28 -0500 Subject: [PATCH 0992/1247] kfdtest: Remove TEMP_GFX10_BLACKLIST With LLVM-based assembly these shaders are now valid for GFX10, with the exception of KFDSVMEvictTest. Signed-off-by: Graham Sider Change-Id: Idc872139176bbc1cc8d7ae61a8e4572360ecb5d5 --- tests/kfdtest/scripts/kfdtest.exclude | 36 ++++++++------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 392dba3cde..885054ba50 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -224,26 +224,10 @@ FILTER[aldebaran]=\ "KFDMemoryTest.PtraceAccess:"\ "KFDMemoryTest.DeviceHdpFlush" -# SP3 Compiler needs to be updated for GFX10. Temporarily disable all tests -# that require shader compiler -# Adding KFDSVMEvictTest as SVM/HMM was never validated on GFX10 -TEMP_GFX10_BLACKLIST=\ -"KFDMemoryTest.FlatScratchAccess:"\ -"KFDMemoryTest.PtraceAccessInvisibleVram:"\ -"KFDQMTest.QueuePriorityOnDifferentPipe:"\ -"KFDQMTest.QueuePriorityOnSamePipe:"\ -"KFDCWSRTest.BasicTest:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDEvictTest.QueueTest:"\ -"KFDMemoryTest.MapUnmapToNodes:"\ -"KFDMemoryTest.HostHdpFlush:"\ -"KFDMemoryTest.DeviceHdpFlush:"\ -"KFDSVMEvictTest.*" - FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ -"$TEMP_GFX10_BLACKLIST:"\ -"KFDMemoryTest.MMBench" +"KFDMemoryTest.MMBench:"\ +"KFDSVMEvictTest.*" # Need to verify the following failed tests on another machine: # Exceptions not being received during exception tests @@ -254,42 +238,42 @@ FILTER[navi12]=\ "KFDExceptionTest.*:"\ "KFDPerfCountersTest.*:"\ "KFDPerformanceTest.P2PBandWidthTest:"\ -"$TEMP_GFX10_BLACKLIST" +"KFDSVMEvictTest.*" FILTER[navi14]=\ "$BLACKLIST_ALL_ASICS:"\ -"$TEMP_GFX10_BLACKLIST" +"KFDSVMEvictTest.*" FILTER[sienna_cichlid]=\ "$BLACKLIST_ALL_ASICS:"\ -"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ +"KFDSVMEvictTest.*" FILTER[navy_flounder]=\ "$BLACKLIST_ALL_ASICS:"\ -"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ +"KFDSVMEvictTest.*" FILTER[dimgrey_cavefish]=\ "$BLACKLIST_ALL_ASICS:"\ -"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ +"KFDSVMEvictTest.*" FILTER[beige_goby]=\ "$BLACKLIST_ALL_ASICS:"\ -"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ +"KFDSVMEvictTest.*" FILTER[yellow_carp]=\ "$BLACKLIST_ALL_ASICS:"\ -"$TEMP_GFX10_BLACKLIST:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDIPCTest.CMABasicTest" +"KFDIPCTest.CMABasicTest:"\ +"KFDSVMEvictTest.*" From 7eeba830f88ad9f60fce7a1fbf940764fe54c057 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 9 Mar 2022 10:47:27 -0500 Subject: [PATCH 0993/1247] kfdtest: Add KFDASMTest Includes a simple AssembleShader test which loops through all shaders for all supported targets, dispatching a RunAssemble call for each shader. Also adds extra safety on a couple shaders that only work on gfx9/gfx90a. Signed-off-by: Graham Sider Change-Id: I3ca1c92136f3871eb62fcb9645694f22287aaeec --- tests/kfdtest/CMakeLists.txt | 1 + tests/kfdtest/src/Assemble.hpp | 2 + tests/kfdtest/src/KFDASMTest.cpp | 73 +++++++++++++++++++++++++++++++ tests/kfdtest/src/KFDASMTest.hpp | 39 +++++++++++++++++ tests/kfdtest/src/ShaderStore.cpp | 48 +++++++++++++++----- tests/kfdtest/src/ShaderStore.hpp | 5 +++ 6 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 tests/kfdtest/src/KFDASMTest.cpp create mode 100644 tests/kfdtest/src/KFDASMTest.hpp diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index b1208f54c2..a2b122d42b 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -169,6 +169,7 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/KFDDBGTest.cpp src/KFDGWSTest.cpp src/KFDIPCTest.cpp + src/KFDASMTest.cpp src/KFDEvictTest.cpp src/KFDHWSTest.cpp diff --git a/tests/kfdtest/src/Assemble.hpp b/tests/kfdtest/src/Assemble.hpp index d61229a5a5..46fb946a84 100644 --- a/tests/kfdtest/src/Assemble.hpp +++ b/tests/kfdtest/src/Assemble.hpp @@ -43,6 +43,8 @@ #ifndef _ASSEMBLE_H_ #define _ASSEMBLE_H_ +#include "OSWrapper.hpp" + #define ASM_MCPU_LEN 16 class Assembler { diff --git a/tests/kfdtest/src/KFDASMTest.cpp b/tests/kfdtest/src/KFDASMTest.cpp new file mode 100644 index 0000000000..4b9f5d69c8 --- /dev/null +++ b/tests/kfdtest/src/KFDASMTest.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2022 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "GoogleTestExtension.hpp" +#include "KFDASMTest.hpp" +#include "ShaderStore.hpp" +#include "Assemble.hpp" + +void KFDASMTest::SetUp() {} +void KFDASMTest::TearDown() {} + +static const std::vector TargetList = { + 0x080001, + 0x080002, + 0x080003, + 0x080005, + 0x080100, + 0x090000, + 0x090002, + 0x090004, + 0x090006, + 0x090008, + 0x090009, + 0x09000a, + 0x09000c, + 0x0a0100, + 0x0a0101, + 0x0a0102, + 0x0a0103, + 0x0a0300, + 0x0a0301, + 0x0a0302, + 0x0a0303, + 0x0a0304, + 0x0a0305, + 0x0a0306, +}; + +TEST_F(KFDASMTest, AssembleShaders) { + TEST_START(TESTPROFILE_RUNALL) + + for (auto &t : TargetList) { + Assembler asmblr(t); + + LOG() << "Running ASM test for target " << asmblr.GetTargetAsic() << std::endl; + + for (auto &s : ShaderList) { + EXPECT_SUCCESS(asmblr.RunAssemble(s)); + } + } + + TEST_END +} diff --git a/tests/kfdtest/src/KFDASMTest.hpp b/tests/kfdtest/src/KFDASMTest.hpp new file mode 100644 index 0000000000..5f601e165a --- /dev/null +++ b/tests/kfdtest/src/KFDASMTest.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2022 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_ASM_TEST__H__ +#define __KFD_ASM_TEST__H__ + +#include + +class KFDASMTest : public testing::Test { + public: + KFDASMTest() {} + ~KFDASMTest() {} + + protected: + virtual void SetUp(); + virtual void TearDown(); +}; + +#endif // __KFD_ASM_TEST__H__ diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 63cd68a063..8b40351f04 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -21,6 +21,30 @@ * */ +#include "ShaderStore.hpp" + +/** + * KFDASMTest List + */ + +const std::vector ShaderList = { + NoopIsa, + CopyDwordIsa, + InfiniteLoopIsa, + AtomicIncIsa, + ScratchCopyDwordIsa, + PollMemoryIsa, + CopyOnSignalIsa, + PollAndCopyIsa, + WriteFlagAndValueIsa, + WriteAndSignalIsa, + LoopIsa, + IterateIsa, + ReadMemoryIsa, + GwsInitIsa, + GwsAtomicIncreaseIsa, +}; + /** * Macros */ @@ -251,7 +275,7 @@ const char *PollAndCopyIsa = R"( s_store_dword s17, s[2:3], 0x0 glc s_waitcnt vmcnt(0) & lgkmcnt(0) buffer_wbl2 - .else + .elseif (.amdgcn.gfx_generation_number == 9) s_movk_i32 s18, 0x1 LOOP: s_load_dword s16, s[0:1], 0x0 glc @@ -277,16 +301,18 @@ const char *PollAndCopyIsa = R"( const char *WriteFlagAndValueIsa = R"( .text // Assume two inputs buffer in s[0:1] and s[2:3] - v_mov_b32 v0, s0 - v_mov_b32 v1, s1 - s_load_dword s18, s[2:3], 0x0 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - s_store_dword s18, s[0:1], 0x4 glc - s_waitcnt vmcnt(0) & lgkmcnt(0) - buffer_wbl2 - s_waitcnt vmcnt(0) & lgkmcnt(0) - v_mov_b32 v16, 0x1 - flat_store_dword v[0:1], v16 glc + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_stepping == 10) + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + s_load_dword s18, s[2:3], 0x0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s18, s[0:1], 0x4 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) + buffer_wbl2 + s_waitcnt vmcnt(0) & lgkmcnt(0) + v_mov_b32 v16, 0x1 + flat_store_dword v[0:1], v16 glc + .endif s_endpgm )"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index 231e7f73d6..e0151a6537 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -24,6 +24,11 @@ #ifndef _SHADERSTORE_H_ #define _SHADERSTORE_H_ +#include + +/* KFDASMTest List */ +extern const std::vector ShaderList; + /* Common */ extern const char *NoopIsa; extern const char *CopyDwordIsa; From b9e8bc1f523d93e33a9cf16582e716980d6ebd5c Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Mon, 25 Apr 2022 22:06:57 -0500 Subject: [PATCH 0994/1247] kfdtest: add log message to KFDExceptionTest [WHY] These tests force HW exceptions in the GPU driver. Some of these exceptions might print page fault error messages at kernel level. These are expected errors due to the nature of the tests, but still might cause confusion to users. [HOW] Add log message to warn the user about these kernel error messages. Signed-off-by: Alex Sierra Change-Id: I8eef87b83939e37230da0c374c2f77d2d484baa9 --- tests/kfdtest/src/KFDExceptionTest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 9fa15e0969..ddcf8ceb16 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -33,6 +33,9 @@ void KFDExceptionTest::SetUp() { KFDBaseComponentTest::SetUp(); + LOG() << "This Exception test might cause expected page fault " + "error logs at kernel level." << std::endl; + ROUTINE_END } From 4b041a8ad97c46bbe266a98bc93eb7490a94bb03 Mon Sep 17 00:00:00 2001 From: David Francis Date: Mon, 2 May 2022 09:13:20 -0400 Subject: [PATCH 0995/1247] kfdtest: Use correct isa buffer in GPU coherency test In the LLVM rework, a line was accidentally changed from isaBuffer1 to isaBuffer, causing VramCacheCoherenceWithRemoteGPU to fail. Change it back. Signed-off-by: David Francis Change-Id: Ie1f3465b5c46556f18682d1b3d1f086bb790c648 --- tests/kfdtest/src/KFDMemoryTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 4e9cb5fd19..57ea6dc1a2 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2154,7 +2154,7 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) { ASSERT_SUCCESS(queue1.Create(nondefaultNode)); HsaMemoryBuffer isaBuffer1(PAGE_SIZE, nondefaultNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteFlagAndValueIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteFlagAndValueIsa, isaBuffer1.As())); Dispatch dispatch1(isaBuffer1); dispatch1.SetArgs(buffer.As(), buffer.As()+dwSource); From c05c66e75662e0ee292c1f2673db3a7300a2ac8d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 27 Apr 2022 11:10:01 -0400 Subject: [PATCH 0996/1247] kfdtest: SVM mGPU tests skip GPUs prior to gfx9 Otherwise the test failed because KFD returns failure to map or prefetch the SVM range to those GPUs which don't support SVM. Change-Id: Ideaec2a686bfabb2fd6ae41b1a690da144121a89 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 283a567ff4..5bdcf5b80f 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -997,9 +997,18 @@ TEST_F(KFDSVMRangeTest, MultiGPUMigrationTest) { return; } - const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + const std::vector gpuNodesAll = m_NodeInfo.GetNodesWithGPU(); + std::vector gpuNodes; + + for (int i : gpuNodesAll) { + const HsaNodeProperties *pNodeProperties; + + pNodeProperties = m_NodeInfo.GetNodeProperties(gpuNodesAll.at(i)); + if (pNodeProperties->Capability.ui32.SVMAPISupported) + gpuNodes.push_back(gpuNodesAll.at(i)); + } if (gpuNodes.size() < 2) { - LOG() << "Skipping test: at least two GPUs needed." << std::endl; + LOG() << "Skipping test: at least two SVM supported GPUs needed." << std::endl; return; } @@ -1072,9 +1081,18 @@ TEST_F(KFDSVMRangeTest, MultiGPUAccessInPlaceTest) { return; } - const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + const std::vector gpuNodesAll = m_NodeInfo.GetNodesWithGPU(); + std::vector gpuNodes; + + for (int i : gpuNodesAll) { + const HsaNodeProperties *pNodeProperties; + + pNodeProperties = m_NodeInfo.GetNodeProperties(gpuNodesAll.at(i)); + if (pNodeProperties->Capability.ui32.SVMAPISupported) + gpuNodes.push_back(gpuNodesAll.at(i)); + } if (gpuNodes.size() < 2) { - LOG() << "Skipping test: at least two GPUs needed." << std::endl; + LOG() << "Skipping test: at least two SVM supported GPUs needed." << std::endl; return; } From ffa6c95858404d93a35ecea6d260bd1855fbebf1 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 4 May 2022 16:13:20 -0400 Subject: [PATCH 0997/1247] kfdtest: Hotfix wrong isaBuffer used in DeviceHdpFlush In DeviceHdpFlush, isaBuffer was accidentally used instead of isaBuffer0 during LLVM re-work--revert. Signed-off-by: Graham Sider Change-Id: I98d2322e772f821a39505bb336ceb4e6cd8722ef --- tests/kfdtest/src/KFDMemoryTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 57ea6dc1a2..2d29c00129 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1879,7 +1879,7 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { ASSERT_SUCCESS(queue0.Create(nodes[1])); HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteAndSignalIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteAndSignalIsa, isaBuffer0.As())); Dispatch dispatch0(isaBuffer0); dispatch0.SetArgs(buffer, mmioBase); From fe2098e011342ce0ed9017949cdae1df5211b84b Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 4 May 2022 18:13:23 -0400 Subject: [PATCH 0998/1247] kfdtest: Add DeviceHdpFlush to gfx10 blacklist DeviceHdpFlush was part of the prior TEMP_GFX10_BLACKLIST and is soft-hanging after being re-added to the test list. Temporarily move back to gfx10 blacklists until root-caused (). Signed-off-by: Graham Sider Change-Id: Id37b7105566e8243465e4b21ebc2e2e8c4c79923 --- tests/kfdtest/scripts/kfdtest.exclude | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 885054ba50..f48049cbfd 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -247,6 +247,7 @@ FILTER[navi14]=\ FILTER[sienna_cichlid]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ +"KFDMemoryTest.DeviceHdpFlush:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ "KFDSVMEvictTest.*" @@ -254,6 +255,7 @@ FILTER[sienna_cichlid]=\ FILTER[navy_flounder]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ +"KFDMemoryTest.DeviceHdpFlush:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ "KFDSVMEvictTest.*" @@ -261,6 +263,7 @@ FILTER[navy_flounder]=\ FILTER[dimgrey_cavefish]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ +"KFDMemoryTest.DeviceHdpFlush:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ "KFDSVMEvictTest.*" @@ -268,6 +271,7 @@ FILTER[dimgrey_cavefish]=\ FILTER[beige_goby]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ +"KFDMemoryTest.DeviceHdpFlush:"\ "KFDDBGTest.*:"\ "KFDPerfCountersTest.*:"\ "KFDSVMEvictTest.*" @@ -275,5 +279,6 @@ FILTER[beige_goby]=\ FILTER[yellow_carp]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDQMTest.BasicCuMaskingEven:"\ +"KFDMemoryTest.DeviceHdpFlush:"\ "KFDIPCTest.CMABasicTest:"\ "KFDSVMEvictTest.*" From 0955f350e30a9b05bdafb4854b3e57317ee385f4 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 5 May 2022 11:18:06 -0400 Subject: [PATCH 0999/1247] kfdtest: Clean up kfdtest.exclude Add BLACKLIST_GFX10 and BLACKLIST_GFX10_NV2X to reduce redundant individual blacklists. Remove KFDDBGTest from kfdtest.exclude and CMakeLists.txt as it is deprecated. Signed-off-by: Graham Sider Change-Id: I5ffebb012ff8ef4bba5aadfeb3c7478fe43c65f1 --- tests/kfdtest/CMakeLists.txt | 1 - tests/kfdtest/scripts/kfdtest.exclude | 49 +++++++++++---------------- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index a2b122d42b..55fb4c8c51 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -166,7 +166,6 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/KFDExceptionTest.cpp src/KFDGraphicsInterop.cpp src/KFDPerfCounters.cpp - src/KFDDBGTest.cpp src/KFDGWSTest.cpp src/KFDIPCTest.cpp src/KFDASMTest.cpp diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index f48049cbfd..7148c552a4 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -69,7 +69,6 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDQMTest.mGPUShareBO:"\ "KFDQMTest.SdmaEventInterrupt:"\ "KFDMemoryTest.CacheInvalidateOnRemoteWrite:"\ -"KFDDBGTest.BasicDebuggerSuspendResume:"\ "KFDEvictTest.BurstyTest:"\ "KFDHWSTest.*:"\ "KFDSVMRangeTest.ReadOnlyRangeTest" @@ -106,6 +105,15 @@ KV_QUEUE_BLACKLIST=\ "KFDQMTest.Atomics:"\ "KFDQMTest.GPUDoorbellWrite" +BLACKLIST_GFX10=\ +"KFDMemoryTest.DeviceHdpFlush:"\ +"KFDSVMEvictTest.*" + +BLACKLIST_GFX10_NV2X=\ +"$BLACKLIST_GFX10:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDPerfCountersTest.*" + # KFDQMTest.CpuWriteCoherence fails. 0 dwordsAvailable (KFD-338) # KFDMemoryTest.MemoryRegister fails on SDMA queue creation (KFD-337) FILTER[kaveri]=\ @@ -226,8 +234,8 @@ FILTER[aldebaran]=\ FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDMemoryTest.MMBench:"\ -"KFDSVMEvictTest.*" +"$BLACKLIST_GFX10:"\ +"KFDMemoryTest.MMBench" # Need to verify the following failed tests on another machine: # Exceptions not being received during exception tests @@ -235,50 +243,31 @@ FILTER[navi10]=\ # P2PBandwidth failing (wait times out) on node-to-multiple-nodes by [push, NONE] FILTER[navi12]=\ "$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX10:"\ "KFDExceptionTest.*:"\ "KFDPerfCountersTest.*:"\ -"KFDPerformanceTest.P2PBandWidthTest:"\ -"KFDSVMEvictTest.*" +"KFDPerformanceTest.P2PBandWidthTest" FILTER[navi14]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDSVMEvictTest.*" +"$BLACKLIST_GFX10" FILTER[sienna_cichlid]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.DeviceHdpFlush:"\ -"KFDDBGTest.*:"\ -"KFDPerfCountersTest.*:"\ -"KFDSVMEvictTest.*" +"$BLACKLIST_GFX10_NV2X" FILTER[navy_flounder]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.DeviceHdpFlush:"\ -"KFDDBGTest.*:"\ -"KFDPerfCountersTest.*:"\ -"KFDSVMEvictTest.*" +"$BLACKLIST_GFX10_NV2X" FILTER[dimgrey_cavefish]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.DeviceHdpFlush:"\ -"KFDDBGTest.*:"\ -"KFDPerfCountersTest.*:"\ -"KFDSVMEvictTest.*" +"$BLACKLIST_GFX10_NV2X" FILTER[beige_goby]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.DeviceHdpFlush:"\ -"KFDDBGTest.*:"\ -"KFDPerfCountersTest.*:"\ -"KFDSVMEvictTest.*" +"$BLACKLIST_GFX10_NV2X" FILTER[yellow_carp]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ -"KFDMemoryTest.DeviceHdpFlush:"\ -"KFDIPCTest.CMABasicTest:"\ -"KFDSVMEvictTest.*" +"$BLACKLIST_GFX10_NV2X" From 5f2c75b7d62b6a1b33f7ac6f40d55df5098812d9 Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Wed, 23 Feb 2022 16:15:40 -0800 Subject: [PATCH 1000/1247] File Reorganization with backward compatibility Header installed in /opt/rocm/include/hsakmt Wrapper header files for backward compatibility Change-Id: I5f6ccac996a43a7c20e0f75e48b1c55b610f0a16 --- CMakeLists.txt | 10 ++++- hsakmt-backward-compat.cmake | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 hsakmt-backward-compat.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 273335d7cd..cf37aa5d2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -143,7 +143,7 @@ target_sources ( ${HSAKMT_TARGET} PRIVATE ${HSAKMT_SRC} ) target_include_directories( ${HSAKMT_TARGET} PUBLIC $ - $ + $ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) @@ -214,9 +214,15 @@ install ( TARGETS ${HSAKMT_TARGET} EXPORT ${HSAKMT_TARGET}Targets #install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel ) # Install public headers -install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_TARGET} COMPONENT devel PATTERN "linux" EXCLUDE ) +# Option to build header path migration helpers. +option(INCLUDE_PATH_COMPATIBILITY "Generate backward compatible headers and include paths. Use of these headers will warn when included." ON) +if(INCLUDE_PATH_COMPATIBILITY) + include(hsakmt-backward-compat.cmake) +endif() + # Record our usage data for clients find_package calls. install ( EXPORT ${HSAKMT_TARGET}Targets FILE ${HSAKMT_TARGET}Targets.cmake diff --git a/hsakmt-backward-compat.cmake b/hsakmt-backward-compat.cmake new file mode 100644 index 0000000000..89f356c552 --- /dev/null +++ b/hsakmt-backward-compat.cmake @@ -0,0 +1,71 @@ +# Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +set(HSAKMT_WRAPPER_DIR ${CMAKE_CURRENT_BINARY_DIR}/wrapper_dir) +set(HSAKMT_WRAPPER_INC_DIR ${HSAKMT_WRAPPER_DIR}/include) +#Function to generate header template file +function(create_header_template) + file(WRITE ${HSAKMT_WRAPPER_DIR}/header.hpp.in "/* + Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the \"Software\"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + */\n\n#ifndef @include_guard@\n#define @include_guard@ \n\n#pragma message(\"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\")\n@include_statements@\n\n#endif") +endfunction() + +#use header template file and generate wrapper header files +function(generate_wrapper_header) + file(MAKE_DIRECTORY ${HSAKMT_WRAPPER_INC_DIR}) + #find all header files from include folder + file(GLOB include_files ${CMAKE_CURRENT_SOURCE_DIR}/include/*.h) + #generate wrapper header files + foreach(header_file ${include_files}) + # set include guard + get_filename_component(INC_GUARD_NAME ${header_file} NAME_WE) + string(TOUPPER ${INC_GUARD_NAME} INC_GUARD_NAME) + set(include_guard "${include_guard}HSAKMT_WRAPPER_INCLUDE_${INC_GUARD_NAME}_H") + # set include statements + get_filename_component(file_name ${header_file} NAME) + set(include_statements "${include_statements}#include \"hsakmt/${file_name}\"\n") + configure_file(${HSAKMT_WRAPPER_DIR}/header.hpp.in ${HSAKMT_WRAPPER_INC_DIR}/${file_name}) + unset(include_guard) + unset(include_statements) + endforeach() +endfunction() + +#Creater a template for header file +create_header_template() +#Use template header file and generater wrapper header files +generate_wrapper_header() +install(DIRECTORY ${HSAKMT_WRAPPER_INC_DIR} DESTINATION . COMPONENT devel PATTERN "linux" EXCLUDE) From 30bba898300fb21b7f6cfaa7b91f847668b2d2d7 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Sun, 8 May 2022 15:14:51 -0400 Subject: [PATCH 1001/1247] kfdtest: Move BasicCuMaskingEven to gfx10 blacklist KFDQMTest.BasicCuMaskingEven was previously in TEMP_GFX10_BLACKLIST and was later added to BLACKLIST_GFX10_NV2X. shows this test also has problems on NV1X, so move to BLACKLIST_GFX10 to blacklist for all GFX10 until resolved. Signed-off-by: Graham Sider Change-Id: I75426780a52172ed0f2468f0297241902da5002a --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 7148c552a4..2c702b8213 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -107,11 +107,11 @@ KV_QUEUE_BLACKLIST=\ BLACKLIST_GFX10=\ "KFDMemoryTest.DeviceHdpFlush:"\ +"KFDQMTest.BasicCuMaskingEven:"\ "KFDSVMEvictTest.*" BLACKLIST_GFX10_NV2X=\ "$BLACKLIST_GFX10:"\ -"KFDQMTest.BasicCuMaskingEven:"\ "KFDPerfCountersTest.*" # KFDQMTest.CpuWriteCoherence fails. 0 dwordsAvailable (KFD-338) From d5f074902c46e1326b6f53a01a0b975f8cec8aa3 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 11 May 2022 13:39:49 -0400 Subject: [PATCH 1002/1247] kfdtest: Update CMake message on LLVM build search Old CMake message was misleading, update to reflect what it's actually doing. Signed-off-by: Graham Sider Change-Id: I8fd18a9095ea350cc10494b4382706027b447ba3 --- tests/kfdtest/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 55fb4c8c51..6eeffa5585 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -105,8 +105,8 @@ find_path( LIGHTNING_CMAKE_DIR NAMES LLVMConfig.cmake if ( DEFINED LIGHTNING_CMAKE_DIR AND EXISTS ${LIGHTNING_CMAKE_DIR} ) set ( LLVM_DIR ${LIGHTNING_CMAKE_DIR} ) else() - message( WARNING "Couldn't find Lightning build. " - "Attempting to use system LLVM install..." ) + message( STATUS "Couldn't find Lightning build in compute directory. " + "Searching LLVM_DIR then defaulting to system LLVM install if still not found..." ) endif() find_package( LLVM REQUIRED CONFIG ) From 65aec53bca6d479a72cce24dc4bcb900f5e949af Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 8 Feb 2022 16:24:46 -0500 Subject: [PATCH 1003/1247] CMakeLists.txt: Remove unneeded dependencies These were required back due to dependency issues in earlier ROCm releases. With thunk being static now and with better dependency definitions being used, we can remove these Signed-off-by: Kent Russell Change-Id: I266a783993edf32811caf027f4289ede0cbfcb16 --- CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf37aa5d2d..195266b2ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,11 +191,6 @@ if(NOT DISTRO_ID MATCHES "ubuntu") find_library(LIBGCC NAMES libgcc_s.so.1 REQUIRED) message(STATUS "LIBGCC:" ${LIBGCC}) target_link_libraries( ${HSAKMT_TARGET} PRIVATE ${LIBGCC} ) -else() - find_library(UDEV NAMES libudev.so libudev.a REQUIRED) - message(STATUS "UDEV:" ${UDEV}) - find_package(ZLIB REQUIRED) - target_link_libraries( ${HSAKMT_TARGET} PRIVATE ${ZLIB} ${UDEV} ) endif() ## Define default paths and packages. From cee64d48257fd00b086284d2e0589b7a936364c6 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 18 May 2022 10:41:22 -0400 Subject: [PATCH 1004/1247] kfdtest: Add gfx11 to kfdtest.exclude With this patch gfx11 is supported in amd-staging. Signed-off-by: Graham Sider Change-Id: I4bbfc87345a486dd8f2e0091ea7b82c255a8ad15 --- tests/kfdtest/scripts/kfdtest.exclude | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 2c702b8213..eb1a6c75a8 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -271,3 +271,15 @@ FILTER[beige_goby]=\ FILTER[yellow_carp]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X" + +FILTER[plum_bonito]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX10_NV2X" + +FILTER[wheat_nas]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX10_NV2X" + +FILTER[hotpink_bonefish]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX10_NV2X" From c198b91c94f2211191d43dc01725c89a1f1b729f Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 19 May 2022 12:32:02 -0400 Subject: [PATCH 1005/1247] kfdtest: Add case for gfx11 in FamilyIdFromNode GFX11 set to FAMILY_NV. Signed-off-by: Graham Sider Change-Id: I545241c39bbe739e39a8943b242b9fc49a65a7e1 --- tests/kfdtest/src/KFDTestUtil.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 2eddc8857b..fe647fb853 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -192,6 +192,7 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { familyId = FAMILY_AL; break; case 10: + case 11: familyId = FAMILY_NV; break; } From 707200e26e0af1ad9f7b9102e77d8d9b066f4d0a Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Wed, 1 Jun 2022 10:28:47 -0700 Subject: [PATCH 1006/1247] : Use GNUInstallDirs Use GNUInstallDirs variables in post install scripts License file installed in CMAKE_INSTALL_DOCDIR Change-Id: I182ca292e03787a6c189e8de31d32244b65b5687 --- CMakeLists.txt | 2 +- DEBIAN/postinst.in | 2 +- DEBIAN/prerm.in | 2 +- RPM/post.in | 2 +- RPM/postun.in | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 195266b2ff..113b83c720 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,7 +290,7 @@ set(CPACK_COMPONENT_DESCRIPTION "HSAKMT development package.\n This package incl set ( ENABLE_LDCONFIG ON CACHE BOOL "Set library links and caches using ldconfig.") # Install License file -install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR}/${CPACK_PACKAGE_NAME} COMPONENT devel) +install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel) # Make proper version for appending # Default Value is 99999, setting it first diff --git a/DEBIAN/postinst.in b/DEBIAN/postinst.in index 04198dc7bb..e8a7917d40 100755 --- a/DEBIAN/postinst.in +++ b/DEBIAN/postinst.in @@ -5,7 +5,7 @@ set -e # left-hand term originates from ENABLE_LDCONFIG = ON/OFF at package build do_ldconfig() { if [ "@ENABLE_LDCONFIG@" == "ON" ]; then - echo @CPACK_PACKAGING_INSTALL_PREFIX@/lib > /etc/ld.so.conf.d/x86_64-libhsakmt.conf + echo @CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ > /@CMAKE_INSTALL_SYSCONFDIR@/ld.so.conf.d/x86_64-libhsakmt.conf ldconfig fi } diff --git a/DEBIAN/prerm.in b/DEBIAN/prerm.in index f16a1ab3b7..329420ca88 100755 --- a/DEBIAN/prerm.in +++ b/DEBIAN/prerm.in @@ -5,7 +5,7 @@ set -e # left-hand term originates from ENABLE_LDCONFIG = ON/OFF at package build rm_ldconfig() { if [ "@ENABLE_LDCONFIG@" == "ON" ]; then - rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig + rm -f /@CMAKE_INSTALL_SYSCONFDIR@/ld.so.conf.d/x86_64-libhsakmt.conf && ldconfig fi } diff --git a/RPM/post.in b/RPM/post.in index 5fc16b48a2..4a723b3761 100644 --- a/RPM/post.in +++ b/RPM/post.in @@ -1,5 +1,5 @@ # left-hand term originates from ENABLE_LDCONFIG = ON/OFF at package build if [ "@ENABLE_LDCONFIG@" == "ON" ]; then - echo -e "@CPACK_PACKAGING_INSTALL_PREFIX@/lib\n@CPACK_PACKAGING_INSTALL_PREFIX@/lib64" > /etc/ld.so.conf.d/x86_64-libhsakmt.conf + echo -e "@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@" > /@CMAKE_INSTALL_SYSCONFDIR@/ld.so.conf.d/x86_64-libhsakmt.conf ldconfig fi diff --git a/RPM/postun.in b/RPM/postun.in index 0691ac8617..b321d76fa4 100644 --- a/RPM/postun.in +++ b/RPM/postun.in @@ -1,6 +1,6 @@ # second term originates from ENABLE_LDCONFIG = ON/OFF at package build if [ $1 -le 1 ] && [ "@ENABLE_LDCONFIG@" == "ON" ]; then # perform the below actions for rpm remove($1=0) or upgrade($1=1) operations - rm -f /etc/ld.so.conf.d/x86_64-libhsakmt.conf + rm -f /@CMAKE_INSTALL_SYSCONFDIR@/ld.so.conf.d/x86_64-libhsakmt.conf ldconfig fi From 6da6058d4a30ef8106e91368b506c54402a70939 Mon Sep 17 00:00:00 2001 From: Daniel Phillips Date: Tue, 31 May 2022 10:02:36 -0400 Subject: [PATCH 1007/1247] libhsakmt: Add support for hsaKmtAvailableMemory New API function to report available memory per GPU Signed-off-by: Daniel Phillips Change-Id: I63c1e4ca0020c657977ab3635947ab0ed0a81440 --- include/hsakmt.h | 11 +++++++++++ include/linux/kfd_ioctl.h | 18 ++++++++++++++++-- src/libhsakmt.ver | 1 + src/memory.c | 24 ++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index ff2d023760..abc617f42b 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -374,6 +374,17 @@ hsaKmtFreeMemory( HSAuint64 SizeInBytes //IN ); +/** + Inquires memory available for allocation as a memory buffer +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtAvailableMemory( + HSAuint32 Node, + HSAuint64 *AvailableBytes + ); + /** Registers with KFD a memory buffer that may be accessed by the GPU */ diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 8a0ed49d70..ba8de4b211 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -34,9 +34,10 @@ * - 1.6 - Query clear flags in SVM get_attr API * - 1.7 - Checkpoint Restore (CRIU) API * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs + * - 1.9 - Add available_memory ioctl */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 8 +#define KFD_IOCTL_MINOR_VERSION 9 /* * Debug revision change log @@ -769,6 +770,16 @@ struct kfd_ioctl_free_memory_of_gpu_args { __u64 handle; /* to KFD */ }; +/* Inquire available memory with kfd_ioctl_get_available_memory + * + * @available: memory available for alloc + */ +struct kfd_ioctl_get_available_memory_args { + __u64 available; /* from KFD */ + __u32 gpu_id; /* to KFD */ + __u32 pad; +}; + /* Map memory to one or more GPUs * * @handle: memory handle returned by alloc @@ -1327,8 +1338,11 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_CRIU_OP \ AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args) +#define AMDKFD_IOC_AVAILABLE_MEMORY \ + AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x23 +#define AMDKFD_COMMAND_END 0x24 /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 2d63d22d15..50c309dc0b 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -24,6 +24,7 @@ hsaKmtSetQueueCUMask; hsaKmtSetMemoryPolicy; hsaKmtAllocMemory; hsaKmtFreeMemory; +hsaKmtAvailableMemory; hsaKmtRegisterMemory; hsaKmtRegisterMemoryToNodes; hsaKmtRegisterMemoryWithFlags; diff --git a/src/memory.c b/src/memory.c index 6d2a4f4450..5eae74a92c 100644 --- a/src/memory.c +++ b/src/memory.c @@ -199,6 +199,30 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, return fmm_release(MemoryAddress); } +HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node, + HSAuint64 *AvailableBytes) +{ + struct kfd_ioctl_get_available_memory_args args = {}; + HSAKMT_STATUS result; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(9); + + pr_debug("[%s] node %d\n", __func__, Node); + + result = validate_nodeid(Node, &args.gpu_id); + if (result != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, Node); + return result; + } + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY, &args)) + return HSAKMT_STATUS_ERROR; + + *AvailableBytes = args.available; + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress, HSAuint64 MemorySizeInBytes) { From 852d29bca2c15f069dfaa8c13dc5d30d9d64752a Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 8 Jun 2022 16:49:32 -0400 Subject: [PATCH 1008/1247] kfdtest: Add temporary blacklist for GFX11 Add blacklist for tests on GFX11 that are under debug/not functional. Signed-off-by: Graham Sider Change-Id: I32f375f29036a2fc1a54f2792b31ebc45f4d668b --- tests/kfdtest/scripts/kfdtest.exclude | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index eb1a6c75a8..fcd5d0fcbb 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -114,6 +114,24 @@ BLACKLIST_GFX10_NV2X=\ "$BLACKLIST_GFX10:"\ "KFDPerfCountersTest.*" +# GFX11 still undergoing debug. Ticket links: +# KFDMemoryTest.FlatScratchAccess - SWDEV-329877 +# KFDEvictTest.QueueTest - SWDEV-325064 +# KFDQMTest.QueuePriorityOn*/EmptyDispatch - SWDEV-340963 +# KFDQMTest.MultipleCpQueuesStressDispatch - SWDEV-340965 +# KFDExceptionTest.* - SWDEV-340972 +TEMPORARY_BLACKLIST_GFX11=\ +"KFDQMTest.CreateAqlCpQueue:"\ +"KFDQMTest.QueuePriorityOn*:"\ +"KFDQMTest.MultipleCpQueuesStressDispatch:"\ +"KFDQMTest.EmptyDispatch:"\ +"KFDCWSRTest.InterruptRestore:"\ +"KFDExceptionTest.*:"\ +"KFDEvictTest.QueueTest:"\ +"KFDSVMRangeTest.*Migrate*:"\ +"KFDSVMRangeTest.*Migration*:"\ +"KFDMemoryTest.FlatScratchAccess" + # KFDQMTest.CpuWriteCoherence fails. 0 dwordsAvailable (KFD-338) # KFDMemoryTest.MemoryRegister fails on SDMA queue creation (KFD-337) FILTER[kaveri]=\ @@ -274,12 +292,15 @@ FILTER[yellow_carp]=\ FILTER[plum_bonito]=\ "$BLACKLIST_ALL_ASICS:"\ -"$BLACKLIST_GFX10_NV2X" +"$BLACKLIST_GFX10_NV2X:"\ +"$TEMPORARY_BLACKLIST_GFX11" FILTER[wheat_nas]=\ "$BLACKLIST_ALL_ASICS:"\ -"$BLACKLIST_GFX10_NV2X" +"$BLACKLIST_GFX10_NV2X:"\ +"$TEMPORARY_BLACKLIST_GFX11" FILTER[hotpink_bonefish]=\ "$BLACKLIST_ALL_ASICS:"\ -"$BLACKLIST_GFX10_NV2X" +"$BLACKLIST_GFX10_NV2X:"\ +"$TEMPORARY_BLACKLIST_GFX11" From 0e908f05bb50db78eee087699281c081aad639c6 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 9 Jun 2022 11:14:41 -0500 Subject: [PATCH 1009/1247] libhsakmt: add env var to set max VA alignment order This env variable sets the max VA alignment order size as "PAGE_SIZE * 2^alignment order" during mapping. By default the order size is set to 9(2MB). Signed-off-by: Alex Sierra Change-Id: I01ae4e0963f4d21c7c367464e60f865bc58d7fac --- src/fmm.c | 22 +++++++++++++++++++--- src/libhsakmt.h | 1 - 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 2319513c29..35da3b8cf3 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -214,6 +214,9 @@ typedef struct { /* whether all memory is coherent (GPU cache disabled) */ bool disable_cache; + + /* specifies the alignment size as PAGE_SIZE * 2^alignment_order */ + uint32_t alignment_order; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -705,6 +708,7 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, uint64_t size, uint64_t align) { uint64_t aligned_padded_size, guard_size; + uint64_t alignment_size = PAGE_SIZE << svm.alignment_order; void *addr, *aligned_addr, *aligned_end, *mapping_end; if (address) @@ -715,10 +719,14 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, return NULL; } - /* Align big buffers to the next power-of-2 up to 1GB - * size for memory allocation optimization + /* Align big buffers to the next power-of-2. By default, the max alignment + * size is set to 2MB. This can be modified by the env variable + * HSA_MAX_VA_ALIGN. This variable sets the order of the alignment size as + * PAGE_SIZE * 2^HSA_MAX_VA_ALIGN. Setting HSA_MAX_VA_ALIGN = 18 (1GB), + * improves the time for memory allocation and mapping. But it might lose + * performance when GFX access it, specially for big allocations (>3GB). */ - while (align < GPU_GIANT_PAGE_SIZE && size >= (align << 1)) + while (align < alignment_size && size >= (align << 1)) align <<= 1; /* Add padding to guarantee proper alignment and leave guard @@ -2168,6 +2176,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; + char *maxVaAlignStr; unsigned int guardPages = 1; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2199,6 +2208,13 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (!guardPagesStr || sscanf(guardPagesStr, "%u", &guardPages) != 1) guardPages = 1; + /* Sets the max VA alignment order size during mapping. By default the order + * size is set to 9(2MB) + */ + maxVaAlignStr = getenv("HSA_MAX_VA_ALIGN"); + if (!maxVaAlignStr || sscanf(maxVaAlignStr, "%u", &svm.alignment_order) != 1) + svm.alignment_order = 9; + gpu_mem_count = 0; g_first_gpu_mem = NULL; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index e0c42de22d..e4246e0b4b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -67,7 +67,6 @@ extern int PAGE_SHIFT; /* 2MB huge page size for 4-level page tables on Vega10 and later GPUs */ #define GPU_HUGE_PAGE_SIZE (2 << 20) -#define GPU_GIANT_PAGE_SIZE (1 << 30) #define CHECK_PAGE_MULTIPLE(x) \ do { if ((uint64_t)PORT_VPTR_TO_UINT64(x) % PAGE_SIZE) return HSAKMT_STATUS_INVALID_PARAMETER; } while(0) From f7978b1ff6c31eb5eb48f2a560281bf5c7109413 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 19 May 2022 12:10:05 -0400 Subject: [PATCH 1010/1247] run_kfdtest.sh: Add --exclude flag to exclude additional subtests Currently, using --gtest_filter will override any exclusions from kfdtest.exclude. To add an additional test (or set of tests) to the exclusion list dynamically, the --exclude (-e) flag will allow the user to pass in a string of GTest-style exclusions to add to the list generated by kfdtest.exclude e.g. run_kfdtest.sh -e "*KFDLocalMemoryTest.*:KFDEventTest.*" will use kfdtest.exclude, but will also exclude all LocalMemory and Event tests Change-Id: Ic23ec271ba2cd2240d2e98558c0117ff2a064ed2 --- tests/kfdtest/scripts/run_kfdtest.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index 6311e1b9a5..f6a5a9b95d 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -80,6 +80,7 @@ GDB="" NODE="" FORCE_HIGH="" RUN_IN_DOCKER="" +ADDITIONAL_EXCLUDE="" printUsage() { echo @@ -95,6 +96,7 @@ printUsage() { echo " -l , --list List available nodes" echo " --high Force clocks to high for test execution" echo " -d , --docker Run in docker container" + echo " -e , --exclude Additional tests to exclude, in addition to kfdtest.exclude (colon-separated, single quoted string as an argument)" echo " -h , --help Prints this help" echo echo "Gtest arguments will be forwarded to the app" @@ -122,6 +124,9 @@ getFilter() { gtestFilter="--gtest_filter=${FILTER[$platform]}" ;; esac + if [ -n "$ADDITIONAL_EXCLUDE" ]; then + gtestFilter="$gtestFilter:$ADDITIONAL_EXCLUDE" + fi } TOPOLOGY_SYSFS_DIR=/sys/devices/virtual/kfd/kfd/topology/nodes @@ -168,6 +173,11 @@ runKfdTest() { PKG_ROOT="$(getPackageRoot)" fi + if [ -n "$GTEST_ARGS" ] && [ -n "$ADDITIONAL_EXCLUDE" ]; then + echo "Cannot use -e and --gtest_filter flags together" + exit 0 + fi + if [ "$NODE" == "" ]; then hsaNodes=$(getHsaNodes) @@ -242,6 +252,8 @@ while [ "$1" != "" ]; do FORCE_HIGH="true" ;; -d | --docker ) RUN_IN_DOCKER="true" ;; + -e | --exclude ) + ADDITIONAL_EXCLUDE="$2" ; shift ;; -h | --help ) printUsage; exit 0 ;; *) From 88778e13dc1d62f5a3b4b3f66b4286a0f970bba9 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 7 Apr 2022 10:14:03 -0400 Subject: [PATCH 1011/1247] libhsakmt: hsaKmtGetNodeProperties add gpu_id Add KFDGpuID to HsaNodeProperties to return gpu_id to upper layer, gpu_id is hash ID generated by KFD to distinguish GPUs on the system. ROCr and ROCProfiler will use gpu_id to analyze SMI event message. Change-Id: I6eabe6849230e04120674f5bc55e6ea254a532d6 Signed-off-by: Philip Yang --- include/hsakmttypes.h | 4 +++- src/fmm.c | 7 +++---- src/libhsakmt.h | 1 - src/topology.c | 26 ++++++++++++-------------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 9063f851bd..ab2591bf8d 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -328,7 +328,9 @@ typedef struct _HsaNodeProperties HSAuint32 VGPRSizePerCU; // VGPR size in bytes per CU HSAuint32 SGPRSizePerCU; // SGPR size in bytes per CU - HSAuint8 Reserved[12]; + + HSAuint32 KFDGpuID; // GPU Hash ID generated by KFD + HSAuint8 Reserved[8]; } HsaNodeProperties; diff --git a/src/fmm.c b/src/fmm.c index 35da3b8cf3..92b76e1a19 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2170,7 +2170,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i; int32_t gpu_mem_id = 0; - uint32_t gpu_id; HsaNodeProperties props; struct kfd_process_device_apertures *process_apertures; uint32_t num_of_sysfs_nodes; @@ -2235,14 +2234,14 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); - ret = topology_sysfs_get_node_props(i, &props, &gpu_id, NULL, NULL); + ret = topology_sysfs_get_node_props(i, &props, NULL, NULL); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; topology_setup_is_dgpu_param(&props); /* Skip non-GPU nodes */ - if (gpu_id != 0) { + if (props.KFDGpuID) { int fd = open_drm_render_device(props.DrmRenderMinor); if (fd <= 0) { ret = HSAKMT_STATUS_ERROR; @@ -2254,7 +2253,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_count].EngineId.ui32.Stepping = props.EngineId.ui32.Stepping; gpu_mem[gpu_mem_count].drm_render_fd = fd; - gpu_mem[gpu_mem_count].gpu_id = gpu_id; + gpu_mem[gpu_mem_count].gpu_id = props.KFDGpuID; gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; gpu_mem[gpu_mem_count].node_id = i; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index e4246e0b4b..822744b6b7 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -173,7 +173,6 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - uint32_t *gpu_id, bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); void topology_setup_is_dgpu_param(HsaNodeProperties *props); diff --git a/src/topology.c b/src/topology.c index 81ff62f572..99a6a03c89 100644 --- a/src/topology.c +++ b/src/topology.c @@ -56,7 +56,6 @@ #define KFD_SYSFS_PATH_NODES "/sys/devices/virtual/kfd/kfd/topology/nodes" typedef struct { - uint32_t gpu_id; HsaNodeProperties node; HsaMemoryProperties *mem; /* node->NumBanks elements */ HsaCacheProperties *cache; @@ -1037,7 +1036,6 @@ static int topology_get_marketing_name(int minor, uint16_t *marketing_name) HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - uint32_t *gpu_id, bool *p2p_links, uint32_t *num_p2pLinks) { @@ -1056,13 +1054,14 @@ HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; assert(props); - assert(gpu_id); ret = topology_sysfs_map_node_id(node_id, &sys_node_id); if (ret != HSAKMT_STATUS_SUCCESS) return ret; /* Retrieve the GPU ID */ - ret = topology_sysfs_get_gpu_id(sys_node_id, gpu_id); + ret = topology_sysfs_get_gpu_id(sys_node_id, &props->KFDGpuID); + if (ret != HSAKMT_STATUS_SUCCESS) + return ret; read_buf = malloc(PAGE_SIZE); if (!read_buf) @@ -1723,7 +1722,7 @@ static int32_t gpu_get_direct_link_cpu(uint32_t gpu_node, node_props_t *node_pro HsaIoLinkProperties *props = node_props[gpu_node].link; uint32_t i; - if (!node_props[gpu_node].gpu_id || !props || + if (!node_props[gpu_node].node.KFDGpuID || !props || node_props[gpu_node].node.NumIOLinks == 0) return -1; @@ -1776,7 +1775,7 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, return HSAKMT_STATUS_INVALID_PARAMETER; /* CPU->CPU is not an indirect link */ - if (!node_props[node1].gpu_id && !node_props[node2].gpu_id) + if (!node_props[node1].node.KFDGpuID && !node_props[node2].node.KFDGpuID) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (node_props[node1].node.HiveID && @@ -1784,16 +1783,16 @@ static HSAKMT_STATUS get_indirect_iolink_info(uint32_t node1, uint32_t node2, node_props[node1].node.HiveID == node_props[node2].node.HiveID) return HSAKMT_STATUS_INVALID_PARAMETER; - if (node_props[node1].gpu_id) + if (node_props[node1].node.KFDGpuID) dir_cpu1 = gpu_get_direct_link_cpu(node1, node_props); - if (node_props[node2].gpu_id) + if (node_props[node2].node.KFDGpuID) dir_cpu2 = gpu_get_direct_link_cpu(node2, node_props); if (dir_cpu1 < 0 && dir_cpu2 < 0) return HSAKMT_STATUS_ERROR; /* if the node2(dst) is GPU , it need to be large bar for host access*/ - if (node_props[node2].gpu_id) { + if (node_props[node2].node.KFDGpuID) { for (i = 0; i < node_props[node2].node.NumMemoryBanks; ++i) if (node_props[node2].mem[i].HeapType == HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC) @@ -1922,7 +1921,6 @@ retry: for (i = 0; i < sys_props.NumNodes; i++) { ret = topology_sysfs_get_node_props(i, &temp_props[i].node, - &temp_props[i].gpu_id, &p2p_links, &num_p2pLinks); if (ret != HSAKMT_STATUS_SUCCESS) { free_properties(temp_props, i); @@ -1963,7 +1961,7 @@ retry: goto err; } } - } else if (!temp_props[i].gpu_id) { /* a CPU node */ + } else if (!temp_props[i].node.KFDGpuID) { /* a CPU node */ ret = topology_get_cpu_cache_props( i, cpuinfo, &temp_props[i]); if (ret != HSAKMT_STATUS_SUCCESS) { @@ -2104,7 +2102,7 @@ HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) if (!g_props || !g_system || g_system->NumNodes <= nodeid) return HSAKMT_STATUS_INVALID_NODE_UNIT; if (gpu_id) - *gpu_id = g_props[nodeid].gpu_id; + *gpu_id = g_props[nodeid].node.KFDGpuID; return HSAKMT_STATUS_SUCCESS; } @@ -2114,7 +2112,7 @@ HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id) uint64_t node_idx; for (node_idx = 0; node_idx < g_system->NumNodes; node_idx++) { - if (g_props[node_idx].gpu_id == gpu_id) { + if (g_props[node_idx].node.KFDGpuID == gpu_id) { *node_id = node_idx; return HSAKMT_STATUS_SUCCESS; } @@ -2383,7 +2381,7 @@ uint16_t get_device_id_by_gpu_id(HSAuint32 gpu_id) return 0; for (i = 0; i < g_system->NumNodes; i++) { - if (g_props[i].gpu_id == gpu_id) + if (g_props[i].node.KFDGpuID == gpu_id) return g_props[i].node.DeviceId; } From 405fbd6f93780eb24907f64c41134ebc79be1c3e Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 14 Jan 2022 16:25:06 -0500 Subject: [PATCH 1012/1247] libhsakmt: add open SMI event handle System Management Interface event is read from anonymous file handle, this helper wrap the ioctl interface to get anonymous file handle for GPU nodeid. Define SMI event IDs, event triggers, copy the same value from kfd_ioctl.h to avoid translation. Change-Id: I5c8ba5301473bb3b80bb4e2aa33a9f675bedb001 Signed-off-by: Philip Yang --- include/hsakmt.h | 16 ++++++++++++++ include/hsakmttypes.h | 49 +++++++++++++++++++++++++++++++++++++++++++ src/events.c | 27 ++++++++++++++++++++++++ src/libhsakmt.ver | 1 + 4 files changed, 93 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index abc617f42b..ca586ba36b 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -877,6 +877,22 @@ hsaKmtGetXNACKMode( HSAint32 * enable // OUT: returns XNACK value. ); +/** + Open anonymous file handle to enable events and read SMI events. + + To enable events, write 64bit events mask to fd, event enums as bit index. + for example, event mask (HSA_SMI_EVENT_MASK_FROM_INDEX(HSA_SMI_EVENT_INDEX_MAX) - 1) to enable all events + + Read event from fd is not blocking, use poll with timeout value to check if event is available. + Event is dropped if kernel event fifo is full. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtOpenSMI( + HSAuint32 NodeId, // IN: GPU node_id to receive the SMI event from + int *fd // OUT: anonymous file handle +); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index ab2591bf8d..690e001b2b 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1354,6 +1354,55 @@ typedef struct _HSA_SVM_ATTRIBUTE { HSAuint32 value; // attribute value } HSA_SVM_ATTRIBUTE; +typedef enum _HSA_SMI_EVENT { + HSA_SMI_EVENT_NONE = 0, /* not used */ + HSA_SMI_EVENT_VMFAULT = 1, /* event start counting at 1 */ + HSA_SMI_EVENT_THERMAL_THROTTLE = 2, + HSA_SMI_EVENT_GPU_PRE_RESET = 3, + HSA_SMI_EVENT_GPU_POST_RESET = 4, + HSA_SMI_EVENT_MIGRATE_START = 5, + HSA_SMI_EVENT_MIGRATE_END = 6, + HSA_SMI_EVENT_PAGE_FAULT_START = 7, + HSA_SMI_EVENT_PAGE_FAULT_END = 8, + HSA_SMI_EVENT_QUEUE_EVICTION = 9, + HSA_SMI_EVENT_QUEUE_RESTORE = 10, + HSA_SMI_EVENT_UNMAP_FROM_GPU = 11, + HSA_SMI_EVENT_INDEX_MAX = 12, + + /* + * max event number, as a flag bit to get events from all processes, + * this requires super user permission, otherwise will not be able to + * receive event from any process. Without this flag to receive events + * from same process. + */ + HSA_SMI_EVENT_ALL_PROCESS = 64 +} HSA_EVENT_TYPE; + +typedef enum _HSA_MIGRATE_TRIGGERS { + HSA_MIGRATE_TRIGGER_PREFETCH, + HSA_MIGRATE_TRIGGER_PAGEFAULT_GPU, + HSA_MIGRATE_TRIGGER_PAGEFAULT_CPU, + HSA_MIGRATE_TRIGGER_TTM_EVICTION +} HSA_MIGRATE_TRIGGERS; + +typedef enum _HSA_QUEUE_EVICTION_TRIGGERS { + HSA_QUEUE_EVICTION_TRIGGER_SVM, + HSA_QUEUE_EVICTION_TRIGGER_USERPTR, + HSA_QUEUE_EVICTION_TRIGGER_TTM, + HSA_QUEUE_EVICTION_TRIGGER_SUSPEND, + HSA_QUEUE_EVICTION_CRIU_CHECKPOINT, + HSA_QUEUE_EVICTION_CRIU_RESTORE +} HSA_QUEUE_EVICTION_TRIGGERS; + +typedef enum _HSA_SVM_UNMAP_TRIGGERS { + HSA_SVM_UNMAP_TRIGGER_MMU_NOTIFY, + HSA_SVM_UNMAP_TRIGGER_MMU_NOTIFY_MIGRATE, + HSA_SVM_UNMAP_TRIGGER_UNMAP_FROM_CPU +} HSA_SVM_UNMAP_TRIGGERS; + +#define HSA_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1)) +#define HSA_SMI_EVENT_MSG_SIZE 96 + #pragma pack(pop, hsakmttypes_h) diff --git a/src/events.c b/src/events.c index d4c751c0cc..06d3959532 100644 --- a/src/events.c +++ b/src/events.c @@ -339,3 +339,30 @@ out: return result; } + +HSAKMT_STATUS HSAKMTAPI hsaKmtOpenSMI(HSAuint32 NodeId, int *fd) +{ + struct kfd_ioctl_smi_events_args args; + HSAKMT_STATUS result; + uint32_t gpuid; + + CHECK_KFD_OPEN(); + + pr_debug("[%s] node %d\n", __func__, NodeId); + + result = validate_nodeid(NodeId, &gpuid); + if (result != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); + return result; + } + + args.gpuid = gpuid; + result = kmtIoctl(kfd_fd, AMDKFD_IOC_SMI_EVENTS, &args); + if (result) { + pr_debug("open SMI event fd failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + *fd = args.anon_fd; + return HSAKMT_STATUS_SUCCESS; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 50c309dc0b..46370c6eee 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -69,6 +69,7 @@ hsaKmtSVMSetAttr; hsaKmtSVMGetAttr; hsaKmtSetXNACKMode; hsaKmtGetXNACKMode; +hsaKmtOpenSMI; local: *; }; From e17b15923050e87d54218ceaeb8cfbe10df8a8df Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 20 May 2022 12:02:19 -0400 Subject: [PATCH 1013/1247] libhsakmt: Make queue memory allocation non-paged Non-paged allocation for queue memory necessary for binding wptr to GART. Required to support usermode queue oversubscription with MES on GFX11. Change ensures queue memory does not specify ATS. Signed-off-by: Graham Sider Change-Id: I10b23b0205c90dad902c711a88cfb5e9b4979617 --- src/queues.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/queues.c b/src/queues.c index c83dd93038..d38ea0c6f1 100644 --- a/src/queues.c +++ b/src/queues.c @@ -364,12 +364,13 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) static void *allocate_exec_aligned_memory(uint32_t size, bool use_ats, uint32_t NodeId, + bool nonPaged, bool DeviceLocal, bool Uncached) { if (!use_ats) return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId, - DeviceLocal, DeviceLocal, + nonPaged, DeviceLocal, Uncached); return allocate_exec_aligned_memory_cpu(size); } @@ -412,7 +413,7 @@ static int handle_concrete_asic(struct queue *q, if (q->eop_buffer_size > 0) { q->eop_buffer = allocate_exec_aligned_memory(q->eop_buffer_size, q->use_ats, - NodeId, true, /* Unused for VRAM */false); + NodeId, true, true, /* Unused for VRAM */false); if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; @@ -436,7 +437,7 @@ static int handle_concrete_asic(struct queue *q, q->debug_memory_size; q->ctx_save_restore = allocate_exec_aligned_memory(total_mem_alloc_size, - q->use_ats, NodeId, false, false); + q->use_ats, NodeId, false, false, false); if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; @@ -477,7 +478,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, int err; HsaNodeProperties props; uint32_t cu_num, i; - bool use_ats; CHECK_KFD_OPEN(); @@ -489,18 +489,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, if (result != HSAKMT_STATUS_SUCCESS) return result; - use_ats = prefer_ats(NodeId); - struct queue *q = allocate_exec_aligned_memory(sizeof(*q), - use_ats, - NodeId, false, true); + false, NodeId, true, false, true); if (!q) return HSAKMT_STATUS_NO_MEMORY; memset(q, 0, sizeof(*q)); q->gfxv = get_gfxv_by_node_id(NodeId); - q->use_ats = use_ats; + q->use_ats = false; q->eop_buffer_size = EOP_BUFFER_SIZE(q->gfxv); /* By default, CUs are all turned on. Initialize cu_mask to '1 From 350eba3a074a214f6ff8888069af7c9ed9d5bfcc Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 16 Jun 2022 17:16:16 -0400 Subject: [PATCH 1014/1247] kfdtest: Update gpuName logic for ip discovery Kernel amd-staging-drm-next branch changes GFX11 fish_colour sysfs naming to "ip discovery". Update run_kfdtest.sh to use sysfs gfx_target_version for ASICs that have transitioned to IP discovery topology. Signed-off-by: Graham Sider Change-Id: If202a0ceeed7324364539a33661f0abcf0973f07 --- tests/kfdtest/scripts/kfdtest.exclude | 5 +++-- tests/kfdtest/scripts/run_kfdtest.sh | 12 ++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index fcd5d0fcbb..2680e7b352 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -290,17 +290,18 @@ FILTER[yellow_carp]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X" -FILTER[plum_bonito]=\ +FILTER[gfx1100]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X:"\ "$TEMPORARY_BLACKLIST_GFX11" +# TODO: Update to gfx version when kernel moves to staging-drm-next FILTER[wheat_nas]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X:"\ "$TEMPORARY_BLACKLIST_GFX11" -FILTER[hotpink_bonefish]=\ +FILTER[gfx1102]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X:"\ "$TEMPORARY_BLACKLIST_GFX11" diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index f6a5a9b95d..f84ed0e760 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -144,7 +144,8 @@ getHsaNodes() { } -# Prints GPU Name for the given Node ID +# Prints GPU Name for the given Node ID. If transitioned to IP discovery, +# use target gfx version # param - Node ID getNodeName() { local nodeId=$1; shift; @@ -153,7 +154,14 @@ getNodeName() { local CpuCoresCount=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep cpu_cores_count | awk '{print $2}') local SimdCount=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep simd_count | awk '{print $2}') if [ "$CpuCoresCount" -eq 0 ] && [ "$SimdCount" -gt 0 ]; then - gpuName="raven_dgpuFallback" + gpuName="raven_dgpuFallback" + fi + elif [ "$gpuName" == "ip discovery" ]; then + if [ -n "$HSA_OVERRIDE_GFX_VERSION" ]; then + gpuName="gfx$(echo "$HSA_OVERRIDE_GFX_VERSION" | awk 'BEGIN {FS="."; RS=""} {printf "%d%x%x", $1, $2, $3 }')" + else + local GfxVersionDec=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep gfx_target_version | awk '{print $2}') + gpuName="gfx$(printf "$GfxVersionDec" | fold -w2 | awk 'BEGIN {FS="\n"; RS=""} {printf "%d%x%x", $1, $2, $3}')" fi fi echo "$gpuName" From 79cd63fab6096990b4ddafdaa1ab12be56f2d228 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 9 Jun 2022 13:41:31 -0400 Subject: [PATCH 1015/1247] libhsakmt: permit runtime enable version for new hw mode set restrictions The KFD no longer allow debug ops that modify HW state prior to trap activation so permit bump in major version. Signed-off-by: Jonathan Kim Change-Id: I072d3998b7b043df9a67f0f6762b0afdfa9382c6 --- src/debug.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 2703722919..a7cd6a9c76 100644 --- a/src/debug.c +++ b/src/debug.c @@ -297,6 +297,7 @@ hsaKmtGetKernelDebugTrapVersionInfo( } #define HSA_RUNTIME_ENABLE_MIN_MAJOR 10 +#define HSA_RUNTIME_ENABLE_MAX_MAJOR 11 #define HSA_RUNTIME_ENABLE_MIN_MINOR 0 HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, bool setupTtmp) @@ -310,8 +311,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, if (result) return HSAKMT_STATUS_NOT_SUPPORTED; - if (kMajor != HSA_RUNTIME_ENABLE_MIN_MAJOR || - (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR) + if (kMajor < HSA_RUNTIME_ENABLE_MIN_MAJOR || kMajor > HSA_RUNTIME_ENABLE_MAX_MAJOR || + (kMajor == HSA_RUNTIME_ENABLE_MIN_MAJOR && + (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR)) return HSAKMT_STATUS_NOT_SUPPORTED; memset(&args, 0x00, sizeof(args)); From 7799611c016ba7bdb7551ee63eb24fc54c6a6eb8 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 14 Jan 2022 16:30:41 -0500 Subject: [PATCH 1016/1247] kfdtest: add KFDSVMRangeTest.HMMProfilingEvent Open SMI event file handle, prefetch to migrate svm range to GPU, read HMM profiling events, then check event_id, address, size, pid, event triggers are the expected value. Start separate thread to read SMI event, the same way applications use. Use thread barrier to ensure no event is dropped. Change-Id: I0683969d18d1579847e125d86aa4257602adb13f Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 83 +++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 5bdcf5b80f..34484801ac 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -21,6 +21,7 @@ * */ #include "KFDSVMRangeTest.hpp" +#include #include #include #include "PM4Queue.hpp" @@ -1401,3 +1402,85 @@ queue_fail: TEST_END } + +/* + * Test SMI HMM SVM profiling event + * Use separate thread to read event the same way as ROCr and ROCProfiler + */ +struct ReadEventThreadParams { + int nodeid; + HSAuint64 *pBuf; + int BufSize; + pthread_barrier_t *barrier; +}; + +unsigned int ReadSMIEventThread(void* p) { + struct ReadEventThreadParams *pArgs = (struct ReadEventThreadParams *)p; + char msg[HSA_SMI_EVENT_MSG_SIZE]; + struct pollfd fds = {0}; + HSAuint64 events; + int fd; + + EXPECT_SUCCESS(hsaKmtOpenSMI(pArgs->nodeid, &fd)); + events = HSA_SMI_EVENT_MASK_FROM_INDEX(HSA_SMI_EVENT_INDEX_MAX) - 1; + EXPECT_EQ(write(fd, &events, sizeof(events)), sizeof(events)); + + pthread_barrier_wait(pArgs->barrier); + + fds.fd = fd; + fds.events = POLLIN; + EXPECT_GE(poll(&fds, 1, 1000), 0); + + memset(msg, 0, sizeof(msg)); + EXPECT_GE(read(fd, msg, HSA_SMI_EVENT_MSG_SIZE), 0); + + int event_id, pid, size, trigger, unused; + HSAuint64 timestamp; + HSAuint64 addr; + EXPECT_EQ(sscanf(msg, "%x %ld -%d @%lx(%d) %d->%x %x:%d %d\n", &event_id, ×tamp, &pid, + &addr, &size, &unused, &unused, &unused, &unused, &trigger), 10); + EXPECT_EQ(event_id, HSA_SMI_EVENT_MIGRATE_START); + EXPECT_EQ((HSAuint64 *)(addr << PAGE_SHIFT), pArgs->pBuf); + EXPECT_EQ(size << PAGE_SHIFT, pArgs->BufSize); + EXPECT_EQ(pid, getpid()); + EXPECT_EQ(trigger, HSA_MIGRATE_TRIGGER_PREFETCH); + close(fd); +} + +TEST_F(KFDSVMRangeTest, HMMProfilingEvent) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + if (m_VersionInfo.KernelInterfaceMinorVersion < 10) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (!GetVramSize(defaultGPUNode)) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + pthread_barrier_t barrier; + ASSERT_SUCCESS(pthread_barrier_init(&barrier, NULL, 2)); + + int BufSize = 16 << 10; + HsaSVMRange SysBuffer(BufSize, defaultGPUNode); + HSAuint64 *pBuf = SysBuffer.As(); + + struct ReadEventThreadParams pArgs = {defaultGPUNode, pBuf, BufSize, &barrier}; + uint64_t threadId; + ASSERT_EQ(true, StartThread(&ReadSMIEventThread, &pArgs, threadId)); + + pthread_barrier_wait(&barrier); + + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufSize, defaultGPUNode)); + + WaitForThread(threadId); + + TEST_END +} From e1d1a6fbb0e060183ea7a0c5fbbc7b95950827ab Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Mon, 27 Jun 2022 13:22:36 -0400 Subject: [PATCH 1017/1247] libhsakmt: add new flag for svm It is to add new option for always keeping gpu mapping and bump KFD version for the feature of unified save restore memory. Signed-off-by: Eric Huang Change-Id: Iebee35e6de4d52fa29f82dd19f6bbf5640249492 --- include/linux/kfd_ioctl.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index ba8de4b211..4898451b22 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -35,9 +35,11 @@ * - 1.7 - Checkpoint Restore (CRIU) API * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs * - 1.9 - Add available_memory ioctl + * - 1.10 - Add SMI profiler event log + * - 1.11 - Add unified memory for ctx save/restore area */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 9 +#define KFD_IOCTL_MINOR_VERSION 11 /* * Debug revision change log @@ -1080,6 +1082,8 @@ struct kfd_ioctl_cross_memory_copy_args { #define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010 /* GPUs mostly read, may allow similar optimizations as RO, but writes fault */ #define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020 +/* Keep GPU memory mapping always valid as if XNACK is disable */ +#define KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED 0x00000040 /** * kfd_ioctl_svm_op - SVM ioctl operations From 37be876cadcbed1261a80254edef65d2b50d7a02 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Mon, 6 Jun 2022 13:57:36 -0400 Subject: [PATCH 1018/1247] libhsakmt: allocate unified memory for ctx save restore area To improve performance on queue preemption, allocate ctx s/r area in VRAM instead of system memory, and migrate it back to system memory when VRAM is full. Signed-off-by: Eric Huang Change-Id: If775782027188dbe84b6868260e429373675434c --- include/hsakmttypes.h | 1 + src/queues.c | 109 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 95 insertions(+), 15 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 690e001b2b..65f23dec51 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1331,6 +1331,7 @@ typedef enum _HSA_SVM_FLAGS { HSA_SVM_FLAG_GPU_RO = 0x00000008, // GPUs only read, allows replication HSA_SVM_FLAG_GPU_EXEC = 0x00000010, // Allow execution on GPU HSA_SVM_FLAG_GPU_READ_MOSTLY = 0x00000020, // GPUs mostly read, may allow similar optimizations as RO, but writes fault + HSA_SVM_FLAG_GPU_ALWAYS_MAPPED = 0x00000040, // Keep GPU memory mapping always valid as if XNACK is disable } HSA_SVM_FLAGS; typedef enum _HSA_SVM_ATTR_TYPE { diff --git a/src/queues.c b/src/queues.c index d38ea0c6f1..5702c95329 100644 --- a/src/queues.c +++ b/src/queues.c @@ -68,6 +68,7 @@ struct queue { uint32_t eop_buffer_size; uint32_t gfxv; bool use_ats; + bool unified_ctx_save_restore; /* This queue structure is allocated from GPU with page aligned size * but only small bytes are used. We use the extra space in the end for * cu_mask bits array. @@ -384,13 +385,49 @@ static void free_exec_aligned_memory(void *addr, uint32_t size, uint32_t align, munmap(addr, size); } +static HSAKMT_STATUS register_svm_range(void *mem, uint32_t size, + uint32_t gpuNode, uint32_t prefetchNode, + uint32_t preferredNode, bool alwaysMapped) +{ + HSA_SVM_ATTRIBUTE *attrs; + HSAuint64 s_attr; + HSAuint32 nattr; + HSAuint32 flags; + + flags = HSA_SVM_FLAG_HOST_ACCESS; + + if (alwaysMapped) { + CHECK_KFD_MINOR_VERSION(11); + flags |= HSA_SVM_FLAG_GPU_ALWAYS_MAPPED; + } + + nattr = 5; + s_attr = sizeof(*attrs) * nattr; + attrs = (HSA_SVM_ATTRIBUTE *)alloca(s_attr); + + attrs[0].type = HSA_SVM_ATTR_PREFETCH_LOC; + attrs[0].value = prefetchNode; + attrs[1].type = HSA_SVM_ATTR_PREFERRED_LOC; + attrs[1].value = preferredNode; + attrs[2].type = HSA_SVM_ATTR_CLR_FLAGS; + attrs[2].value = ~flags; + attrs[3].type = HSA_SVM_ATTR_SET_FLAGS; + attrs[3].value = flags; + attrs[4].type = HSA_SVM_ATTR_ACCESS; + attrs[4].value = gpuNode; + + return hsaKmtSVMSetAttr(mem, size, nattr, attrs); +} + static void free_queue(struct queue *q) { if (q->eop_buffer) free_exec_aligned_memory(q->eop_buffer, q->eop_buffer_size, PAGE_SIZE, q->use_ats); - if (q->ctx_save_restore) + if (q->unified_ctx_save_restore) + free(q->ctx_save_restore); + else if (q->ctx_save_restore) free_exec_aligned_memory(q->ctx_save_restore, q->ctx_save_restore_size, PAGE_SIZE, q->use_ats); @@ -398,6 +435,20 @@ static void free_queue(struct queue *q) free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->use_ats); } +static inline void fill_cwsr_header(struct queue *q, void *addr, + HsaEvent *Event, volatile HSAint64 *ErrPayload) +{ + HsaUserContextSaveAreaHeader *header = + (HsaUserContextSaveAreaHeader *)addr; + + header->ErrorEventId = 0; + if (Event) + header->ErrorEventId = Event->EventId; + header->ErrorReason = ErrPayload; + header->DebugOffset = q->ctx_save_restore_size; + header->DebugSize = q->debug_memory_size; +} + static int handle_concrete_asic(struct queue *q, struct kfd_ioctl_create_queue_args *args, uint32_t NodeId, @@ -425,7 +476,8 @@ static int handle_concrete_asic(struct queue *q, if (ret) { uint32_t total_mem_alloc_size = 0; - HsaUserContextSaveAreaHeader *header; + HsaNodeProperties node; + bool svm_api; args->ctx_save_restore_size = q->ctx_save_restore_size; args->ctl_stack_size = q->ctl_stack_size; @@ -435,22 +487,49 @@ static int handle_concrete_asic(struct queue *q, */ total_mem_alloc_size = q->ctx_save_restore_size + q->debug_memory_size; - q->ctx_save_restore = - allocate_exec_aligned_memory(total_mem_alloc_size, - q->use_ats, NodeId, false, false, false); - if (!q->ctx_save_restore) - return HSAKMT_STATUS_NO_MEMORY; + if (hsaKmtGetNodeProperties(NodeId, &node)) + svm_api = false; + else + svm_api = node.Capability.ui32.SVMAPISupported; + + /* Allocate unified memory for context save restore + * area on dGPU. + */ + if (!q->use_ats && svm_api) { + uint32_t size = PAGE_ALIGN_UP(total_mem_alloc_size); + void *addr; + HSAKMT_STATUS r = HSAKMT_STATUS_ERROR; + + if (posix_memalign(&addr, GPU_HUGE_PAGE_SIZE, size)) + pr_err("[%s] posix_memalign failed:\n", __func__); + else { + fill_cwsr_header(q, addr, Event, ErrPayload); + + r = register_svm_range(addr, size, + NodeId, NodeId, 0, true); + + if (r == HSAKMT_STATUS_SUCCESS) { + q->ctx_save_restore = addr; + q->unified_ctx_save_restore = true; + } else + free(addr); + } + } + + if (!q->unified_ctx_save_restore) { + q->ctx_save_restore = allocate_exec_aligned_memory( + total_mem_alloc_size, + q->use_ats, NodeId, + false, false, false); + + if (!q->ctx_save_restore) + return HSAKMT_STATUS_NO_MEMORY; + + fill_cwsr_header(q, q->ctx_save_restore, Event, ErrPayload); + } args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; - - header = (HsaUserContextSaveAreaHeader *)q->ctx_save_restore; - header->ErrorEventId = 0; - if (Event) - header->ErrorEventId = Event->EventId; - header->ErrorReason = ErrPayload; - header->DebugOffset = q->ctx_save_restore_size; - header->DebugSize = q->debug_memory_size; } return HSAKMT_STATUS_SUCCESS; From 9745db30538e129305da74627f92ec4fc8ef8ebf Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 3 May 2022 11:06:25 -0400 Subject: [PATCH 1019/1247] KFDTest: Use default packaging name for kfdtest Previously we omitted the version and arch in the filenames. By adding this, as well as the ROCM build variable, this will allow for easy version version detection on systems. Instead of kfdtest = v1.0.0, now it will feature the build number, allowing for easier identification as to which version is installed. Change-Id: I311ed7010486e7c70af669d282910fe29ee8db45 --- tests/kfdtest/CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 6eeffa5585..5c5db26940 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -33,10 +33,24 @@ set ( CPACK_PACKAGE_FILE_NAME "kfdtest" ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION "This package includes kfdtest, the list of excluded tests for each ASIC, and a convenience script to run the test suite" ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "Test suite for ROCK/KFD" ) + +# Make proper version for appending +# Default Value is 99999, setting it first +set(ROCM_VERSION_FOR_PACKAGE "99999") +if(DEFINED ENV{ROCM_LIBPATCH_VERSION}) + set(ROCM_VERSION_FOR_PACKAGE $ENV{ROCM_LIBPATCH_VERSION}) +endif() + set ( CPACK_PACKAGE_VERSION_MAJOR "1" ) set ( CPACK_PACKAGE_VERSION_MINOR "0" ) set ( CPACK_PACKAGE_VERSION_PATCH "0" ) set ( CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) +set ( CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") +set ( CPACK_RPM_FILE_NAME "RPM-DEFAULT") + +set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${ROCM_VERSION_FOR_PACKAGE}") +set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") + ## Define default variable and variables for the optional build target hsakmt-dev set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) From a31206e98b3d483beb60f472d4504674e1ca8b10 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 12 Jul 2022 18:25:32 -0400 Subject: [PATCH 1020/1247] libhsakmt: fix runtime disable check Kernel debug IOCTL got version bumped to v11. Updated runtime enable but missed runtime disable check update. Signed-off-by: Jonathan Kim Change-Id: I71a8970ccfe7dc517abe7b4ad962369aea6a0496 --- src/debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index a7cd6a9c76..14bcb884e9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -346,8 +346,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) if (result) return HSAKMT_STATUS_NOT_SUPPORTED; - if (kMajor != HSA_RUNTIME_ENABLE_MIN_MAJOR || - (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR) + if (kMajor < HSA_RUNTIME_ENABLE_MIN_MAJOR || kMajor > HSA_RUNTIME_ENABLE_MAX_MAJOR || + (kMajor == HSA_RUNTIME_ENABLE_MIN_MAJOR && + (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR)) return HSAKMT_STATUS_NOT_SUPPORTED; memset(&args, 0x00, sizeof(args)); From 4b8c74bf04a72009a610e39cffda14b087300de5 Mon Sep 17 00:00:00 2001 From: David Francis Date: Wed, 6 Jul 2022 09:13:00 -0400 Subject: [PATCH 1021/1247] kfdtest: Change KFDQMTest.EmptyDispatch to NoopIsa When the shaders were moved to ShaderStore, KFDQMTest.EmptyDispatch was erroneously changed to use LoopIsa instead of NoopIsa. Change it back. Signed-off-by: David Francis Change-Id: Iaf7d0d107e3bf3bd8b7d616b137a1740e309cf91 --- tests/kfdtest/src/KFDQMTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index ffc568ebdf..2f156ad23b 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1034,7 +1034,7 @@ TEST_F(KFDQMTest, EmptyDispatch) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(NoopIsa, isaBuffer.As())); SyncDispatch(isaBuffer, NULL, NULL); From 39e8a85aaccc5b45b5bf13672e4a16f84bb1f38a Mon Sep 17 00:00:00 2001 From: David Francis Date: Wed, 6 Jul 2022 09:18:50 -0400 Subject: [PATCH 1022/1247] kfdtest: Remove strange load from LoopIsa LoopIsa is a shader that performs a variety of intensive calculations in a loop. It is used by tests such as KFDQMTest.QueuePriorityOn* It contained a scalar load, despite not having any buffer to read from. This load causes page faults on GFX11. It is unclear why it did not cause page faults on earlier ASICs. Remove the load. Signed-off-by: David Francis Change-Id: I7426d0db48e933f3bb870467ea88476f7a283040 --- tests/kfdtest/src/ShaderStore.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 8b40351f04..06a6c0f95a 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -386,7 +386,6 @@ const char *LoopIsa = R"( s_mov_b32 s11, s7 s_cmp_le_i32 s1, s0 s_cbranch_scc1 END_OF_PGM - s_buffer_load_dwordx8 s[8:15], s[8:11], 0x10 v_add_f32 v0, 2.0, v0 v_cvt_f32_i32 v17, s1 s_waitcnt lgkmcnt(0) From aa25cb1accba432c576162f5d9af227f2246c904 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Tue, 28 Jun 2022 18:20:49 -0400 Subject: [PATCH 1023/1247] Fix numa linking We should link against numa without hardcoding the path to it. CMake should determine how to link numa automatically, similar to how rt and pthread is linked. Fixes Change-Id: Ifb9ac30e200c66cbd7f1cf80d25fffef1dcf8d2f Signed-off-by: Jeremy Newton --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 113b83c720..d00248ee83 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -160,7 +160,7 @@ get_os_info() find_package(PkgConfig) # Check for libraries required for building find_library(LIBC NAMES libc.so.6 REQUIRED) -find_library(NUMA NAMES libnuma.so REQUIRED) +find_library(NUMA NAMES numa REQUIRED) message(STATUS "LIBC:" ${LIBC}) message(STATUS "NUMA:" ${NUMA}) @@ -183,7 +183,7 @@ endif() include_directories(${DRM_INCLUDE_DIRS}) target_link_libraries ( ${HSAKMT_TARGET} - PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt ${LIBC} ${NUMA} + PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt ${LIBC} numa ) target_compile_options(${HSAKMT_TARGET} PRIVATE ${DRM_CFLAGS} ${HSAKMT_C_FLAGS}) From c6da7d1353ad0f662b046be18b81690d1488e436 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 13 Jul 2022 15:16:11 -0400 Subject: [PATCH 1024/1247] kfdtest: Remove fixed tests from GFX11 blacklist Signed-off-by: Graham Sider Change-Id: I2dc785ea88a1a9222c6b7886fb75b6c7d699036a --- tests/kfdtest/scripts/kfdtest.exclude | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 2680e7b352..efd93892dc 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -117,14 +117,11 @@ BLACKLIST_GFX10_NV2X=\ # GFX11 still undergoing debug. Ticket links: # KFDMemoryTest.FlatScratchAccess - SWDEV-329877 # KFDEvictTest.QueueTest - SWDEV-325064 -# KFDQMTest.QueuePriorityOn*/EmptyDispatch - SWDEV-340963 # KFDQMTest.MultipleCpQueuesStressDispatch - SWDEV-340965 # KFDExceptionTest.* - SWDEV-340972 TEMPORARY_BLACKLIST_GFX11=\ "KFDQMTest.CreateAqlCpQueue:"\ -"KFDQMTest.QueuePriorityOn*:"\ "KFDQMTest.MultipleCpQueuesStressDispatch:"\ -"KFDQMTest.EmptyDispatch:"\ "KFDCWSRTest.InterruptRestore:"\ "KFDExceptionTest.*:"\ "KFDEvictTest.QueueTest:"\ From 412b24137edbfd54669fe0094ea6a1be2b842567 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 14 Jul 2022 16:45:12 -0400 Subject: [PATCH 1025/1247] kfdtest: Full TearDown and SetUp in child process With the next patch, child processes need to fully reinitialize the topology in order to recreate the process apertures. Just calling hsaKmtOpenKFD is no longer sufficient. Tests based on KFDMultiProcessTest already did this correctly (KFDHWSTest, eviction tests). This patch fixes KFDExceptionTest and KFDIPCTest. Signed-off-by: Felix Kuehling Change-Id: Iaad24e88ddd29c1105bf791a77891cc55a6072ff --- tests/kfdtest/src/KFDExceptionTest.cpp | 30 +++++++++----------------- tests/kfdtest/src/KFDExceptionTest.hpp | 5 ++++- tests/kfdtest/src/KFDIPCTest.cpp | 9 +++++--- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index ddcf8ceb16..67ccc68336 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -184,11 +184,8 @@ TEST_F(KFDExceptionTest, AddressFault) { m_ChildPid = fork(); if (m_ChildPid == 0) { - m_ChildStatus = hsaKmtOpenKFD(); - if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { - WARN() << "KFD open failed in child process" << std::endl; - return; - } + KFDBaseComponentTest::TearDown(); + KFDBaseComponentTest::SetUp(); HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode, false); @@ -228,11 +225,8 @@ TEST_F(KFDExceptionTest, PermissionFault) { m_ChildPid = fork(); if (m_ChildPid == 0) { - m_ChildStatus = hsaKmtOpenKFD(); - if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { - WARN() << "KFD open failed in child process" << std::endl; - return; - } + KFDBaseComponentTest::TearDown(); + KFDBaseComponentTest::SetUp(); HsaMemoryBuffer readOnlyBuffer(PAGE_SIZE, defaultGPUNode, false /*zero*/, false /*isLocal*/, true /*isExec*/, @@ -278,11 +272,8 @@ TEST_F(KFDExceptionTest, FaultStorm) { m_ChildPid = fork(); if (m_ChildPid == 0) { - m_ChildStatus = hsaKmtOpenKFD(); - if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { - WARN() << "KFD open failed in child process" << std::endl; - return; - } + KFDBaseComponentTest::TearDown(); + KFDBaseComponentTest::SetUp(); TestMemoryException(defaultGPUNode, 0x12345678, 0x76543210, 1024, 1024, 1); } else { @@ -321,11 +312,10 @@ TEST_F(KFDExceptionTest, SdmaQueueException) { if (m_ChildPid == 0) { unsigned int* pDb = NULL; unsigned int *nullPtr = NULL; - m_ChildStatus = hsaKmtOpenKFD(); - if (m_ChildStatus != HSAKMT_STATUS_SUCCESS) { - WARN() << "KFD open failed in child process" << std::endl; - return; - } + + KFDBaseComponentTest::TearDown(); + KFDBaseComponentTest::SetUp(); + m_MemoryFlags.ui32.NonPaged = 1; ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, m_MemoryFlags, reinterpret_cast(&pDb))); diff --git a/tests/kfdtest/src/KFDExceptionTest.hpp b/tests/kfdtest/src/KFDExceptionTest.hpp index df57649e2c..f544d95100 100644 --- a/tests/kfdtest/src/KFDExceptionTest.hpp +++ b/tests/kfdtest/src/KFDExceptionTest.hpp @@ -42,8 +42,11 @@ class KFDExceptionTest : public KFDBaseComponentTest { * child process finishes, gtest assumes the test has finished and * starts the next test while the parent is still active. */ - if (m_ChildPid == 0) + if (m_ChildPid == 0) { + if (!m_ChildStatus && HasFatalFailure()) + m_ChildStatus = HSAKMT_STATUS_ERROR; exit(m_ChildStatus); + } } protected: diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index c18b471307..77a94b6eb7 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -70,8 +70,8 @@ void KFDIPCTest::BasicTestChildProcess(int defaultGPUNode, int *pipefd, HsaMemFl /* Open KFD device for child process. This needs to called before * any memory definitions */ - if (HSAKMT_STATUS_SUCCESS != hsaKmtOpenKFD()) - exit(1); + TearDown(); + SetUp(); SDMAQueue sdmaQueue; HsaSharedMemoryHandle sharedHandleLM; @@ -635,7 +635,10 @@ TEST_F(KFDIPCTest, CrossMemoryAttachTest) { /* Create a child process and share the above Local Memory with it */ m_ChildPid = fork(); - if (m_ChildPid == 0 && hsaKmtOpenKFD() == HSAKMT_STATUS_SUCCESS) { + if (m_ChildPid == 0) { + TearDown(); + SetUp(); + /* Child Process */ status = CrossMemoryAttachChildProcess(defaultGPUNode, pipeCtoP[1], pipePtoC[0], CMA_READ_TEST); From 87aca673e8a9da2bd93a49dc34db2a08e613b008 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 13 Jul 2022 16:51:56 -0400 Subject: [PATCH 1026/1247] libhsakmt: Init apertures in AcquireSystemProperties This allows init_process_apertures to use the whole consistent topoology instead of taking its own partial snapshot. Signed-off-by: Felix Kuehling Change-Id: Ia13e7aa7fcd090ea8d6cacd4babb29a27c20207f --- src/fmm.c | 2 +- src/libhsakmt.h | 4 +-- src/openclose.c | 14 ---------- src/topology.c | 73 ++++++++++++++++++++++++++++++++----------------- 4 files changed, 51 insertions(+), 42 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 92b76e1a19..97d03cfdbd 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2234,7 +2234,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) for (i = 0; i < NumNodes; i++) { memset(&props, 0, sizeof(props)); - ret = topology_sysfs_get_node_props(i, &props, NULL, NULL); + ret = topology_get_node_props(i, &props); if (ret != HSAKMT_STATUS_SUCCESS) goto sysfs_parse_failed; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 822744b6b7..b3cbea7b4f 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -172,9 +172,9 @@ int get_drm_render_fd_by_gpu_id(HSAuint32 gpu_id); HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, uint32_t NumberOfNodes, uint32_t *NodeArray); -HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, HsaNodeProperties *props, - bool *p2p_links, uint32_t *num_p2pLinks); HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); +HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId, + HsaNodeProperties *NodeProperties); void topology_setup_is_dgpu_param(HsaNodeProperties *props); bool topology_is_svm_needed(HSA_ENGINE_ID EngineId); diff --git a/src/openclose.c b/src/openclose.c index 19dbf65894..d52868e963 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -179,14 +179,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (result != HSAKMT_STATUS_SUCCESS) goto topology_sysfs_failed; - result = fmm_init_process_apertures(sys_props.NumNodes); - if (result != HSAKMT_STATUS_SUCCESS) - goto init_process_aperture_failed; - - result = init_process_doorbells(sys_props.NumNodes); - if (result != HSAKMT_STATUS_SUCCESS) - goto init_doorbell_failed; - kfd_open_count = 1; if (init_device_debugging_memory(sys_props.NumNodes) != HSAKMT_STATUS_SUCCESS) @@ -212,10 +204,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) pthread_mutex_unlock(&hsakmt_mutex); return result; - -init_doorbell_failed: - fmm_destroy_process_apertures(); -init_process_aperture_failed: topology_sysfs_failed: kfd_version_failed: close(fd); @@ -235,8 +223,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void) if (--kfd_open_count == 0) { destroy_counter_props(); destroy_device_debugging_memory(); - destroy_process_doorbells(); - fmm_destroy_process_apertures(); if (kfd_fd) { close(kfd_fd); kfd_fd = 0; diff --git a/src/topology.c b/src/topology.c index 99a6a03c89..ed8f0d9dd7 100644 --- a/src/topology.c +++ b/src/topology.c @@ -87,7 +87,7 @@ static const char *supported_processor_vendor_name[] = { }; static HSAKMT_STATUS topology_take_snapshot(void); -static HSAKMT_STATUS topology_drop_snapshot(void); +static void topology_drop_snapshot(void); static const struct hsa_gfxip_table gfxip_lookup_table[] = { /* Kaveri Family */ @@ -1034,10 +1034,10 @@ static int topology_get_marketing_name(int minor, uint16_t *marketing_name) return 0; } -HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, - HsaNodeProperties *props, - bool *p2p_links, - uint32_t *num_p2pLinks) +static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, + HsaNodeProperties *props, + bool *p2p_links, + uint32_t *num_p2pLinks) { FILE *fd; char *read_buf, *p, *envvar, dummy; @@ -2066,15 +2066,10 @@ err: } /* Drop the Snashot of the HSA topology information. Assume lock is held. */ -HSAKMT_STATUS topology_drop_snapshot(void) +void topology_drop_snapshot(void) { - HSAKMT_STATUS err; - - if (!!g_system != !!g_props) { + if (!!g_system != !!g_props) pr_warn("Probably inconsistency?\n"); - err = HSAKMT_STATUS_SUCCESS; - goto out; - } if (g_props) { /* Remove state */ @@ -2090,11 +2085,6 @@ HSAKMT_STATUS topology_drop_snapshot(void) map_user_to_sysfs_node_id = NULL; map_user_to_sysfs_node_id_size = 0; } - - err = HSAKMT_STATUS_SUCCESS; - -out: - return err; } HSAKMT_STATUS validate_nodeid(uint32_t nodeid, uint32_t *gpu_id) @@ -2124,7 +2114,7 @@ HSAKMT_STATUS gpuid_to_nodeid(uint32_t gpu_id, uint32_t *node_id) HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties(HsaSystemProperties *SystemProperties) { - HSAKMT_STATUS err; + HSAKMT_STATUS err = HSAKMT_STATUS_SUCCESS; CHECK_KFD_OPEN(); @@ -2133,14 +2123,36 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAcquireSystemProperties(HsaSystemProperties *Syste pthread_mutex_lock(&hsakmt_mutex); + /* We already have a valid snapshot. Avoid double initialization that + * would leak memory. + */ + if (g_system) { + *SystemProperties = *g_system; + goto out; + } + err = topology_take_snapshot(); if (err != HSAKMT_STATUS_SUCCESS) goto out; assert(g_system); + err = fmm_init_process_apertures(g_system->NumNodes); + if (err != HSAKMT_STATUS_SUCCESS) + goto init_process_apertures_failed; + + err = init_process_doorbells(g_system->NumNodes); + if (err != HSAKMT_STATUS_SUCCESS) + goto init_doorbells_failed; + *SystemProperties = *g_system; - err = HSAKMT_STATUS_SUCCESS; + + goto out; + +init_doorbells_failed: + fmm_destroy_process_apertures(); +init_process_apertures_failed: + topology_drop_snapshot(); out: pthread_mutex_unlock(&hsakmt_mutex); @@ -2149,15 +2161,25 @@ out: HSAKMT_STATUS HSAKMTAPI hsaKmtReleaseSystemProperties(void) { - HSAKMT_STATUS err; - pthread_mutex_lock(&hsakmt_mutex); - err = topology_drop_snapshot(); + destroy_process_doorbells(); + fmm_destroy_process_apertures(); + topology_drop_snapshot(); pthread_mutex_unlock(&hsakmt_mutex); - return err; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId, + HsaNodeProperties *NodeProperties) +{ + if (!g_system || !g_props || NodeId >= g_system->NumNodes) + return HSAKMT_STATUS_ERROR; + + *NodeProperties = g_props[NodeId].node; + return HSAKMT_STATUS_SUCCESS; } HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, @@ -2176,7 +2198,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, if (err != HSAKMT_STATUS_SUCCESS) goto out; - *NodeProperties = g_props[NodeId].node; + err = topology_get_node_props(NodeId, NodeProperties); + if (err != HSAKMT_STATUS_SUCCESS) + goto out; /* For CPU only node don't add any additional GPU memory banks. */ if (gpu_id) { uint64_t base, limit; @@ -2188,7 +2212,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeProperties(HSAuint32 NodeId, &limit) == HSAKMT_STATUS_SUCCESS) NodeProperties->NumMemoryBanks += 1; } - err = HSAKMT_STATUS_SUCCESS; out: pthread_mutex_unlock(&hsakmt_mutex); From 9ac2c7517100c257e746e4ea99b9b20b96cee8c3 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 13 Jul 2022 20:40:43 -0400 Subject: [PATCH 1027/1247] libhsakmt: Map VRAM only on supported peer GPUs hsaKmtMapMemoryToGPU should not try to map VRAM on peer GPUs that don't have an IO-Link to the memory. The new P2P mapping code in KFD will fail otherwise. Signed-off-by: Felix Kuehling Change-Id: I6d59b55651b98756865a0f69eafef3e386372cf3 --- src/fmm.c | 91 ++++++++++++++++++++++++++++++++++++++++++------- src/libhsakmt.h | 3 ++ src/topology.c | 22 ++++++++---- 3 files changed, 96 insertions(+), 20 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 97d03cfdbd..0dea2a8c4c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -184,6 +185,8 @@ typedef struct { */ manageable_aperture_t gpuvm_aperture; /* used for GPUVM on APU, outsidethe canonical address range */ int drm_render_fd; + uint32_t usable_peer_id_num; + uint32_t *usable_peer_id_array; } gpu_mem_t; enum svm_aperture_type { @@ -832,6 +835,17 @@ static int32_t gpu_mem_find_by_gpu_id(uint32_t gpu_id) return -1; } +static int32_t gpu_mem_find_by_node_id(uint32_t node_id) +{ + uint32_t i; + + for (i = 0 ; i < gpu_mem_count ; i++) + if (gpu_mem[i].node_id == node_id) + return i; + + return -1; +} + static manageable_aperture_t *fmm_get_aperture(HsaApertureInfo info) { switch (info.type) { @@ -2170,7 +2184,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i; int32_t gpu_mem_id = 0; - HsaNodeProperties props; struct kfd_process_device_apertures *process_apertures; uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -2233,10 +2246,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) is_dgpu = false; for (i = 0; i < NumNodes; i++) { - memset(&props, 0, sizeof(props)); + HsaNodeProperties props; + ret = topology_get_node_props(i, &props); if (ret != HSAKMT_STATUS_SUCCESS) - goto sysfs_parse_failed; + goto gpu_mem_init_failed; topology_setup_is_dgpu_param(&props); @@ -2245,9 +2259,18 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) int fd = open_drm_render_device(props.DrmRenderMinor); if (fd <= 0) { ret = HSAKMT_STATUS_ERROR; - goto sysfs_parse_failed; + goto gpu_mem_init_failed; } + gpu_mem[gpu_mem_count].usable_peer_id_array = + calloc(NumNodes, sizeof(uint32_t)); + if (!gpu_mem[gpu_mem_count].usable_peer_id_array) { + ret = HSAKMT_STATUS_NO_MEMORY; + goto gpu_mem_init_failed; + } + gpu_mem[gpu_mem_count].usable_peer_id_array[0] = props.KFDGpuID; + gpu_mem[gpu_mem_count].usable_peer_id_num = 1; + gpu_mem[gpu_mem_count].EngineId.ui32.Major = props.EngineId.ui32.Major; gpu_mem[gpu_mem_count].EngineId.ui32.Minor = props.EngineId.ui32.Minor; gpu_mem[gpu_mem_count].EngineId.ui32.Stepping = props.EngineId.ui32.Stepping; @@ -2311,6 +2334,11 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) } for (i = 0 ; i < num_of_sysfs_nodes ; i++) { + HsaNodeProperties nodeProps; + HsaIoLinkProperties linkProps[NumNodes]; + uint32_t nodeId; + uint32_t j; + /* Map Kernel process device data node i <--> gpu_mem_id which * indexes into gpu_mem[] based on gpu_id */ @@ -2320,10 +2348,37 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (all_gpu_id_array_size == gpu_mem_count) { ret = HSAKMT_STATUS_ERROR; - goto invalid_gpu_id; + goto aperture_init_failed; } all_gpu_id_array[all_gpu_id_array_size++] = process_apertures[i].gpu_id; + /* Add this GPU to the usable_peer_id_arrays of all GPUs that + * this GPU has an IO link to. This GPU can map memory + * allocated on those GPUs. + */ + nodeId = gpu_mem[gpu_mem_id].node_id; + ret = topology_get_node_props(nodeId, &nodeProps); + if (ret != HSAKMT_STATUS_SUCCESS) + goto aperture_init_failed; + assert(nodeProps.NumIOLinks <= NumNodes); + ret = topology_get_iolink_props(nodeId, nodeProps.NumIOLinks, + linkProps); + if (ret != HSAKMT_STATUS_SUCCESS) + goto aperture_init_failed; + for (j = 0; j < nodeProps.NumIOLinks; j++) { + int32_t to_gpu_mem_id = + gpu_mem_find_by_node_id(linkProps[j].NodeTo); + uint32_t peer; + + if (to_gpu_mem_id < 0) + continue; + + assert(gpu_mem[to_gpu_mem_id].usable_peer_id_num < NumNodes); + peer = gpu_mem[to_gpu_mem_id].usable_peer_id_num++; + gpu_mem[to_gpu_mem_id].usable_peer_id_array[peer] = + gpu_mem[gpu_mem_id].gpu_id; + } + gpu_mem[gpu_mem_id].lds_aperture.base = PORT_UINT64_TO_VPTR(process_apertures[i].lds_base); gpu_mem[gpu_mem_id].lds_aperture.limit = @@ -2374,7 +2429,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) ret = acquire_vm(gpu_mem[gpu_mem_id].gpu_id, gpu_mem[gpu_mem_id].drm_render_fd); if (ret != HSAKMT_STATUS_SUCCESS) - goto acquire_vm_failed; + goto aperture_init_failed; } all_gpu_id_array_size *= sizeof(uint32_t); @@ -2438,15 +2493,15 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) free(process_apertures); return ret; -invalid_gpu_id: +aperture_init_failed: init_svm_failed: -acquire_vm_failed: set_memory_policy_failed: free(all_gpu_id_array); all_gpu_id_array = NULL; get_aperture_ioctl_failed: free(process_apertures); sysfs_parse_failed: +gpu_mem_init_failed: fmm_destroy_process_apertures(); return ret; } @@ -2455,6 +2510,8 @@ void fmm_destroy_process_apertures(void) { release_mmio(); if (gpu_mem) { + while (gpu_mem_count-- > 0) + free(gpu_mem[gpu_mem_count].usable_peer_id_array); free(gpu_mem); gpu_mem = NULL; } @@ -2635,8 +2692,18 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, sizeof(uint32_t); } else { /* not specified, not registered: map all GPUs */ - args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; - args.n_devices = all_gpu_id_array_size / sizeof(uint32_t); + int32_t gpu_mem_id = gpu_mem_find_by_node_id(obj->node_id); + + if (!obj->userptr && get_device_id_by_node_id(obj->node_id) && + gpu_mem_id >= 0) { + args.device_ids_array_ptr = (uint64_t) + gpu_mem[gpu_mem_id].usable_peer_id_array; + args.n_devices = + gpu_mem[gpu_mem_id].usable_peer_id_num; + } else { + args.device_ids_array_ptr = (uint64_t)all_gpu_id_array; + args.n_devices = all_gpu_id_array_size / sizeof(uint32_t); + } } args.n_success = 0; @@ -3736,7 +3803,5 @@ void fmm_clear_all_mem(void) fmm_clear_aperture(&gpu_mem[i].scratch_physical); } - gpu_mem_count = 0; - free(gpu_mem); - gpu_mem = NULL; + fmm_destroy_process_apertures(); } diff --git a/src/libhsakmt.h b/src/libhsakmt.h index b3cbea7b4f..c9d6fcf46b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -175,6 +175,9 @@ HSAKMT_STATUS validate_nodeid_array(uint32_t **gpu_id_array, HSAKMT_STATUS topology_sysfs_get_system_props(HsaSystemProperties *props); HSAKMT_STATUS topology_get_node_props(HSAuint32 NodeId, HsaNodeProperties *NodeProperties); +HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId, + HSAuint32 NumIoLinks, + HsaIoLinkProperties *IoLinkProperties); void topology_setup_is_dgpu_param(HsaNodeProperties *props); bool topology_is_svm_needed(HSA_ENGINE_ID EngineId); diff --git a/src/topology.c b/src/topology.c index ed8f0d9dd7..da09c193f2 100644 --- a/src/topology.c +++ b/src/topology.c @@ -2339,12 +2339,24 @@ out: return err; } +HSAKMT_STATUS topology_get_iolink_props(HSAuint32 NodeId, + HSAuint32 NumIoLinks, + HsaIoLinkProperties *IoLinkProperties) +{ + if (!g_system || !g_props || NodeId >= g_system->NumNodes) + return HSAKMT_STATUS_ERROR; + + memcpy(IoLinkProperties, g_props[NodeId].link, + NumIoLinks * sizeof(*IoLinkProperties)); + + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, HSAuint32 NumIoLinks, HsaIoLinkProperties *IoLinkProperties) { HSAKMT_STATUS err; - uint32_t i; if (!IoLinkProperties) return HSAKMT_STATUS_INVALID_PARAMETER; @@ -2364,12 +2376,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetNodeIoLinkProperties(HSAuint32 NodeId, goto out; } - for (i = 0; i < MIN(g_props[NodeId].node.NumIOLinks, NumIoLinks); i++) { - assert(g_props[NodeId].link); - IoLinkProperties[i] = g_props[NodeId].link[i]; - } - - err = HSAKMT_STATUS_SUCCESS; + assert(g_props[NodeId].link); + err = topology_get_iolink_props(NodeId, NumIoLinks, IoLinkProperties); out: pthread_mutex_unlock(&hsakmt_mutex); From cdaaf8236aa56f4d45c9cf140dc74f032fc2679b Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 19 Jul 2022 21:46:42 -0400 Subject: [PATCH 1028/1247] kfdtest: Remove CMA tests The CMA feature is deprecated and about to be removed from the DKMS branch. It was never supported upstream. Signed-off-by: Felix Kuehling Change-Id: I64b6213eb3adbdc550542e51181cd8ba6ca4cb45 --- tests/kfdtest/src/KFDIPCTest.cpp | 537 ------------------------------- 1 file changed, 537 deletions(-) diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/tests/kfdtest/src/KFDIPCTest.cpp index 77a94b6eb7..7f8cf21215 100644 --- a/tests/kfdtest/src/KFDIPCTest.cpp +++ b/tests/kfdtest/src/KFDIPCTest.cpp @@ -216,540 +216,3 @@ TEST_F(KFDIPCTest, BasicTest) { TEST_END } - -/* Cross Memory Attach Test. Memory Descriptor Array. - * The following 2 2D-arrays describe the source and destination memory arrays used - * by CMA test. The entry is only valid if Size != 0. Each of these buffers will be - * filled intially with "FillPattern". After the test the srcRange is still expected - * to have the same pattern. The dstRange is expected to have srcRange pattern. - * - * For e.g. for TEST_COUNT = 1, - * srcRange has 2 buffers of size 0x1800. Buf1 filled with 0xA5A5A5A5 and Buf2 - * filled with 0xAAAAAAAA - * dstRange has 3 buffers of size 0x1000. All of them filled 0xFFFFFFFF. - * After Copy: dstBuf1[0-0x1000] is expected to be 0xA5A5A5A5 - * dstBuf2[0-0x800] is expected to be 0xA5A5A5A5 - * dstBuf3[0x800-0x1000] is expected to be 0xAAAAAAAA - * and dstBuf4[0x0-0x1000] is expected to be 0xAAAAAAAA - * - * For this CMA test, after copying only the first and the last of dstBuf is checked - */ - -static testMemoryDescriptor srcRange[CMA_TEST_COUNT][CMA_MEMORY_TEST_ARRAY_SIZE] = { - { /* Memory Type Size FillPattern FirstItem Last item */ - { CMA_MEM_TYPE_USERPTR, 0x801800, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_USERPTR, 0x1800, 0xAAAAAAAA, 0xAAAAAAAA, 0xAAAAAAAA }, - { CMA_MEM_TYPE_USERPTR, 0x0, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_USERPTR, 0x0, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - }, - { - { CMA_MEM_TYPE_SYSTEM, 0x208000, 0xDEADBEEF, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_SYSTEM, 0x4000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_SYSTEM, 0x6000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_SYSTEM, 0x2000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - }, - { - { CMA_MEM_TYPE_LOCAL_MEM, 0x800000, 0xDEADBEEF, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xA5A5A5A5, 0xA5A5A5A5, 0xA5A5A5A5 }, - } -}; - -static testMemoryDescriptor dstRange[CMA_TEST_COUNT][CMA_MEMORY_TEST_ARRAY_SIZE] = { - { - /* Memory Type Size FillPattern FirstItem Last item */ - { CMA_MEM_TYPE_USERPTR, 0x801000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_USERPTR, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xAAAAAAAA }, - { CMA_MEM_TYPE_USERPTR, 0x1000, 0xFFFFFFFF, 0xAAAAAAAA, 0xAAAAAAAA }, - { CMA_MEM_TYPE_USERPTR, 0x0, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, - }, - { - { CMA_MEM_TYPE_SYSTEM, 0x202000, 0xFFFFFFFF, 0xDEADBEEF, 0xDEADBEEF }, - { CMA_MEM_TYPE_SYSTEM, 0x4000, 0xFFFFFFFF, 0xDEADBEEF, 0xDEADBEEF }, - { CMA_MEM_TYPE_SYSTEM, 0x8000, 0xFFFFFFFF, 0xDEADBEEF, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_SYSTEM, 0x6000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, - }, - { - { CMA_MEM_TYPE_LOCAL_MEM, 0x800000, 0xFFFFFFFF, 0xDEADBEEF, 0xDEADBEEF }, - { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, - { CMA_MEM_TYPE_LOCAL_MEM, 0x1000, 0xFFFFFFFF, 0xA5A5A5A5, 0xA5A5A5A5 }, - } -}; - -KFDCMAArray::KFDCMAArray() : m_ValidCount(0), m_QueueArray(HSA_QUEUE_SDMA) { - memset(m_MemArray, 0, sizeof(m_MemArray)); - memset(m_HsaMemoryRange, 0, sizeof(m_HsaMemoryRange)); -} - -CMA_TEST_STATUS KFDCMAArray::Destroy() { - for (int i = 0; i < m_ValidCount; i++) { - if (m_MemArray[i]) { - void *userPtr; - - userPtr = m_MemArray[i]->GetUserPtr(); - delete m_MemArray[i]; - - if (userPtr) - free(userPtr); - } - } - - memset(m_MemArray, 0, sizeof(m_MemArray)); - memset(m_HsaMemoryRange, 0, sizeof(m_HsaMemoryRange)); - m_ValidCount = 0; - - return CMA_TEST_SUCCESS; -} - -/* Initialize KFDCMAArray based on array of testMemoryDescriptor. Usually testMemoryDescriptor[] is - * statically defined array by the user. Only items with non-zero size are considered valid - */ -CMA_TEST_STATUS KFDCMAArray::Init(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE], int node) { - CMA_TEST_STATUS err = CMA_TEST_SUCCESS; - memset(m_MemArray, 0, sizeof(m_MemArray)); - memset(m_HsaMemoryRange, 0, sizeof(m_HsaMemoryRange)); - - m_ValidCount = 0; - for (int i = 0; i < CMA_MEMORY_TEST_ARRAY_SIZE; i++) { - if ((*memDescriptor)[i].m_MemSize == 0) - continue; - - switch ((*memDescriptor)[i].m_MemType) { - case CMA_MEM_TYPE_SYSTEM: - m_MemArray[i] = new HsaMemoryBuffer((*memDescriptor)[i].m_MemSize, node); - break; - - case CMA_MEM_TYPE_USERPTR: - { - void *userPtr = malloc((*memDescriptor)[i].m_MemSize); - m_MemArray[i] = new HsaMemoryBuffer(userPtr, (*memDescriptor)[i].m_MemSize); - break; - } - - case CMA_MEM_TYPE_LOCAL_MEM: - m_MemArray[i] = new HsaMemoryBuffer((*memDescriptor)[i].m_MemSize, node, false, true); - break; - } - - if (m_MemArray[i]) { - m_HsaMemoryRange[i].MemoryAddress = m_MemArray[i]->As(); - m_HsaMemoryRange[i].SizeInBytes = m_MemArray[i]->Size(); - m_ValidCount++; - } else { - err = CMA_TEST_NOMEM; - break; - } - } - - return err; -} - -/* Fill each buffer of KFDCMAArray with the pattern described by testMemoryDescriptor[] */ -void KFDCMAArray::FillPattern(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE]) { - SDMAQueue sdmaQueue; - bool queueCreated = false; - unsigned int queueNode; - - for (int i = 0; i < m_ValidCount; i++) { - if (m_MemArray[i]->isLocal()) - m_MemArray[i]->Fill((*memDescriptor)[i].m_FillPattern, *m_QueueArray.GetQueue(m_MemArray[i]->Node())); - else - m_MemArray[i]->Fill((*memDescriptor)[i].m_FillPattern); - } -} - -/* Check the first and last item of each buffer in KFDCMAArray with the pattern described by - * testMemoryDescriptor[]. Return 0 on success. - */ -CMA_TEST_STATUS KFDCMAArray::checkPattern(testMemoryDescriptor(*memDescriptor)[CMA_MEMORY_TEST_ARRAY_SIZE]) { - HSAuint64 lastItem; - CMA_TEST_STATUS ret = CMA_TEST_SUCCESS; - unsigned int queueNode = 0; - bool queueCreated = false; - HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); - volatile HSAuint32 *tmp = tmpBuffer.As(); - - for (int i = 0; i < m_ValidCount; i++) { - lastItem = m_MemArray[i]->Size(); - lastItem -= sizeof(HSAuint32); - - if (m_MemArray[i]->isLocal()) { - BaseQueue *sdmaQueue = m_QueueArray.GetQueue(m_MemArray[i]->Node()); - - if (!m_MemArray[i]->IsPattern(0, (*memDescriptor)[i].m_CheckFirstWordPattern, *sdmaQueue, tmp) || - !m_MemArray[i]->IsPattern(lastItem, (*memDescriptor)[i].m_CheckLastWordPattern, *sdmaQueue, tmp)) { - ret = CMA_CHECK_PATTERN_ERROR; - break; - } - - } else { - if (!m_MemArray[i]->IsPattern(0, (*memDescriptor)[i].m_CheckFirstWordPattern) || - !m_MemArray[i]->IsPattern(lastItem, (*memDescriptor)[i].m_CheckLastWordPattern)) { - ret = CMA_CHECK_PATTERN_ERROR; - break; - } - } - } - - return ret; -} - - -/* Non-blocking read and write to avoid Test from hanging (block indefinitely) - * if either server or client process exits due to assert failure - */ -static int write_non_block(int fd, const void *buf, int size) { - int total_bytes = 0, cur_bytes = 0; - int retries = 5; - struct timespec tm = { 0, 10000000ULL }; - const char *ptr = (const char *)buf; - - do { - cur_bytes = write(fd, ptr, (size - total_bytes)); - - if (cur_bytes < 0 && errno != EAGAIN) - return cur_bytes; - - if (cur_bytes > 0) { - total_bytes += cur_bytes; - ptr += cur_bytes; - } - - if (total_bytes < size) - nanosleep(&tm, NULL); - } while (total_bytes < size && retries--); - - /* Check for overflow */ - if (total_bytes > size) - return -1; - - return total_bytes; -} - -static int read_non_block(int fd, void *buf, int size) { - int total_bytes = 0, cur_bytes = 0; - int retries = 5; - struct timespec tm = { 0, 100000000ULL }; - char *ptr = reinterpret_cast(buf); - - do { - cur_bytes = read(fd, ptr, (size - total_bytes)); - - if (cur_bytes < 0 && errno != EAGAIN) - return cur_bytes; - - if (cur_bytes > 0) { - total_bytes += cur_bytes; - ptr += cur_bytes; - } - - if (total_bytes < size) - nanosleep(&tm, NULL); - } while (total_bytes < size && retries--); - - if (total_bytes > size) - return -1; - - return total_bytes; -} - - -/* Send HsaMemoryRange to another process that is connected via writePipe */ -CMA_TEST_STATUS KFDCMAArray::sendCMAArray(int writePipe) { - if (write_non_block(writePipe, reinterpret_cast(&m_HsaMemoryRange), sizeof(m_HsaMemoryRange)) != - sizeof(m_HsaMemoryRange)) - return CMA_IPC_PIPE_ERROR; - return CMA_TEST_SUCCESS; -} - -/* Send HsaMemoryRange from another process and initialize KFDCMAArray */ -CMA_TEST_STATUS KFDCMAArray::recvCMAArray(int readPipe) { - int i; - - if (read_non_block(readPipe, reinterpret_cast(&m_HsaMemoryRange), sizeof(m_HsaMemoryRange)) != - sizeof(m_HsaMemoryRange)) - return CMA_IPC_PIPE_ERROR; - - for (i = 0; i < CMA_MEMORY_TEST_ARRAY_SIZE; i++) { - if (m_HsaMemoryRange[i].SizeInBytes) - m_ValidCount++; - } - return CMA_TEST_SUCCESS; -} - - -CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachChildProcess(int defaultGPUNode, int writePipe, - int readPipe, CMA_TEST_TYPE testType) { - KFDCMAArray cmaLocalArray; - char msg[16]; - int testNo; - CMA_TEST_STATUS status; - - /* Initialize and fill Local Buffer Array with a pattern. - * READ_TEST: Send the Array to parent process. Wait for the parent - * to finish reading and checking. Then move to next text case or - * quit if last one. - * WRITE_TEST: Send Local Buffer Array to parent process and and wait - * for parent to write to it. Check for new pattern. Then move to next - * case or quit if last one. - */ - for (testNo = 0; testNo < CMA_TEST_COUNT; testNo++) { - if (testType == CMA_READ_TEST) { - cmaLocalArray.Init(&srcRange[testNo], defaultGPUNode); - cmaLocalArray.FillPattern(&srcRange[testNo]); - } else { - cmaLocalArray.Init(&dstRange[testNo], defaultGPUNode); - cmaLocalArray.FillPattern(&dstRange[testNo]); - } - - if (cmaLocalArray.sendCMAArray(writePipe) < 0) { - status = CMA_IPC_PIPE_ERROR; - break; - } - - /* Wait until the test is over */ - memset(msg, 0, sizeof(msg)); - if (read_non_block(readPipe, msg, 4) < 0) { - status = CMA_IPC_PIPE_ERROR; - break; - } - - if (!strcmp(msg, "CHCK")) - status = cmaLocalArray.checkPattern(&dstRange[testNo]); - else if (!strcmp(msg, "NEXT")) - status = CMA_TEST_SUCCESS; - else if (!strcmp(msg, "EXIT")) - status = CMA_TEST_ABORT; - else - status = CMA_PARENT_FAIL; - - cmaLocalArray.Destroy(); - if (status != CMA_TEST_SUCCESS) - break; - } - - return status; -} - - -CMA_TEST_STATUS KFDIPCTest::CrossMemoryAttachParentProcess(int defaultGPUNode, pid_t cid, - int writePipe, int readPipe, - CMA_TEST_TYPE testType) { - KFDCMAArray cmaLocalArray, cmaRemoteArray; - HSAuint64 copied = 0; - int testNo; - CMA_TEST_STATUS status; - - /* Receive buffer array from child and then initialize and fill in Local Buffer Array. - * READ_TEST: Copy remote buffer array into Local Buffer Array and then check - * for the new pattern. - * WRITE_TEST: Write Local Buffer Array into remote buffer array. Notify child to - * to check for the new pattern. - */ - for (testNo = 0; testNo < CMA_TEST_COUNT; testNo++) { - status = cmaRemoteArray.recvCMAArray(readPipe); - if (status != CMA_TEST_SUCCESS) - break; - - if (testType == CMA_READ_TEST) { - status = cmaLocalArray.Init(&dstRange[testNo], defaultGPUNode); - if (status != CMA_TEST_SUCCESS) - break; - cmaLocalArray.FillPattern(&dstRange[testNo]); - - if (hsaKmtProcessVMRead(cid, cmaLocalArray.getMemoryRange(), - cmaLocalArray.getValidRangeCount(), - cmaRemoteArray.getMemoryRange(), - cmaRemoteArray.getValidRangeCount(), - &copied) != HSAKMT_STATUS_SUCCESS) { - status = CMA_TEST_HSA_READ_FAIL; - break; - } - - status = cmaLocalArray.checkPattern(&dstRange[testNo]); - if (status != CMA_TEST_SUCCESS) - break; - - cmaLocalArray.Destroy(); - cmaRemoteArray.Destroy(); - - if (write_non_block(writePipe, "NEXT", 4) < 0) { - status = CMA_IPC_PIPE_ERROR; - break; - } - } else { - status = cmaLocalArray.Init(&srcRange[testNo], defaultGPUNode); - if (status != CMA_TEST_SUCCESS) - break; - cmaLocalArray.FillPattern(&srcRange[testNo]); - - if (hsaKmtProcessVMWrite(cid, cmaLocalArray.getMemoryRange(), - cmaLocalArray.getValidRangeCount(), - cmaRemoteArray.getMemoryRange(), - cmaRemoteArray.getValidRangeCount(), - &copied) != HSAKMT_STATUS_SUCCESS) { - status = CMA_TEST_HSA_WRITE_FAIL; - break; - } - - cmaLocalArray.Destroy(); - cmaRemoteArray.Destroy(); - if (write_non_block(writePipe, "CHCK", 4) < 0) { - status = CMA_IPC_PIPE_ERROR; - break; - } - } - } /* for loop */ - - return status; -} - -/* Test Cross Memory Attach - * hsaKmtProcessVMRead and hsaKmtProcessVMWrite are GPU address equivalent to - * process_vm_readv and process_vm_writev. These calls transfer data between - * the address space of the calling process ("the local process") and the process - * identified by pid ("the remote process"). - * - * In the tests parent process will be the local process and child will be - * the remote. - */ -TEST_F(KFDIPCTest, CrossMemoryAttachTest) { - TEST_START(TESTPROFILE_RUNALL) - - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - int pipeCtoP[2], pipePtoC[2]; - int status; - - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - if (!GetVramSize(defaultGPUNode)) { - LOG() << "Skipping test: No VRAM found." << std::endl; - return; - } - - /* Create Pipes for communicating shared handles */ - ASSERT_EQ(pipe2(pipeCtoP, O_NONBLOCK), 0); - ASSERT_EQ(pipe2(pipePtoC, O_NONBLOCK), 0); - - /* Create a child process and share the above Local Memory with it */ - m_ChildPid = fork(); - if (m_ChildPid == 0) { - TearDown(); - SetUp(); - - /* Child Process */ - status = CrossMemoryAttachChildProcess(defaultGPUNode, pipeCtoP[1], - pipePtoC[0], CMA_READ_TEST); - EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Child: Read Test Fail"; - status = CrossMemoryAttachChildProcess(defaultGPUNode, pipeCtoP[1], - pipePtoC[0], CMA_WRITE_TEST); - EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Child: Write Test Fail"; - } else { - int childStatus; - - status = CrossMemoryAttachParentProcess(defaultGPUNode, m_ChildPid, - pipePtoC[1], pipeCtoP[0], CMA_READ_TEST); /* Parent proces */ - EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Read Test Fail"; - status = CrossMemoryAttachParentProcess(defaultGPUNode, m_ChildPid, - pipePtoC[1], pipeCtoP[0], CMA_WRITE_TEST); - EXPECT_EQ(status, CMA_TEST_SUCCESS) << "Parent: Write Test Fail"; - - waitpid(m_ChildPid, &childStatus, 0); - EXPECT_EQ(WIFEXITED(childStatus), true); - EXPECT_EQ(WEXITSTATUS(childStatus), 0); - } - - /* Code path executed by both parent and child with respective fds */ - close(pipeCtoP[1]); - close(pipeCtoP[0]); - close(pipePtoC[1]); - close(pipePtoC[0]); - TEST_END -} - -/* Test Cross Memory Attach - * - * hsaKmtProcessVMRead and hsaKmtProcessVMWrite are GPU address equivalent to - * process_vm_readv and process_vm_writev. These calls are used to transfer data - * between the address space of the calling process ("the local process") and the process - * identified by pid ("the remote process"). However, these functions should also work - * with a single process and single BO. - */ -TEST_F(KFDIPCTest, CMABasicTest) { - TEST_START(TESTPROFILE_RUNALL) - - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - HSAuint64 size = PAGE_SIZE; - SDMAQueue sdmaQueue; - HsaMemoryRange srcRange, dstRange; - HSAuint64 copied; - const int PATTERN1 = 0xA5A5A5A5, PATTERN2 = 0xFFFFFFFF; - HSAKMT_STATUS status; - - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - if (!GetVramSize(defaultGPUNode)) { - LOG() << "Skipping test: No VRAM found." << std::endl; - return; - } - - ASSERT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); - HsaMemoryBuffer tmpBuffer(PAGE_SIZE, 0, true /* zero */); - volatile HSAuint32 *tmp = tmpBuffer.As(); - - /* Initialize test buffer. Fill first half and second half with - * different pattern - */ - HsaMemoryBuffer testLocalBuffer(size, defaultGPUNode, false, true); - testLocalBuffer.Fill(PATTERN1, sdmaQueue, 0, size/2); - testLocalBuffer.Fill(PATTERN2, sdmaQueue, size/2, size/2); - - /* Test1. Copy (or overwrite) buffer onto itself */ - srcRange.MemoryAddress = testLocalBuffer.As(); - srcRange.SizeInBytes = size; - dstRange.MemoryAddress = testLocalBuffer.As(); - dstRange.SizeInBytes = size; - ASSERT_SUCCESS(hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied)); - EXPECT_EQ(copied, size); - - EXPECT_TRUE(testLocalBuffer.IsPattern(0, PATTERN1, sdmaQueue, tmp)); - EXPECT_TRUE(testLocalBuffer.IsPattern(size - 4, PATTERN2, sdmaQueue, tmp)); - - - /* Test2. Test unaligned byte copy. Write 3 bytes to an unaligned destination address */ - const int unaligned_offset = 1; - const int unaligned_size = 3; - const int unaligned_mask = (((1 << (unaligned_size * 8)) - 1) << (unaligned_offset * 8)); - HSAuint32 expected_pattern; - - srcRange.MemoryAddress = testLocalBuffer.As(); - - /* Deliberately set to value > unaligned_size. Only unaligned_size - * should be copied since dstRange.SizeInBytes == unaligned_size - */ - srcRange.SizeInBytes = size; - - dstRange.MemoryAddress = reinterpret_cast(testLocalBuffer.As() + (size / 2) + unaligned_offset); - dstRange.SizeInBytes = unaligned_size; - ASSERT_SUCCESS(hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied)); - EXPECT_EQ(copied, unaligned_size); - - expected_pattern = (PATTERN2 & ~unaligned_mask | (PATTERN1 & unaligned_mask)); - EXPECT_TRUE(testLocalBuffer.IsPattern(size/2, expected_pattern, sdmaQueue, tmp)); - - - /* Test3. Test overflow and expect failure */ - srcRange.MemoryAddress = testLocalBuffer.As(); - srcRange.SizeInBytes = size; - dstRange.MemoryAddress = reinterpret_cast(testLocalBuffer.As() + 4); - dstRange.SizeInBytes = size; /* This should overflow since offset is VA + 4 */ - status = hsaKmtProcessVMRead(getpid(), &dstRange, 1, &srcRange, 1, &copied); - EXPECT_NE(status, HSAKMT_STATUS_SUCCESS); - EXPECT_LE(copied, (size - 4)); - - EXPECT_SUCCESS(sdmaQueue.Destroy()); - - TEST_END -} From 9b2b81e555519ffc65e9317b628518ed53b8e993 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 19 Jul 2022 21:47:30 -0400 Subject: [PATCH 1029/1247] libhsakmt: Remove CMA implementation The CMA feature is deprecated and about to be removed from the DKMS branch. It was never supported upstream. Leave dummy functions in place for now. Signed-off-by: Felix Kuehling Change-Id: I9e51403d753cb91630553aff4f19e931af509740 --- src/memory.c | 65 ++++------------------------------------------------ 1 file changed, 4 insertions(+), 61 deletions(-) diff --git a/src/memory.c b/src/memory.c index 5eae74a92c..7f8e45b6e0 100644 --- a/src/memory.c +++ b/src/memory.c @@ -386,16 +386,6 @@ error: return ret; } -static uint64_t convertHsaToKfdRange(HsaMemoryRange *HsaRange) -{ - if (sizeof(struct kfd_memory_range) != - sizeof(HsaMemoryRange)) { - pr_err("Struct size mismatch in thunk. Cannot cast Hsa Range to KFD IOCTL range\n"); - return 0; - } - return (uint64_t) HsaRange; -} - HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, HsaMemoryRange *LocalMemoryArray, HSAuint64 LocalMemoryArrayCount, @@ -403,31 +393,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMRead(HSAuint32 Pid, HSAuint64 RemoteMemoryArrayCount, HSAuint64 *SizeCopied) { - int ret = HSAKMT_STATUS_SUCCESS; - struct kfd_ioctl_cross_memory_copy_args args = {0}; + pr_err("[%s] Deprecated\n", __func__); - pr_debug("[%s]\n", __func__); - - if (!LocalMemoryArray || !RemoteMemoryArray || - LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) - return HSAKMT_STATUS_ERROR; - - args.flags = 0; - KFD_SET_CROSS_MEMORY_READ(args.flags); - args.pid = Pid; - args.src_mem_range_array = convertHsaToKfdRange(RemoteMemoryArray); - args.src_mem_array_size = RemoteMemoryArrayCount; - args.dst_mem_range_array = convertHsaToKfdRange(LocalMemoryArray); - args.dst_mem_array_size = LocalMemoryArrayCount; - args.bytes_copied = 0; - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args)) - ret = HSAKMT_STATUS_ERROR; - - if (SizeCopied) - *SizeCopied = args.bytes_copied; - - return ret; + return HSAKMT_STATUS_NOT_IMPLEMENTED; } HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, @@ -437,34 +405,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtProcessVMWrite(HSAuint32 Pid, HSAuint64 RemoteMemoryArrayCount, HSAuint64 *SizeCopied) { - int ret = HSAKMT_STATUS_SUCCESS; - struct kfd_ioctl_cross_memory_copy_args args = {0}; + pr_err("[%s] Deprecated\n", __func__); - pr_debug("[%s]\n", __func__); - - if (SizeCopied) - *SizeCopied = 0; - - if (!LocalMemoryArray || !RemoteMemoryArray || - LocalMemoryArrayCount == 0 || RemoteMemoryArrayCount == 0) - return HSAKMT_STATUS_ERROR; - - args.flags = 0; - KFD_SET_CROSS_MEMORY_WRITE(args.flags); - args.pid = Pid; - args.src_mem_range_array = convertHsaToKfdRange(LocalMemoryArray); - args.src_mem_array_size = LocalMemoryArrayCount; - args.dst_mem_range_array = convertHsaToKfdRange(RemoteMemoryArray); - args.dst_mem_array_size = RemoteMemoryArrayCount; - args.bytes_copied = 0; - - if (kmtIoctl(kfd_fd, AMDKFD_IOC_CROSS_MEMORY_COPY, &args)) - ret = HSAKMT_STATUS_ERROR; - - if (SizeCopied) - *SizeCopied = args.bytes_copied; - - return ret; + return HSAKMT_STATUS_NOT_IMPLEMENTED; } From 9d33827a84156ce9116e0188ad924a6dfa05e6e8 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 21 Jul 2022 15:49:57 -0400 Subject: [PATCH 1030/1247] kfdtest: Disable host access for VRAM KFDExceptionTest.SdmaQueueException allocates VRAM with host access. This fails on small-BAR GPUs. This error was incorrectly ignored before 412b24137edb ("kfdtest: Full TearDown and SetUp in child process"). The test doesn't really need host access to the memory. Therefore the fix is to disable the HostAccess flag. Signed-off-by: Felix Kuehling Change-Id: Ifec279eeb6c1ecb1160db9b692e6dc8816d761a3 --- tests/kfdtest/src/KFDExceptionTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 67ccc68336..f7c28d8897 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -317,6 +317,7 @@ TEST_F(KFDExceptionTest, SdmaQueueException) { KFDBaseComponentTest::SetUp(); m_MemoryFlags.ui32.NonPaged = 1; + m_MemoryFlags.ui32.HostAccess = 0; ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, m_MemoryFlags, reinterpret_cast(&pDb))); // verify that pDb is not null before it's being used From deb7a20c929ff249413279ae55ebf2a56e23874a Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 25 Jul 2022 17:41:46 -0400 Subject: [PATCH 1031/1247] libhsakmt: Make CWSR area executable The debugger depends on the CWSR area being executable. Set the right flag when registering SVM memory. Signed-off-by: Felix Kuehling Tested-by: Laurent Morichetti Change-Id: I7441e214d1a4da8324d775e777976fabd1c81a6f --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 5702c95329..9518d9ff5b 100644 --- a/src/queues.c +++ b/src/queues.c @@ -394,7 +394,7 @@ static HSAKMT_STATUS register_svm_range(void *mem, uint32_t size, HSAuint32 nattr; HSAuint32 flags; - flags = HSA_SVM_FLAG_HOST_ACCESS; + flags = HSA_SVM_FLAG_HOST_ACCESS | HSA_SVM_FLAG_GPU_EXEC; if (alwaysMapped) { CHECK_KFD_MINOR_VERSION(11); From 17fb40f1f69ebbb411db82d3085207946428af32 Mon Sep 17 00:00:00 2001 From: jie1zhan Date: Tue, 26 Jul 2022 17:24:09 +0800 Subject: [PATCH 1032/1247] Fix allocate memory failed in VRAM : The kernel driver will do align VRAM allocations to 2MB, instead of 4KB. Change-Id: Iea9d8c0f02999b9ea5fd931da82240a33f7bcc69 --- tests/kfdtest/src/KFDMemoryTest.cpp | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 2d29c00129..8c6449ceb7 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -790,6 +790,7 @@ TEST_F(KFDMemoryTest, BigSysBufferStressTest) { TEST_END } +#define VRAM_ALLOCATION_ALIGN (1 << 21) //Align VRAM allocations to 2MB TEST_F(KFDMemoryTest, MMBench) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -900,8 +901,11 @@ TEST_F(KFDMemoryTest, MMBench) { memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 0; memFlags.ui32.NonPaged = 1; - /* Upper limit of buffer number to fit 90% vram size */ - bufLimit = ((vramSizeMB << 20) * 8 / 10) / bufSize ; + + /* Buffer sizes are 2MB aligned to match new allocation policy. + * Upper limit of buffer number to fit 80% vram size. + */ + bufLimit = ((vramSizeMB << 20) * 8 / 10) / ALIGN_UP(bufSize, VRAM_ALLOCATION_ALIGN); if (bufLimit == 0) continue; // skip when bufSize > vram @@ -1558,7 +1562,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); - const unsigned nBufs = 1000; /* measure us, report ns */ + unsigned nBufs = 1000; /* measure us, report ns */ unsigned testIndex, sizeIndex, memType; const unsigned nMemTypes = 2; const char *memTypeStrings[nMemTypes] = {"SysMem", "VRAM"}; @@ -1603,6 +1607,7 @@ TEST_F(KFDMemoryTest, MMBandWidth) { unsigned memType = _TEST_MEMTYPE(testIndex); HSAuint64 mcpRTime, mcpWTime, accessRTime, accessWTime; HSAuint32 allocNode; + unsigned bufLimit; if ((testIndex & (nSizes-1)) == 0) LOG() << "----------------------------------------------------------------------" << std::endl; @@ -1619,6 +1624,16 @@ TEST_F(KFDMemoryTest, MMBandWidth) { memFlags.ui32.PageSize = HSA_PAGE_SIZE_4KB; memFlags.ui32.HostAccess = 1; memFlags.ui32.NonPaged = 1; + + /* Buffer sizes are 2MB aligned to match new allocation policy. + * Upper limit of buffer number to fit 80% vram size. + */ + bufLimit = ((vramSizeMB << 20) * 8 / 10) / ALIGN_UP(bufSize, VRAM_ALLOCATION_ALIGN); + if (bufLimit == 0) + continue; // skip when bufSize > vram + + /* When vram is too small to fit all the buffers, fill 80% vram size*/ + nBufs = std::min(nBufs , bufLimit); } for (i = 0; i < nBufs; i++) From 90ada94141fce67812566cecba96169cfe45cc9d Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 13 Jul 2022 08:44:22 -0400 Subject: [PATCH 1033/1247] kfdtest: Add return statement for ReadSMIEventThread This didn't return anything, so add a "return 0" at the end, since the function expects to return an int value Change-Id: I17c398e431b2ce4571e6ca4abe6d567f110ea2a7 --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 34484801ac..a12d2714e3 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1445,6 +1445,7 @@ unsigned int ReadSMIEventThread(void* p) { EXPECT_EQ(pid, getpid()); EXPECT_EQ(trigger, HSA_MIGRATE_TRIGGER_PREFETCH); close(fd); + return 0; } TEST_F(KFDSVMRangeTest, HMMProfilingEvent) { From c1d8ac84377fc48ea996341c2935408774f144d4 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 13 Jul 2022 16:02:35 -0400 Subject: [PATCH 1034/1247] libhsakmt: bump debug major rev for snapshot and watchpoint changes The debugger can now request snapshot copies with entry size and set/clear watchpoints by device. v3: drop min version check to v10.0 v2: check runtime allowance from v10.3 to 13.x Signed-off-by: Jonathan Kim Change-Id: I9befefb596201a11591de218db29a9317b41e69b --- src/debug.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/debug.c b/src/debug.c index 14bcb884e9..b2021218ca 100644 --- a/src/debug.c +++ b/src/debug.c @@ -297,18 +297,13 @@ hsaKmtGetKernelDebugTrapVersionInfo( } #define HSA_RUNTIME_ENABLE_MIN_MAJOR 10 -#define HSA_RUNTIME_ENABLE_MAX_MAJOR 11 +#define HSA_RUNTIME_ENABLE_MAX_MAJOR 13 #define HSA_RUNTIME_ENABLE_MIN_MINOR 0 -HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, - bool setupTtmp) -{ - struct kfd_ioctl_dbg_trap_args args = {0}; + +static HSAKMT_STATUS checkRuntimeDebugSupport(void) { HSAuint32 kMajor, kMinor; - HSAKMT_STATUS result; - result = hsaKmtGetKernelDebugTrapVersionInfo(&kMajor, &kMinor); - - if (result) + if (hsaKmtGetKernelDebugTrapVersionInfo(&kMajor, &kMinor)) return HSAKMT_STATUS_NOT_SUPPORTED; if (kMajor < HSA_RUNTIME_ENABLE_MIN_MAJOR || kMajor > HSA_RUNTIME_ENABLE_MAX_MAJOR || @@ -316,6 +311,18 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR)) return HSAKMT_STATUS_NOT_SUPPORTED; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, + bool setupTtmp) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + HSAKMT_STATUS result = checkRuntimeDebugSupport(); + + if (result) + return result; + memset(&args, 0x00, sizeof(args)); args.op = KFD_IOC_DBG_TRAP_RUNTIME_ENABLE; args.pid = getpid(); @@ -338,18 +345,10 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) { struct kfd_ioctl_dbg_trap_args args = {0}; - HSAuint32 kMajor, kMinor; - HSAKMT_STATUS result; - - result = hsaKmtGetKernelDebugTrapVersionInfo(&kMajor, &kMinor); + HSAKMT_STATUS result = checkRuntimeDebugSupport(); if (result) - return HSAKMT_STATUS_NOT_SUPPORTED; - - if (kMajor < HSA_RUNTIME_ENABLE_MIN_MAJOR || kMajor > HSA_RUNTIME_ENABLE_MAX_MAJOR || - (kMajor == HSA_RUNTIME_ENABLE_MIN_MAJOR && - (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR)) - return HSAKMT_STATUS_NOT_SUPPORTED; + return result; memset(&args, 0x00, sizeof(args)); args.op = KFD_IOC_DBG_TRAP_RUNTIME_ENABLE; From 0dbac97b75a3850e08aacbb7fc1164219bb65d86 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Sun, 5 Jun 2022 17:58:02 -0400 Subject: [PATCH 1035/1247] kfdtest: Overhaul KFDCWSRTest.BasicTest This patch restructures the CWSR basic test and allows for creating parameterized CWSR tests. This patch introduces four parameterizations. These tests behave as follows: This test dispatches the IterateIsa shader, which continuously increments a vgpr for (num_witems / WAVE_SIZE) waves. While this shader is running, dequeue/requeue requests are sent in a loop to trigger CWSRs. This test defines a CWSR threshold. Once the number of CWSRs triggered reaches the threshold, a known-value is filled into the inputBuf to signal the shader to exit. 4 parameterized tests are defined: KFDCWSRTest.BasicTest/0 KFDCWSRTest.BasicTest/1 KFDCWSRTest.BasicTest/2 KFDCWSRTest.BasicTest/3 0: 1 work-item, CWSR threshold of 10 1: 256 work-items, CWSR threshold of 50 2: 512 work-items, CWSR threshold of 100 3: 1024 work-items, CWSR threshold of 1000 Tuple Format: (num_witems, cwsr_thresh) num_witems: Defines the number of work-items. cwsr_thresh: Defines the number of CWSRs to trigger. Signed-off-by: Graham Sider Change-Id: I639eb7bd75b14ee70e190b4bd19dcf34096fc7bf --- tests/kfdtest/src/KFDCWSRTest.cpp | 138 +++++++++++++++++++----------- tests/kfdtest/src/KFDCWSRTest.hpp | 6 +- tests/kfdtest/src/ShaderStore.cpp | 49 +++++++---- 3 files changed, 120 insertions(+), 73 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 5a4f853ee2..146ae2ae5a 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -29,8 +29,6 @@ void KFDCWSRTest::SetUp() { KFDBaseComponentTest::SetUp(); - wave_number = 1; - ROUTINE_END } @@ -42,14 +40,6 @@ void KFDCWSRTest::TearDown() { ROUTINE_END } -bool isOnEmulator() { - uint32_t isEmuMode = 0; - - fscanf_dec("/sys/module/amdgpu/parameters/emu_mode", &isEmuMode); - - return isEmuMode; -} - static inline uint32_t checkCWSREnabled() { uint32_t cwsr_enable = 0; @@ -61,66 +51,93 @@ static inline uint32_t checkCWSREnabled() { /** * KFDCWSRTest.BasicTest * - * This test dispatches the loop_inc_isa shader and lets it run, ensuring its destination pointer gets incremented. - * It then triggers CWSR and ensures the shader stops running. - * It then resumes the shader, ensures that it's running again and terminates it. + * This test dispatches the IterateIsa shader, which continuously increments a vgpr for + * (num_witems / WAVE_SIZE) waves. While this shader is running, dequeue/requeue requests + * are sent in a loop to trigger CWSRs. + * + * This is a paremeterized test. See the INSTANTIATE_TEST_CASE_P below for an explanation + * on the parameters. + * + * This test defines a CWSR threshold. The shader will continuously loop until inputBuf is + * filled with the known stop value, which occurs once cwsr_thresh CWSRs have been + * successfully triggered. + * + * 4 parameterized tests are defined: + * + * KFDCWSRTest.BasicTest/0 + * KFDCWSRTest.BasicTest/1 + * KFDCWSRTest.BasicTest/2 + * KFDCWSRTest.BasicTest/3 + * + * 0: 1 work-item, CWSR threshold of 10 + * 1: 256 work-items (multi-wave), CWSR threshold of 50 + * 2: 512 work-items (multi-wave), CWSR threshold of 100 + * 3: 1024 work-items (multi-wave), CWSR threshold of 1000 */ -TEST_F(KFDCWSRTest, BasicTest) { +TEST_P(KFDCWSRTest, BasicTest) { TEST_START(TESTPROFILE_RUNALL); + int num_witems = std::get<0>(GetParam()); + int cwsr_thresh = std::get<1>(GetParam()); int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { - HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - HsaMemoryBuffer resultBuf1(PAGE_SIZE, defaultGPUNode, true, false, false); - uint64_t count1 = 400000000; - - if (isOnEmulator()) { - // Divide the iterator times by 10000 so that the test can - // finish in a reasonable time. - count1 /= 10000; - LOG() << "On Emulators" << std::endl; - } - - unsigned int* result1 = resultBuf1.As(); - + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true, false, true); ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(IterateIsa, isaBuffer.As())); - PM4Queue queue1; + unsigned stopval = 0x1234'5678; + unsigned outval = 0x8765'4321; - ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); + HsaMemoryBuffer inputBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + HsaMemoryBuffer outputBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + unsigned int* input = inputBuf.As(); + unsigned int* output = outputBuf.As(); + inputBuf.Fill(0); + outputBuf.Fill(outval); - Dispatch *dispatch1; + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - dispatch1 = new Dispatch(isaBuffer); + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(input, output); + dispatch.SetDim(num_witems, 1, 1); + dispatch.Submit(queue); - dispatch1->SetArgs(reinterpret_cast(count1), result1); - dispatch1->SetDim(wave_number, 1, 1); - - // Submit the shader, queue1 - dispatch1->Submit(queue1); - - //Give time for waves to launch before disabling queue. - Delay(1); - EXPECT_SUCCESS(queue1.Update(0/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false)); Delay(5); - EXPECT_SUCCESS(queue1.Update(100/*percentage*/, BaseQueue::DEFAULT_PRIORITY, false)); - dispatch1->Sync(); - // Ensure all the waves complete as expected - int i; - for (i = 0 ; i < wave_number; ++i) { - if (result1[i] != count1) { - LOG() << "Dispatch 1, work item [" << std::dec << i << "] " - << result1[i] << " != " << count1 << std::endl; - break; + LOG() << "Starting iteration for " << std::dec << num_witems + << " work items(s) (targeting " << std::dec << cwsr_thresh + << " CWSRs)" << std::endl; + + for (int num_cwsrs = 0; num_cwsrs < cwsr_thresh; num_cwsrs++) { + + // Send dequeue request + EXPECT_SUCCESS(queue.Update(0, BaseQueue::DEFAULT_PRIORITY, false)); + + Delay(5); + + // Send requeue request + EXPECT_SUCCESS(queue.Update(100, BaseQueue::DEFAULT_PRIORITY, false)); + + Delay(50); + + // Check for reg mangling + for (int i = 0; i < num_witems; i++) { + EXPECT_EQ(outval, output[i]); } } - EXPECT_EQ(i, wave_number); - EXPECT_SUCCESS(queue1.Destroy()); + LOG() << "Successful completion for " << std::dec << num_witems + << " work item(s) (CWSRs triggered: " << std::dec << cwsr_thresh + << ")" << std::endl; + LOG() << "Signalling shader stop..." << std::endl; - delete dispatch1; + inputBuf.Fill(stopval); + + // Wait for shader to finish or timeout if shader has vm page fault + EXPECT_EQ(0, dispatch.SyncWithStatus(180000)); + + EXPECT_SUCCESS(queue.Destroy()); } else { LOG() << "Skipping test: No CWSR present for family ID 0x" << m_FamilyId << "." << std::endl; } @@ -128,6 +145,23 @@ TEST_F(KFDCWSRTest, BasicTest) { TEST_END } +/** + * Instantiates various KFDCWSRTest.BasicTest parameterizations + * Tuple Format: (num_witems, cwsr_thresh) + * + * num_witems: Defines the number of work-items. + * cwsr_thresh: Defines the number of CWSRs to trigger. + */ +INSTANTIATE_TEST_CASE_P( + , KFDCWSRTest, + ::testing::Values( + std::make_tuple(1, 10), /* Single Wave Test, 10 CWSR Triggers */ + std::make_tuple(256, 50), /* Multi Wave Test, 50 CWSR Triggers */ + std::make_tuple(512, 100), /* Multi Wave Test, 100 CWSR Triggers */ + std::make_tuple(1024, 1000) /* Multi Wave Test, 1000 CWSR Triggers */ + ) +); + /** * KFDCWSRTest.InterruptRestore * diff --git a/tests/kfdtest/src/KFDCWSRTest.hpp b/tests/kfdtest/src/KFDCWSRTest.hpp index 53c925b0aa..789411352f 100644 --- a/tests/kfdtest/src/KFDCWSRTest.hpp +++ b/tests/kfdtest/src/KFDCWSRTest.hpp @@ -29,7 +29,8 @@ #include "PM4Queue.hpp" #include "KFDBaseComponentTest.hpp" -class KFDCWSRTest : public KFDBaseComponentTest { +class KFDCWSRTest : public KFDBaseComponentTest, + public ::testing::WithParamInterface> { public: KFDCWSRTest() {} ~KFDCWSRTest() {} @@ -37,9 +38,6 @@ class KFDCWSRTest : public KFDBaseComponentTest { protected: virtual void SetUp(); virtual void TearDown(); - - protected: // Members - unsigned wave_number; }; #endif // __KFD_CWSR_TEST__H__ diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 06a6c0f95a..42b723f7fb 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -453,35 +453,50 @@ const char *LoopIsa = R"( */ /* Initial state: - * s[0:1] - 64 bits iteration number; only the lower 32 bits are useful. - * s[2:3] - result buffer base address + * s[0:1] - input buffer base address + * s[2:3] - output buffer base address * s4 - workgroup id - * v0 - workitem id, always 0 because - * NUM_THREADS_X(number of threads) in workgroup set to 1 + * v0 - workitem id * Registers: * v0 - calculated workitem = v0 + s4 * NUM_THREADS_X, which is s4 - * v2 - = s0, 32 bits iteration number + * v[2:3] - corresponding input buf address: s[0:1] + v0 * 8 * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 - * v6 - counter + * v6 - register storing known-value output for mangle testing + * v7 - counter */ const char *IterateIsa = SHADER_MACROS R"( - // Copy the parameters from scalar registers to vector registers - v_mov_b32 v2, s0 // v[2:3] = s[0:1] - v_mov_b32 v3, s1 // v[2:3] = s[0:1] + // Compute address of output buffer v_mov_b32 v0, s4 // use workgroup id as index v_lshlrev_b32 v0, 2, v0 // v0 *= 4 V_ADD_CO_U32 v4, s2, v0 // v[4:5] = s[2:3] + v0 * 4 v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 V_ADD_CO_CI_U32 v5, v5, 0 // v[4:5] = s[2:3] + v0 * 4 - v_mov_b32 v6, 0 - LOOP: - V_ADD_CO_U32 v6, 1, v6 - // Compare the result value (v6) to iteration value (v2), and - // jump if equal (i.e. if VCC is not zero after the comparison) - V_CMP_LT_U32 v6, v2 - s_cbranch_vccnz LOOP - flat_store_dword v[4:5], v6 + // Compute address of input buffer + v_lshlrev_b32 v0, 1, v0 // v0 *= 8 + V_ADD_CO_U32 v2, s0, v0 // v[2:3] = s[0:1] + v0 * 8 + v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v0 * 8 + V_ADD_CO_CI_U32 v3, v3, 0 // v[2:3] = s[0:1] + v0 * 8 + + // Store known-value output in register + flat_load_dword v6, v[4:5] glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + + // Initialize counter + v_mov_b32 v7, 0 + + LOOP: + flat_store_dword v[4:5], v6 // store known-val in output + V_ADD_CO_U32 v7, 1, v7 // increment counter + + s_load_dword s6, s[0:1], 0 glc + s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish + s_cmp_eq_i32 s6, 0x12345678 // compare input buf to stopval + s_cbranch_scc1 L_QUIT // branch if notified to quit by host + + s_branch LOOP + + L_QUIT: s_waitcnt vmcnt(0) & lgkmcnt(0) s_endpgm )"; From 0055ef46c4054bbc1e29be698a0474e24fc15b8b Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 23 Aug 2022 16:00:49 -0400 Subject: [PATCH 1036/1247] kfdtest: Add KFDCWSRTest.BasicTest* to GFX10 blacklist Failure with new CWSR tests reported for GFX10, for now add to blacklist. Signed-off-by: Graham Sider Change-Id: I5b2bd9ec61c64ad66e1c34ba2c192bece808f56f --- tests/kfdtest/scripts/kfdtest.exclude | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index efd93892dc..7b4a8befd4 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -105,10 +105,12 @@ KV_QUEUE_BLACKLIST=\ "KFDQMTest.Atomics:"\ "KFDQMTest.GPUDoorbellWrite" +# KFDCWSRTest.BasicTest*: SWDEV-353206 BLACKLIST_GFX10=\ "KFDMemoryTest.DeviceHdpFlush:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDSVMEvictTest.*" +"KFDSVMEvictTest.*:"\ +"KFDCWSRTest.BasicTest*" BLACKLIST_GFX10_NV2X=\ "$BLACKLIST_GFX10:"\ From 4267c4b52470748c4cfcda6d5c5526df4588fca5 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 23 Aug 2022 16:05:36 -0400 Subject: [PATCH 1037/1247] kfdtest: Bump C++ compiler to gnu++17 Required due to LLVM retirement of llvm::apply_tuple, instead using std::apply which was introduced in C++17. Signed-off-by: Graham Sider Change-Id: I6646ebcca7d71d3e1bcf340ccfa3db2c15a3110a --- tests/kfdtest/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 5c5db26940..8976a584c1 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -203,7 +203,7 @@ message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) if ( "${CMAKE_C_COMPILER_VERSION}" STRGREATER "4.8.0") ## Add --enable-new-dtags to generate DT_RUNPATH -set ( CMAKE_CXX_FLAGS "-std=gnu++14 -Wl,--enable-new-dtags" ) +set ( CMAKE_CXX_FLAGS "-std=gnu++17 -Wl,--enable-new-dtags" ) endif() if ( "${CMAKE_BUILD_TYPE}" STREQUAL Release ) set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2" ) From 66e9e97e0db8237d730faa392afe8639a175d3e9 Mon Sep 17 00:00:00 2001 From: Lang Yu Date: Wed, 17 Aug 2022 11:24:16 +0800 Subject: [PATCH 1038/1247] libhsakmt: add FamilyID info into HsaNodeProperties Query family id info from drm render node, then ROCr can query this info directly from Thunk instead of parsing the info by itself. Signed-off-by: Lang Yu Change-Id: I030bd27ab2379fbf87f3d787302c3b8613456278 --- include/hsakmttypes.h | 4 +++- src/topology.c | 50 +++++++++++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 65f23dec51..b0e066d605 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -330,7 +330,9 @@ typedef struct _HsaNodeProperties HSAuint32 SGPRSizePerCU; // SGPR size in bytes per CU HSAuint32 KFDGpuID; // GPU Hash ID generated by KFD - HSAuint8 Reserved[8]; + + HSAuint32 FamilyID; // GPU family id + HSAuint8 Reserved[4]; } HsaNodeProperties; diff --git a/src/topology.c b/src/topology.c index da09c193f2..00deabc7d7 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1004,34 +1004,48 @@ exit: return ret; } -static int topology_get_marketing_name(int minor, uint16_t *marketing_name) +static int topology_get_node_props_from_drm(HsaNodeProperties *props) { int drm_fd; uint32_t major_version; uint32_t minor_version; amdgpu_device_handle device_handle; + struct amdgpu_gpu_info gpu_info; const char *name; - int i; + int i, ret = 0; - if (marketing_name == NULL) + if (props->MarketingName == NULL) return -1; - drm_fd = drmOpenRender(minor); + + drm_fd = drmOpenRender(props->DrmRenderMinor); if (drm_fd < 0) return -1; + if (amdgpu_device_initialize(drm_fd, &major_version, &minor_version, &device_handle) < 0) { - drmClose(drm_fd); - return -1; + ret = -1; + goto err_device_initialize; } + name = amdgpu_get_marketing_name(device_handle); if (name != NULL) { for (i = 0; name[i] != 0 && i < HSA_PUBLIC_NAME_SIZE - 1; i++) - marketing_name[i] = name[i]; - marketing_name[i] = '\0'; + props->MarketingName[i] = name[i]; + props->MarketingName[i] = '\0'; } + + if (amdgpu_query_gpu_info(device_handle, &gpu_info)) { + ret = -1; + goto err_query_gpu_info; + } + + props->FamilyID = gpu_info.family_id; + +err_query_gpu_info: amdgpu_device_deinitialize(device_handle); +err_device_initialize: drmClose(drm_fd); - return 0; + return ret; } static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, @@ -1170,6 +1184,10 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, gfxv = (uint32_t)prop_val; } + /* Bail out early, if a CPU node */ + if (props->NumCPUCores) + goto err; + gfxv_major = HSA_GET_GFX_VERSION_MAJOR(gfxv); gfxv_minor = HSA_GET_GFX_VERSION_MINOR(gfxv); gfxv_stepping = HSA_GET_GFX_VERSION_STEP(gfxv); @@ -1210,15 +1228,11 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, snprintf((char *)props->AMDName, sizeof(props->AMDName)-1, "GFX%06x", HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32)); - if (!props->NumCPUCores) { - /* Is dGPU Node, not APU - * Retrieve the marketing name of the node. - */ - if (topology_get_marketing_name(props->DrmRenderMinor, - props->MarketingName) != 0) - pr_info("failed to get marketing name for device ID 0x%x\n", - props->DeviceId); - } + /* Is dGPU Node, not APU + * Retrieve the marketing name of the node. + */ + if (topology_get_node_props_from_drm(props)) + pr_info("failed to get marketing name for device ID 0x%x\n", props->DeviceId); /* Get VGPR/SGPR size in byte per CU */ props->SGPRSizePerCU = SGPR_SIZE_PER_CU; From ea4d4917c1c4167f24707be97e07db6057cea409 Mon Sep 17 00:00:00 2001 From: "kent.russell@amd.com" Date: Mon, 29 Aug 2022 08:56:41 -0400 Subject: [PATCH 1039/1247] src/topology.c: Fix NULL check Now that HsaNodeProperties is passed in to topology_get_node_props_from_drm, check that pointer instead of the pointer for MarketingName (which throws a compiler warning) Signed-off-by: kent.russell@amd.com Change-Id: If76b24e1bab5a62e514ab440b6316c7b7cd264c1 --- src/topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 00deabc7d7..4aea77e276 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1014,7 +1014,7 @@ static int topology_get_node_props_from_drm(HsaNodeProperties *props) const char *name; int i, ret = 0; - if (props->MarketingName == NULL) + if (props == NULL) return -1; drm_fd = drmOpenRender(props->DrmRenderMinor); From 1c385fb257c8318384d5065603e6391b1607be59 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Tue, 23 Aug 2022 11:34:19 -0400 Subject: [PATCH 1040/1247] libhsakmt: Add check for invalid aperture Avoiding segfault when an invalid SharedMemoryHandle is passed in when calling fmm_register_shared_memory. Change-Id: I0e0bbed01487fc10afcbb170eb9330e70b209d14 Signed-off-by: David Yat Sin --- src/fmm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index 0dea2a8c4c..cf6c2e36c7 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3411,6 +3411,8 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo importArgs.gpu_id = SharedMemoryStruct->ExportGpuId; aperture = fmm_get_aperture(SharedMemoryStruct->ApeInfo); + if (!aperture) + return HSAKMT_STATUS_INVALID_PARAMETER; pthread_mutex_lock(&aperture->fmm_mutex); reservedMem = aperture_allocate_area(aperture, NULL, From 3dbf5feffe25a3037fae5372e4b048ac9cb9eae8 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 16 Aug 2022 13:56:09 -0400 Subject: [PATCH 1041/1247] libhsakmt: Fix queue leaking debug memory Queue ctx_save_restore memory is allocated with size ctx_save_restore_size + debug_memory_size, use the same size in free_queue to free ctx_save_restore memory. Change-Id: I4902ff15fb82ddea64b8342b89776a1bf5c38d13 Signed-off-by: Philip Yang --- src/queues.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 9518d9ff5b..20a1da4f61 100644 --- a/src/queues.c +++ b/src/queues.c @@ -429,7 +429,7 @@ static void free_queue(struct queue *q) free(q->ctx_save_restore); else if (q->ctx_save_restore) free_exec_aligned_memory(q->ctx_save_restore, - q->ctx_save_restore_size, + q->ctx_save_restore_size + q->debug_memory_size, PAGE_SIZE, q->use_ats); free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->use_ats); From 4951495fcaec1dc7635b2f23400a7b6f3fe4eded Mon Sep 17 00:00:00 2001 From: Shikai Guo Date: Thu, 1 Sep 2022 17:02:02 +0800 Subject: [PATCH 1042/1247] libhsakmt: add filter node for new chip When running kfdtest test case, because the filter node of the new chip is missing in libhsakmt, the test case is not supported, so a new test node is added in order to spporting kfdtest case. Signed-off-by: shikaguo Change-Id: I0cd9ffd7d4387129cfb0f8de6b669f431949ab49 --- tests/kfdtest/scripts/kfdtest.exclude | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 7b4a8befd4..4a078ae2d1 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -304,3 +304,7 @@ FILTER[gfx1102]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X:"\ "$TEMPORARY_BLACKLIST_GFX11" + +FILTER[gfx1036]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX10_NV2X" From 57a1c6f3ff43f5d0d7ce857ba0b1e4d9de15e480 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Thu, 1 Sep 2022 16:37:02 -0400 Subject: [PATCH 1043/1247] libhsakmt: Remove potential double free in queue creation Remove potential double free condition when free_queue() is called after hsaKmtDestroyQueue() if mapping doorbell fails during queue creation. Signed-off-by: Mukul Joshi Change-Id: If2aa19c455b30d2940b232dbafb9cc1eaad721a5 --- src/queues.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 20a1da4f61..6ea17b4a7d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -662,7 +662,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, err = map_doorbell(NodeId, gpu_id, doorbell_mmap_offset); if (err != HSAKMT_STATUS_SUCCESS) { hsaKmtDestroyQueue(q->queue_id); - free_queue(q); return HSAKMT_STATUS_ERROR; } From d98c729ff9c09895e6bef26b0926d6114ad03d26 Mon Sep 17 00:00:00 2001 From: jie1zhan Date: Thu, 1 Sep 2022 17:06:23 +0800 Subject: [PATCH 1044/1247] libhsakmt: Add check for the runntime debuuger Avoiding the segfault, runtime debugger enable is not supported if the firmware of gpu doesn't support debug exceptions. Signed-off-by: jie1zhan Change-Id: Ifad57a6e78cb1c92b1f8927355ece8c64e89c51b --- src/debug.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/debug.c b/src/debug.c index b2021218ca..0b73c6ce8f 100644 --- a/src/debug.c +++ b/src/debug.c @@ -302,6 +302,25 @@ hsaKmtGetKernelDebugTrapVersionInfo( static HSAKMT_STATUS checkRuntimeDebugSupport(void) { HSAuint32 kMajor, kMinor; + HsaNodeProperties node = {0}; + HsaSystemProperties props = {0}; + + memset(&node, 0x00, sizeof(node)); + memset(&props, 0x00, sizeof(props)); + if (hsaKmtAcquireSystemProperties(&props)) + return HSAKMT_STATUS_ERROR; + + //the firmware of gpu node doesn't support the debugger, disable it. + for (uint32_t i = 0; i < props.NumNodes; i++) { + if (hsaKmtGetNodeProperties(i, &node)) + return HSAKMT_STATUS_ERROR; + + //ignore cpu node + if (node.NumCPUCores) + continue; + if (!node.Capability.ui32.DebugSupportedFirmware) + return HSAKMT_STATUS_NOT_SUPPORTED; + } if (hsaKmtGetKernelDebugTrapVersionInfo(&kMajor, &kMinor)) return HSAKMT_STATUS_NOT_SUPPORTED; From 06a90612e9646e958e8d1ecb75d79f24de07ada6 Mon Sep 17 00:00:00 2001 From: Aaron Liu Date: Thu, 25 Aug 2022 10:33:54 +0800 Subject: [PATCH 1045/1247] libhsakmt: expand control stack size limit for all gfx103x GFX1036(ISA version) is not included in the previous range. This patch can really include all gfx10 series ASICs. Signed-off-by: Aaron Liu Change-Id: I0e28dbfc031c216166b306b9fb39f644f75a330f --- src/queues.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/queues.c b/src/queues.c index 6ea17b4a7d..9a9c528351 100644 --- a/src/queues.c +++ b/src/queues.c @@ -279,8 +279,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) wg_data_size = cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(q->gfxv); q->ctl_stack_size = PAGE_ALIGN_UP(sizeof(HsaUserContextSaveAreaHeader) + ctl_stack_size); - if (q->gfxv >= GFX_VERSION_NAVI10 && - q->gfxv <= GFX_VERSION_YELLOW_CARP) { + if ((q->gfxv & 0x3f0000) == 0xA0000) { /* HW design limits control stack size to 0x7000. * This is insufficient for theoretical PM4 cases * but sufficient for AQL, limited by SPI events. From 3190f189b4add24018ebc805853b31dc2c90386d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 22 Jul 2022 15:38:54 -0400 Subject: [PATCH 1046/1247] kfdtest: Add KFDSVMRangeTest.VramOvercommitTest Change-Id: Id5b23d5efd4f6a9717d1ca196c8635846493f77d Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index a12d2714e3..246ef4e0c5 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1485,3 +1485,63 @@ TEST_F(KFDSVMRangeTest, HMMProfilingEvent) { TEST_END } + +/* + * Test SVM support VRAM overcommitment + * + * Prefetch total VRAM size plus overCommitSize SVM range to VRAM. after VRAM is full, + * KFD should support VRAM overcommitment by evicting SVM ranges to system memory to alloc + * VRAM for new ranges. + */ +TEST_F(KFDSVMRangeTest, VramOvercommitTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + if (!vramSize) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + unsigned long overCommitSize = 1UL << 30; + + /* With XNACK off, KFD checks that all SVM memory will fit into system memory */ + if (vramSize + overCommitSize > GetSysMemSize() / 2) { + LOG() << "Skipping test: no enough system memory." << std::endl; + return; + } + + unsigned long BufSize = 512UL << 20; + unsigned long numBufs = (vramSize + overCommitSize) / BufSize; + HSAKMT_STATUS ret; + + void *pBuf[numBufs]; + unsigned long i; + + for (i = 0; i < numBufs; i++) { + pBuf[i] = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + ASSERT_NOTNULL(pBuf[i]); + + ret = RegisterSVMRange(defaultGPUNode, pBuf[i], BufSize, defaultGPUNode, 0); + if (ret != HSAKMT_STATUS_SUCCESS) + break; + } + + EXPECT_EQ(numBufs, i); + + while (i--) + munmap(pBuf[i], BufSize); + + TEST_END +} From 115f1f8d1fd5edf648863dd7ea6b2b8102d2c89f Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 25 Jul 2022 08:11:02 -0400 Subject: [PATCH 1047/1247] kfdtest: Add KFDSVMRangeTest.VramOvercommitGiantRangeTest Change-Id: I5f6b3b6a910ff6646bf4b0c48ae3e94ad243cf88 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 246ef4e0c5..716a10fcfa 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1545,3 +1545,53 @@ TEST_F(KFDSVMRangeTest, VramOvercommitTest) { TEST_END } + +/* + * Test SVM support VRAM overcommitment + * + * Prefetch giant overcommit SVM range to VRAM, KFD should support VRAM overcommitment + * by spliting giant range into smaller ranges, evicting SVM ranges to system memory to + * alloc VRAM for overcommitment ranges. + */ +TEST_F(KFDSVMRangeTest, VramOvercommitGiantRangeTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + HSAuint64 vramSize = GetVramSize(defaultGPUNode); + if (!vramSize) { + LOG() << "Skipping test: No VRAM found." << std::endl; + return; + } + + unsigned long overCommitSize = 1UL << 30; + + /* With XNACK off, KFD checks that all SVM memory will fit into system memory */ + if (vramSize + overCommitSize > GetSysMemSize() / 2) { + LOG() << "Skipping test: no enough system memory." << std::endl; + return; + } + + unsigned long BufSize = vramSize + overCommitSize; + HSAKMT_STATUS ret; + void *pBuf; + + pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + ASSERT_NOTNULL(pBuf); + + ret = RegisterSVMRange(defaultGPUNode, pBuf, BufSize, defaultGPUNode, 0); + EXPECT_EQ (HSAKMT_STATUS_SUCCESS, ret); + + munmap(pBuf, BufSize); + TEST_END +} From 2230d01c8a8320901ccb3dd4b3e47e24dfe6ac63 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 15 Aug 2022 23:40:27 -0400 Subject: [PATCH 1048/1247] kfdtest: Add KFDSVMRangeTest.PrefaultPartialRangeTest Change-Id: I00617dd5a2216fab90c2b0d116825ec274d14d13 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 716a10fcfa..2f78263078 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1595,3 +1595,56 @@ TEST_F(KFDSVMRangeTest, VramOvercommitGiantRangeTest) { munmap(pBuf, BufSize); TEST_END } + +/* + * Test partial range prefault + * + * mmap alloc 4 pages range, memset middle 2 pages, prefetch entire range to VRAM, + * use sdma to memset the rest 2 pages, each page has different value 0x1, 0x2, 0x3, 0x4 + * then check if all page have the specific value after migrating 4 pages to system memory. + */ +TEST_F(KFDSVMRangeTest, PrefaultPartialRangeTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (!SVMAPISupported()) + return; + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId < FAMILY_AI) { + LOG() << std::hex << "Skipping test: No svm range support for family ID 0x" << m_FamilyId << "." << std::endl; + return; + } + + unsigned long BufSize = 4 * PAGE_SIZE; + HSAKMT_STATUS ret; + char *pBuf; + + pBuf = (char *)mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + ASSERT_NOTNULL(pBuf); + + memset(pBuf + PAGE_SIZE, 0x2, PAGE_SIZE); + memset(pBuf + 2 * PAGE_SIZE, 0x3, PAGE_SIZE); + + EXPECT_SUCCESS(RegisterSVMRange(defaultGPUNode, pBuf, BufSize, 0, 0)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufSize, defaultGPUNode)); + + SDMAQueue sdmaQueue; + EXPECT_SUCCESS(sdmaQueue.Create(defaultGPUNode)); + + sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(sdmaQueue.GetFamilyId(), + pBuf, 0x01010101, PAGE_SIZE)); + sdmaQueue.PlaceAndSubmitPacket(SDMAFillDataPacket(sdmaQueue.GetFamilyId(), + pBuf + 3 * PAGE_SIZE, 0x04040404, PAGE_SIZE)); + sdmaQueue.Wait4PacketConsumption(); + + EXPECT_SUCCESS(sdmaQueue.Destroy()); + + for (int i = 0; i < 4; i++) + EXPECT_EQ(pBuf[i * PAGE_SIZE], i + 1); + + munmap(pBuf, BufSize); + TEST_END +} From b7710a1dda17b5aca850cef0693d0e4470e20ba8 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 17 Aug 2022 11:19:56 -0400 Subject: [PATCH 1049/1247] libhsakmt: Add mmap alloc aligned helper function mmap alloc larger address range with align padding page plus guard pages, then unmap the padding and guard pages at beginning and end of the range, return aligned address range. Change-Id: Iaf3c711a079c744289efbafee9b5e63aaf724765 Signed-off-by: Philip Yang --- src/fmm.c | 83 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index cf6c2e36c7..e769cbfe87 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -706,13 +706,60 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, return start; } +static void *mmap_allocate_aligned(int prot, int flags, uint64_t size, + uint64_t align, uint64_t guard_size, + void *aper_base, void *aper_limit) +{ + void *addr, *aligned_addr, *aligned_end, *mapping_end; + uint64_t aligned_padded_size; + + aligned_padded_size = size + guard_size * 2 + (align - PAGE_SIZE); + + /* Map memory PROT_NONE to alloc address space only */ + addr = mmap(0, aligned_padded_size, PROT_NONE, flags, -1, 0); + if (addr == MAP_FAILED) { + pr_err("mmap failed: %s\n", strerror(errno)); + return NULL; + } + + /* Adjust for alignment and guard pages */ + aligned_addr = (void *)ALIGN_UP((uint64_t)addr + guard_size, align); + if (aligned_addr < aper_base || + VOID_PTR_ADD(aligned_addr, size - 1) > aper_limit) { + pr_err("mmap returned %p, out of range %p-%p\n", aligned_addr, + aper_base, aper_limit); + munmap(addr, aligned_padded_size); + return NULL; + } + + /* Unmap padding and guard pages */ + if (aligned_addr > addr) + munmap(addr, VOID_PTRS_SUB(aligned_addr, addr)); + + aligned_end = VOID_PTR_ADD(aligned_addr, size); + mapping_end = VOID_PTR_ADD(addr, aligned_padded_size); + if (mapping_end > aligned_end) + munmap(aligned_end, VOID_PTRS_SUB(mapping_end, aligned_end)); + + if (prot == PROT_NONE) + return aligned_addr; + + /* MAP_FIXED to the aligned address with required prot */ + addr = mmap(aligned_addr, size, prot, flags | MAP_FIXED, -1, 0); + if (addr == MAP_FAILED) { + pr_err("mmap failed: %s\n", strerror(errno)); + return NULL; + } + + return addr; +} + static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, void *address, uint64_t size, uint64_t align) { - uint64_t aligned_padded_size, guard_size; uint64_t alignment_size = PAGE_SIZE << svm.alignment_order; - void *addr, *aligned_addr, *aligned_end, *mapping_end; + uint64_t guard_size; if (address) return NULL; @@ -736,37 +783,9 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, * pages on both sides */ guard_size = (uint64_t)aper->guard_pages * PAGE_SIZE; - aligned_padded_size = size + align + - 2*guard_size - PAGE_SIZE; - /* Map memory */ - addr = mmap(0, aligned_padded_size, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); - if (addr == MAP_FAILED) { - pr_err("mmap failed: %s\n", strerror(errno)); - return NULL; - } - - /* Adjust for alignment and guard pages, range-check the reslt */ - aligned_addr = (void *)ALIGN_UP((uint64_t)addr + guard_size, align); - if (aligned_addr < aper->base || - VOID_PTR_ADD(aligned_addr, size - 1) > aper->limit) { - pr_err("mmap returned %p, out of range %p-%p\n", aligned_addr, - aper->base, aper->limit); - munmap(addr, aligned_padded_size); - return NULL; - } - - /* Unmap padding and guard pages */ - if (aligned_addr > addr) - munmap(addr, VOID_PTRS_SUB(aligned_addr, addr)); - - aligned_end = VOID_PTR_ADD(aligned_addr, size); - mapping_end = VOID_PTR_ADD(addr, aligned_padded_size); - if (mapping_end > aligned_end) - munmap(aligned_end, VOID_PTRS_SUB(mapping_end, aligned_end)); - - return aligned_addr; + return mmap_allocate_aligned(PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, + size, align, guard_size, aper->base, aper->limit); } static void mmap_aperture_release(manageable_aperture_t *aper, From b2691c359d73b660d8e8d040935fd2f2f22336a2 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 17 Aug 2022 13:33:12 -0400 Subject: [PATCH 1050/1247] libhsakmt: Use mmap aligned for scratch allocation To remove duplicate mmap aligned allocation code. Change-Id: Ibc05cc4aaf6d190bd2382e33bdeca1496960c5f2 Signed-off-by: Philip Yang --- src/fmm.c | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index e769cbfe87..8378620106 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1283,28 +1283,13 @@ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeIn aligned_size, SCRATCH_ALIGN); pthread_mutex_unlock(&svm.dgpu_aperture->fmm_mutex); } else { - uint64_t aligned_padded_size = aligned_size + - SCRATCH_ALIGN - PAGE_SIZE; - void *padded_end, *aligned_start, *aligned_end; - if (address) return NULL; - mem = mmap(0, aligned_padded_size, - PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); - if (!mem) - return NULL; - /* align start and unmap padding */ - padded_end = VOID_PTR_ADD(mem, aligned_padded_size); - aligned_start = (void *)ALIGN_UP((uint64_t)mem, SCRATCH_ALIGN); - aligned_end = VOID_PTR_ADD(aligned_start, aligned_size); - if (aligned_start > mem) - munmap(mem, VOID_PTRS_SUB(aligned_start, mem)); - if (aligned_end < padded_end) - munmap(aligned_end, - VOID_PTRS_SUB(padded_end, aligned_end)); - mem = aligned_start; + mem = mmap_allocate_aligned(PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + aligned_size, SCRATCH_ALIGN, 0, + 0, (void *)LONG_MAX); } /* Remember scratch backing aperture for later */ From 093cf898fb77f471301ca76defc2597622fad38d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 16 Aug 2022 09:36:11 -0400 Subject: [PATCH 1051/1247] libhsakmt: Set CWSR SVM range MADV_DONTFORK fork process copy-on-write MMU nitifier on CWSR range will evict user queues, and then update GPU mapping and resume queues, use MADV_DONTFORK to avoid COW MMU notifier callback on CWSR SVM range. Use mmap to alloc SVM range for CWSR because posix_memalign don't alloc new range in child process, this fails to register svm range as range is invalid address in forked child process. Change-Id: Ibaea56a691dd6f577ed2e1f2d43f4a3500b8316f Signed-off-by: Philip Yang --- src/fmm.c | 5 ++--- src/fmm.h | 3 +++ src/queues.c | 26 ++++++++++++++++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 8378620106..9b45caed35 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -706,9 +706,8 @@ static void *reserved_aperture_allocate_aligned(manageable_aperture_t *app, return start; } -static void *mmap_allocate_aligned(int prot, int flags, uint64_t size, - uint64_t align, uint64_t guard_size, - void *aper_base, void *aper_limit) +void *mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align, + uint64_t guard_size, void *aper_base, void *aper_limit) { void *addr, *aligned_addr, *aligned_end, *mapping_end; uint64_t aligned_padded_size; diff --git a/src/fmm.h b/src/fmm.h index b49e73f7bf..498366e7a0 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -90,4 +90,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address); int open_drm_render_device(int minor); +void *mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align, + uint64_t guard_size, void *aper_base, void *aper_limit); + #endif /* FMM_H_ */ diff --git a/src/queues.c b/src/queues.c index 9a9c528351..fa3ed564f8 100644 --- a/src/queues.c +++ b/src/queues.c @@ -425,7 +425,8 @@ static void free_queue(struct queue *q) q->eop_buffer_size, PAGE_SIZE, q->use_ats); if (q->unified_ctx_save_restore) - free(q->ctx_save_restore); + munmap(q->ctx_save_restore, + PAGE_ALIGN_UP(q->ctx_save_restore_size + q->debug_memory_size)); else if (q->ctx_save_restore) free_exec_aligned_memory(q->ctx_save_restore, q->ctx_save_restore_size + q->debug_memory_size, @@ -500,9 +501,21 @@ static int handle_concrete_asic(struct queue *q, void *addr; HSAKMT_STATUS r = HSAKMT_STATUS_ERROR; - if (posix_memalign(&addr, GPU_HUGE_PAGE_SIZE, size)) - pr_err("[%s] posix_memalign failed:\n", __func__); - else { + addr = mmap_allocate_aligned(PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + size, GPU_HUGE_PAGE_SIZE, 0, + 0, (void *)LONG_MAX); + if (!addr) { + pr_err("mmap failed to alloc ctx area size 0x%x: %s\n", + size, strerror(errno)); + } else { + /* + * To avoid fork child process COW MMU notifier + * callback evict parent process queues. + */ + if (madvise(addr, size, MADV_DONTFORK)) + pr_err("madvise failed -%d\n", errno); + fill_cwsr_header(q, addr, Event, ErrPayload); r = register_svm_range(addr, size, @@ -511,8 +524,9 @@ static int handle_concrete_asic(struct queue *q, if (r == HSAKMT_STATUS_SUCCESS) { q->ctx_save_restore = addr; q->unified_ctx_save_restore = true; - } else - free(addr); + } else { + munmap(addr, size); + } } } From 82a41c7e4d79723cb0deda18fad62330dd31a52b Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 8 Sep 2022 16:56:42 -0400 Subject: [PATCH 1052/1247] kfdtest: Remove unnecessary v_ ops in IterateIsa IterateIsa had some leftover instructions from when the shader was getting updated for KFDCWSRTest.BasicTest. Signed-off-by: Graham Sider Change-Id: I41ae7b7948cbe2aff8bf61b170b9a7d498b836a3 --- tests/kfdtest/src/ShaderStore.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 42b723f7fb..4ae13a17ca 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -459,7 +459,6 @@ const char *LoopIsa = R"( * v0 - workitem id * Registers: * v0 - calculated workitem = v0 + s4 * NUM_THREADS_X, which is s4 - * v[2:3] - corresponding input buf address: s[0:1] + v0 * 8 * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 * v6 - register storing known-value output for mangle testing * v7 - counter @@ -472,12 +471,6 @@ const char *IterateIsa = SHADER_MACROS R"( v_mov_b32 v5, s3 // v[4:5] = s[2:3] + v0 * 4 V_ADD_CO_CI_U32 v5, v5, 0 // v[4:5] = s[2:3] + v0 * 4 - // Compute address of input buffer - v_lshlrev_b32 v0, 1, v0 // v0 *= 8 - V_ADD_CO_U32 v2, s0, v0 // v[2:3] = s[0:1] + v0 * 8 - v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v0 * 8 - V_ADD_CO_CI_U32 v3, v3, 0 // v[2:3] = s[0:1] + v0 * 8 - // Store known-value output in register flat_load_dword v6, v[4:5] glc s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish From 14aa475905445e1fd9b0ece7fb48c0796e497792 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 11 Aug 2022 12:23:25 -0400 Subject: [PATCH 1053/1247] kfdtest: Update exclude for gfx1101 Signed-off-by: Graham Sider Change-Id: I78c1130c4a85a98a265a090e40390e56d3be2819 --- tests/kfdtest/scripts/kfdtest.exclude | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 4a078ae2d1..8d88ae6b61 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -294,8 +294,7 @@ FILTER[gfx1100]=\ "$BLACKLIST_GFX10_NV2X:"\ "$TEMPORARY_BLACKLIST_GFX11" -# TODO: Update to gfx version when kernel moves to staging-drm-next -FILTER[wheat_nas]=\ +FILTER[gfx1101]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X:"\ "$TEMPORARY_BLACKLIST_GFX11" From 169673a4354250271d6ea838097276bb278244ca Mon Sep 17 00:00:00 2001 From: Daniel Phillips Date: Thu, 9 Jun 2022 14:57:40 -0700 Subject: [PATCH 1054/1247] kfdtest: Add thunk test for KFD memory availability ioctl Signed-off-by: Daniel Phillips Change-Id: Ic4c1ffefdc3570718a1fce4e53ca5f1ebde8c479 --- tests/kfdtest/src/KFDMemoryTest.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 8c6449ceb7..2bcc7524f5 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -249,6 +249,31 @@ TEST_F(KFDMemoryTest, MemoryAlloc) { TEST_END } +// Basic test for hsaKmtAllocMemory +TEST_F(KFDMemoryTest, MemoryAllocAll) { + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + HsaMemFlags memFlags = {0}; + memFlags.ui32.NonPaged = 1; // sys mem vs vram + HSAuint64 available; + void *object = NULL; + int shrink = 21, success = HSAKMT_STATUS_NO_MEMORY; + + EXPECT_SUCCESS(hsaKmtAvailableMemory(defaultGPUNode, &available)); + LOG() << "Available: " << available << " bytes" << std::endl; + for (int i = 0; i < available >> shrink; i++) { + HSAuint64 size = available - ((HSAuint64)i << shrink); + if (hsaKmtAllocMemory(defaultGPUNode, size, memFlags, &object) == HSAKMT_STATUS_SUCCESS) { + LOG() << "Allocated: " << size << " bytes" << std::endl; + success = hsaKmtFreeMemory(object, available); + break; + } + } + EXPECT_SUCCESS(success); + TEST_END +} + TEST_F(KFDMemoryTest, AccessPPRMem) { TEST_START(TESTPROFILE_RUNALL) From 79279e860f042938e2c3c4525a328e56809559f0 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 20 Sep 2022 11:32:37 -0400 Subject: [PATCH 1055/1247] libhsakmt: Skip hsa_gfxip_table search for GFX11+ Prior to launch some ASICs may re-use PCI DIDs from older generations. This can cause issues during topology initialization as hsa_gfxip_table lookups will override sysfs-provided gfx versions, causing incorrect gfxip selection. Since no new entries will be added to hsa_gfxip_table, limit its search only to pre-GFX11 ASICs. Signed-off-by: Graham Sider Change-Id: I53eaefac5db2650a36a6ce9f21daf750f50cfd26 --- src/topology.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/topology.c b/src/topology.c index 4aea77e276..2887c661bc 100644 --- a/src/topology.c +++ b/src/topology.c @@ -800,8 +800,11 @@ err1: return ret; } -static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id) +static const struct hsa_gfxip_table *find_hsa_gfxip_device(uint16_t device_id, uint8_t gfxv_major) { + if (gfxv_major > 10) + return NULL; + uint32_t i, table_size; table_size = sizeof(gfxip_lookup_table)/sizeof(struct hsa_gfxip_table); @@ -1192,7 +1195,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, gfxv_minor = HSA_GET_GFX_VERSION_MINOR(gfxv); gfxv_stepping = HSA_GET_GFX_VERSION_STEP(gfxv); - hsa_gfxip = find_hsa_gfxip_device(props->DeviceId); + hsa_gfxip = find_hsa_gfxip_device(props->DeviceId, gfxv_major); if (hsa_gfxip || gfxv) { envvar = getenv("HSA_OVERRIDE_GFX_VERSION"); if (envvar) { From 590fd531c0f6ba181cd642dfbf8725685d6dfb0c Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 20 Sep 2022 10:57:09 -0400 Subject: [PATCH 1056/1247] kfdtest: Correct mmap return value checking On error mmap returns value MAP_FAILED, which is (void *)-1, not NULL pointer. Change-Id: I81b187266c943fa0aa4fab21b529d4c2989b12ad Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 2 +- tests/kfdtest/src/KFDSVMRangeTest.cpp | 8 ++++---- tests/kfdtest/src/KFDTestUtil.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index 319b054a64..22953d01ab 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -88,7 +88,7 @@ void KFDSVMEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HS for (HSAuint32 i = 0; i < count; i++) { m_pBuf = mmap(0, vramBufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - EXPECT_NOTNULL(m_pBuf); + ASSERT_NE(MAP_FAILED, m_pBuf); m_Flags = (HSA_SVM_FLAGS)0; retry: diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 2f78263078..f020529441 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1254,7 +1254,7 @@ TEST_F(KFDSVMRangeTest, MigrateFileBackedRangeTest) { ASSERT_EQ(size, write(fd, buf, size)); void *MmapedFile = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - ASSERT_NE(nullptr, MmapedFile); + ASSERT_NE(MAP_FAILED, MmapedFile); HsaSVMRange filebackedRange(MmapedFile, size, defaultGPUNode, defaultGPUNode); @@ -1531,7 +1531,7 @@ TEST_F(KFDSVMRangeTest, VramOvercommitTest) { for (i = 0; i < numBufs; i++) { pBuf[i] = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - ASSERT_NOTNULL(pBuf[i]); + ASSERT_NE(MAP_FAILED, pBuf[i]); ret = RegisterSVMRange(defaultGPUNode, pBuf[i], BufSize, defaultGPUNode, 0); if (ret != HSAKMT_STATUS_SUCCESS) @@ -1587,7 +1587,7 @@ TEST_F(KFDSVMRangeTest, VramOvercommitGiantRangeTest) { void *pBuf; pBuf = mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - ASSERT_NOTNULL(pBuf); + ASSERT_NE(MAP_FAILED, pBuf); ret = RegisterSVMRange(defaultGPUNode, pBuf, BufSize, defaultGPUNode, 0); EXPECT_EQ (HSAKMT_STATUS_SUCCESS, ret); @@ -1623,7 +1623,7 @@ TEST_F(KFDSVMRangeTest, PrefaultPartialRangeTest) { char *pBuf; pBuf = (char *)mmap(0, BufSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - ASSERT_NOTNULL(pBuf); + ASSERT_NE(MAP_FAILED, pBuf); memset(pBuf + PAGE_SIZE, 0x2, PAGE_SIZE); memset(pBuf + 2 * PAGE_SIZE, 0x3, PAGE_SIZE); diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index fe647fb853..103b144b05 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -862,7 +862,7 @@ HsaSVMRange::HsaSVMRange(void *addr, HSAuint64 size, HSAuint32 GPUNode, HSAuint3 m_SelfAllocated(false) { if (!m_pUser) { m_pUser = mmap(0, m_Size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - EXPECT_NOTNULL(m_pUser); + EXPECT_NE(MAP_FAILED, m_pUser); m_SelfAllocated = true; } From e44952d8a68e22cf96659cc0427701f7de1fe373 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 23 Sep 2022 16:28:25 -0400 Subject: [PATCH 1057/1247] kfdtest: Add dedicated FamilyId for GFX11 Add FAMILY_GFX11 to KfdFamilyId enum. Signed-off-by: Graham Sider Change-Id: Ib2aa3f8cf31041f7b4cdd9a2f3e36489dde5554c --- tests/kfdtest/src/KFDTestFlags.hpp | 19 ++++++++++--------- tests/kfdtest/src/KFDTestUtil.cpp | 3 ++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index 9087ba23f8..7fd49c7013 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -52,15 +52,16 @@ enum ENVCAPS{ enum KfdFamilyId { FAMILY_UNKNOWN = 0, - FAMILY_CI, // Sea Islands: Hawaii (P), Maui (P), Bonaire (M) - FAMILY_KV, // Fusion Kaveri: Spectre, Spooky; Fusion Kabini: Kalindi - FAMILY_VI, // Volcanic Islands: Iceland (V), Tonga (M) - FAMILY_CZ, // Carrizo, Nolan, Amur - FAMILY_AI, // Arctic Islands - FAMILY_RV, // Raven - FAMILY_AR, // Arcturus - FAMILY_AL, // Aldebaran - FAMILY_NV, // Navi10 + FAMILY_CI, // Sea Islands: Hawaii (P), Maui (P), Bonaire (M) + FAMILY_KV, // Fusion Kaveri: Spectre, Spooky; Fusion Kabini: Kalindi + FAMILY_VI, // Volcanic Islands: Iceland (V), Tonga (M) + FAMILY_CZ, // Carrizo, Nolan, Amur + FAMILY_AI, // Arctic Islands + FAMILY_RV, // Raven + FAMILY_AR, // Arcturus + FAMILY_AL, // Aldebaran + FAMILY_NV, // Navi10 + FAMILY_GFX11, // GFX11 }; #endif // __KFD_TEST_FLAGS__H__ diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 103b144b05..feeb6fd19d 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -192,8 +192,9 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { familyId = FAMILY_AL; break; case 10: - case 11: familyId = FAMILY_NV; + case 11: + familyId = FAMILY_GFX11; break; } From 6294ef564b723b2fd2799d862f2f3dd616c88ddf Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 11 Jul 2022 13:23:58 -0400 Subject: [PATCH 1058/1247] kfdtest: Update COMPUTE_PGM_RSRC1 for software trap For software trap in GFX11, COMPUTE_PGM_RSRC1 must have PRIV = 1. Signed-off-by: Graham Sider Change-Id: Id504889c3ca2588b6c8cefdebaec00dcfc217995 --- tests/kfdtest/src/Dispatch.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index 3aa3892f38..87eb659c7f 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -138,10 +138,11 @@ void Dispatch::BuildIb() { pgmRsrc2 |= (1 << COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT) & COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK; + const bool priv = (m_FamilyId == FAMILY_GFX11); const unsigned int COMPUTE_PGM_RSRC[] = { - // PGM_RSRC1 = { VGPRS: 16 SGPRS: 16 PRIORITY: m_SpiPriority FLOAT_MODE: c0 PRIV: 0 - // DX10_CLAMP: 0 DEBUG_MODE: 0 IEEE_MODE: 0 BULKY: 0 CDBG_USER: 0 } - 0x000c0084 | ((m_SpiPriority & 3) << 10), + // PGM_RSRC1 = { VGPRS: 16 SGPRS: 16 PRIORITY: m_SpiPriority FLOAT_MODE: c0 + // PRIV: 0 (1 for GFX11) DX10_CLAMP: 0 DEBUG_MODE: 0 IEEE_MODE: 0 BULKY: 0 CDBG_USER: 0 } + 0x000c0084 | ((m_SpiPriority & 3) << 10) | (priv << 20), pgmRsrc2 }; From af48352d9acda0d00ff9b5d1ef89d7c52389a0d3 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 29 Sep 2022 09:19:38 -0400 Subject: [PATCH 1059/1247] kfdtest: Add missing break in FamilyIdFromNode Add missing break after GFX11 FamilyId patch. Signed-off-by: Graham Sider Change-Id: I8fd727054442d74a95f69724ab07c4095f8ae77e --- tests/kfdtest/src/KFDTestUtil.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index feeb6fd19d..fc8bd0248e 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -193,6 +193,7 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { break; case 10: familyId = FAMILY_NV; + break; case 11: familyId = FAMILY_GFX11; break; From 73adbdee2c1391950e5ff66e3527a6e450967367 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 8 Sep 2022 16:44:51 -0400 Subject: [PATCH 1060/1247] kfdtest: Make KFDCWSRTest.BasicTest buffer sizes dynamic KFDCWSRTest.BasicTest is parameterized to allow an easy method of tweaking the number of work-items (and save/restores). The input/output buffers were previously hardcoded to a single page, which would cause a segmentation fault if the number of work-items specified is greater than 1024 for wave32. Signed-off-by: Graham Sider Change-Id: Ieefc819a5d81c77cee88081a287fd383e6378e74 --- tests/kfdtest/src/KFDCWSRTest.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 146ae2ae5a..da66efda19 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -88,8 +88,11 @@ TEST_P(KFDCWSRTest, BasicTest) { unsigned stopval = 0x1234'5678; unsigned outval = 0x8765'4321; - HsaMemoryBuffer inputBuf(PAGE_SIZE, defaultGPUNode, true, false, false); - HsaMemoryBuffer outputBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + // 4B per work-item ==> 1 page per 1024 work-items (take ceiling) + unsigned bufSize = PAGE_SIZE * ((num_witems / 1024) + (num_witems % 1024 != 0)); + + HsaMemoryBuffer inputBuf(bufSize, defaultGPUNode, true, false, false); + HsaMemoryBuffer outputBuf(bufSize, defaultGPUNode, true, false, false); unsigned int* input = inputBuf.As(); unsigned int* output = outputBuf.As(); inputBuf.Fill(0); From 4b9461dd42f8f0d5c99a885ebcb1f9731980b617 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Wed, 12 Oct 2022 10:06:53 +0800 Subject: [PATCH 1061/1247] kfdtest: add blacklist for gfx1103 Signed-off-by: Yifan Zhang Change-Id: I9ffe4dd8add505d0a6cfd3ed974fab6cef05f039 --- tests/kfdtest/scripts/kfdtest.exclude | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 8d88ae6b61..fcb9ba8f58 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -304,6 +304,11 @@ FILTER[gfx1102]=\ "$BLACKLIST_GFX10_NV2X:"\ "$TEMPORARY_BLACKLIST_GFX11" +FILTER[gfx1103]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX10_NV2X:"\ +"$TEMPORARY_BLACKLIST_GFX11" + FILTER[gfx1036]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X" From 178a619b80ef9ecf5753c7232e71e5e0e1e61fb3 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Wed, 31 Aug 2022 12:16:04 -0500 Subject: [PATCH 1062/1247] src: use SVM mechanism to register userptr memory Register and map userptrs through Shared Virtual Memory(SVM) API at the Kernel level when available. Using this approach, performance will be improve as register/unregister memory will not trigger any system call to KFD driver. Signed-off-by: Alex Sierra Change-Id: I20723cbeb340bf48b95e1115f0102c031397bc14 --- src/fmm.c | 147 ++++++++++++++++++++++------ tests/kfdtest/src/KFDMemoryTest.cpp | 10 +- 2 files changed, 125 insertions(+), 32 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 9b45caed35..b2ce7da43e 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -220,6 +220,9 @@ typedef struct { /* specifies the alignment size as PAGE_SIZE * 2^alignment_order */ uint32_t alignment_order; + + /* whether to check all dGPUs in the topology support SVM API */ + bool is_svm_api_supported; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -239,7 +242,8 @@ static svm_t svm = { .dgpu_alt_aperture = NULL, .userptr_for_paged_mem = false, .check_userptr = false, - .disable_cache = false + .disable_cache = false, + .is_svm_api_supported = false }; /* On APU, for memory allocated on the system memory that GPU doesn't access @@ -962,6 +966,72 @@ static HsaMemFlags fmm_translate_ioc_to_hsa_flags(uint32_t ioc_flags) return mflags; } +static HSAKMT_STATUS fmm_register_mem_svm_api(void *address, + uint64_t size, + bool coarse_grain) +{ + struct kfd_ioctl_svm_args *args; + size_t s_attr; + HSAuint32 page_offset = (HSAuint64)address & (PAGE_SIZE-1); + HSAuint64 aligned_addr = (HSAuint64)address - page_offset; + HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); + + if (!g_first_gpu_mem) + return HSAKMT_STATUS_ERROR; + + s_attr = sizeof(struct kfd_ioctl_svm_attribute); + args = alloca(sizeof(*args) + s_attr); + args->start_addr = aligned_addr; + args->size = aligned_size; + args->op = KFD_IOCTL_SVM_OP_SET_ATTR; + args->nattr = 1; + args->attrs[0].type = coarse_grain ? + HSA_SVM_ATTR_CLR_FLAGS : HSA_SVM_ATTR_SET_FLAGS; + args->attrs[0].value = HSA_SVM_FLAG_COHERENT; + pr_debug("Registering to SVM %p size: %ld\n", (void*)aligned_addr, + aligned_size); + /* Driver does one copy_from_user, with extra attrs size */ + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { + pr_debug("op set range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS fmm_map_mem_svm_api(void *address, + uint64_t size, + uint32_t *nodes_to_map, + uint32_t nodes_array_size) +{ + struct kfd_ioctl_svm_args *args; + size_t s_attr; + uint32_t i, nattr; + + if (!g_first_gpu_mem) + return HSAKMT_STATUS_ERROR; + + nattr = nodes_array_size; + s_attr = sizeof(struct kfd_ioctl_svm_attribute) * nattr; + args = alloca(sizeof(*args) + s_attr); + + args->start_addr = (uint64_t)address; + args->size = size; + args->op = KFD_IOCTL_SVM_OP_SET_ATTR; + args->nattr = nattr; + for (i = 0; i < nodes_array_size; i++) { + args->attrs[i].type = HSA_SVM_ATTR_ACCESS_IN_PLACE; + args->attrs[i].value = nodes_to_map[i]; + } + /* Driver does one copy_from_user, with extra attrs size */ + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { + pr_debug("op set range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + /* After allocating the memory, return the vm_object created for this memory. * Return NULL if any failure. */ @@ -1755,7 +1825,9 @@ HSAKMT_STATUS fmm_release(void *address) object = vm_find_object(address, 0, &aperture); if (!object) - return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + return svm.is_svm_api_supported ? + HSAKMT_STATUS_SUCCESS : + HSAKMT_STATUS_MEMORY_NOT_REGISTERED; if (aperture == &cpuvm_aperture) { /* APU system memory */ @@ -2247,6 +2319,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) */ is_dgpu = false; + svm.is_svm_api_supported = true; for (i = 0; i < NumNodes; i++) { HsaNodeProperties props; @@ -2283,6 +2356,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; gpu_mem[gpu_mem_count].node_id = i; + svm.is_svm_api_supported &= props.Capability.ui32.SVMAPISupported; gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; gpu_mem[gpu_mem_count].scratch_physical.ops = &reserved_aperture_ops; @@ -2807,23 +2881,36 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, - uint64_t *gpuvm_addr, vm_object_t *object) + uint64_t *gpuvm_addr, vm_object_t *object, + uint32_t *nodes_to_map, uint32_t nodes_array_size) { manageable_aperture_t *aperture; void *svm_addr; - HSAuint64 svm_size; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); int ret; aperture = svm.dgpu_aperture; - svm_addr = object->start; - svm_size = object->size; - /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, object, NULL, 0); + if (svm.is_svm_api_supported) { + svm_addr = (void*)((HSAuint64)addr - page_offset); + if (!nodes_to_map) { + nodes_to_map = all_gpu_id_array; + nodes_array_size = all_gpu_id_array_size; + } + pr_debug("%s Mapping Address %p size aligned: %ld offset: %x\n", + __func__, svm_addr, PAGE_ALIGN_UP(page_offset + size), page_offset); + ret = fmm_map_mem_svm_api(svm_addr, + PAGE_ALIGN_UP(page_offset + size), + nodes_to_map, + nodes_array_size / sizeof(uint32_t)); + + } else { + svm_addr = object->start; + ret = _fmm_map_to_gpu(aperture, svm_addr, object->size, object, NULL, 0); + } if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -2847,7 +2934,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) address, size); object = vm_find_object(address, size, &aperture); - if (!object) { + if (!object && !svm.is_svm_api_supported ) { if (!is_dgpu) { /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); @@ -2862,8 +2949,8 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); ret = 0; - } else if (object->userptr) { - ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); + } else if ((svm.is_svm_api_supported && !object) || object->userptr) { + ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object, NULL, 0); } else { ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0); /* Update alternate GPUVM address only for @@ -2873,7 +2960,8 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); } - pthread_mutex_unlock(&aperture->fmm_mutex); + if (object) + pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } @@ -2938,7 +3026,6 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, print_device_id_array((void *)args.device_ids_array_ptr, args.n_devices * sizeof(uint32_t)); - ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); remove_device_ids_from_mapped_array(object, @@ -3040,7 +3127,7 @@ int fmm_unmap_from_gpu(void *address) object = vm_find_object(address, 0, &aperture); if (!object) /* On APUs GPU unmapping of system memory is a no-op */ - return is_dgpu ? -EINVAL : 0; + return (!is_dgpu || svm.is_svm_api_supported) ? 0 : -EINVAL; /* Successful vm_find_object returns with the aperture locked */ if (aperture == &cpuvm_aperture) @@ -3128,7 +3215,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | - (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), &obj); + (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), + &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; @@ -3180,6 +3268,9 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, return HSAKMT_STATUS_SUCCESS; /* Register a new user ptr */ + if (svm.is_svm_api_supported) + return fmm_register_mem_svm_api(address, size_in_bytes, coarse_grain); + ret = fmm_register_user_memory(address, size_in_bytes, &object, coarse_grain); if (ret != HSAKMT_STATUS_SUCCESS) return ret; @@ -3497,9 +3588,9 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) /* On APUs we assume it's a random system memory address * where registration and dergistration is a no-op */ - return is_dgpu ? - HSAKMT_STATUS_MEMORY_NOT_REGISTERED : - HSAKMT_STATUS_SUCCESS; + return (!is_dgpu || svm.is_svm_api_supported) ? + HSAKMT_STATUS_SUCCESS : + HSAKMT_STATUS_MEMORY_NOT_REGISTERED; /* Successful vm_find_object returns with aperture locked */ if (aperture == &cpuvm_aperture) { @@ -3550,7 +3641,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address) { - manageable_aperture_t *aperture; + manageable_aperture_t *aperture = NULL; vm_object_t *object; uint32_t i; uint32_t *registered_node_id_array, registered_node_id_array_size; @@ -3561,24 +3652,22 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_INVALID_PARAMETER; object = vm_find_object(address, size, &aperture); - if (!object) + if (!object && !svm.is_svm_api_supported) return HSAKMT_STATUS_ERROR; /* Successful vm_find_object returns with aperture locked */ /* APU memory is not supported by this function */ - if (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible) { + if (aperture && + (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible)) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } - /* For userptr, we ignore the nodes array and map all registered nodes. - * This is to simply the implementation of allowing the same memory - * region to be registered multiple times. - */ - if (object->userptr) { - retcode = _fmm_map_to_gpu_userptr(address, size, - gpuvm_address, object); - pthread_mutex_unlock(&aperture->fmm_mutex); + if ((svm.is_svm_api_supported && !object) || object->userptr) { + retcode = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, + object, nodes_to_map, num_of_nodes * sizeof(uint32_t)); + if (object) + pthread_mutex_unlock(&aperture->fmm_mutex); return retcode ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 2bcc7524f5..b4cb37a5a6 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -443,7 +443,6 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { EXPECT_SUCCESS(hsaKmtRegisterMemory((void *)&mem[0], sizeof(HSAuint32))); EXPECT_SUCCESS(hsaKmtMapMemoryToGPU((void *)&mem[0], sizeof(HSAuint32), &gpuva2)); - EXPECT_TRUE(gpuva1 != gpuva2); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva1))); EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(gpuva1))); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva2))); @@ -1095,7 +1094,12 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { static volatile HSAuint32 mem[4]; // 8 bytes for register only and // 8 bytes for register to nodes HsaMemoryBuffer hsaBuffer((void *)(&mem[0]), sizeof(HSAuint32)*2); - if (is_dgpu()) { // APU doesn't use userptr + /* + * APU doesn't use userptr. + * User pointers registered with SVM API, does not create vm_object_t. + * Therefore, pointer info can not be queried. + */ + if (is_dgpu() && mem != hsaBuffer.As()) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[0]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); @@ -1122,7 +1126,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo(reinterpret_cast(address), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED); EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As()); - if (is_dgpu()) { + if (is_dgpu() && &mem[1] != hsaBuffer.As() + 1) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[1]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); From 45fad297528572deebbf4ddee2fba8251e241980 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 29 Sep 2022 17:37:39 -0500 Subject: [PATCH 1063/1247] libhsakmt: query svm info from userptrs at fault events Get more debug information about user pointers that were registered through SVM API, and triggered by memory exception events. A new kfdtest with this use case was also included inside KFDExceptionTest. Signed-off-by: Alex Sierra Change-Id: I8e9df3c1c6c3f42d7b9235d12406d80d31746443 --- src/events.c | 79 +++++++++++++++++++++++++- tests/kfdtest/src/KFDExceptionTest.cpp | 48 ++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/events.c b/src/events.c index 06d3959532..1c5b03dcf2 100644 --- a/src/events.c +++ b/src/events.c @@ -220,6 +220,80 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event, return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); } +static HSAKMT_STATUS get_mem_info_svm_api(uint64_t address, uint32_t gpu_id) +{ + struct kfd_ioctl_svm_args *args; + uint32_t node_id; + HSAuint32 s_attr; + HSAuint32 i; + HSA_SVM_ATTRIBUTE attrs[] = { + {HSA_SVM_ATTR_PREFERRED_LOC, 0}, + {HSA_SVM_ATTR_PREFETCH_LOC, 0}, + {HSA_SVM_ATTR_ACCESS, gpu_id}, + {HSA_SVM_ATTR_SET_FLAGS, 0}, + }; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(5); + + s_attr = sizeof(attrs); + args = alloca(sizeof(*args) + s_attr); + args->start_addr = address; + args->size = PAGE_SIZE; + args->op = KFD_IOCTL_SVM_OP_GET_ATTR; + args->nattr = s_attr / sizeof(*attrs); + memcpy(args->attrs, attrs, s_attr); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { + pr_debug("op get range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + pr_err("GPU address 0x%lx, is Unified memory\n", address); + for (i = 0; i < args->nattr; i++) { + if (args->attrs[i].value == KFD_IOCTL_SVM_LOCATION_SYSMEM || + args->attrs[i].value == KFD_IOCTL_SVM_LOCATION_UNDEFINED) + node_id = args->attrs[i].value; + else + gpuid_to_nodeid(args->attrs[i].value, &node_id); + switch (args->attrs[i].type) { + case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: + pr_err("Preferred location for address 0x%lx is Node id %d\n", + address, node_id); + break; + case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: + pr_err("Prefetch location for address 0x%lx is Node id %d\n", + address, node_id); + break; + case KFD_IOCTL_SVM_ATTR_ACCESS: + pr_err("Node id %d has access to address 0x%lx\n", + node_id, address); + break; + case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: + pr_err("Node id %d has access in place to address 0x%lx\n", + node_id, address); + break; + case KFD_IOCTL_SVM_ATTR_NO_ACCESS: + pr_err("Node id %d has no access to address 0x%lx\n", + node_id, address); + break; + case KFD_IOCTL_SVM_ATTR_SET_FLAGS: + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_COHERENT) + pr_err("Fine grained coherency between devices\n"); + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_RO) + pr_err("Read only\n"); + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_EXEC) + pr_err("GPU exec allowed\n"); + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) + pr_err("GPU always mapped\n"); + break; + default: + pr_debug("get invalid attr type 0x%x\n", args->attrs[i].type); + return HSAKMT_STATUS_ERROR; + } + } + + return HSAKMT_STATUS_SUCCESS; +} //Analysis memory exception data, print debug messages static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * memory_exception_data) @@ -241,9 +315,10 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * pr_err("Execute to none-executable page\n"); ret = fmm_get_mem_info((const void *)addr, &info); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("Address does not belong to a known buffer\n"); + ret = get_mem_info_svm_api(addr, memory_exception_data->gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) + pr_err("Address does not belong to a known buffer\n"); return; } diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index f7c28d8897..2ca5c6777d 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -27,6 +27,7 @@ #include "SDMAPacket.hpp" #include "SDMAQueue.hpp" #include "Dispatch.hpp" +#include void KFDExceptionTest::SetUp() { ROUTINE_START @@ -253,6 +254,53 @@ TEST_F(KFDExceptionTest, PermissionFault) { TEST_END } +/* Allocate Read Only user pointer buffer. Test Memory Exception failure by + * attempting to write to that buffer in the child process. + */ +TEST_F(KFDExceptionTest, PermissionFaultUserPointer) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; + return; + } + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + KFDBaseComponentTest::TearDown(); + KFDBaseComponentTest::SetUp(); + + void *pBuf = mmap(NULL, PAGE_SIZE, PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + ASSERT_NE(pBuf, MAP_FAILED); + EXPECT_SUCCESS(hsaKmtRegisterMemory(pBuf, PAGE_SIZE)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(pBuf, PAGE_SIZE, NULL)); + HsaMemoryBuffer srcSysBuffer(PAGE_SIZE, defaultGPUNode, false); + + srcSysBuffer.Fill(0xAA55AA55); + + TestMemoryException(defaultGPUNode, srcSysBuffer.As(), + (HSAuint64)pBuf); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + if (is_dgpu()) { + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } else { + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + } + + TEST_END +} + /* Test VM fault storm handling by copying to/from invalid pointers * with lots of work items at the same time */ From 75428364a7da039539513bb8fef9bed99524bac7 Mon Sep 17 00:00:00 2001 From: Ramesh Errabolu Date: Mon, 12 Sep 2022 16:27:14 -0500 Subject: [PATCH 1064/1247] Add support for CRIU testing Change-Id: I8945a078ee8ae491245da6091e64b118584a48ab --- tests/kfdtest/src/KFDMemoryTest.cpp | 30 ++++++++++++++++++++++++++++ tests/kfdtest/src/KFDTestMain.cpp | 9 +++++++++ tests/kfdtest/src/LinuxOSWrapper.cpp | 11 ++++++++++ tests/kfdtest/src/OSWrapper.hpp | 2 ++ 4 files changed, 52 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index b4cb37a5a6..5ee8537f37 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -39,6 +39,9 @@ #include "SDMAPacket.hpp" #include "linux/kfd_ioctl.h" +/* Captures user specified time (seconds) to sleep */ +extern unsigned int g_SleepTime; + void KFDMemoryTest::SetUp() { ROUTINE_START @@ -93,6 +96,15 @@ TEST_F(KFDMemoryTest, MMapLarge) { int i = 0; /* Allocate 1024GB, aka 1TB*/ for (; i < nObjects; i++) { + + /* Code snippet to allow CRIU checkpointing */ + if (i == (1 << 6)) { + if (g_SleepTime > 0) { + LOG() << "Pause for: " << g_SleepTime << " seconds" << std::endl; + sleep(g_SleepTime); + } + } + if (hsaKmtRegisterMemory(addr + i, s - i)) break; if (hsaKmtMapMemoryToGPUNodes(addr + i, s - i, @@ -633,6 +645,7 @@ void KFDMemoryTest::BinarySearchLargestBuffer(int allocNode, const HsaMemFlags & HSAuint64 highMB, int nodeToMap, HSAuint64 *lastSizeMB) { int ret; + int iter = 0; HsaMemMapFlags mapFlags = {0}; HSAuint64 granularityMB = highMB > 512 ? 128 : 16; @@ -656,6 +669,15 @@ void KFDMemoryTest::BinarySearchLargestBuffer(int allocNode, const HsaMemFlags & continue; } + /* Code snippet to allow CRIU checkpointing */ + iter++; + if (iter == 3) { + if (g_SleepTime > 0) { + LOG() << "Pause for: " << g_SleepTime << " seconds" << std::endl; + sleep(g_SleepTime); + } + } + ret = hsaKmtMapMemoryToGPUNodes(pDb, size, NULL, mapFlags, 1, reinterpret_cast(&nodeToMap)); if (ret) { @@ -908,6 +930,14 @@ TEST_F(KFDMemoryTest, MMBench) { HSAuint64 allocTime, map1Time, unmap1Time, mapAllTime, unmapAllTime, freeTime; HSAuint32 allocNode; + /* Code snippet to allow CRIU checkpointing */ + if (testIndex == 3) { + if (g_SleepTime > 0) { + LOG() << "Pause for: " << g_SleepTime << " seconds" << std::endl; + sleep(g_SleepTime); + } + } + if ((testIndex % nSizes) == 0) LOG() << "--------------------------------------------------------------------------" << std::endl; diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index 2b33ac153d..59c0dc806f 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -53,6 +53,7 @@ unsigned int g_TestTimeOut; int g_TestNodeId; int g_TestDstNodeId; bool g_IsChildProcess; +unsigned int g_SleepTime; unsigned int g_TestGPUFamilyId; class KFDBaseComponentTest *g_baseTest; @@ -80,12 +81,20 @@ GTEST_API_ int main(int argc, char **argv) { if ( args.TimeOut > 0 ) g_TestTimeOut = args.TimeOut; + g_SleepTime = 0x00; + if (args.SleepTime > 0) { + g_SleepTime = args.SleepTime; + } + // If --node is not specified, then args.NodeId == -1 g_TestNodeId = args.NodeId; g_TestDstNodeId = args.DstNodeId; LOG() << "Profile: " << (TESTPROFILE)g_TestRunProfile << std::endl; LOG() << "HW capabilities: 0x" << std::hex << g_TestENVCaps << std::endl; + if (g_SleepTime > 0) { + LOG() << "Sleep time in seconds as specified by user: " << std::dec << g_SleepTime << std::endl; + } return RUN_ALL_TESTS(); } diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/tests/kfdtest/src/LinuxOSWrapper.cpp index f5308249bf..b8855061d5 100644 --- a/tests/kfdtest/src/LinuxOSWrapper.cpp +++ b/tests/kfdtest/src/LinuxOSWrapper.cpp @@ -109,6 +109,7 @@ void ComandLineArgumentsUsage() { printf("\t--child arg\t - Child Process\n"); printf("\t--timeout arg\t - Time Out\n"); printf("\t--dst_node\t - For testing multiple nodes"); + printf("\t--sleep_time\t - For testing CRIU, etc"); } bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs) { @@ -123,6 +124,7 @@ bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs) { "timeout", required_argument, 0, 0}, { "node", required_argument, 0, 0 }, { "dst_node", required_argument, 0, 0 }, + { "sleep_time", required_argument, 0, 0 }, { 0, 0, 0, 0 } }; @@ -132,6 +134,7 @@ bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs) rArgs.TimeOut = 0; rArgs.NodeId = -1; rArgs.DstNodeId = -1; + rArgs.SleepTime = 0; while (true) { int c = getopt_long(argc, argv, "", long_options, &option_index); @@ -200,6 +203,14 @@ bool GetCommandLineArguments(int argc, char **argv, CommandLineArguments& rArgs) rArgs.DstNodeId = dstNodeId; } break; + /* Sleep time - used in testing CRIU */ + case 6: + { + int sleepTime = atoi(optarg); + if (sleepTime >= 0) + rArgs.SleepTime = sleepTime; + } + break; } } diff --git a/tests/kfdtest/src/OSWrapper.hpp b/tests/kfdtest/src/OSWrapper.hpp index 4ded18e0e0..d7d69b7f73 100644 --- a/tests/kfdtest/src/OSWrapper.hpp +++ b/tests/kfdtest/src/OSWrapper.hpp @@ -64,6 +64,8 @@ struct CommandLineArguments { unsigned int TimeOut; int NodeId; int DstNodeId; + /* Time in units of seconds */ + unsigned int SleepTime; }; // It is either MEM_NONE or the bitwise OR of one or more of the following flags From 4fa9404fe2e783578bbf02dc22cb2aebf792469a Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Fri, 4 Nov 2022 17:31:08 -0400 Subject: [PATCH 1065/1247] Rework libdrm requires/recommends To be more alligned with ROCr, libdrm dev package appears to be required, but we don't care if it's ours or the distro's. So require either but recommend our package to get the latest version. Change-Id: I744ce4861644a83ba94c39e0bf4230eab58cc68a Signed-off-by: Jeremy Newton --- CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d00248ee83..32ce214514 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,19 +358,20 @@ if ( PROC_RESULT EQUAL "0" AND NOT EVAL_RESULT STREQUAL "" ) endif() # Setting devel package dependendent version -set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-dev, rocm-core") -set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-amdgpu1") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-amdgpu-dev | libdrm-dev, rocm-core") +set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-dev") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") -set(CPACK_RPM_PACKAGE_REQUIRES "libdrm-devel, rocm-core") +set(CPACK_RPM_PACKAGE_REQUIRES "rocm-core") # NOTE: CentOS/RHEL <8 and SUSE <10 do not have support for rpm recommends, # so HSAKMT_REQUIRES_LIBDRM should be defined when building EL7 to avoid # producing binaries which do not install on CentOS7 or RHEL7 if(HSAKMT_REQUIRES_LIBDRM) - string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu" ) + string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu-devel" ) else() - set(CPACK_RPM_PACKAGE_RECOMMENDS "libdrm-amdgpu" ) + string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", (libdrm-amdgpu-devel or libdrm-devel)" ) + set(CPACK_RPM_PACKAGE_RECOMMENDS "libdrm-amdgpu-devel" ) endif() # Set the names now using CPACK utility From a63d53ad8bfd05e14652353cc9bbe31605ac187e Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Thu, 13 Oct 2022 10:42:21 -0400 Subject: [PATCH 1066/1247] Fix libc/gcc hardpath issues Don't use the full path to link against libc, but rather let cmake find it. Regarding gcc_s, it doesn't seem like this is still needed, so I've removed it instead. Change-Id: I1dc594f10c647b2abfdab7c5e0de90c331c6eeaf Signed-off-by: Jeremy Newton --- CMakeLists.txt | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 32ce214514..7e78bfbe79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -159,7 +159,7 @@ get_os_info() find_package(PkgConfig) # Check for libraries required for building -find_library(LIBC NAMES libc.so.6 REQUIRED) +find_library(LIBC NAMES c REQUIRED) find_library(NUMA NAMES numa REQUIRED) message(STATUS "LIBC:" ${LIBC}) message(STATUS "NUMA:" ${NUMA}) @@ -183,15 +183,10 @@ endif() include_directories(${DRM_INCLUDE_DIRS}) target_link_libraries ( ${HSAKMT_TARGET} - PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt ${LIBC} numa + PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt c numa ) target_compile_options(${HSAKMT_TARGET} PRIVATE ${DRM_CFLAGS} ${HSAKMT_C_FLAGS}) -if(NOT DISTRO_ID MATCHES "ubuntu") - find_library(LIBGCC NAMES libgcc_s.so.1 REQUIRED) - message(STATUS "LIBGCC:" ${LIBGCC}) - target_link_libraries( ${HSAKMT_TARGET} PRIVATE ${LIBGCC} ) -endif() ## Define default paths and packages. if( CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT ) From 6467664ec7207a7708cd9f09b0c8473e72dc8e53 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 5 Oct 2022 16:04:11 -0400 Subject: [PATCH 1067/1247] kfdtest: Rename IterateIsa to PersistentIterateIsa To avoid confusion since this shader has changed to be persistent (original IterateIsa may be re-used for debugger tests). Signed-off-by: Graham Sider Change-Id: I4643692765fc7665933257e89d5b922e779ad2e5 --- tests/kfdtest/src/KFDCWSRTest.cpp | 4 ++-- tests/kfdtest/src/ShaderStore.cpp | 4 ++-- tests/kfdtest/src/ShaderStore.hpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index da66efda19..743b03d1b3 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -51,7 +51,7 @@ static inline uint32_t checkCWSREnabled() { /** * KFDCWSRTest.BasicTest * - * This test dispatches the IterateIsa shader, which continuously increments a vgpr for + * This test dispatches the PersistentIterateIsa shader, which continuously increments a vgpr for * (num_witems / WAVE_SIZE) waves. While this shader is running, dequeue/requeue requests * are sent in a loop to trigger CWSRs. * @@ -83,7 +83,7 @@ TEST_P(KFDCWSRTest, BasicTest) { if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true, false, true); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(IterateIsa, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PersistentIterateIsa, isaBuffer.As())); unsigned stopval = 0x1234'5678; unsigned outval = 0x8765'4321; diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 4ae13a17ca..fc97d6490e 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -39,7 +39,7 @@ const std::vector ShaderList = { WriteFlagAndValueIsa, WriteAndSignalIsa, LoopIsa, - IterateIsa, + PersistentIterateIsa, ReadMemoryIsa, GwsInitIsa, GwsAtomicIncreaseIsa, @@ -463,7 +463,7 @@ const char *LoopIsa = R"( * v6 - register storing known-value output for mangle testing * v7 - counter */ -const char *IterateIsa = SHADER_MACROS R"( +const char *PersistentIterateIsa = SHADER_MACROS R"( // Compute address of output buffer v_mov_b32 v0, s4 // use workgroup id as index v_lshlrev_b32 v0, 2, v0 // v0 *= 4 diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index e0151a6537..c655030f9b 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -48,7 +48,7 @@ extern const char *WriteAndSignalIsa; extern const char *LoopIsa; /* KFDCWSRTest */ -extern const char *IterateIsa; +extern const char *PersistentIterateIsa; /* KFDEvictTest */ extern const char *ReadMemoryIsa; From e0476629fe021d6a4589490b4e8060c895eab412 Mon Sep 17 00:00:00 2001 From: Nirmal Unnikrishnan Date: Fri, 26 Aug 2022 18:26:55 -0500 Subject: [PATCH 1068/1247] : updating the kfdtest packaging version Change-Id: I4132d1106bd997b64b1496ea268961172a545102 --- tests/kfdtest/CMakeLists.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 8976a584c1..e56643d3d7 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -29,7 +29,7 @@ cmake_minimum_required(VERSION 3.5 FATAL_ERROR) project(KFDTest) # For DEB/RPM generation -set ( CPACK_PACKAGE_FILE_NAME "kfdtest" ) +set ( CPACK_PACKAGE_NAME "kfdtest" ) set ( CPACK_PACKAGE_CONTACT "Advanced Micro Devices Inc." ) set ( CPACK_PACKAGE_DESCRIPTION "This package includes kfdtest, the list of excluded tests for each ASIC, and a convenience script to run the test suite" ) set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "Test suite for ROCK/KFD" ) @@ -48,10 +48,20 @@ set ( CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/RadeonOpenCompute/ROCT-Thun set ( CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") set ( CPACK_RPM_FILE_NAME "RPM-DEFAULT") +## Debian package values +set ( CPACK_DEBIAN_PACKAGE_RELEASE "local" ) +if( DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE} ) + set ( CPACK_DEBIAN_PACKAGE_RELEASE $ENV{CPACK_DEBIAN_PACKAGE_RELEASE} ) +endif() +## RPM package variables +set ( CPACK_RPM_PACKAGE_RELEASE "local" ) +if( DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE} ) + set ( CPACK_RPM_PACKAGE_RELEASE $ENV{CPACK_RPM_PACKAGE_RELEASE} ) +endif() + set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${ROCM_VERSION_FOR_PACKAGE}") set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") - ## Define default variable and variables for the optional build target hsakmt-dev set ( SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} CACHE STRING "Location of hsakmt source code." ) set ( CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE STRING "Default installation directory." ) @@ -65,7 +75,6 @@ set (CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core") # RPM package specific variables set (CPACK_RPM_PACKAGE_REQUIRES "rocm-core") - #set ( CMAKE_VERBOSE_MAKEFILE on ) find_package(PkgConfig) From 88934cec2c2b4bea905291451bb65e48985393dd Mon Sep 17 00:00:00 2001 From: David Francis Date: Wed, 16 Nov 2022 10:39:07 -0500 Subject: [PATCH 1069/1247] libhsakmt: Don't close kfd_fd When hsa is closed, it would close open fds for /dev/kfd but not for /dev/dri/renderD*. This caused issues with CRIU checkpoint, which expects that /dev/kfd will be open if /dev/dri/renderD* is. As a workaround for the CRIU behaviour, leave /dev/kfd open when closing hsa. Signed-off-by: David Francis Change-Id: Ie1b2d5b1d8986750b0e560ae2934b7c73cff942e --- src/globals.c | 2 +- src/openclose.c | 20 +++++++++----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/globals.c b/src/globals.c index 3d75b9f1b7..851853aad0 100644 --- a/src/globals.c +++ b/src/globals.c @@ -27,7 +27,7 @@ // HSAKMT global data -int kfd_fd; +int kfd_fd = -1; unsigned long kfd_open_count; unsigned long system_properties_count; pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; diff --git a/src/openclose.c b/src/openclose.c index d52868e963..4da5ac6d5b 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -99,7 +99,7 @@ static void clear_after_fork(void) destroy_device_debugging_memory(); if (kfd_fd) { close(kfd_fd); - kfd_fd = 0; + kfd_fd = -1; } kfd_open_count = 0; parent_pid = -1; @@ -160,15 +160,17 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (result != HSAKMT_STATUS_SUCCESS) goto open_failed; - fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); + if (kfd_fd < 0) { + fd = open(kfd_device_name, O_RDWR | O_CLOEXEC); - if (fd == -1) { - result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; - goto open_failed; + if (fd == -1) { + result = HSAKMT_STATUS_KERNEL_IO_CHANNEL_NOT_OPENED; + goto open_failed; + } + + kfd_fd = fd; } - kfd_fd = fd; - init_page_size(); result = init_kfd_version(); @@ -223,10 +225,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCloseKFD(void) if (--kfd_open_count == 0) { destroy_counter_props(); destroy_device_debugging_memory(); - if (kfd_fd) { - close(kfd_fd); - kfd_fd = 0; - } } result = HSAKMT_STATUS_SUCCESS; From 01fd84db5e11102e4b0e9ee6ac145cd7f2f663c0 Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Thu, 17 Nov 2022 17:14:41 -0800 Subject: [PATCH 1070/1247] Change pragma message to warning File reorganization feature was implemented with backward compatibility The backward compatibility support will be deprecated in future release. Changed the #pragma message to #warning for a smooth transition Change-Id: I21025f4cefb40721f095130263b4247877979d36 --- hsakmt-backward-compat.cmake | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/hsakmt-backward-compat.cmake b/hsakmt-backward-compat.cmake index 89f356c552..6e1b3a9e97 100644 --- a/hsakmt-backward-compat.cmake +++ b/hsakmt-backward-compat.cmake @@ -41,7 +41,21 @@ function(create_header_template) LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */\n\n#ifndef @include_guard@\n#define @include_guard@ \n\n#pragma message(\"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\")\n@include_statements@\n\n#endif") + */ + +#ifndef @include_guard@ +#define @include_guard@ + + +#if defined(__GNUC__) +#warning \"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\" +#else +#pragma message(\"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\") +#endif + +@include_statements@ + +#endif") endfunction() #use header template file and generate wrapper header files From f46ddb7eadc9046e073b71961b5c32d2b44472a8 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Fri, 25 Nov 2022 15:01:53 +0000 Subject: [PATCH 1071/1247] libhsakmt: Initialize fd to -1 Fix compile error due to warning in some environments Change-Id: Ie5fcfabb872c27c0de349eb215345b997fae7201 --- src/openclose.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openclose.c b/src/openclose.c index 4da5ac6d5b..df98e46219 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -141,7 +141,7 @@ static HSAKMT_STATUS init_vars_from_env(void) HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) { HSAKMT_STATUS result; - int fd; + int fd = -1; HsaSystemProperties sys_props; pthread_mutex_lock(&hsakmt_mutex); From 8e8aa024fd5716f7dce075f60e2937ce66ea06eb Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Fri, 25 Nov 2022 11:18:30 -0500 Subject: [PATCH 1072/1247] kfdtest: remove scc test in MapUnmapToNodes for gfx90a A+A Modifier scc is disabled from gfx90a's asm, so remove the shader for gfx90a A+A and keep it for newer asics with scc support. Signed-off-by: Eric Huang Change-Id: Iec3c7ccd5156a855adb2b02feb3db0761876aa2f --- tests/kfdtest/src/KFDMemoryTest.cpp | 2 +- tests/kfdtest/src/ShaderStore.cpp | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 5ee8537f37..082c2e60dc 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -177,7 +177,7 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); const char *pReadMemory; - if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode)) + if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode) && (m_FamilyId != FAMILY_AL)) /* On A+A system memory is mapped as NC */ pReadMemory = PollNCMemoryIsa; else diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index fc97d6490e..c676502a85 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -189,10 +189,6 @@ const char *PollMemoryIsa = R"( * polled can be Non-coherant memory. SCC system-level * cache coherence is not supported in scalar (smem) path. * Use vmem operations with scc - * - * Note: Only works on Aldebaran, and even then the scc modifier - * has been defeatured. This shader is more or less - * deprecated. */ const char *PollNCMemoryIsa = R"( .text From 8e69b9c70e67fbee551272017759459ac5c7c1fc Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Mon, 21 Nov 2022 22:10:30 -0500 Subject: [PATCH 1073/1247] libhsakmt: Fix use of uninitialized variable When is hsaKmtCreateQueue called first time for node doorbells[NodeId].size is initialized to zero in init_process_doorbells but used to calculate the doorbell offset. It works just by accident because doorbells[NodeId].size is uint32_t so -1 will be 0xFFFFFFFF which is zero extended into 0x00000000FFFFFFFF and it will work as long as mmap offset bits are not within lower 32 bits. Bug: https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/issues/78 Signed-off-by: Felix Kuehling Change-Id: Ia791adfc51363d4704cb50fa4f01137b7dd48a75 --- src/queues.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/queues.c b/src/queues.c index fa3ed564f8..3b4104b62d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -654,16 +654,16 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, q->queue_id = args.queue_id; if (IS_SOC15(q->gfxv)) { + HSAuint64 mask = DOORBELLS_PAGE_SIZE(DOORBELL_SIZE(q->gfxv)) - 1; + /* On SOC15 chips, the doorbell offset within the * doorbell page is included in the doorbell offset * returned by KFD. This allows CP queue doorbells to be * allocated dynamically (while SDMA queue doorbells fixed) * rather than based on the its process queue ID. */ - doorbell_mmap_offset = args.doorbell_offset & - ~(HSAuint64)(doorbells[NodeId].size - 1); - doorbell_offset = args.doorbell_offset & - (doorbells[NodeId].size - 1); + doorbell_mmap_offset = args.doorbell_offset & ~mask; + doorbell_offset = args.doorbell_offset & mask; } else { /* On older chips, the doorbell offset within the * doorbell page is based on the queue ID. From e71eb13784b67344942fa432a0afeb987f53b6c0 Mon Sep 17 00:00:00 2001 From: Daniel Phillips Date: Mon, 31 Oct 2022 07:30:12 -0700 Subject: [PATCH 1074/1247] kfdtest: Also detect under-reporting of available memory Detect under-reporting of available memory by initially attempting to allocate substantially more than reported available memory, and ensure that the allocation fails. Continue shrinking the attempted allocation until it succeeds, then fail the test if the successful allocation is either too much more than or too much less than reported available. Signed-off-by: Daniel Phillips Change-Id: Ib418f0aa26e8db80590a6c5f2578da56a4b60f2b --- tests/kfdtest/src/KFDMemoryTest.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 082c2e60dc..be8d8b6bec 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -264,23 +264,32 @@ TEST_F(KFDMemoryTest, MemoryAlloc) { // Basic test for hsaKmtAllocMemory TEST_F(KFDMemoryTest, MemoryAllocAll) { TEST_START(TESTPROFILE_RUNALL) - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); HsaMemFlags memFlags = {0}; memFlags.ui32.NonPaged = 1; // sys mem vs vram HSAuint64 available; void *object = NULL; int shrink = 21, success = HSAKMT_STATUS_NO_MEMORY; - EXPECT_SUCCESS(hsaKmtAvailableMemory(defaultGPUNode, &available)); LOG() << "Available: " << available << " bytes" << std::endl; + HSAuint64 leeway = (10 << shrink), size = available + leeway; for (int i = 0; i < available >> shrink; i++) { - HSAuint64 size = available - ((HSAuint64)i << shrink); if (hsaKmtAllocMemory(defaultGPUNode, size, memFlags, &object) == HSAKMT_STATUS_SUCCESS) { - LOG() << "Allocated: " << size << " bytes" << std::endl; success = hsaKmtFreeMemory(object, available); break; } + size -= (1 << shrink); + } + if (success == HSAKMT_STATUS_SUCCESS) { + LOG() << "Allocated: " << size << " bytes" << std::endl; + if (size > available + leeway) { + LOG() << "Under-reported available memory!" << std::endl; + success = HSAKMT_STATUS_ERROR; + } + if (size < available - leeway) { + LOG() << "Over-reported available memory!" << std::endl; + success = HSAKMT_STATUS_NO_MEMORY; + } } EXPECT_SUCCESS(success); TEST_END From 8a746bdaed664ef7884f0a2726b6843d1c39b1c0 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Mon, 31 Oct 2022 21:29:39 +0000 Subject: [PATCH 1075/1247] libhsakmt: add env var to en/dis registration through SVM Setting this variable to '0' will force to disable memory registration/allocation through SVM API mechanism. Not setting this or setting to '1', SVM API will be used only if all GPUs support it. Signed-off-by: Alex Sierra Change-Id: Icdf7656de09aa9988b567ec6c024953398e9bb48 --- src/fmm.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index b2ce7da43e..67d0d0bb09 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2263,7 +2263,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; - char *maxVaAlignStr; + char *maxVaAlignStr, *useSvmStr; unsigned int guardPages = 1; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2302,6 +2302,9 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (!maxVaAlignStr || sscanf(maxVaAlignStr, "%u", &svm.alignment_order) != 1) svm.alignment_order = 9; + useSvmStr = getenv("HSA_USE_SVM"); + svm.is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")); + gpu_mem_count = 0; g_first_gpu_mem = NULL; @@ -2319,7 +2322,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) */ is_dgpu = false; - svm.is_svm_api_supported = true; for (i = 0; i < NumNodes; i++) { HsaNodeProperties props; From 2d53430ce3f332cd7c2f05f6a9606d75fb6d76b3 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 25 Nov 2022 13:53:31 -0500 Subject: [PATCH 1076/1247] libhsakmt: Fix and simplify debug_get_reg_status The NULL pointer check was the only way for that function to fail. And it was done after the pointer was accessed. Simplify this by just returning the result as a return value instead of using a pointer as output parameter. This way the function can never fail and the caller doesn't need to do any error handling. Declare the function in libhsakmt.h instead of duplicating the declaration in fmm.c. Signed-off-by: Felix Kuehling Change-Id: I91b90d66166fd3b5cdc47c73a9bbc369c45b51fe --- src/debug.c | 17 +++++------------ src/fmm.c | 5 ++--- src/libhsakmt.h | 1 + 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/debug.c b/src/debug.c index 0b73c6ce8f..d225e0cc9a 100644 --- a/src/debug.c +++ b/src/debug.c @@ -33,8 +33,6 @@ static bool *is_device_debugged; -int debug_get_reg_status(uint32_t node_id, bool *is_debugged); - HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes) { unsigned int i; @@ -57,6 +55,11 @@ void destroy_device_debugging_memory(void) } } +bool debug_get_reg_status(uint32_t node_id) +{ + return is_device_debugged[node_id]; +} + HSAKMT_STATUS HSAKMTAPI hsaKmtDbgRegister(HSAuint32 NodeId) { HSAKMT_STATUS result; @@ -262,16 +265,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(HSAuint32 NodeId, return HSAKMT_STATUS_SUCCESS; } -int debug_get_reg_status(uint32_t node_id, bool *is_debugged) -{ - *is_debugged = NULL; - if (!is_device_debugged) - return -1; - - *is_debugged = is_device_debugged[node_id]; - return 0; -} - /* Get the major and minor version of the kernel debugger support. */ HSAKMT_STATUS HSAKMTAPI diff --git a/src/fmm.c b/src/fmm.c index 67d0d0bb09..847bd0ecfa 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -296,7 +296,6 @@ static inline HsaSharedMemoryHandle *to_hsa_shared_memory_handle( return (HsaSharedMemoryHandle *)SharedMemoryStruct; } -extern int debug_get_reg_status(uint32_t node_id, bool *is_debugged); static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture); static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, @@ -2841,9 +2840,9 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert VOID_PTR_ADD(address, size - 1) > aperture->limit) return -1; - ret = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id, &is_debugger); + is_debugger = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id); /* allocate object within the scratch backing aperture */ - if (!ret && !is_debugger) { + if (!is_debugger) { obj = fmm_allocate_memory_object( gpu_id, address, size, aperture, &mmap_offset, KFD_IOC_ALLOC_MEM_FLAGS_VRAM | diff --git a/src/libhsakmt.h b/src/libhsakmt.h index c9d6fcf46b..2e4b455cbf 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -191,6 +191,7 @@ HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes); void destroy_process_doorbells(void); HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes); void destroy_device_debugging_memory(void); +bool debug_get_reg_status(uint32_t node_id); HSAKMT_STATUS init_counter_props(unsigned int NumNodes); void destroy_counter_props(void); uint32_t *convert_queue_ids(HSAuint32 NumQueues, HSA_QUEUEID *Queues); From 73b0fb3d7ce82be2c3c5f15f5bf4f5973da28dc9 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 7 Sep 2022 15:32:56 -0400 Subject: [PATCH 1077/1247] libhsakmt: Refactor and clean up CPU mappings Use a common helper for CPU mappings to reduce duplicate code. Consistently use MAP_SHARED for all render_fd mappings. Remove double-mapping for AQL queue buffers on the CPU. This workaround is only needed on the GPU. Signed-off-by: Felix Kuehling Change-Id: Iff86c8cc9f1e5c982614b3f11129bc2cf8cbba02 --- src/fmm.c | 113 +++++++++++++++++++++--------------------------------- 1 file changed, 44 insertions(+), 69 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 847bd0ecfa..5acf106d54 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1417,6 +1417,23 @@ static void *__fmm_allocate_device(uint32_t gpu_id, void *address, uint64_t Memo return mem; } +static void *fmm_map_to_cpu(void *mem, uint64_t size, bool host_access, + int fd, uint64_t mmap_offset) { + int flag = MAP_SHARED | MAP_FIXED; + int prot = host_access ? PROT_READ | PROT_WRITE : PROT_NONE; + void *ret = mmap(mem, size, prot, flag, fd, mmap_offset); + + if (ret != MAP_FAILED) + /* This madvise() call is needed to avoid additional references + * to mapped BOs in child processes that can prevent freeing + * memory in the parent process and lead to out-of-memory + * conditions. + */ + madvise(mem, size, MADV_DONTFORK); + + return ret; +} + void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags mflags) { @@ -1465,25 +1482,15 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, } if (mem) { - int map_fd = gpu_mem[gpu_mem_id].drm_render_fd; - int prot = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : - PROT_NONE; - int flag = mflags.ui32.HostAccess ? MAP_SHARED | MAP_FIXED : - MAP_PRIVATE|MAP_FIXED; - void *ret = mmap(mem, MemorySizeInBytes, prot, flag, - map_fd, mmap_offset); + void *ret = fmm_map_to_cpu(mem, MemorySizeInBytes, + mflags.ui32.HostAccess, + gpu_mem[gpu_mem_id].drm_render_fd, + mmap_offset); if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); return NULL; } - /* - * This madvise() call is needed to avoid additional references - * to mapped BOs in child processes that can prevent freeing - * memory in the parent process and lead to out-of-memory - * conditions. - */ - madvise(mem, MemorySizeInBytes, MADV_DONTFORK); } return mem; @@ -1718,29 +1725,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, &mmap_offset, ioc_flags, &vm_obj); if (mem && mflags.ui32.HostAccess) { - int map_fd = gpu_drm_fd; - void *ret = mmap(mem, MemorySizeInBytes, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); + void *ret = fmm_map_to_cpu(mem, MemorySizeInBytes, + mflags.ui32.HostAccess, + gpu_drm_fd, mmap_offset); + if (ret == MAP_FAILED) { __fmm_release(vm_obj, aperture); return NULL; } - - madvise(ret, MemorySizeInBytes, MADV_DONTFORK); - - if (mflags.ui32.AQLQueueMemory) { - uint64_t my_buf_size = size / 2; - - memset(ret, 0, MemorySizeInBytes); - mmap(VOID_PTR_ADD(mem, my_buf_size), MemorySizeInBytes, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); - - madvise(VOID_PTR_ADD(mem, my_buf_size), - MemorySizeInBytes, - MADV_DONTFORK); - } } } @@ -2822,9 +2814,9 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert int32_t gpu_mem_id; int ret; bool is_debugger = 0; + uint32_t flags; void *mmap_ret = NULL; uint64_t mmap_offset = 0; - int map_fd; vm_object_t *obj; /* Retrieve gpu_mem id according to gpu_id */ @@ -2841,37 +2833,22 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert return -1; is_debugger = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id); + flags = is_debugger ? KFD_IOC_ALLOC_MEM_FLAGS_GTT : + KFD_IOC_ALLOC_MEM_FLAGS_VRAM; + flags |= KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE; /* allocate object within the scratch backing aperture */ - if (!is_debugger) { - obj = fmm_allocate_memory_object( - gpu_id, address, size, aperture, &mmap_offset, - KFD_IOC_ALLOC_MEM_FLAGS_VRAM | - KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); - if (!obj) - return -1; - /* Create a CPU mapping for the debugger */ - map_fd = gpu_mem[gpu_mem_id].drm_render_fd; - mmap_ret = mmap(address, size, PROT_NONE, - MAP_PRIVATE | MAP_FIXED, map_fd, mmap_offset); - if (mmap_ret == MAP_FAILED) { - __fmm_release(obj, aperture); - return -1; - } - } else { - obj = fmm_allocate_memory_object( - gpu_id, address, size, aperture, &mmap_offset, - KFD_IOC_ALLOC_MEM_FLAGS_GTT | - KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE); - map_fd = gpu_mem[gpu_mem_id].drm_render_fd; - mmap_ret = mmap(address, size, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, map_fd, mmap_offset); - if (mmap_ret == MAP_FAILED) { - __fmm_release(obj, aperture); - return -1; - } + obj = fmm_allocate_memory_object(gpu_id, address, size, + aperture, &mmap_offset, flags); + if (!obj) + return -1; + /* Create a CPU mapping for the debugger */ + mmap_ret = fmm_map_to_cpu(address, size, is_debugger, + gpu_mem[gpu_mem_id].drm_render_fd, + mmap_offset); + if (mmap_ret == MAP_FAILED) { + __fmm_release(obj, aperture); + return -1; } - madvise(mmap_ret, size, MADV_DONTFORK); /* map to GPU */ ret = _fmm_map_to_gpu(aperture, address, size, NULL, &gpu_id, sizeof(uint32_t)); @@ -3537,7 +3514,6 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo if (importArgs.mmap_offset) { int32_t gpu_mem_id = gpu_mem_find_by_gpu_id(importArgs.gpu_id); - int map_fd; void *ret; if (gpu_mem_id < 0) { @@ -3545,15 +3521,14 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo goto err_free_obj; } obj->node_id = gpu_mem[gpu_mem_id].node_id; - map_fd = gpu_mem[gpu_mem_id].drm_render_fd; - ret = mmap(reservedMem, (SizeInPages << PAGE_SHIFT), - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_FIXED, map_fd, importArgs.mmap_offset); + ret = fmm_map_to_cpu(reservedMem, (SizeInPages << PAGE_SHIFT), + true, gpu_mem[gpu_mem_id].drm_render_fd, + importArgs.mmap_offset); + if (ret == MAP_FAILED) { err = HSAKMT_STATUS_ERROR; goto err_free_obj; } - madvise(ret, (SizeInPages << PAGE_SHIFT), MADV_DONTFORK); } *MemoryAddress = reservedMem; From 7787a039bd53cff9cfc424ae4344515201053a51 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 7 Sep 2022 15:53:31 -0400 Subject: [PATCH 1078/1247] libhsakmt: Remove unnecessary CPU unmap This is handled by __fmm_release calling aperture_release_area. Signed-off-by: Felix Kuehling Change-Id: Ib8ed300e1734f03aeb9dfc8074897ece310b8af9 --- src/fmm.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 5acf106d54..df97680c04 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3060,11 +3060,6 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, args.n_success = 0; ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); - /* unmap from CPU while keeping the address space reserved */ - mmap(address, object->size, PROT_NONE, - MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, - -1, 0); - remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, args.n_success * sizeof(uint32_t)); From 7db29c47979fd4ad9a317d80f8c5420e28b8a470 Mon Sep 17 00:00:00 2001 From: James Zhu Date: Tue, 29 Nov 2022 17:08:00 -0500 Subject: [PATCH 1079/1247] kfdtest: track Test Status in syslog Track Test Status in syslog, it will help understand sys log assoicated with test cases. Change-Id: I7c0749102db9bc73d6ae3a237ec347a8fefb12e9 Signed-off-by: James Zhu --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 5618945505..bb510e040b 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -21,6 +21,8 @@ * */ +#include + #include "KFDBaseComponentTest.hpp" #include "KFDTestUtil.hpp" @@ -70,6 +72,14 @@ void KFDBaseComponentTest::SetUp() { m_pAsm = new Assembler(GetGfxVersion(nodeProperties)); + const testing::TestInfo* curr_test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + + openlog("KFDTEST", LOG_CONS , LOG_USER); + syslog(LOG_INFO, "[Node#%03d] STARTED ========== %s.%s ==========", + m_NodeInfo.HsaDefaultGPUNode(), + curr_test_info->test_case_name(), curr_test_info->name()); + ROUTINE_END } @@ -92,6 +102,20 @@ void KFDBaseComponentTest::TearDown() { delete m_pAsm; m_pAsm = nullptr; + const testing::TestInfo* curr_test_info = + ::testing::UnitTest::GetInstance()->current_test_info(); + + if (curr_test_info->result()->Passed()) + syslog(LOG_INFO, "[Node#%03d] PASSED ========== %s.%s ==========", + m_NodeInfo.HsaDefaultGPUNode(), + curr_test_info->test_case_name(), curr_test_info->name()); + else + syslog(LOG_WARNING, "[Node#%03d] FAILED ========== %s.%s ==========", + m_NodeInfo.HsaDefaultGPUNode(), + curr_test_info->test_case_name(), curr_test_info->name()); + + closelog(); + ROUTINE_END } From 21e95a4f2a4c74cf95c1b6cedc419c2cab949a8e Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 8 Dec 2022 16:24:27 -0600 Subject: [PATCH 1080/1247] Revert "libhsakmt: add env var to en/dis registration through SVM" This reverts commit 8a746bdaed664ef7884f0a2726b6843d1c39b1c0. There are some openMP issues that were introduced after SVM userptr feature was added. Signed-off-by: Alex Sierra Change-Id: Ib01046571d2c84fa0fd228ecba0dee0eae3f994d --- src/fmm.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index df97680c04..6c1bc17dca 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2254,7 +2254,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; - char *maxVaAlignStr, *useSvmStr; + char *maxVaAlignStr; unsigned int guardPages = 1; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2293,9 +2293,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (!maxVaAlignStr || sscanf(maxVaAlignStr, "%u", &svm.alignment_order) != 1) svm.alignment_order = 9; - useSvmStr = getenv("HSA_USE_SVM"); - svm.is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")); - gpu_mem_count = 0; g_first_gpu_mem = NULL; @@ -2313,6 +2310,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) */ is_dgpu = false; + svm.is_svm_api_supported = true; for (i = 0; i < NumNodes; i++) { HsaNodeProperties props; From d9f86ae02b8e95feb19974f4ae6a26e093e2abdb Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 8 Dec 2022 17:31:35 -0600 Subject: [PATCH 1081/1247] Revert "libhsakmt: query svm info from userptrs at fault events" This reverts commit 45fad297528572deebbf4ddee2fba8251e241980. There are some openMP issues that were introduced after SVM userptr feature was added. Signed-off-by: Alex Sierra Change-Id: I6566c9f0d39d05ecb92f38159880763f432939a5 --- src/events.c | 79 +------------------------- tests/kfdtest/src/KFDExceptionTest.cpp | 48 ---------------- 2 files changed, 2 insertions(+), 125 deletions(-) diff --git a/src/events.c b/src/events.c index 1c5b03dcf2..06d3959532 100644 --- a/src/events.c +++ b/src/events.c @@ -220,80 +220,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event, return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); } -static HSAKMT_STATUS get_mem_info_svm_api(uint64_t address, uint32_t gpu_id) -{ - struct kfd_ioctl_svm_args *args; - uint32_t node_id; - HSAuint32 s_attr; - HSAuint32 i; - HSA_SVM_ATTRIBUTE attrs[] = { - {HSA_SVM_ATTR_PREFERRED_LOC, 0}, - {HSA_SVM_ATTR_PREFETCH_LOC, 0}, - {HSA_SVM_ATTR_ACCESS, gpu_id}, - {HSA_SVM_ATTR_SET_FLAGS, 0}, - }; - - CHECK_KFD_OPEN(); - CHECK_KFD_MINOR_VERSION(5); - - s_attr = sizeof(attrs); - args = alloca(sizeof(*args) + s_attr); - args->start_addr = address; - args->size = PAGE_SIZE; - args->op = KFD_IOCTL_SVM_OP_GET_ATTR; - args->nattr = s_attr / sizeof(*attrs); - memcpy(args->attrs, attrs, s_attr); - if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { - pr_debug("op get range attrs failed %s\n", strerror(errno)); - return HSAKMT_STATUS_ERROR; - } - - pr_err("GPU address 0x%lx, is Unified memory\n", address); - for (i = 0; i < args->nattr; i++) { - if (args->attrs[i].value == KFD_IOCTL_SVM_LOCATION_SYSMEM || - args->attrs[i].value == KFD_IOCTL_SVM_LOCATION_UNDEFINED) - node_id = args->attrs[i].value; - else - gpuid_to_nodeid(args->attrs[i].value, &node_id); - switch (args->attrs[i].type) { - case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: - pr_err("Preferred location for address 0x%lx is Node id %d\n", - address, node_id); - break; - case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: - pr_err("Prefetch location for address 0x%lx is Node id %d\n", - address, node_id); - break; - case KFD_IOCTL_SVM_ATTR_ACCESS: - pr_err("Node id %d has access to address 0x%lx\n", - node_id, address); - break; - case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: - pr_err("Node id %d has access in place to address 0x%lx\n", - node_id, address); - break; - case KFD_IOCTL_SVM_ATTR_NO_ACCESS: - pr_err("Node id %d has no access to address 0x%lx\n", - node_id, address); - break; - case KFD_IOCTL_SVM_ATTR_SET_FLAGS: - if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_COHERENT) - pr_err("Fine grained coherency between devices\n"); - if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_RO) - pr_err("Read only\n"); - if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_EXEC) - pr_err("GPU exec allowed\n"); - if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) - pr_err("GPU always mapped\n"); - break; - default: - pr_debug("get invalid attr type 0x%x\n", args->attrs[i].type); - return HSAKMT_STATUS_ERROR; - } - } - - return HSAKMT_STATUS_SUCCESS; -} //Analysis memory exception data, print debug messages static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * memory_exception_data) @@ -315,10 +241,9 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * pr_err("Execute to none-executable page\n"); ret = fmm_get_mem_info((const void *)addr, &info); + if (ret != HSAKMT_STATUS_SUCCESS) { - ret = get_mem_info_svm_api(addr, memory_exception_data->gpu_id); - if (ret != HSAKMT_STATUS_SUCCESS) - pr_err("Address does not belong to a known buffer\n"); + pr_err("Address does not belong to a known buffer\n"); return; } diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index 2ca5c6777d..f7c28d8897 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -27,7 +27,6 @@ #include "SDMAPacket.hpp" #include "SDMAQueue.hpp" #include "Dispatch.hpp" -#include void KFDExceptionTest::SetUp() { ROUTINE_START @@ -254,53 +253,6 @@ TEST_F(KFDExceptionTest, PermissionFault) { TEST_END } -/* Allocate Read Only user pointer buffer. Test Memory Exception failure by - * attempting to write to that buffer in the child process. - */ -TEST_F(KFDExceptionTest, PermissionFaultUserPointer) { - TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); - TEST_START(TESTPROFILE_RUNALL) - - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - if (m_FamilyId == FAMILY_RV) { - LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; - return; - } - - m_ChildPid = fork(); - if (m_ChildPid == 0) { - KFDBaseComponentTest::TearDown(); - KFDBaseComponentTest::SetUp(); - - void *pBuf = mmap(NULL, PAGE_SIZE, PROT_READ, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - ASSERT_NE(pBuf, MAP_FAILED); - EXPECT_SUCCESS(hsaKmtRegisterMemory(pBuf, PAGE_SIZE)); - EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(pBuf, PAGE_SIZE, NULL)); - HsaMemoryBuffer srcSysBuffer(PAGE_SIZE, defaultGPUNode, false); - - srcSysBuffer.Fill(0xAA55AA55); - - TestMemoryException(defaultGPUNode, srcSysBuffer.As(), - (HSAuint64)pBuf); - } else { - int childStatus; - - waitpid(m_ChildPid, &childStatus, 0); - if (is_dgpu()) { - EXPECT_EQ(WIFEXITED(childStatus), true); - EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); - } else { - EXPECT_EQ(WIFSIGNALED(childStatus), true); - EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); - } - } - - TEST_END -} - /* Test VM fault storm handling by copying to/from invalid pointers * with lots of work items at the same time */ From f2bda56d04390b025bfa18a512246af2e1f2923f Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 8 Dec 2022 17:32:35 -0600 Subject: [PATCH 1082/1247] Revert "src: use SVM mechanism to register userptr memory" This reverts commit 178a619b80ef9ecf5753c7232e71e5e0e1e61fb3. There are some openMP issues that were introduced after SVM userptr feature was added. Signed-off-by: Alex Sierra Change-Id: I7ef87c5232a3bcbe594c743fa4b4958601845ba5 --- src/fmm.c | 147 ++++++---------------------- tests/kfdtest/src/KFDMemoryTest.cpp | 10 +- 2 files changed, 32 insertions(+), 125 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 6c1bc17dca..d5b9704fc5 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -220,9 +220,6 @@ typedef struct { /* specifies the alignment size as PAGE_SIZE * 2^alignment_order */ uint32_t alignment_order; - - /* whether to check all dGPUs in the topology support SVM API */ - bool is_svm_api_supported; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -242,8 +239,7 @@ static svm_t svm = { .dgpu_alt_aperture = NULL, .userptr_for_paged_mem = false, .check_userptr = false, - .disable_cache = false, - .is_svm_api_supported = false + .disable_cache = false }; /* On APU, for memory allocated on the system memory that GPU doesn't access @@ -965,72 +961,6 @@ static HsaMemFlags fmm_translate_ioc_to_hsa_flags(uint32_t ioc_flags) return mflags; } -static HSAKMT_STATUS fmm_register_mem_svm_api(void *address, - uint64_t size, - bool coarse_grain) -{ - struct kfd_ioctl_svm_args *args; - size_t s_attr; - HSAuint32 page_offset = (HSAuint64)address & (PAGE_SIZE-1); - HSAuint64 aligned_addr = (HSAuint64)address - page_offset; - HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); - - if (!g_first_gpu_mem) - return HSAKMT_STATUS_ERROR; - - s_attr = sizeof(struct kfd_ioctl_svm_attribute); - args = alloca(sizeof(*args) + s_attr); - args->start_addr = aligned_addr; - args->size = aligned_size; - args->op = KFD_IOCTL_SVM_OP_SET_ATTR; - args->nattr = 1; - args->attrs[0].type = coarse_grain ? - HSA_SVM_ATTR_CLR_FLAGS : HSA_SVM_ATTR_SET_FLAGS; - args->attrs[0].value = HSA_SVM_FLAG_COHERENT; - pr_debug("Registering to SVM %p size: %ld\n", (void*)aligned_addr, - aligned_size); - /* Driver does one copy_from_user, with extra attrs size */ - if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { - pr_debug("op set range attrs failed %s\n", strerror(errno)); - return HSAKMT_STATUS_ERROR; - } - - return HSAKMT_STATUS_SUCCESS; -} - -static HSAKMT_STATUS fmm_map_mem_svm_api(void *address, - uint64_t size, - uint32_t *nodes_to_map, - uint32_t nodes_array_size) -{ - struct kfd_ioctl_svm_args *args; - size_t s_attr; - uint32_t i, nattr; - - if (!g_first_gpu_mem) - return HSAKMT_STATUS_ERROR; - - nattr = nodes_array_size; - s_attr = sizeof(struct kfd_ioctl_svm_attribute) * nattr; - args = alloca(sizeof(*args) + s_attr); - - args->start_addr = (uint64_t)address; - args->size = size; - args->op = KFD_IOCTL_SVM_OP_SET_ATTR; - args->nattr = nattr; - for (i = 0; i < nodes_array_size; i++) { - args->attrs[i].type = HSA_SVM_ATTR_ACCESS_IN_PLACE; - args->attrs[i].value = nodes_to_map[i]; - } - /* Driver does one copy_from_user, with extra attrs size */ - if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { - pr_debug("op set range attrs failed %s\n", strerror(errno)); - return HSAKMT_STATUS_ERROR; - } - - return HSAKMT_STATUS_SUCCESS; -} - /* After allocating the memory, return the vm_object created for this memory. * Return NULL if any failure. */ @@ -1816,9 +1746,7 @@ HSAKMT_STATUS fmm_release(void *address) object = vm_find_object(address, 0, &aperture); if (!object) - return svm.is_svm_api_supported ? - HSAKMT_STATUS_SUCCESS : - HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; if (aperture == &cpuvm_aperture) { /* APU system memory */ @@ -2310,7 +2238,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) */ is_dgpu = false; - svm.is_svm_api_supported = true; for (i = 0; i < NumNodes; i++) { HsaNodeProperties props; @@ -2347,7 +2274,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; gpu_mem[gpu_mem_count].node_id = i; - svm.is_svm_api_supported &= props.Capability.ui32.SVMAPISupported; gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; gpu_mem[gpu_mem_count].scratch_physical.ops = &reserved_aperture_ops; @@ -2857,36 +2783,23 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, - uint64_t *gpuvm_addr, vm_object_t *object, - uint32_t *nodes_to_map, uint32_t nodes_array_size) + uint64_t *gpuvm_addr, vm_object_t *object) { manageable_aperture_t *aperture; void *svm_addr; + HSAuint64 svm_size; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); int ret; aperture = svm.dgpu_aperture; + svm_addr = object->start; + svm_size = object->size; + /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - if (svm.is_svm_api_supported) { - svm_addr = (void*)((HSAuint64)addr - page_offset); - if (!nodes_to_map) { - nodes_to_map = all_gpu_id_array; - nodes_array_size = all_gpu_id_array_size; - } - pr_debug("%s Mapping Address %p size aligned: %ld offset: %x\n", - __func__, svm_addr, PAGE_ALIGN_UP(page_offset + size), page_offset); - ret = fmm_map_mem_svm_api(svm_addr, - PAGE_ALIGN_UP(page_offset + size), - nodes_to_map, - nodes_array_size / sizeof(uint32_t)); - - } else { - svm_addr = object->start; - ret = _fmm_map_to_gpu(aperture, svm_addr, object->size, object, NULL, 0); - } + ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, object, NULL, 0); if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -2910,7 +2823,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) address, size); object = vm_find_object(address, size, &aperture); - if (!object && !svm.is_svm_api_supported ) { + if (!object) { if (!is_dgpu) { /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); @@ -2925,8 +2838,8 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); ret = 0; - } else if ((svm.is_svm_api_supported && !object) || object->userptr) { - ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object, NULL, 0); + } else if (object->userptr) { + ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); } else { ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0); /* Update alternate GPUVM address only for @@ -2936,8 +2849,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); } - if (object) - pthread_mutex_unlock(&aperture->fmm_mutex); + pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } @@ -3002,6 +2914,7 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, print_device_id_array((void *)args.device_ids_array_ptr, args.n_devices * sizeof(uint32_t)); + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); remove_device_ids_from_mapped_array(object, @@ -3098,7 +3011,7 @@ int fmm_unmap_from_gpu(void *address) object = vm_find_object(address, 0, &aperture); if (!object) /* On APUs GPU unmapping of system memory is a no-op */ - return (!is_dgpu || svm.is_svm_api_supported) ? 0 : -EINVAL; + return is_dgpu ? -EINVAL : 0; /* Successful vm_find_object returns with the aperture locked */ if (aperture == &cpuvm_aperture) @@ -3186,8 +3099,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | - (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), - &obj); + (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; @@ -3239,9 +3151,6 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, return HSAKMT_STATUS_SUCCESS; /* Register a new user ptr */ - if (svm.is_svm_api_supported) - return fmm_register_mem_svm_api(address, size_in_bytes, coarse_grain); - ret = fmm_register_user_memory(address, size_in_bytes, &object, coarse_grain); if (ret != HSAKMT_STATUS_SUCCESS) return ret; @@ -3557,9 +3466,9 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) /* On APUs we assume it's a random system memory address * where registration and dergistration is a no-op */ - return (!is_dgpu || svm.is_svm_api_supported) ? - HSAKMT_STATUS_SUCCESS : - HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + return is_dgpu ? + HSAKMT_STATUS_MEMORY_NOT_REGISTERED : + HSAKMT_STATUS_SUCCESS; /* Successful vm_find_object returns with aperture locked */ if (aperture == &cpuvm_aperture) { @@ -3610,7 +3519,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address) { - manageable_aperture_t *aperture = NULL; + manageable_aperture_t *aperture; vm_object_t *object; uint32_t i; uint32_t *registered_node_id_array, registered_node_id_array_size; @@ -3621,22 +3530,24 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_INVALID_PARAMETER; object = vm_find_object(address, size, &aperture); - if (!object && !svm.is_svm_api_supported) + if (!object) return HSAKMT_STATUS_ERROR; /* Successful vm_find_object returns with aperture locked */ /* APU memory is not supported by this function */ - if (aperture && - (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible)) { + if (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } - if ((svm.is_svm_api_supported && !object) || object->userptr) { - retcode = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, - object, nodes_to_map, num_of_nodes * sizeof(uint32_t)); - if (object) - pthread_mutex_unlock(&aperture->fmm_mutex); + /* For userptr, we ignore the nodes array and map all registered nodes. + * This is to simply the implementation of allowing the same memory + * region to be registered multiple times. + */ + if (object->userptr) { + retcode = _fmm_map_to_gpu_userptr(address, size, + gpuvm_address, object); + pthread_mutex_unlock(&aperture->fmm_mutex); return retcode ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index be8d8b6bec..584c1fef87 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -464,6 +464,7 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { EXPECT_SUCCESS(hsaKmtRegisterMemory((void *)&mem[0], sizeof(HSAuint32))); EXPECT_SUCCESS(hsaKmtMapMemoryToGPU((void *)&mem[0], sizeof(HSAuint32), &gpuva2)); + EXPECT_TRUE(gpuva1 != gpuva2); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva1))); EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(gpuva1))); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva2))); @@ -1133,12 +1134,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { static volatile HSAuint32 mem[4]; // 8 bytes for register only and // 8 bytes for register to nodes HsaMemoryBuffer hsaBuffer((void *)(&mem[0]), sizeof(HSAuint32)*2); - /* - * APU doesn't use userptr. - * User pointers registered with SVM API, does not create vm_object_t. - * Therefore, pointer info can not be queried. - */ - if (is_dgpu() && mem != hsaBuffer.As()) { + if (is_dgpu()) { // APU doesn't use userptr EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[0]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); @@ -1165,7 +1161,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo(reinterpret_cast(address), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED); EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As()); - if (is_dgpu() && &mem[1] != hsaBuffer.As() + 1) { + if (is_dgpu()) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[1]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); From 505287412f476d0a223327453d66b7a92279eb2f Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Fri, 6 Jan 2023 17:15:43 -0500 Subject: [PATCH 1083/1247] Revert "libhsakmt: Remove unnecessary CPU unmap" This reverts commit 7787a039bd53cff9cfc424ae4344515201053a51. It causes a regression in pytorch benchmark. Change-Id: I96173dbd061cf38d6f451c02cb181ae51b7f625e Signed-off-by: Eric Huang --- src/fmm.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fmm.c b/src/fmm.c index d5b9704fc5..9f099c5a34 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2971,6 +2971,11 @@ static int _fmm_unmap_from_gpu_scratch(uint32_t gpu_id, args.n_success = 0; ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); + /* unmap from CPU while keeping the address space reserved */ + mmap(address, object->size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, + -1, 0); + remove_device_ids_from_mapped_array(object, (uint32_t *)args.device_ids_array_ptr, args.n_success * sizeof(uint32_t)); From b25867c4b8dea987f9bfc12c00373151526929eb Mon Sep 17 00:00:00 2001 From: David Belanger Date: Wed, 11 Jan 2023 09:18:54 -0500 Subject: [PATCH 1084/1247] libhsakmt: Disabled allocation of CWSR with SVM for GFX11. This is a temporary work around for GPU hang issues observed on GFX11. Change-Id: I98fbedbbd1c51fe402c2116b35ca548931a390c9 Signed-off-by: David Belanger --- src/libhsakmt.h | 2 ++ src/queues.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 2e4b455cbf..7f62c512de 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -116,6 +116,8 @@ extern int hsakmt_debug_level; #define HSA_GET_GFX_VERSION_MINOR(gfxv) (((gfxv) / 100) % 100) #define HSA_GET_GFX_VERSION_STEP(gfxv) ((gfxv) % 100) +#define HSA_GET_GFX_VERSION_MAJOR_HEX(gfxv) (((gfxv) >> 16) & 0xff) + /* Expects HSA_ENGINE_ID.ui32, returns gfxv (full) in hex */ #define HSA_GET_GFX_VERSION_FULL(ui32) \ (((ui32.Major) << 16) | ((ui32.Minor) << 8) | (ui32.Stepping)) diff --git a/src/queues.c b/src/queues.c index 3b4104b62d..7e28ba751a 100644 --- a/src/queues.c +++ b/src/queues.c @@ -490,6 +490,8 @@ static int handle_concrete_asic(struct queue *q, if (hsaKmtGetNodeProperties(NodeId, &node)) svm_api = false; + else if (HSA_GET_GFX_VERSION_MAJOR_HEX(q->gfxv) == 11) + svm_api = false; else svm_api = node.Capability.ui32.SVMAPISupported; From bc40579f96ece6a0015713d20ad3577429888d88 Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Sun, 22 Jan 2023 23:54:22 -0800 Subject: [PATCH 1085/1247] Added OS details to kfdtest rpm packge name Change-Id: I600e094c364e1c7219ae3db12f0c4e1f7598c132 --- tests/kfdtest/CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index e56643d3d7..ba1d99be51 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -59,6 +59,19 @@ if( DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE} ) set ( CPACK_RPM_PACKAGE_RELEASE $ENV{CPACK_RPM_PACKAGE_RELEASE} ) endif() +## Note: rpm --eval %{?dist} will evaluate to NULL in Debian +## So Debian distros won't append dist tag to CPACK_RPM_PACKAGE_RELEASE. +## Also for debian package name , the dist tag is added from build env +execute_process( COMMAND rpm --eval %{?dist} + RESULT_VARIABLE PROC_RESULT + OUTPUT_VARIABLE EVAL_RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE ) +message("RESULT_VARIABLE ${PROC_RESULT} OUTPUT_VARIABLE: ${EVAL_RESULT}") +## Add distribution tag to rpm package name +if ( PROC_RESULT EQUAL "0" AND NOT EVAL_RESULT STREQUAL "" ) + string ( APPEND CPACK_RPM_PACKAGE_RELEASE "%{?dist}" ) +endif() + set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}.${ROCM_VERSION_FOR_PACKAGE}") set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") From a847a7b80edda2103c7094c17b26b547c04957c2 Mon Sep 17 00:00:00 2001 From: David Belanger Date: Tue, 24 Jan 2023 09:55:04 -0500 Subject: [PATCH 1086/1247] libhsakmt: Fixed VGPR memory size for GFX11.0 and GFX11.1. Fixed VGPR memory size, size was too small for some GPU, causing a memory overflow. Refactored macro code into a function. Thanks to Jay Cornwall for locating the problem and proposing the fix. Change-Id: Iffedea1c4f341967f02c56d810ff048225b02c16 Signed-off-by: David Belanger --- src/libhsakmt.h | 6 +++--- src/queues.c | 24 +++++++++++++++++++++++- src/topology.c | 2 +- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 7f62c512de..0f80e5eb3b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -149,6 +149,8 @@ enum full_gfx_versions { GFX_VERSION_VANGOGH = 0x0A0303, GFX_VERSION_BEIGE_GOBY = 0x0A0304, GFX_VERSION_YELLOW_CARP = 0x0A0305, + GFX_VERSION_PLUM_BONITO = 0x0B0000, + GFX_VERSION_WHEAT_NAS = 0x0B0001, }; struct hsa_gfxip_table { @@ -222,8 +224,6 @@ uint32_t get_num_sysfs_nodes(void); bool is_forked_child(void); /* Calculate VGPR and SGPR register file size per CU */ -#define VGPR_SIZE_PER_CU(gfxv) \ - (((gfxv) == GFX_VERSION_ARCTURUS || \ - (gfxv) == GFX_VERSION_ALDEBARAN) ? 0x80000 : 0x40000) +uint32_t get_vgpr_size_per_cu(uint32_t gfxv); #define SGPR_SIZE_PER_CU 0x4000 #endif diff --git a/src/queues.c b/src/queues.c index 7e28ba751a..c53d219c75 100644 --- a/src/queues.c +++ b/src/queues.c @@ -45,7 +45,7 @@ (((gfxv) >= 0x80000) ? 4096 : 0)) #define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv) \ - (VGPR_SIZE_PER_CU(gfxv) + SGPR_SIZE_PER_CU + \ + (get_vgpr_size_per_cu(gfxv) + SGPR_SIZE_PER_CU + \ LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) #define CNTL_STACK_BYTES_PER_WAVE(gfxv) \ @@ -87,6 +87,28 @@ struct process_doorbells { static unsigned int num_doorbells; static struct process_doorbells *doorbells; +uint32_t get_vgpr_size_per_cu(uint32_t gfxv) +{ + uint32_t vgpr_size = 0; + + switch (gfxv) + { + case GFX_VERSION_ARCTURUS: + case GFX_VERSION_ALDEBARAN: + vgpr_size = 0x80000; + break; + case GFX_VERSION_PLUM_BONITO: + case GFX_VERSION_WHEAT_NAS: + vgpr_size = 0x60000; + break; + default: + vgpr_size = 0x40000; + break; + } + + return vgpr_size; +} + HSAKMT_STATUS init_process_doorbells(unsigned int NumNodes) { unsigned int i; diff --git a/src/topology.c b/src/topology.c index 2887c661bc..7e99ec10d5 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1239,7 +1239,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, /* Get VGPR/SGPR size in byte per CU */ props->SGPRSizePerCU = SGPR_SIZE_PER_CU; - props->VGPRSizePerCU = VGPR_SIZE_PER_CU(HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32)); + props->VGPRSizePerCU = get_vgpr_size_per_cu(HSA_GET_GFX_VERSION_FULL(props->EngineId.ui32)); } else if (props->DeviceId) /* still return success */ From 0eb0bae38b535c6fd048b9e82cdfbac39a3a3d44 Mon Sep 17 00:00:00 2001 From: David Belanger Date: Tue, 24 Jan 2023 11:35:31 -0500 Subject: [PATCH 1087/1247] Revert "libhsakmt: Disabled allocation of CWSR with SVM for GFX11." This reverts commit b25867c4b8dea987f9bfc12c00373151526929eb. Change-Id: I05bf82266f563c63c0b794a24b0926e7652ce42d Signed-off-by: David Belanger --- src/libhsakmt.h | 2 -- src/queues.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 0f80e5eb3b..55fc5a465d 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -116,8 +116,6 @@ extern int hsakmt_debug_level; #define HSA_GET_GFX_VERSION_MINOR(gfxv) (((gfxv) / 100) % 100) #define HSA_GET_GFX_VERSION_STEP(gfxv) ((gfxv) % 100) -#define HSA_GET_GFX_VERSION_MAJOR_HEX(gfxv) (((gfxv) >> 16) & 0xff) - /* Expects HSA_ENGINE_ID.ui32, returns gfxv (full) in hex */ #define HSA_GET_GFX_VERSION_FULL(ui32) \ (((ui32.Major) << 16) | ((ui32.Minor) << 8) | (ui32.Stepping)) diff --git a/src/queues.c b/src/queues.c index c53d219c75..f59c3da7bf 100644 --- a/src/queues.c +++ b/src/queues.c @@ -512,8 +512,6 @@ static int handle_concrete_asic(struct queue *q, if (hsaKmtGetNodeProperties(NodeId, &node)) svm_api = false; - else if (HSA_GET_GFX_VERSION_MAJOR_HEX(q->gfxv) == 11) - svm_api = false; else svm_api = node.Capability.ui32.SVMAPISupported; From 3fb1496fb3a03c9a4ee040c09b8044c77a1237a9 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 26 Jan 2023 15:01:24 -0500 Subject: [PATCH 1088/1247] kfdtest: Remove redundant SGPR/VGPR size checks in KFDTopologyTest KFDTopologyTest.BasicTest duplicates Thunk logic to calculate VGPR size, meaning it will always be the same, and SGPR size is a constant. Since no benefit, remove comparisons. Signed-off-by: Graham Sider Change-Id: I99e7ff6fb69ed07bc0716fdf43946b19c67b9268 --- tests/kfdtest/src/KFDTopologyTest.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index b4b406a24a..b4c2ebfca6 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -59,11 +59,7 @@ TEST_F(KFDTopologyTest , BasicTest) { EXPECT_GE(pNodeProperties->EngineId.ui32.Major, 7) << "Major Version is less than 7"; EXPECT_LT(pNodeProperties->EngineId.ui32.Minor, 10) << "Minor Version is greater than 9"; EXPECT_GT(pNodeProperties->uCodeEngineVersions.uCodeSDMA, 0) << "sDMA firmware version is 0"; - HSAuint32 VGPRSize = (FamilyIdFromNode(pNodeProperties) == FAMILY_AR || - FamilyIdFromNode(pNodeProperties) == FAMILY_AL) ? - 0x80000 : 0x40000; - EXPECT_EQ(pNodeProperties->VGPRSizePerCU, VGPRSize) << "VGPR Size Per CU is not correct"; - EXPECT_EQ(pNodeProperties->SGPRSizePerCU, 0x4000) << "SGPR Size Per CU is not correct"; + LOG() << "VGPR Size is " << pNodeProperties->VGPRSizePerCU << " SGPR Size is " << pNodeProperties->SGPRSizePerCU << std::endl; } From efcc9b275b992cab7b9abe13984e9718dd303ef6 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Tue, 7 Feb 2023 01:55:06 -0600 Subject: [PATCH 1089/1247] libhsakmt: Correct reporting of Shader Engines number. The Shader Engines number should be shadder array_count divided by simd_arrays_per_engine not array_count. Signed-off-by: Xiaogang Chen Change-Id: I808d1fedd6b9843500719e902ecf759f5668a7d1 --- src/topology.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/topology.c b/src/topology.c index 7e99ec10d5..0f5c0f9706 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1067,6 +1067,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, uint32_t sys_node_id; uint32_t gfxv = 0; uint8_t gfxv_major, gfxv_minor, gfxv_stepping; + uint32_t simd_arrays_count; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; @@ -1140,7 +1141,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, else if (strcmp(prop_name, "wave_front_size") == 0) props->WaveFrontSize = (uint32_t)prop_val; else if (strcmp(prop_name, "array_count") == 0) - props->NumShaderBanks = (uint32_t)prop_val; + simd_arrays_count = (uint32_t)prop_val; else if (strcmp(prop_name, "simd_arrays_per_engine") == 0) props->NumArrays = (uint32_t)prop_val; else if (strcmp(prop_name, "cu_per_simd_array") == 0) @@ -1191,6 +1192,9 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if (props->NumCPUCores) goto err; + if (props->NumArrays != 0) + props->NumShaderBanks = simd_arrays_count/props->NumArrays; + gfxv_major = HSA_GET_GFX_VERSION_MAJOR(gfxv); gfxv_minor = HSA_GET_GFX_VERSION_MINOR(gfxv); gfxv_stepping = HSA_GET_GFX_VERSION_STEP(gfxv); From fb8f42233d09f46115471766d31aacbb1ad4d851 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Tue, 24 Jan 2023 18:32:47 +0000 Subject: [PATCH 1090/1247] Fix unitialized variable warning in valgrind Change-Id: I91e70d67671a8f7289b734407011380b6b97238a --- src/topology.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/topology.c b/src/topology.c index 0f5c0f9706..d808a2d84f 100644 --- a/src/topology.c +++ b/src/topology.c @@ -539,6 +539,8 @@ static int get_cpu_cache_info(const char *prefix, struct proc_cpuinfo *cpuinfo, fscanf_dec(path, &this_cache->CacheLevel); /* CacheType */ snprintf(path, 256, "%s/index%d/type", prefix, idx); + + memset(str, 0, sizeof(str)); fscanf_str(path, str); if (!strcmp(str, "Data")) this_cache->CacheType.ui32.Data = 1; From 053b89414a4f62382a8309120ee98f693c7bf152 Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Wed, 8 Feb 2023 14:19:49 -0800 Subject: [PATCH 1091/1247] File reorg backward compatibility message changed to #error Change-Id: I70b6f06b5e82242b3f50e7d1f0dac8a1eb8add11 --- hsakmt-backward-compat.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hsakmt-backward-compat.cmake b/hsakmt-backward-compat.cmake index 6e1b3a9e97..68b81a803a 100644 --- a/hsakmt-backward-compat.cmake +++ b/hsakmt-backward-compat.cmake @@ -48,7 +48,7 @@ function(create_header_template) #if defined(__GNUC__) -#warning \"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\" +#error \"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\" #else #pragma message(\"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\") #endif From 53d53655d743e4960b8ada7d2b5de488d268975e Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Tue, 14 Feb 2023 14:19:31 +0000 Subject: [PATCH 1092/1247] Fix for unitialized variables Change-Id: Ie8a004db699248d0cde4213077520ea503754399 --- src/topology.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/topology.c b/src/topology.c index d808a2d84f..f2b2075e3e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1059,17 +1059,17 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, uint32_t *num_p2pLinks) { FILE *fd; - char *read_buf, *p, *envvar, dummy; + char *read_buf, *p, *envvar, dummy = '\0'; char prop_name[256]; char path[256]; - unsigned long long prop_val; - uint32_t prog, major, minor, step; + unsigned long long prop_val = 0; + uint32_t prog, major = 0, minor = 0, step = 0; int read_size; const struct hsa_gfxip_table *hsa_gfxip; uint32_t sys_node_id; uint32_t gfxv = 0; uint8_t gfxv_major, gfxv_minor, gfxv_stepping; - uint32_t simd_arrays_count; + uint32_t simd_arrays_count = 0; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; From 60831e86b2c089b1d0631692306ad9ffbb16ec9c Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 5 Oct 2022 15:45:50 -0400 Subject: [PATCH 1093/1247] kfdtest: Update GFX11 blacklists Remove BLACKLIST_GFX10_NV2X from GFX11 blacklists, update BLACKLIST_GFX11 as needed. Signed-off-by: Graham Sider Change-Id: I84bd91ba20a5d3df27478fb4c97afa12f8a3e76a --- tests/kfdtest/scripts/kfdtest.exclude | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index fcb9ba8f58..268a3f4dec 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -116,19 +116,13 @@ BLACKLIST_GFX10_NV2X=\ "$BLACKLIST_GFX10:"\ "KFDPerfCountersTest.*" -# GFX11 still undergoing debug. Ticket links: # KFDMemoryTest.FlatScratchAccess - SWDEV-329877 -# KFDEvictTest.QueueTest - SWDEV-325064 # KFDQMTest.MultipleCpQueuesStressDispatch - SWDEV-340965 -# KFDExceptionTest.* - SWDEV-340972 -TEMPORARY_BLACKLIST_GFX11=\ +BLACKLIST_GFX11=\ "KFDQMTest.CreateAqlCpQueue:"\ "KFDQMTest.MultipleCpQueuesStressDispatch:"\ "KFDCWSRTest.InterruptRestore:"\ -"KFDExceptionTest.*:"\ -"KFDEvictTest.QueueTest:"\ -"KFDSVMRangeTest.*Migrate*:"\ -"KFDSVMRangeTest.*Migration*:"\ +"KFDPerfCountersTest.*:"\ "KFDMemoryTest.FlatScratchAccess" # KFDQMTest.CpuWriteCoherence fails. 0 dwordsAvailable (KFD-338) @@ -291,23 +285,21 @@ FILTER[yellow_carp]=\ FILTER[gfx1100]=\ "$BLACKLIST_ALL_ASICS:"\ -"$BLACKLIST_GFX10_NV2X:"\ -"$TEMPORARY_BLACKLIST_GFX11" +"$BLACKLIST_GFX11" +# SWDEV-384028 FILTER[gfx1101]=\ "$BLACKLIST_ALL_ASICS:"\ -"$BLACKLIST_GFX10_NV2X:"\ -"$TEMPORARY_BLACKLIST_GFX11" +"$BLACKLIST_GFX11:"\ +"KFDExceptionTest.SdmaQueueException" FILTER[gfx1102]=\ "$BLACKLIST_ALL_ASICS:"\ -"$BLACKLIST_GFX10_NV2X:"\ -"$TEMPORARY_BLACKLIST_GFX11" +"$BLACKLIST_GFX11" FILTER[gfx1103]=\ "$BLACKLIST_ALL_ASICS:"\ -"$BLACKLIST_GFX10_NV2X:"\ -"$TEMPORARY_BLACKLIST_GFX11" +"$BLACKLIST_GFX11" FILTER[gfx1036]=\ "$BLACKLIST_ALL_ASICS:"\ From 64aa9009e11f2dbc181e3c05d98d019725963486 Mon Sep 17 00:00:00 2001 From: "kent.russell@amd.com" Date: Wed, 18 Jan 2023 13:44:04 -0500 Subject: [PATCH 1094/1247] Add check for available_memory API If the KFD IOCTL version doesn't support available_memory, don't run the test. Just skip the test Change-Id: Iebf526d4563ab9f3c054bbfb38c214a1b893fcb5 --- tests/kfdtest/src/KFDMemoryTest.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 584c1fef87..459ffc8f3a 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -268,6 +268,12 @@ TEST_F(KFDMemoryTest, MemoryAllocAll) { HsaMemFlags memFlags = {0}; memFlags.ui32.NonPaged = 1; // sys mem vs vram HSAuint64 available; + + if (m_VersionInfo.KernelInterfaceMinorVersion < 9) { + LOG() << "Available memory IOCTL not present in KFD. Exiting." << std::endl; + return; + } + void *object = NULL; int shrink = 21, success = HSAKMT_STATUS_NO_MEMORY; EXPECT_SUCCESS(hsaKmtAvailableMemory(defaultGPUNode, &available)); From 564913526a17318e9ce2c4077c54be76b1c1503f Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 23 Feb 2023 13:51:51 +0800 Subject: [PATCH 1095/1247] kfdtest: add MES judging API in test utility. Signed-off-by: Yifan Zhang Change-Id: I978fc85b7c81ea65b97953a50d2d0312bcba95bf --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 5 +++++ tests/kfdtest/src/KFDBaseComponentTest.hpp | 1 + tests/kfdtest/src/KFDTestUtil.cpp | 5 +++++ tests/kfdtest/src/KFDTestUtil.hpp | 1 + 4 files changed, 12 insertions(+) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index bb510e040b..a286f21983 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -164,6 +164,11 @@ unsigned int KFDBaseComponentTest::GetFamilyIdFromNodeId(unsigned int nodeId) return FamilyIdFromNode(m_NodeInfo.GetNodeProperties(nodeId)); } +bool KFDBaseComponentTest::isMES(unsigned int nodeId) +{ + return CheckMES(m_NodeInfo.GetNodeProperties(nodeId)); +} + int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) { HsaNodeProperties *nodeProperties; _HSAKMT_STATUS status; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index cc87465ad5..2f9d8350e2 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -58,6 +58,7 @@ class KFDBaseComponentTest : public testing::Test { // @return DRM Render Node if successful or -1 on failure int FindDRMRenderNode(int gpuNode); unsigned int GetFamilyIdFromNodeId(unsigned int nodeId); + bool isMES(unsigned int nodeId); unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; } // @brief Executed before the first test that uses KFDBaseComponentTest. diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index fc8bd0248e..5a0401dd15 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -234,6 +234,11 @@ bool isTonga(const HsaNodeProperties *props) { return false; } +bool CheckMES(const HsaNodeProperties *props) { + /* MES devices need GART mappings and therefore need non-paged memory */ + return (props->EngineId.ui32.Major >= 11) ? true : false; +} + const uint32_t GetGfxVersion(const HsaNodeProperties *props) { return ((props->EngineId.ui32.Major << 16) | (props->EngineId.ui32.Minor << 8) | diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 938ff8bf69..a9f6c9d9be 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -50,6 +50,7 @@ HSAKMT_STATUS CreateQueueTypeEvent(bool ManualReset, bool IsSignaled, unsigned i bool is_dgpu(); bool isTonga(const HsaNodeProperties *props); +bool CheckMES(const HsaNodeProperties *props); bool hasPciAtomicsSupport(int node); unsigned int FamilyIdFromNode(const HsaNodeProperties *props); const uint32_t GetGfxVersion(const HsaNodeProperties *props); From e40ae8481e144c4825957310734b6d11b40f2b75 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 23 Feb 2023 14:08:21 +0800 Subject: [PATCH 1096/1247] =?UTF-8?q?kfdtest:=20Using=20non-paged=20memory?= =?UTF-8?q?=20allocation=20only=20on=20devices=20that=C2=A0have=20MES=20sc?= =?UTF-8?q?heduler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I9181b353aac791f546aa7679ffd7cb8d9f8ef765 --- tests/kfdtest/src/BaseQueue.cpp | 2 +- tests/kfdtest/src/KFDQMTest.cpp | 2 +- tests/kfdtest/src/KFDTestUtil.cpp | 4 ++-- tests/kfdtest/src/KFDTestUtil.hpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index dd1620168c..0809c1f97a 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -49,7 +49,7 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 memset(&m_Resources, 0, sizeof(m_Resources)); m_QueueBuf = new HsaMemoryBuffer(size, NodeId, true/*zero*/, false/*local*/, true/*exec*/, - /*isScratch */ false, /* isReadOnly */false, /* isUncached */true); + /*isScratch */ false, /* isReadOnly */false, /* isUncached */true, /* NonPaged */g_baseTest->isMES(NodeId)); if (type == HSA_QUEUE_COMPUTE_AQL) { m_Resources.Queue_read_ptr_aql = &pointers[0]; diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 2f156ad23b..b3567f7847 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1174,7 +1174,7 @@ TEST_F(KFDQMTest, CreateAqlCpQueue) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HsaMemoryBuffer pointers(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer pointers(PAGE_SIZE, defaultGPUNode, /*zero*/true, /*local*/false, /*exec*/false, /*isScratch */false, /* isReadOnly */false, /* isUncached */false, /* NonPaged */g_baseTest->isMES(defaultGPUNode)); ASSERT_SUCCESS(queue.Create(defaultGPUNode, PAGE_SIZE, pointers.As())); diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 5a0401dd15..7a1e888a58 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -254,7 +254,7 @@ HSAuint64 GetSystemTickCountInMicroSec() { const HsaMemoryBuffer HsaMemoryBuffer::Null; HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, - bool isScratch, bool isReadOnly, bool isUncached) + bool isScratch, bool isReadOnly, bool isUncached, bool NonPaged) :m_Size(size), m_pUser(NULL), m_pBuf(NULL), @@ -278,7 +278,7 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b EXPECT_EQ(isUncached, 0) << "Uncached flag is relevant only for system or host memory"; } else { m_Flags.ui32.HostAccess = 1; - m_Flags.ui32.NonPaged = 0; + m_Flags.ui32.NonPaged = NonPaged ? 1 : 0; m_Flags.ui32.CoarseGrain = 0; m_Flags.ui32.NoNUMABind = 1; m_Flags.ui32.Uncached = isUncached; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index a9f6c9d9be..c8d4de4051 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -69,7 +69,7 @@ class HsaMemoryBuffer { public: HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero = true, bool isLocal = false, - bool isExec = false, bool isScratch = false, bool isReadOnly = false, bool isUncached = false); + bool isExec = false, bool isScratch = false, bool isReadOnly = false, bool isUncached = false, bool NonPaged = false); HsaMemoryBuffer(void *addr, HSAuint64 size); template RetType As() { From 332f59eb2ad804498ecd522c8d6abf0bcb2e3a3c Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 16 Nov 2021 23:13:11 -0500 Subject: [PATCH 1097/1247] libhsakmt: Implement dmabuf export for RDMA Implement hsaKmtExportDMABufHandle, which can be used for a new upstreamable RDMA solution. It exports a DMABuf handle for an arbitrary virtual address along with the offset of the address within the allocation. It also checks that the size of the intended export does not exceed the allocation. This uses the new AMDKFD_IOC_EXPORT_DMABUF, which requires KFD ioctl API version 1.12. Signed-off-by: Felix Kuehling Change-Id: Ie5fdb1f73ab3c7fa36c315ce326b1fb89eacc8b6 --- include/hsakmt.h | 20 +++++++++++++++++++ include/linux/kfd_ioctl.h | 14 +++++++++++-- src/fmm.c | 42 +++++++++++++++++++++++++++++++++++++++ src/fmm.h | 4 ++++ src/libhsakmt.ver | 1 + src/memory.c | 14 +++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index ca586ba36b..63ac299511 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -436,6 +436,26 @@ hsaKmtRegisterGraphicsHandleToNodes( HSAuint32* NodeArray //IN ); +/** + * Export a dmabuf handle and offset for a given memory address + * + * Validates that @MemoryAddress belongs to a valid allocation and that the + * @MemorySizeInBytes doesn't exceed the end of that allocation. Returns a + * dmabuf fd of the allocation and the offset of MemoryAddress within that + * allocation. The memory will remain allocated even after the allocation is + * freed by hsaKmtFreeMemory for as long as a dmabuf fd remains open or any + * importer of that fd maintains an active reference to the memory. + */ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtExportDMABufHandle( + void *MemoryAddress, //IN + HSAuint64 MemorySizeInBytes, //IN + int *DMABufFd, //OUT + HSAuint64 *Offset //OUT + ); + /** Export a memory buffer for sharing with other processes diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 4898451b22..8dc670a4d6 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -37,9 +37,10 @@ * - 1.9 - Add available_memory ioctl * - 1.10 - Add SMI profiler event log * - 1.11 - Add unified memory for ctx save/restore area + * - 1.12 - Add DMA buf export ioctl */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 11 +#define KFD_IOCTL_MINOR_VERSION 12 /* * Debug revision change log @@ -847,6 +848,12 @@ struct kfd_ioctl_import_dmabuf_args { __u32 dmabuf_fd; /* to KFD */ }; +struct kfd_ioctl_export_dmabuf_args { + __u64 handle; /* to KFD */ + __u32 flags; /* to KFD */ + __u32 dmabuf_fd; /* from KFD */ +}; + /* * KFD SMI(System Management Interface) events */ @@ -1345,8 +1352,11 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_AVAILABLE_MEMORY \ AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args) +#define AMDKFD_IOC_EXPORT_DMABUF \ + AMDKFD_IOWR(0x24, struct kfd_ioctl_export_dmabuf_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x24 +#define AMDKFD_COMMAND_END 0x25 /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ diff --git a/src/fmm.c b/src/fmm.c index 9f099c5a34..774f9e0cab 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3309,6 +3309,48 @@ error_free_metadata: return status; } +HSAKMT_STATUS fmm_export_dma_buf_fd(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + int *DMABufFd, + HSAuint64 *Offset) +{ + struct kfd_ioctl_export_dmabuf_args exportArgs = {0}; + manageable_aperture_t *aperture; + HsaApertureInfo ApeInfo; + vm_object_t *obj; + HSAuint64 offset; + int r; + + aperture = fmm_find_aperture(MemoryAddress, &ApeInfo); + if (!aperture) + return HSAKMT_STATUS_INVALID_PARAMETER; + + pthread_mutex_lock(&aperture->fmm_mutex); + obj = vm_find_object_by_address_range(aperture, MemoryAddress); + if (obj) { + offset = VOID_PTRS_SUB(MemoryAddress, obj->start); + if (offset + MemorySizeInBytes <= obj->size) { + exportArgs.handle = obj->handle; + exportArgs.flags = O_CLOEXEC; + exportArgs.dmabuf_fd = 0; + } else { + obj = NULL; + } + } + pthread_mutex_unlock(&aperture->fmm_mutex); + if (!obj) + return HSAKMT_STATUS_INVALID_PARAMETER; + + r = kmtIoctl(kfd_fd, AMDKFD_IOC_EXPORT_DMABUF, (void *)&exportArgs); + if (r) + return HSAKMT_STATUS_ERROR; + + *DMABufFd = exportArgs.dmabuf_fd; + *Offset = offset; + + return HSAKMT_STATUS_SUCCESS; +} + HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, HSAuint64 SizeInBytes, HsaSharedMemoryHandle *SharedMemoryHandle) diff --git a/src/fmm.h b/src/fmm.h index 498366e7a0..d0146d74cd 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -78,6 +78,10 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, uint32_t *gpu_id_array, uint32_t gpu_id_array_size); HSAKMT_STATUS fmm_deregister_memory(void *address); +HSAKMT_STATUS fmm_export_dma_buf_fd(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + int *DMABufFd, + HSAuint64 *Offset); HSAKMT_STATUS fmm_share_memory(void *MemoryAddress, HSAuint64 SizeInBytes, HsaSharedMemoryHandle *SharedMemoryHandle); diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 46370c6eee..50564de31b 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -70,6 +70,7 @@ hsaKmtSVMGetAttr; hsaKmtSetXNACKMode; hsaKmtGetXNACKMode; hsaKmtOpenSMI; +hsaKmtExportDMABufHandle; local: *; }; diff --git a/src/memory.c b/src/memory.c index 7f8e45b6e0..4cb05057ad 100644 --- a/src/memory.c +++ b/src/memory.c @@ -318,6 +318,20 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes(HSAuint64 GraphicsRe return ret; } +HSAKMT_STATUS HSAKMTAPI hsaKmtExportDMABufHandle(void *MemoryAddress, + HSAuint64 MemorySizeInBytes, + int *DMABufFd, + HSAuint64 *Offset) +{ + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(12); + + pr_debug("[%s] address %p\n", __func__, MemoryAddress); + + return fmm_export_dma_buf_fd(MemoryAddress, MemorySizeInBytes, + DMABufFd, Offset); +} + HSAKMT_STATUS HSAKMTAPI hsaKmtShareMemory(void *MemoryAddress, HSAuint64 SizeInBytes, HsaSharedMemoryHandle *SharedMemoryHandle) From e5ab87ede72ca79abad76c9e6a0569565b5f9077 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Wed, 17 Nov 2021 22:31:19 -0500 Subject: [PATCH 1098/1247] kfdtest: Add test for hsaKmtExportDMABufHandle Signed-off-by: Felix Kuehling Change-Id: Ia87377c1d4201fecfa00c2e0ca53b507608df2b3 --- tests/kfdtest/src/KFDMemoryTest.cpp | 80 +++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 459ffc8f3a..ae6688dfa9 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2466,3 +2466,83 @@ TEST_F(KFDMemoryTest, MultiThreadRegisterUserptrTest) { TEST_END } + +TEST_F(KFDMemoryTest, ExportDMABufTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (m_VersionInfo.KernelInterfaceMinorVersion < 12) { + LOG() << "Skipping test, requires KFD ioctl version 1.12 or newer" << std::endl; + return; + } + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + // Use a GTT BO for export because it's conveniently CPU accessible. + // On multi-GPU systems this also checks for interactions with driver- + // internal DMA buf use for DMA attachment to multiple GPUs + HsaMemFlags memFlags = m_MemoryFlags; + memFlags.ui32.NonPaged = 1; + + HSAuint32 *buf; + ASSERT_SUCCESS(hsaKmtAllocMemory(0, PAGE_SIZE, memFlags, + reinterpret_cast(&buf))); + ASSERT_SUCCESS(hsaKmtMapMemoryToGPU(buf, PAGE_SIZE, NULL)); + + for (int i = 0; i < PAGE_SIZE/4; i++) + buf[i] = i; + const HSAuint64 INDEX = 25; + const HSAuint64 SIZE = 25; + HSAuint64 offset; + int fd; + + // Expected error: address out of range (not a BO) + ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, + hsaKmtExportDMABufHandle(buf + PAGE_SIZE/4, SIZE*4, &fd, &offset)); + // Expected error: size out of range + ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, + hsaKmtExportDMABufHandle(buf + INDEX, PAGE_SIZE, &fd, &offset)); + + // For real this time. Check that the offset matches + ASSERT_SUCCESS(hsaKmtExportDMABufHandle(buf + INDEX, SIZE*4, &fd, &offset)); + ASSERT_EQ(INDEX*4, offset); + + // Free the original BO. The memory should persist as long as the DMA buf + // handle exists. + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(buf)); + ASSERT_SUCCESS(hsaKmtFreeMemory(buf, PAGE_SIZE)); + + // Import the BO using the Interop API and check the contents. It doesn't + // map the import for CPU access, which gives us an excuse to test GPU + // mapping of the imported BO as well. + HsaGraphicsResourceInfo info; + ASSERT_SUCCESS(hsaKmtRegisterGraphicsHandleToNodes(fd, &info, 1, &defaultGPUNode)); + buf = reinterpret_cast(info.MemoryAddress); + ASSERT_EQ(info.SizeInBytes, PAGE_SIZE); + + HsaMemMapFlags mapFlags = {0}; + ASSERT_SUCCESS(hsaKmtMapMemoryToGPUNodes(buf, PAGE_SIZE, NULL, mapFlags, 1, + &defaultGPUNode)); + + PM4Queue pm4Queue; + ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); + HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(CopyDwordIsa, isaBuffer.As())); + for (int i = 0; i < PAGE_SIZE/4; i++) { + Dispatch dispatch(isaBuffer); + dispatch.SetArgs(&buf[i], dstBuffer.As()); + dispatch.Submit(pm4Queue); + dispatch.Sync(g_TestTimeOut); + ASSERT_EQ(i, *dstBuffer.As()); + } + ASSERT_SUCCESS(pm4Queue.Destroy()); + + ASSERT_SUCCESS(hsaKmtUnmapMemoryToGPU(buf)); + ASSERT_SUCCESS(hsaKmtDeregisterMemory(buf)); + + ASSERT_EQ(0, close(fd)); + + TEST_END +} From 9f0f7741de0f41b9671da68ac2fadd5710aa64da Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Wed, 1 Mar 2023 20:53:00 +0800 Subject: [PATCH 1099/1247] gfx11 is able to perform atomic ops even PCI reports no atomic support. Signed-off-by: Yifan Zhang Change-Id: Ie0d8af5a64ed717b140ac14db654c65ec7aa5ebb --- tests/kfdtest/src/KFDTestUtil.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 7a1e888a58..cbc1d4104a 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -140,6 +140,10 @@ bool hasPciAtomicsSupport(int node) { if (pNodeProperties->NumCPUCores && pNodeProperties->NumFComputeCores) return true; + /* gfx11 is able to perform aotmic ops even PCI reports no atomic support. */ + if (pNodeProperties->EngineId.ui32.Major >= 11) + return true; + HsaIoLinkProperties *IolinkProperties = new HsaIoLinkProperties[pNodeProperties->NumIOLinks]; if (hsaKmtGetNodeIoLinkProperties(node, pNodeProperties->NumIOLinks, IolinkProperties)) { LOG() << "Unable to get Node IO Link Information for node " << node << std::endl; From 3f55ba9fb8ca33c62b11bf772e759ab3fe7aec65 Mon Sep 17 00:00:00 2001 From: Eric Huang Date: Wed, 15 Feb 2023 11:44:49 -0500 Subject: [PATCH 1100/1247] kfdtest: add the check for svm usage limit Since KFD counts svm allocation as system memory usage, KFDSVMEvictTest will fail on the case of small system memory, adding check is to skip test. Signed-off-by: Eric Huang Change-Id: I040f16f2dd0d4092d069a632cfba9c28293f781b --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index 22953d01ab..cd61de3c3e 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -69,6 +69,12 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf */ size = sysMemSize / 3 + vramSize; size = size > vramSize << 1 ? vramSize << 1 : size; + /* Check if there is enough system memory to pass test, + * KFD system memory limit is 15/16. + */ + if (size > (sysMemSize - (sysMemSize >> 4))) + return 0; + sizeInPages = size >> PAGE_SHIFT; count = sizeInPages / (vramBufSizeInPages * N_PROCESSES); From d3bb1ca4af6af76127b32ac593c2ab340a706731 Mon Sep 17 00:00:00 2001 From: Daniel Phillips Date: Fri, 3 Mar 2023 15:01:03 -0800 Subject: [PATCH 1101/1247] kfdtests: Relax MemoryAllocAll failure criteria The MemoryAllocAll test in kfdtests exercises the new KFD memory availability API by trying to allocate a single buffer object that exactly fills all of vram. Desired object size is determined using the memory availility KFD ioctl via libhsakmt, then an object is allocated slightly larger than that size. If the allocation attempt fails then the test tries to allocate a slightly smaller object, and continues trying with smaller sizes until the allocation succeeds. The test succeeds if the successfully allocated object is within some specified tolerance of the available memory reported. There are a number of known issues that can cause the successfully allocated object to be significantly smaller than reported availability. Until these issues are addressed, we should not fail the test, but just log the actual divergence between the size of the object we thought we could allocate, and what was actually possible. Signed-off-by: Daniel Phillips Change-Id: I165a30865ffbb2353286dcc896ad8e24af124615 --- tests/kfdtest/src/KFDMemoryTest.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index ae6688dfa9..df394a43e1 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -290,11 +290,9 @@ TEST_F(KFDMemoryTest, MemoryAllocAll) { LOG() << "Allocated: " << size << " bytes" << std::endl; if (size > available + leeway) { LOG() << "Under-reported available memory!" << std::endl; - success = HSAKMT_STATUS_ERROR; } if (size < available - leeway) { LOG() << "Over-reported available memory!" << std::endl; - success = HSAKMT_STATUS_NO_MEMORY; } } EXPECT_SUCCESS(success); From c9118482421b32ed9458e404b362a8744735c60c Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Fri, 10 Mar 2023 13:08:33 -0800 Subject: [PATCH 1102/1247] Compile time flag to switch between #warning and #error message Using backward compatibility paths will provide an #error message. Compile time option added to enable/disable the #error message. Disabling the same will provide a #warning message Change-Id: Ibb84241ba35aefb7a8450d68231e52242a634ed3 --- CMakeLists.txt | 14 ++++++++++++++ hsakmt-backward-compat.cmake | 10 ++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e78bfbe79..c447d6de98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,6 +210,20 @@ install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INS # Option to build header path migration helpers. option(INCLUDE_PATH_COMPATIBILITY "Generate backward compatible headers and include paths. Use of these headers will warn when included." ON) if(INCLUDE_PATH_COMPATIBILITY) + # To enable/disable #error in wrapper header files + if(NOT DEFINED ROCM_HEADER_WRAPPER_WERROR) + if(DEFINED ENV{ROCM_HEADER_WRAPPER_WERROR}) + set(ROCM_HEADER_WRAPPER_WERROR "$ENV{ROCM_HEADER_WRAPPER_WERROR}" + CACHE STRING "Header wrapper warnings as errors.") + else() + set(ROCM_HEADER_WRAPPER_WERROR "ON" CACHE STRING "Header wrapper warnings as errors.") + endif() + endif() + if(ROCM_HEADER_WRAPPER_WERROR) + set(deprecated_error 1) + else() + set(deprecated_error 0) + endif() include(hsakmt-backward-compat.cmake) endif() diff --git a/hsakmt-backward-compat.cmake b/hsakmt-backward-compat.cmake index 68b81a803a..3c5adabb42 100644 --- a/hsakmt-backward-compat.cmake +++ b/hsakmt-backward-compat.cmake @@ -46,12 +46,18 @@ function(create_header_template) #ifndef @include_guard@ #define @include_guard@ - -#if defined(__GNUC__) +#ifndef ROCM_HEADER_WRAPPER_WERROR +#define ROCM_HEADER_WRAPPER_WERROR @deprecated_error@ +#endif +#if ROCM_HEADER_WRAPPER_WERROR /* ROCM_HEADER_WRAPPER_WERROR 1 */ #error \"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\" +#else /* ROCM_HEADER_WRAPPER_WERROR 0 */ +#if defined(__GNUC__) +#warning \"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\" #else #pragma message(\"@file_name@ has moved to @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/hsakmt and package include paths have changed.\\nInclude as \\\"hsakmt/@file_name@\\\" when using cmake packages.\") #endif +#endif /* ROCM_HEADER_WRAPPER_WERROR */ @include_statements@ From 0f977fd1d84d5185b0c7f6109cd31775b584a8fb Mon Sep 17 00:00:00 2001 From: AravindanC Date: Fri, 3 Feb 2023 08:15:54 -0800 Subject: [PATCH 1103/1247] ASAN Packaging for libhsakmt Change-Id: I0a6232cdb61742aa81394bb49d2b5e890b6ada6f --- CMakeLists.txt | 82 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c447d6de98..95b90a045d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,10 +196,21 @@ set ( CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX} CACHE STRING "Default install set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "Default packaging prefix." ) set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) +## set components +if ( ENABLE_ASAN_PACKAGING ) + # ASAN Package requires only asan component with libraries and license file + set ( CPACK_COMPONENTS_ALL asan ) +else() + set ( CPACK_COMPONENTS_ALL devel ) +endif() + # Installs binaries and exports the library usage data to ${HSAKMT_TARGET}Targets install ( TARGETS ${HSAKMT_TARGET} EXPORT ${HSAKMT_TARGET}Targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel ) + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT asan + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT asan ) +install ( TARGETS ${HSAKMT_TARGET} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT devel ) #install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel ) @@ -208,7 +219,10 @@ install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INS COMPONENT devel PATTERN "linux" EXCLUDE ) # Option to build header path migration helpers. -option(INCLUDE_PATH_COMPATIBILITY "Generate backward compatible headers and include paths. Use of these headers will warn when included." ON) +if( NOT ENABLE_ASAN_PACKAGING ) + option(INCLUDE_PATH_COMPATIBILITY "Generate backward compatible headers and include paths. Use of these headers will warn when included." ON) +endif() + if(INCLUDE_PATH_COMPATIBILITY) # To enable/disable #error in wrapper header files if(NOT DEFINED ROCM_HEADER_WRAPPER_WERROR) @@ -226,7 +240,6 @@ if(INCLUDE_PATH_COMPATIBILITY) endif() include(hsakmt-backward-compat.cmake) endif() - # Record our usage data for clients find_package calls. install ( EXPORT ${HSAKMT_TARGET}Targets FILE ${HSAKMT_TARGET}Targets.cmake @@ -246,27 +259,28 @@ add_library( ${HSAKMT_TARGET}::${HSAKMT_TARGET} ALIAS ${HSAKMT_TARGET} ) include(CMakePackageConfigHelpers) configure_package_config_file(${HSAKMT_TARGET}-config.cmake.in - ${HSAKMT_TARGET}-config.cmake - INSTALL_DESTINATION - ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} ) + ${HSAKMT_TARGET}-config.cmake + INSTALL_DESTINATION + ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} ) write_basic_package_version_file(${HSAKMT_TARGET}-config-version.cmake VERSION ${BUILD_VERSION_STRING} - COMPATIBILITY - AnyNewerVersion) + COMPATIBILITY + AnyNewerVersion) install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_TARGET}-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_TARGET}-config-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_TARGET}-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${HSAKMT_TARGET}-config-version.cmake DESTINATION - ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} + ${CMAKE_INSTALL_LIBDIR}/cmake/${HSAKMT_TARGET} COMPONENT devel) # Optionally record the package's find module in the user's package cache. if ( NOT DEFINED EXPORT_TO_USER_PACKAGE_REGISTRY ) set ( EXPORT_TO_USER_PACKAGE_REGISTRY "off" ) endif() -set ( EXPORT_TO_USER_PACKAGE_REGISTRY ${EXPORT_TO_USER_PACKAGE_REGISTRY} CACHE BOOL "Add cmake package config location to the user's cmake package registry.") +set ( EXPORT_TO_USER_PACKAGE_REGISTRY ${EXPORT_TO_USER_PACKAGE_REGISTRY} + CACHE BOOL "Add cmake package config location to the user's cmake package registry.") if(${EXPORT_TO_USER_PACKAGE_REGISTRY}) # Enable writing to the registry set(CMAKE_EXPORT_PACKAGE_REGISTRY ON) @@ -300,6 +314,7 @@ set ( ENABLE_LDCONFIG ON CACHE BOOL "Set library links and caches using ldconfig # Install License file install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel) +install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR}-asan COMPONENT asan) # Make proper version for appending # Default Value is 99999, setting it first @@ -330,6 +345,15 @@ set(CPACK_DEBIAN_PACKAGE_PROVIDES "hsakmt-roct") set(CPACK_RPM_PACKAGE_OBSOLETES "hsakmt-roct") set(CPACK_RPM_PACKAGE_PROVIDES "hsakmt-roct") +# Debian devel component specific flags +set(CPACK_DEBIAN_DEVEL_PACKAGE_NAME "hsakmt-roct-dev") +set(CPACK_DEBIAN_DEVEL_PACKAGE_REPLACES "hsakmt-roct") +set(CPACK_DEBIAN_DEVEL_PACKAGE_PROVIDES "hsakmt-roct") +set(CPACK_RPM_DEVEL_PACKAGE_PROVIDES "hsakmt-roct") + +# Debian package specific variable for ASAN +set(CPACK_DEBIAN_ASAN_PACKAGE_NAME "hsakmt-roct-asan" ) + ## Process the Rpm install/remove scripts to update the CPACK variables configure_file ( "${CMAKE_CURRENT_SOURCE_DIR}/RPM/post.in" RPM/post @ONLY ) configure_file ( "${CMAKE_CURRENT_SOURCE_DIR}/RPM/postun.in" RPM/postun @ONLY ) @@ -342,9 +366,17 @@ if(DEFINED CPACK_PACKAGING_INSTALL_PREFIX) endif() set(CPACK_RPM_PACKAGE_NAME "hsakmt-roct-devel") set(CPACK_RPM_PACKAGE_LICENSE "MIT") + +# RPM package specific variable for ASAN +set(CPACK_RPM_ASAN_PACKAGE_NAME "hsakmt-roct-asan" ) + +# RPM package devel specific variable +set(CPACK_RPM_DEVEL_PACKAGE_NAME "hsakmt-roct-devel") # Since we changed the package name to match RPM specs, take care of older builds that had -dev installed # Also cover the fact that this now replaces the old binary package hsakmt-roct set(CPACK_RPM_PACKAGE_OBSOLETES "hsakmt-roct-dev hsakmt-roct") +set(CPACK_RPM_DEVEL_PACKAGE_OBSOLETES "hsakmt-roct-dev hsakmt-roct") + if(DEFINED ENV{CPACK_RPM_PACKAGE_RELEASE}) set(CPACK_RPM_PACKAGE_RELEASE $ENV{CPACK_RPM_PACKAGE_RELEASE}) else() @@ -370,17 +402,33 @@ endif() set(CPACK_DEBIAN_PACKAGE_DEPENDS "libdrm-amdgpu-dev | libdrm-dev, rocm-core") set(CPACK_DEBIAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-dev") set(CPACK_DEBIAN_PACKAGE_REPLACES "hsakmt-roct") +set(CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS "libdrm-amdgpu-dev | libdrm-dev, rocm-core") +set(CPACK_DEBIAN_DEVEL_PACKAGE_RECOMMENDS "libdrm-amdgpu-dev") +set(CPACK_DEBIAN_DEVEL_PACKAGE_REPLACES "hsakmt-roct") + +# Debian package specific variable for ASAN +set(CPACK_DEBIAN_ASAN_PACKAGE_DEPENDS "libdrm-amdgpu-dev | libdrm-dev, rocm-core-asan") +set(CPACK_DEBIAN_ASAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-dev") set(CPACK_RPM_PACKAGE_REQUIRES "rocm-core") +set(CPACK_RPM_DEVEL_PACKAGE_REQUIRES "rocm-core") +# RPM package specific variable for ASAN +set(CPACK_RPM_ASAN_PACKAGE_REQUIRES "rocm-core-asan") # NOTE: CentOS/RHEL <8 and SUSE <10 do not have support for rpm recommends, # so HSAKMT_REQUIRES_LIBDRM should be defined when building EL7 to avoid # producing binaries which do not install on CentOS7 or RHEL7 if(HSAKMT_REQUIRES_LIBDRM) string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", libdrm-amdgpu-devel" ) + string ( APPEND CPACK_RPM_DEVEL_PACKAGE_REQUIRES ", libdrm-amdgpu-devel" ) + string ( APPEND CPACK_RPM_ASAN_PACKAGE_REQUIRES ", libdrm-amdgpu-devel" ) else() string ( APPEND CPACK_RPM_PACKAGE_REQUIRES ", (libdrm-amdgpu-devel or libdrm-devel)" ) + string ( APPEND CPACK_RPM_DEVEL_PACKAGE_REQUIRES ", (libdrm-amdgpu-devel or libdrm-devel)" ) + string ( APPEND CPACK_RPM_ASAN_PACKAGE_REQUIRES ", (libdrm-amdgpu-devel or libdrm-devel)" ) set(CPACK_RPM_PACKAGE_RECOMMENDS "libdrm-amdgpu-devel" ) + set(CPACK_RPM_DEVEL_PACKAGE_RECOMMENDS "libdrm-amdgpu-devel" ) + set(CPACK_RPM_ASAN_PACKAGE_RECOMMENDS "libdrm-amdgpu-devel" ) endif() # Set the names now using CPACK utility @@ -391,6 +439,14 @@ set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") if(NOT ROCM_DEP_ROCMCORE) string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_PACKAGE_REQUIRES ${CPACK_RPM_PACKAGE_REQUIRES}) string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_RPM_DEVEL_PACKAGE_REQUIRES ${CPACK_RPM_DEVEL_PACKAGE_REQUIRES}) + string(REGEX REPLACE ",? ?rocm-core" "" CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS ${CPACK_DEBIAN_DEVEL_PACKAGE_DEPENDS}) + string(REGEX REPLACE ",? ?rocm-core-asan" "" CPACK_RPM_ASAN_PACKAGE_REQUIRES ${CPACK_RPM_ASAN_PACKAGE_REQUIRES}) + string(REGEX REPLACE ",? ?rocm-core-asan" "" CPACK_DEBIAN_ASAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_ASAN_PACKAGE_DEPENDS}) endif() include(CPack) + +cpack_add_component(asan + DISPLAY_NAME "ASAN" + DESCRIPTION "ASAN libraries for the LIBHSAKMT") From 63c8cf115a4c793c4e7dfa6cb5f7bf1c6a1d0e8a Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Wed, 31 Aug 2022 12:16:04 -0500 Subject: [PATCH 1104/1247] src: use SVM mechanism to register userptr memory Register and map userptrs through Shared Virtual Memory(SVM) API at the Kernel level when available. Using this approach, performance will be improve as register/unregister memory will not trigger any system call to KFD driver. Signed-off-by: Alex Sierra Change-Id: I3726b4b5e1c6a52a83786fbe0af6322eb29ae7c9 --- src/fmm.c | 151 ++++++++++++++++++++++------ tests/kfdtest/src/KFDMemoryTest.cpp | 10 +- 2 files changed, 128 insertions(+), 33 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 774f9e0cab..436af68223 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -220,6 +220,9 @@ typedef struct { /* specifies the alignment size as PAGE_SIZE * 2^alignment_order */ uint32_t alignment_order; + + /* whether to check all dGPUs in the topology support SVM API */ + bool is_svm_api_supported; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -239,7 +242,8 @@ static svm_t svm = { .dgpu_alt_aperture = NULL, .userptr_for_paged_mem = false, .check_userptr = false, - .disable_cache = false + .disable_cache = false, + .is_svm_api_supported = false }; /* On APU, for memory allocated on the system memory that GPU doesn't access @@ -961,6 +965,72 @@ static HsaMemFlags fmm_translate_ioc_to_hsa_flags(uint32_t ioc_flags) return mflags; } +static HSAKMT_STATUS fmm_register_mem_svm_api(void *address, + uint64_t size, + bool coarse_grain) +{ + struct kfd_ioctl_svm_args *args; + size_t s_attr; + HSAuint32 page_offset = (HSAuint64)address & (PAGE_SIZE-1); + HSAuint64 aligned_addr = (HSAuint64)address - page_offset; + HSAuint64 aligned_size = PAGE_ALIGN_UP(page_offset + size); + + if (!g_first_gpu_mem) + return HSAKMT_STATUS_ERROR; + + s_attr = sizeof(struct kfd_ioctl_svm_attribute); + args = alloca(sizeof(*args) + s_attr); + args->start_addr = aligned_addr; + args->size = aligned_size; + args->op = KFD_IOCTL_SVM_OP_SET_ATTR; + args->nattr = 1; + args->attrs[0].type = coarse_grain ? + HSA_SVM_ATTR_CLR_FLAGS : HSA_SVM_ATTR_SET_FLAGS; + args->attrs[0].value = HSA_SVM_FLAG_COHERENT; + pr_debug("Registering to SVM %p size: %ld\n", (void*)aligned_addr, + aligned_size); + /* Driver does one copy_from_user, with extra attrs size */ + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { + pr_debug("op set range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS fmm_map_mem_svm_api(void *address, + uint64_t size, + uint32_t *nodes_to_map, + uint32_t nodes_array_size) +{ + struct kfd_ioctl_svm_args *args; + size_t s_attr; + uint32_t i, nattr; + + if (!g_first_gpu_mem) + return HSAKMT_STATUS_ERROR; + + nattr = nodes_array_size; + s_attr = sizeof(struct kfd_ioctl_svm_attribute) * nattr; + args = alloca(sizeof(*args) + s_attr); + + args->start_addr = (uint64_t)address; + args->size = size; + args->op = KFD_IOCTL_SVM_OP_SET_ATTR; + args->nattr = nattr; + for (i = 0; i < nodes_array_size; i++) { + args->attrs[i].type = HSA_SVM_ATTR_ACCESS_IN_PLACE; + args->attrs[i].value = nodes_to_map[i]; + } + /* Driver does one copy_from_user, with extra attrs size */ + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { + pr_debug("op set range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} + /* After allocating the memory, return the vm_object created for this memory. * Return NULL if any failure. */ @@ -1746,7 +1816,9 @@ HSAKMT_STATUS fmm_release(void *address) object = vm_find_object(address, 0, &aperture); if (!object) - return HSAKMT_STATUS_MEMORY_NOT_REGISTERED; + return svm.is_svm_api_supported ? + HSAKMT_STATUS_SUCCESS : + HSAKMT_STATUS_MEMORY_NOT_REGISTERED; if (aperture == &cpuvm_aperture) { /* APU system memory */ @@ -2182,7 +2254,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; - char *maxVaAlignStr; + char *maxVaAlignStr, *useSvmStr; unsigned int guardPages = 1; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2221,6 +2293,9 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (!maxVaAlignStr || sscanf(maxVaAlignStr, "%u", &svm.alignment_order) != 1) svm.alignment_order = 9; + useSvmStr = getenv("HSA_USE_SVM"); + svm.is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")); + gpu_mem_count = 0; g_first_gpu_mem = NULL; @@ -2274,6 +2349,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; gpu_mem[gpu_mem_count].node_id = i; + svm.is_svm_api_supported &= props.Capability.ui32.SVMAPISupported; gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; gpu_mem[gpu_mem_count].scratch_physical.ops = &reserved_aperture_ops; @@ -2783,23 +2859,36 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert } static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, - uint64_t *gpuvm_addr, vm_object_t *object) + uint64_t *gpuvm_addr, vm_object_t *object, + uint32_t *nodes_to_map, uint32_t nodes_array_size) { manageable_aperture_t *aperture; void *svm_addr; - HSAuint64 svm_size; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); int ret; aperture = svm.dgpu_aperture; - svm_addr = object->start; - svm_size = object->size; - /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - ret = _fmm_map_to_gpu(aperture, svm_addr, svm_size, object, NULL, 0); + if (svm.is_svm_api_supported) { + svm_addr = (void*)((HSAuint64)addr - page_offset); + if (!nodes_to_map) { + nodes_to_map = all_gpu_id_array; + nodes_array_size = all_gpu_id_array_size; + } + pr_debug("%s Mapping Address %p size aligned: %ld offset: %x\n", + __func__, svm_addr, PAGE_ALIGN_UP(page_offset + size), page_offset); + ret = fmm_map_mem_svm_api(svm_addr, + PAGE_ALIGN_UP(page_offset + size), + nodes_to_map, + nodes_array_size / sizeof(uint32_t)); + + } else { + svm_addr = object->start; + ret = _fmm_map_to_gpu(aperture, svm_addr, object->size, object, NULL, 0); + } if (ret == 0 && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; @@ -2823,7 +2912,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) address, size); object = vm_find_object(address, size, &aperture); - if (!object) { + if (!object && !svm.is_svm_api_supported ) { if (!is_dgpu) { /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); @@ -2838,8 +2927,8 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); ret = 0; - } else if (object->userptr) { - ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object); + } else if ((svm.is_svm_api_supported && !object) || object->userptr) { + ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object, NULL, 0); } else { ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0); /* Update alternate GPUVM address only for @@ -2849,7 +2938,8 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); } - pthread_mutex_unlock(&aperture->fmm_mutex); + if (object) + pthread_mutex_unlock(&aperture->fmm_mutex); return ret; } @@ -2914,7 +3004,6 @@ static int _fmm_unmap_from_gpu(manageable_aperture_t *aperture, void *address, print_device_id_array((void *)args.device_ids_array_ptr, args.n_devices * sizeof(uint32_t)); - ret = kmtIoctl(kfd_fd, AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, &args); remove_device_ids_from_mapped_array(object, @@ -3016,7 +3105,7 @@ int fmm_unmap_from_gpu(void *address) object = vm_find_object(address, 0, &aperture); if (!object) /* On APUs GPU unmapping of system memory is a no-op */ - return is_dgpu ? -EINVAL : 0; + return (!is_dgpu || svm.is_svm_api_supported) ? 0 : -EINVAL; /* Successful vm_find_object returns with the aperture locked */ if (aperture == &cpuvm_aperture) @@ -3104,7 +3193,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | - (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), &obj); + (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), + &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; @@ -3156,6 +3246,9 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, return HSAKMT_STATUS_SUCCESS; /* Register a new user ptr */ + if (svm.is_svm_api_supported) + return fmm_register_mem_svm_api(address, size_in_bytes, coarse_grain); + ret = fmm_register_user_memory(address, size_in_bytes, &object, coarse_grain); if (ret != HSAKMT_STATUS_SUCCESS) return ret; @@ -3513,9 +3606,9 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) /* On APUs we assume it's a random system memory address * where registration and dergistration is a no-op */ - return is_dgpu ? - HSAKMT_STATUS_MEMORY_NOT_REGISTERED : - HSAKMT_STATUS_SUCCESS; + return (!is_dgpu || svm.is_svm_api_supported) ? + HSAKMT_STATUS_SUCCESS : + HSAKMT_STATUS_MEMORY_NOT_REGISTERED; /* Successful vm_find_object returns with aperture locked */ if (aperture == &cpuvm_aperture) { @@ -3566,7 +3659,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address) { - manageable_aperture_t *aperture; + manageable_aperture_t *aperture = NULL; vm_object_t *object; uint32_t i; uint32_t *registered_node_id_array, registered_node_id_array_size; @@ -3577,24 +3670,22 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_INVALID_PARAMETER; object = vm_find_object(address, size, &aperture); - if (!object) + if (!object && !svm.is_svm_api_supported) return HSAKMT_STATUS_ERROR; /* Successful vm_find_object returns with aperture locked */ /* APU memory is not supported by this function */ - if (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible) { + if (aperture && + (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible)) { pthread_mutex_unlock(&aperture->fmm_mutex); return HSAKMT_STATUS_ERROR; } - /* For userptr, we ignore the nodes array and map all registered nodes. - * This is to simply the implementation of allowing the same memory - * region to be registered multiple times. - */ - if (object->userptr) { - retcode = _fmm_map_to_gpu_userptr(address, size, - gpuvm_address, object); - pthread_mutex_unlock(&aperture->fmm_mutex); + if ((svm.is_svm_api_supported && !object) || object->userptr) { + retcode = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, + object, nodes_to_map, num_of_nodes * sizeof(uint32_t)); + if (object) + pthread_mutex_unlock(&aperture->fmm_mutex); return retcode ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; } diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index df394a43e1..3e0612aa05 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -468,7 +468,6 @@ TEST_F(KFDMemoryTest, MemoryRegisterSamePtr) { EXPECT_SUCCESS(hsaKmtRegisterMemory((void *)&mem[0], sizeof(HSAuint32))); EXPECT_SUCCESS(hsaKmtMapMemoryToGPU((void *)&mem[0], sizeof(HSAuint32), &gpuva2)); - EXPECT_TRUE(gpuva1 != gpuva2); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva1))); EXPECT_SUCCESS(hsaKmtDeregisterMemory(reinterpret_cast(gpuva1))); EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(reinterpret_cast(gpuva2))); @@ -1138,7 +1137,12 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { static volatile HSAuint32 mem[4]; // 8 bytes for register only and // 8 bytes for register to nodes HsaMemoryBuffer hsaBuffer((void *)(&mem[0]), sizeof(HSAuint32)*2); - if (is_dgpu()) { // APU doesn't use userptr + /* + * APU doesn't use userptr. + * User pointers registered with SVM API, does not create vm_object_t. + * Therefore, pointer info can not be queried. + */ + if (is_dgpu() && mem != hsaBuffer.As()) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[0]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); @@ -1165,7 +1169,7 @@ TEST_F(KFDMemoryTest, QueryPointerInfo) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo(reinterpret_cast(address), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_ALLOCATED); EXPECT_EQ(ptrInfo.CPUAddress, hostBuffer.As()); - if (is_dgpu()) { + if (is_dgpu() && &mem[1] != hsaBuffer.As() + 1) { EXPECT_SUCCESS(hsaKmtQueryPointerInfo((void *)(&mem[1]), &ptrInfo)); EXPECT_EQ(ptrInfo.Type, HSA_POINTER_REGISTERED_USER); EXPECT_EQ(ptrInfo.CPUAddress, &mem[0]); From 2a1d6ee8b5bc1043b5a03dc63af5297906c2aedc Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 29 Sep 2022 17:37:39 -0500 Subject: [PATCH 1105/1247] libhsakmt: query svm info from userptrs at fault events Get more debug information about user pointers that were registered through SVM API, and triggered by memory exception events. A new kfdtest with this use case was also included inside KFDExceptionTest. Signed-off-by: Alex Sierra Change-Id: I0ef4929afe0625b9b5cbbbebef11ede66dda60ab --- src/events.c | 79 +++++++++++++++++++++++++- tests/kfdtest/src/KFDExceptionTest.cpp | 48 ++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/events.c b/src/events.c index 06d3959532..1c5b03dcf2 100644 --- a/src/events.c +++ b/src/events.c @@ -220,6 +220,80 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event, return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); } +static HSAKMT_STATUS get_mem_info_svm_api(uint64_t address, uint32_t gpu_id) +{ + struct kfd_ioctl_svm_args *args; + uint32_t node_id; + HSAuint32 s_attr; + HSAuint32 i; + HSA_SVM_ATTRIBUTE attrs[] = { + {HSA_SVM_ATTR_PREFERRED_LOC, 0}, + {HSA_SVM_ATTR_PREFETCH_LOC, 0}, + {HSA_SVM_ATTR_ACCESS, gpu_id}, + {HSA_SVM_ATTR_SET_FLAGS, 0}, + }; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(5); + + s_attr = sizeof(attrs); + args = alloca(sizeof(*args) + s_attr); + args->start_addr = address; + args->size = PAGE_SIZE; + args->op = KFD_IOCTL_SVM_OP_GET_ATTR; + args->nattr = s_attr / sizeof(*attrs); + memcpy(args->attrs, attrs, s_attr); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_SVM + (s_attr << _IOC_SIZESHIFT), args)) { + pr_debug("op get range attrs failed %s\n", strerror(errno)); + return HSAKMT_STATUS_ERROR; + } + + pr_err("GPU address 0x%lx, is Unified memory\n", address); + for (i = 0; i < args->nattr; i++) { + if (args->attrs[i].value == KFD_IOCTL_SVM_LOCATION_SYSMEM || + args->attrs[i].value == KFD_IOCTL_SVM_LOCATION_UNDEFINED) + node_id = args->attrs[i].value; + else + gpuid_to_nodeid(args->attrs[i].value, &node_id); + switch (args->attrs[i].type) { + case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: + pr_err("Preferred location for address 0x%lx is Node id %d\n", + address, node_id); + break; + case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: + pr_err("Prefetch location for address 0x%lx is Node id %d\n", + address, node_id); + break; + case KFD_IOCTL_SVM_ATTR_ACCESS: + pr_err("Node id %d has access to address 0x%lx\n", + node_id, address); + break; + case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE: + pr_err("Node id %d has access in place to address 0x%lx\n", + node_id, address); + break; + case KFD_IOCTL_SVM_ATTR_NO_ACCESS: + pr_err("Node id %d has no access to address 0x%lx\n", + node_id, address); + break; + case KFD_IOCTL_SVM_ATTR_SET_FLAGS: + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_COHERENT) + pr_err("Fine grained coherency between devices\n"); + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_RO) + pr_err("Read only\n"); + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_EXEC) + pr_err("GPU exec allowed\n"); + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) + pr_err("GPU always mapped\n"); + break; + default: + pr_debug("get invalid attr type 0x%x\n", args->attrs[i].type); + return HSAKMT_STATUS_ERROR; + } + } + + return HSAKMT_STATUS_SUCCESS; +} //Analysis memory exception data, print debug messages static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * memory_exception_data) @@ -241,9 +315,10 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * pr_err("Execute to none-executable page\n"); ret = fmm_get_mem_info((const void *)addr, &info); - if (ret != HSAKMT_STATUS_SUCCESS) { - pr_err("Address does not belong to a known buffer\n"); + ret = get_mem_info_svm_api(addr, memory_exception_data->gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) + pr_err("Address does not belong to a known buffer\n"); return; } diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/tests/kfdtest/src/KFDExceptionTest.cpp index f7c28d8897..2ca5c6777d 100644 --- a/tests/kfdtest/src/KFDExceptionTest.cpp +++ b/tests/kfdtest/src/KFDExceptionTest.cpp @@ -27,6 +27,7 @@ #include "SDMAPacket.hpp" #include "SDMAQueue.hpp" #include "Dispatch.hpp" +#include void KFDExceptionTest::SetUp() { ROUTINE_START @@ -253,6 +254,53 @@ TEST_F(KFDExceptionTest, PermissionFault) { TEST_END } +/* Allocate Read Only user pointer buffer. Test Memory Exception failure by + * attempting to write to that buffer in the child process. + */ +TEST_F(KFDExceptionTest, PermissionFaultUserPointer) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL) + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (m_FamilyId == FAMILY_RV) { + LOG() << "Skipping test: IOMMU issues on Raven." << std::endl; + return; + } + + m_ChildPid = fork(); + if (m_ChildPid == 0) { + KFDBaseComponentTest::TearDown(); + KFDBaseComponentTest::SetUp(); + + void *pBuf = mmap(NULL, PAGE_SIZE, PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + ASSERT_NE(pBuf, MAP_FAILED); + EXPECT_SUCCESS(hsaKmtRegisterMemory(pBuf, PAGE_SIZE)); + EXPECT_SUCCESS(hsaKmtMapMemoryToGPU(pBuf, PAGE_SIZE, NULL)); + HsaMemoryBuffer srcSysBuffer(PAGE_SIZE, defaultGPUNode, false); + + srcSysBuffer.Fill(0xAA55AA55); + + TestMemoryException(defaultGPUNode, srcSysBuffer.As(), + (HSAuint64)pBuf); + } else { + int childStatus; + + waitpid(m_ChildPid, &childStatus, 0); + if (is_dgpu()) { + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } else { + EXPECT_EQ(WIFSIGNALED(childStatus), true); + EXPECT_EQ(WTERMSIG(childStatus), SIGSEGV); + } + } + + TEST_END +} + /* Test VM fault storm handling by copying to/from invalid pointers * with lots of work items at the same time */ From 5d80a4d214a4bcd29ca3ade098bcfe534ab4237d Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 23 Mar 2023 11:14:58 -0400 Subject: [PATCH 1106/1247] kfdtest: Add KFDQMTest.BasicCuMaskingEven to GFX11 blacklist Test is inconsistent across ASICs. Add to blacklist to unblock QA. Signed-off-by: Graham Sider Change-Id: I31e5aa2450165227107536bef8402db2c0dc6d7f --- tests/kfdtest/scripts/kfdtest.exclude | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 268a3f4dec..b2cdd46ad7 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -120,6 +120,7 @@ BLACKLIST_GFX10_NV2X=\ # KFDQMTest.MultipleCpQueuesStressDispatch - SWDEV-340965 BLACKLIST_GFX11=\ "KFDQMTest.CreateAqlCpQueue:"\ +"KFDQMTest.BasicCuMaskingEven:"\ "KFDQMTest.MultipleCpQueuesStressDispatch:"\ "KFDCWSRTest.InterruptRestore:"\ "KFDPerfCountersTest.*:"\ From 0750856d4af44aed944253b5f354d1de47dd4e2f Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 27 Mar 2023 16:18:25 -0400 Subject: [PATCH 1107/1247] =?UTF-8?q?Revert=20"kfdtest:=20Using=20non-page?= =?UTF-8?q?d=20memory=20allocation=20only=20on=20devices=20that=C2=A0have?= =?UTF-8?q?=20MES=20scheduler"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch and the previous made it such that the queue ring buffer was allocated as non-paged for GFX11+. The queue ring buffer should not be mapped as non-paged; the non-paged requirement on GFX11 is only needed for the queue wptr. This patch was causing issues on various tests, such as intermittent CP_INTSRC_BAD_OPCODE interrupts. This reverts commit e40ae8481e144c4825957310734b6d11b40f2b75. Signed-off-by: Graham Sider Change-Id: I55b64aed73dc3b792f0756ae00daf6e10d93ce10 --- tests/kfdtest/src/BaseQueue.cpp | 2 +- tests/kfdtest/src/KFDQMTest.cpp | 2 +- tests/kfdtest/src/KFDTestUtil.cpp | 4 ++-- tests/kfdtest/src/KFDTestUtil.hpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index 0809c1f97a..dd1620168c 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -49,7 +49,7 @@ HSAKMT_STATUS BaseQueue::Create(unsigned int NodeId, unsigned int size, HSAuint6 memset(&m_Resources, 0, sizeof(m_Resources)); m_QueueBuf = new HsaMemoryBuffer(size, NodeId, true/*zero*/, false/*local*/, true/*exec*/, - /*isScratch */ false, /* isReadOnly */false, /* isUncached */true, /* NonPaged */g_baseTest->isMES(NodeId)); + /*isScratch */ false, /* isReadOnly */false, /* isUncached */true); if (type == HSA_QUEUE_COMPUTE_AQL) { m_Resources.Queue_read_ptr_aql = &pointers[0]; diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index b3567f7847..2f156ad23b 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1174,7 +1174,7 @@ TEST_F(KFDQMTest, CreateAqlCpQueue) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HsaMemoryBuffer pointers(PAGE_SIZE, defaultGPUNode, /*zero*/true, /*local*/false, /*exec*/false, /*isScratch */false, /* isReadOnly */false, /* isUncached */false, /* NonPaged */g_baseTest->isMES(defaultGPUNode)); + HsaMemoryBuffer pointers(PAGE_SIZE, defaultGPUNode); ASSERT_SUCCESS(queue.Create(defaultGPUNode, PAGE_SIZE, pointers.As())); diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index cbc1d4104a..afd5e71d99 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -258,7 +258,7 @@ HSAuint64 GetSystemTickCountInMicroSec() { const HsaMemoryBuffer HsaMemoryBuffer::Null; HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, - bool isScratch, bool isReadOnly, bool isUncached, bool NonPaged) + bool isScratch, bool isReadOnly, bool isUncached) :m_Size(size), m_pUser(NULL), m_pBuf(NULL), @@ -282,7 +282,7 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b EXPECT_EQ(isUncached, 0) << "Uncached flag is relevant only for system or host memory"; } else { m_Flags.ui32.HostAccess = 1; - m_Flags.ui32.NonPaged = NonPaged ? 1 : 0; + m_Flags.ui32.NonPaged = 0; m_Flags.ui32.CoarseGrain = 0; m_Flags.ui32.NoNUMABind = 1; m_Flags.ui32.Uncached = isUncached; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index c8d4de4051..a9f6c9d9be 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -69,7 +69,7 @@ class HsaMemoryBuffer { public: HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero = true, bool isLocal = false, - bool isExec = false, bool isScratch = false, bool isReadOnly = false, bool isUncached = false, bool NonPaged = false); + bool isExec = false, bool isScratch = false, bool isReadOnly = false, bool isUncached = false); HsaMemoryBuffer(void *addr, HSAuint64 size); template RetType As() { From 287cb29340c3c343c719fb171e6127fdee8fee71 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 27 Mar 2023 16:18:18 -0400 Subject: [PATCH 1108/1247] Revert "kfdtest: add MES judging API in test utility." See description of previous revert. This reverts commit 564913526a17318e9ce2c4077c54be76b1c1503f. Signed-off-by: Graham Sider Change-Id: I969dc6469e62b50cd7ba0595918538602afa7516 --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 5 ----- tests/kfdtest/src/KFDBaseComponentTest.hpp | 1 - tests/kfdtest/src/KFDTestUtil.cpp | 5 ----- tests/kfdtest/src/KFDTestUtil.hpp | 1 - 4 files changed, 12 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index a286f21983..bb510e040b 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -164,11 +164,6 @@ unsigned int KFDBaseComponentTest::GetFamilyIdFromNodeId(unsigned int nodeId) return FamilyIdFromNode(m_NodeInfo.GetNodeProperties(nodeId)); } -bool KFDBaseComponentTest::isMES(unsigned int nodeId) -{ - return CheckMES(m_NodeInfo.GetNodeProperties(nodeId)); -} - int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) { HsaNodeProperties *nodeProperties; _HSAKMT_STATUS status; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 2f9d8350e2..cc87465ad5 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -58,7 +58,6 @@ class KFDBaseComponentTest : public testing::Test { // @return DRM Render Node if successful or -1 on failure int FindDRMRenderNode(int gpuNode); unsigned int GetFamilyIdFromNodeId(unsigned int nodeId); - bool isMES(unsigned int nodeId); unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; } // @brief Executed before the first test that uses KFDBaseComponentTest. diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index afd5e71d99..c45af06aed 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -238,11 +238,6 @@ bool isTonga(const HsaNodeProperties *props) { return false; } -bool CheckMES(const HsaNodeProperties *props) { - /* MES devices need GART mappings and therefore need non-paged memory */ - return (props->EngineId.ui32.Major >= 11) ? true : false; -} - const uint32_t GetGfxVersion(const HsaNodeProperties *props) { return ((props->EngineId.ui32.Major << 16) | (props->EngineId.ui32.Minor << 8) | diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index a9f6c9d9be..938ff8bf69 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -50,7 +50,6 @@ HSAKMT_STATUS CreateQueueTypeEvent(bool ManualReset, bool IsSignaled, unsigned i bool is_dgpu(); bool isTonga(const HsaNodeProperties *props); -bool CheckMES(const HsaNodeProperties *props); bool hasPciAtomicsSupport(int node); unsigned int FamilyIdFromNode(const HsaNodeProperties *props); const uint32_t GetGfxVersion(const HsaNodeProperties *props); From aab0e36538c4e5128db6e65c281938ad36ce1db2 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 24 Feb 2023 17:24:16 -0500 Subject: [PATCH 1109/1247] README: Update README to point to current documentation Signed-off-by: Kent Russell Change-Id: I3fed80e94edf5ff08a70b2e43450fe8168c5d355 --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b274bb7202..a1b2db2edb 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,8 @@ NOTE: Users of Ubuntu 20.04 will need to add the user to the new "render" group, #### ROCk Driver -The ROCt library is not a standalone product and requires that you have the correct ROCk driver set installed. We recommend reading the full compatibility and installation details which are available in the ROCk github: - -https://github.com/RadeonOpenCompute/ROCK-Radeon-Open-Compute-Kernel-Driver +The ROCt library is not a standalone product and requires that you have the correct ROCk driver installed, or are using a compatible upstream kernel. +Please refer to https://docs.amd.com/ under "Getting Started Guide" for a list of supported Operating Systems and kernel versions, as well as supported hardware. #### Building the Thunk @@ -43,4 +42,4 @@ The information contained herein is for informational purposes only, and is subj AMD, the AMD Arrow logo, and combinations thereof are trademarks of Advanced Micro Devices, Inc. Other product names used in this publication are for identification purposes only and may be trademarks of their respective companies. -Copyright (c) 2014-2020 Advanced Micro Devices, Inc. All rights reserved. +Copyright (c) 2014-2023 Advanced Micro Devices, Inc. All rights reserved. From d0c2770cdecbd5c08a0e9888fe2bc805c75722b3 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Wed, 5 Apr 2023 13:49:01 -0400 Subject: [PATCH 1110/1247] CMakeLists: Use pkgconfig more effectively with DRM_DIR Instead of hard-coding lib64 and other include locations, just prepend the DRM_DIR to the beginning of the CMake prefix path. Then let pkgconfig find the package, the same way that it would if DRM_DIR wasn't set. DRM_DIR takes precedence, but the default paths will be used if DRM_DIR isn't set, or doesn't point to where libdrm is housed Note that /lib and /lib/$ARCH aren't required for DRM_DIR, just the path to the root folder for the package (e.g. /opt/amdgpu instead of /opt/amdgpu/lib or /opt/amdgpu/lib64 or /opt/amdgpu/lib/x86_64-linux-gnu etc) Change-Id: I56767db28476d14e3fa77be1089c3904e2a32450 --- CMakeLists.txt | 17 +++++------------ tests/kfdtest/CMakeLists.txt | 16 ++++------------ 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 95b90a045d..e7a9a361be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,20 +166,13 @@ message(STATUS "NUMA:" ${NUMA}) ## If environment variable DRM_DIR is set, the script ## will pick up the corresponding libraries from that path. -if( DEFINED ENV{DRM_DIR} ) -#assume header files and libraries are under the same path - set ( DRM_DIR $ENV{DRM_DIR} ) - set ( DRM_INCLUDE_DIRS ${DRM_DIR}/include ) - link_directories(${DRM_DIR}/lib64) - set ( DRM_LIBRARIES drm ) - set ( DRM_AMDGPU_LIBRARIES drm_amdgpu ) -else() +list (PREPEND CMAKE_PREFIX_PATH "${DRM_DIR}") + # The module name passed to pkg_check_modules() is determined by the # name of file *.pc - pkg_check_modules(DRM REQUIRED libdrm) - pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) - include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) -endif() +pkg_check_modules(DRM REQUIRED libdrm) +pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) +include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) include_directories(${DRM_INCLUDE_DIRS}) target_link_libraries ( ${HSAKMT_TARGET} diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index ba1d99be51..61c9b84f60 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -92,20 +92,12 @@ set (CPACK_RPM_PACKAGE_REQUIRES "rocm-core") find_package(PkgConfig) -if( DEFINED ENV{DRM_DIR} ) -#assume header files and libraries are under the same path - set ( DRM_DIR $ENV{DRM_DIR} ) - set ( DRM_INCLUDE_DIRS ${DRM_DIR}/include ) - link_directories(${DRM_DIR}/lib64) - set ( DRM_LIBRARIES drm ) - set ( DRM_AMDGPU_LIBRARIES drm_amdgpu ) -else() +list (PREPEND CMAKE_PREFIX_PATH "${DRM_DIR}") # The module name passed to pkg_check_modules() is determined by the # name of file *.pc - pkg_check_modules(DRM REQUIRED libdrm) - pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) - include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) -endif() +pkg_check_modules(DRM REQUIRED libdrm) +pkg_check_modules(DRM_AMDGPU REQUIRED libdrm_amdgpu) +include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) if( DEFINED ENV{LIBHSAKMT_PATH} ) set ( LIBHSAKMT_PATH $ENV{LIBHSAKMT_PATH} ) From 6be4461a0df9ef46cca73361e40a83549fb36d7f Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Fri, 6 May 2022 15:03:41 -0400 Subject: [PATCH 1111/1247] libhsakmt: Add Aqua Vanjaram support Add gfx version for VGPR size per CU calc, add FAMILY_AV to KfdFamilyId, add blacklist filter to kfdtest.exclude. Signed-off-by: Graham Sider Change-Id: I9b8072e45f4d497e0a8fd3f8f97f1425238e8b42 --- src/libhsakmt.h | 1 + src/queues.c | 1 + tests/kfdtest/scripts/kfdtest.exclude | 7 +++++++ tests/kfdtest/src/KFDTestFlags.hpp | 1 + tests/kfdtest/src/KFDTestUtil.cpp | 4 +++- 5 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 55fc5a465d..85e69f6d07 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -136,6 +136,7 @@ enum full_gfx_versions { GFX_VERSION_VEGA20 = 0x090006, GFX_VERSION_ARCTURUS = 0x090008, GFX_VERSION_ALDEBARAN = 0x09000A, + GFX_VERSION_AQUA_VANJARAM = 0x090400, GFX_VERSION_RENOIR = 0x09000C, GFX_VERSION_NAVI10 = 0x0A0100, GFX_VERSION_NAVI12 = 0x0A0101, diff --git a/src/queues.c b/src/queues.c index f59c3da7bf..6ba1e107c6 100644 --- a/src/queues.c +++ b/src/queues.c @@ -95,6 +95,7 @@ uint32_t get_vgpr_size_per_cu(uint32_t gfxv) { case GFX_VERSION_ARCTURUS: case GFX_VERSION_ALDEBARAN: + case GFX_VERSION_AQUA_VANJARAM: vgpr_size = 0x80000; break; case GFX_VERSION_PLUM_BONITO: diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index b2cdd46ad7..672c27ec3a 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -244,6 +244,13 @@ FILTER[aldebaran]=\ "KFDMemoryTest.PtraceAccess:"\ "KFDMemoryTest.DeviceHdpFlush" +FILTER[aqua_vanjaram]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDExceptionTest.FaultStorm:"\ +"KFDEvictTest.BurstyTest:"\ +"KFDMemoryTest.PtraceAccess:"\ +"KFDMemoryTest.DeviceHdpFlush" + FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10:"\ diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index 7fd49c7013..00a74fd6c8 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -60,6 +60,7 @@ enum KfdFamilyId { FAMILY_RV, // Raven FAMILY_AR, // Arcturus FAMILY_AL, // Aldebaran + FAMILY_AV, // Aqua Vanjaram FAMILY_NV, // Navi10 FAMILY_GFX11, // GFX11 }; diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index c45af06aed..b9d9d23d4a 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -188,7 +188,9 @@ unsigned int FamilyIdFromNode(const HsaNodeProperties *props) { break; case 9: familyId = FAMILY_AI; - if (props->EngineId.ui32.Stepping == 2) + if (props->EngineId.ui32.Minor == 4) + familyId = FAMILY_AV; + else if (props->EngineId.ui32.Stepping == 2) familyId = FAMILY_RV; else if (props->EngineId.ui32.Stepping == 8) familyId = FAMILY_AR; From 97a669a979d94cf5023f5703a900cc1fb906fb64 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Mon, 31 Jan 2022 15:57:57 -0500 Subject: [PATCH 1112/1247] libhsakmt: Update context save handling for multi XCC Allocate debug area big enough for all XCCs in the partition. Also, fix the cu_num calculations as driver now reports cu_num as the total number of CUs in the partition. Signed-off-by: Mukul Joshi Change-Id: I6e80d57196b770bb3c2506bc58cb366c0046084b --- include/hsakmttypes.h | 2 +- src/queues.c | 43 ++++++++++++++++++++++++------------------- src/topology.c | 8 ++++++++ 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index b0e066d605..e584fd75a9 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -329,10 +329,10 @@ typedef struct _HsaNodeProperties HSAuint32 VGPRSizePerCU; // VGPR size in bytes per CU HSAuint32 SGPRSizePerCU; // SGPR size in bytes per CU + HSAuint32 NumXcc; // Number of XCC HSAuint32 KFDGpuID; // GPU Hash ID generated by KFD HSAuint32 FamilyID; // GPU family id - HSAuint8 Reserved[4]; } HsaNodeProperties; diff --git a/src/queues.c b/src/queues.c index 6ba1e107c6..357e92a90b 100644 --- a/src/queues.c +++ b/src/queues.c @@ -293,7 +293,7 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) return false; if (node.NumFComputeCores && node.NumSIMDPerCU) { uint32_t ctl_stack_size, wg_data_size; - uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU; + uint32_t cu_num = node.NumFComputeCores / node.NumSIMDPerCU / node.NumXcc; uint32_t wave_num = (q->gfxv < GFX_VERSION_NAVI10) ? MIN(cu_num * 40, node.NumShaderBanks / node.NumArrays * 512) : cu_num * 32; @@ -459,17 +459,21 @@ static void free_queue(struct queue *q) } static inline void fill_cwsr_header(struct queue *q, void *addr, - HsaEvent *Event, volatile HSAint64 *ErrPayload) + HsaEvent *Event, volatile HSAint64 *ErrPayload, HSAuint32 NumXcc) { - HsaUserContextSaveAreaHeader *header = - (HsaUserContextSaveAreaHeader *)addr; + uint32_t i; + HsaUserContextSaveAreaHeader *header; - header->ErrorEventId = 0; - if (Event) - header->ErrorEventId = Event->EventId; - header->ErrorReason = ErrPayload; - header->DebugOffset = q->ctx_save_restore_size; - header->DebugSize = q->debug_memory_size; + for (i = 0; i < NumXcc; i++) { + header = (HsaUserContextSaveAreaHeader *) + ((uintptr_t)addr + (i * q->ctx_save_restore_size)); + header->ErrorEventId = 0; + if (Event) + header->ErrorEventId = Event->EventId; + header->ErrorReason = ErrPayload; + header->DebugOffset = (NumXcc - i) * q->ctx_save_restore_size; + header->DebugSize = q->debug_memory_size * NumXcc; + } } static int handle_concrete_asic(struct queue *q, @@ -502,19 +506,20 @@ static int handle_concrete_asic(struct queue *q, HsaNodeProperties node; bool svm_api; + if (hsaKmtGetNodeProperties(NodeId, &node)) + return HSAKMT_STATUS_ERROR; + args->ctx_save_restore_size = q->ctx_save_restore_size; args->ctl_stack_size = q->ctl_stack_size; /* Total memory to be allocated is = - * (Control Stack size + WG size) + Debug memory area size + * (Control Stack size + WG size) per XCC * num_xcc + + * Debug memory area size */ - total_mem_alloc_size = q->ctx_save_restore_size + - q->debug_memory_size; + total_mem_alloc_size = (q->ctx_save_restore_size + + q->debug_memory_size) * node.NumXcc; - if (hsaKmtGetNodeProperties(NodeId, &node)) - svm_api = false; - else - svm_api = node.Capability.ui32.SVMAPISupported; + svm_api = node.Capability.ui32.SVMAPISupported; /* Allocate unified memory for context save restore * area on dGPU. @@ -539,7 +544,7 @@ static int handle_concrete_asic(struct queue *q, if (madvise(addr, size, MADV_DONTFORK)) pr_err("madvise failed -%d\n", errno); - fill_cwsr_header(q, addr, Event, ErrPayload); + fill_cwsr_header(q, addr, Event, ErrPayload, node.NumXcc); r = register_svm_range(addr, size, NodeId, NodeId, 0, true); @@ -562,7 +567,7 @@ static int handle_concrete_asic(struct queue *q, if (!q->ctx_save_restore) return HSAKMT_STATUS_NO_MEMORY; - fill_cwsr_header(q, q->ctx_save_restore, Event, ErrPayload); + fill_cwsr_header(q, q->ctx_save_restore, Event, ErrPayload, node.NumXcc); } args->ctx_save_restore_address = (uintptr_t)q->ctx_save_restore; diff --git a/src/topology.c b/src/topology.c index f2b2075e3e..cc1231808b 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1186,6 +1186,8 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, props->NumSdmaQueuesPerEngine = prop_val; else if (strcmp(prop_name, "num_cp_queues") == 0) props->NumCpQueues = prop_val; + else if (strcmp(prop_name, "num_xcc") == 0) + props->NumXcc = prop_val; else if (strcmp(prop_name, "gfx_target_version") == 0) gfxv = (uint32_t)prop_val; } @@ -1255,6 +1257,12 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if (props->NumFComputeCores) assert(props->EngineId.ui32.Major && "HSA_OVERRIDE_GFX_VERSION may be needed"); + /* On Older kernels, num_xcc may not be present in system properties. + * Set it to 1 if system properties do not report num_xcc. + */ + if (!props->NumXcc) + props->NumXcc = 1; + err: free(read_buf); fclose(fd); From fd48f14ceb6d5a2fa0b175768988394e21d2b443 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 31 Jan 2022 15:32:12 -0500 Subject: [PATCH 1113/1247] libhsakmt: Update FD creation logic In multi-partition modes, e.g. CPX, we want to create new file descriptor despite using the same render node. Update open_drm_render_device to use a gpu_id to fd map partitioned by render node. Different gpu_id's requesting the same render node will be added to that render node's map list for fetching its fd. Different gpu_id's requesting different render nodes as well as the same gpu_id's requesting the same render node will behave as they did previously. Signed-off-by: Graham Sider Change-Id: Ie153d42355d4d75b1c6ba6ff40fac3295bc87009 --- src/fmm.c | 38 +++++++++++++++++++++++++------------- src/fmm.h | 2 +- src/topology.c | 2 +- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 436af68223..ff02fa9394 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1906,23 +1906,32 @@ static HSAKMT_STATUS get_process_apertures( */ #define DRM_FIRST_RENDER_NODE 128 #define DRM_LAST_RENDER_NODE 255 -static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE]; +#define DRM_MAX_PARTITIONS 8 +static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE][DRM_MAX_PARTITIONS]; +static uint32_t gpu_id_fd_map[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE][DRM_MAX_PARTITIONS]; -int open_drm_render_device(int minor) +int open_drm_render_device(int minor, uint32_t gpu_id) { char path[128]; - int index, fd; + int fd, render_idx, gpu_id_idx; if (minor < DRM_FIRST_RENDER_NODE || minor > DRM_LAST_RENDER_NODE) { pr_err("DRM render minor %d out of range [%d, %d]\n", minor, DRM_FIRST_RENDER_NODE, DRM_LAST_RENDER_NODE); return -EINVAL; } - index = minor - DRM_FIRST_RENDER_NODE; + render_idx = minor - DRM_FIRST_RENDER_NODE; /* If the render node was already opened, keep using the same FD */ - if (drm_render_fds[index]) - return drm_render_fds[index]; + for (gpu_id_idx = 0; gpu_id_idx < DRM_MAX_PARTITIONS && + gpu_id_fd_map[render_idx][gpu_id_idx] != 0; gpu_id_idx++) + if (gpu_id_fd_map[render_idx][gpu_id_idx] == gpu_id) + return drm_render_fds[render_idx][gpu_id_idx]; + + if (gpu_id_idx >= DRM_MAX_PARTITIONS) { + pr_err("Requesting more FDs for same render node than max supported partitions!\n"); + return -EINVAL; + } sprintf(path, "/dev/dri/renderD%d", minor); fd = open(path, O_RDWR | O_CLOEXEC); @@ -1934,7 +1943,8 @@ int open_drm_render_device(int minor) } return -errno; } - drm_render_fds[index] = fd; + gpu_id_fd_map[render_idx][gpu_id_idx] = gpu_id; + drm_render_fds[render_idx][gpu_id_idx] = fd; return fd; } @@ -2325,7 +2335,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) /* Skip non-GPU nodes */ if (props.KFDGpuID) { - int fd = open_drm_render_device(props.DrmRenderMinor); + int fd = open_drm_render_device(props.DrmRenderMinor, props.KFDGpuID); if (fd <= 0) { ret = HSAKMT_STATUS_ERROR; goto gpu_mem_init_failed; @@ -3867,15 +3877,17 @@ static void fmm_clear_aperture(manageable_aperture_t *app) */ void fmm_clear_all_mem(void) { - uint32_t i; + uint32_t i, j; void *map_addr; /* Close render node FDs. The child process needs to open new ones */ - for (i = 0; i <= DRM_LAST_RENDER_NODE - DRM_FIRST_RENDER_NODE; i++) - if (drm_render_fds[i]) { - close(drm_render_fds[i]); - drm_render_fds[i] = 0; + for (i = 0; i <= DRM_LAST_RENDER_NODE - DRM_FIRST_RENDER_NODE; i++) { + for (j = 0; j < DRM_MAX_PARTITIONS && drm_render_fds[i][j] != 0; j++) { + close(drm_render_fds[i][j]); + drm_render_fds[i][j] = 0; + gpu_id_fd_map[i][j] = 0; } + } fmm_clear_aperture(&cpuvm_aperture); fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); diff --git a/src/fmm.h b/src/fmm.h index d0146d74cd..93aa55fb76 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -93,7 +93,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address); -int open_drm_render_device(int minor); +int open_drm_render_device(int minor, uint32_t gpu_id); void *mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align, uint64_t guard_size, void *aper_base, void *aper_limit); diff --git a/src/topology.c b/src/topology.c index cc1231808b..ba5802da86 100644 --- a/src/topology.c +++ b/src/topology.c @@ -694,7 +694,7 @@ static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id, } /* Open DRM Render device */ - ret_value = open_drm_render_device(drm_render_minor); + ret_value = open_drm_render_device(drm_render_minor, gpu_id); if (ret_value > 0) *is_node_supported = true; else if (ret_value != -ENOENT && ret_value != -EPERM) From 8994c3ba0e66ee07760e0daaf0f61cd911fd89d9 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Thu, 4 Aug 2022 17:44:09 -0400 Subject: [PATCH 1114/1247] kfdtest: Program COMPUTE_PGM_RSRC3 for GFX 9.4.3 Program ACCUM_OFFSET to match the number of VGPRS used by the shader as part of Dispatch setup. Signed-off-by: Mukul Joshi Change-Id: Icfa1fbe4de2a62f00743de567f3ed382d3378b17 --- tests/kfdtest/src/Dispatch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index 87eb659c7f..ce3a8f4626 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -221,7 +221,7 @@ void Dispatch::BuildIb() { m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC1, COMPUTE_PGM_RSRC, ARRAY_SIZE(COMPUTE_PGM_RSRC))); - if (m_FamilyId == FAMILY_AL) { + if (m_FamilyId == FAMILY_AL || m_FamilyId == FAMILY_AV) { const unsigned int COMPUTE_PGM_RSRC3[] = {9}; m_IndirectBuf.AddPacket(PM4SetShaderRegPacket(mmCOMPUTE_PGM_RSRC3, COMPUTE_PGM_RSRC3, ARRAY_SIZE(COMPUTE_PGM_RSRC3))); From 252a2cf9598768ee55c42795c994579ed77e3123 Mon Sep 17 00:00:00 2001 From: David Belanger Date: Wed, 13 Apr 2022 10:55:26 -0400 Subject: [PATCH 1115/1247] libhsakmt: EOP Removal Do not allocate the EOP buffer when not required. Signed-off-by: David Belanger Change-Id: I1664a3f0a882219a72278174006cdb8d46fd4f5e --- src/queues.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/queues.c b/src/queues.c index 357e92a90b..715ad853bc 100644 --- a/src/queues.c +++ b/src/queues.c @@ -40,10 +40,6 @@ #define DOORBELL_SIZE(gfxv) (((gfxv) >= 0x90000) ? 8 : 4) #define DOORBELLS_PAGE_SIZE(ds) (1024 * (ds)) -#define EOP_BUFFER_SIZE(gfxv) \ - (((gfxv) == GFX_VERSION_TONGA) ? TONGA_PAGE_SIZE : \ - (((gfxv) >= 0x80000) ? 4096 : 0)) - #define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv) \ (get_vgpr_size_per_cu(gfxv) + SGPR_SIZE_PER_CU + \ LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU) @@ -618,7 +614,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, q->gfxv = get_gfxv_by_node_id(NodeId); q->use_ats = false; - q->eop_buffer_size = EOP_BUFFER_SIZE(q->gfxv); + + if (q->gfxv == GFX_VERSION_TONGA) + q->eop_buffer_size = TONGA_PAGE_SIZE; + else if (q->gfxv == GFX_VERSION_AQUA_VANJARAM) + q->eop_buffer_size = ((Type == HSA_QUEUE_COMPUTE) ? 4096 : 0); + else if (q->gfxv >= 0x80000) + q->eop_buffer_size = 4096; /* By default, CUs are all turned on. Initialize cu_mask to '1 * for all CU bits. From a713fb766ef0c55f51b7a229b5d5e4cce45c7500 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Thu, 1 Sep 2022 16:12:46 -0400 Subject: [PATCH 1116/1247] libhsakmt: Fix memory leak on queue destroy for GFX9.4.3 Currently, on queue destroy, context save restore memory is freed only for a single XCC. Instead, we need to free the entire context save restore memory, which was allocated for all XCCs. Signed-off-by: Mukul Joshi Change-Id: I51ebb12fa8d5ebed41979d68e74f7c5392dca062 --- src/queues.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/queues.c b/src/queues.c index 715ad853bc..9e29fe15f8 100644 --- a/src/queues.c +++ b/src/queues.c @@ -62,6 +62,7 @@ struct queue { uint32_t ctl_stack_size; uint32_t debug_memory_size; uint32_t eop_buffer_size; + uint32_t total_mem_alloc_size; uint32_t gfxv; bool use_ats; bool unified_ctx_save_restore; @@ -448,7 +449,7 @@ static void free_queue(struct queue *q) PAGE_ALIGN_UP(q->ctx_save_restore_size + q->debug_memory_size)); else if (q->ctx_save_restore) free_exec_aligned_memory(q->ctx_save_restore, - q->ctx_save_restore_size + q->debug_memory_size, + q->total_mem_alloc_size, PAGE_SIZE, q->use_ats); free_exec_aligned_memory((void *)q, sizeof(*q), PAGE_SIZE, q->use_ats); @@ -498,7 +499,6 @@ static int handle_concrete_asic(struct queue *q, ret = update_ctx_save_restore_size(NodeId, q); if (ret) { - uint32_t total_mem_alloc_size = 0; HsaNodeProperties node; bool svm_api; @@ -509,10 +509,10 @@ static int handle_concrete_asic(struct queue *q, args->ctl_stack_size = q->ctl_stack_size; /* Total memory to be allocated is = - * (Control Stack size + WG size) per XCC * num_xcc + - * Debug memory area size + * (Control Stack size + WG size + + * Debug memory area size) * num_xcc */ - total_mem_alloc_size = (q->ctx_save_restore_size + + q->total_mem_alloc_size = (q->ctx_save_restore_size + q->debug_memory_size) * node.NumXcc; svm_api = node.Capability.ui32.SVMAPISupported; @@ -521,7 +521,7 @@ static int handle_concrete_asic(struct queue *q, * area on dGPU. */ if (!q->use_ats && svm_api) { - uint32_t size = PAGE_ALIGN_UP(total_mem_alloc_size); + uint32_t size = PAGE_ALIGN_UP(q->total_mem_alloc_size); void *addr; HSAKMT_STATUS r = HSAKMT_STATUS_ERROR; @@ -556,7 +556,7 @@ static int handle_concrete_asic(struct queue *q, if (!q->unified_ctx_save_restore) { q->ctx_save_restore = allocate_exec_aligned_memory( - total_mem_alloc_size, + q->total_mem_alloc_size, q->use_ats, NodeId, false, false, false); From ae659e5427e9e479de03319db862c57e7d4b700f Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 12 Apr 2023 15:04:14 -0400 Subject: [PATCH 1117/1247] libhsakmt: Fix queue destroy SVM path free size Use q->total_mem_alloc_size for munmap in SVM codepath of free_queue. Signed-off-by: Graham Sider Change-Id: I2fecaa1ddb337b1fe71f9cbba45a0c9467eff0c0 --- src/queues.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/queues.c b/src/queues.c index 9e29fe15f8..0f3b71c755 100644 --- a/src/queues.c +++ b/src/queues.c @@ -445,8 +445,7 @@ static void free_queue(struct queue *q) q->eop_buffer_size, PAGE_SIZE, q->use_ats); if (q->unified_ctx_save_restore) - munmap(q->ctx_save_restore, - PAGE_ALIGN_UP(q->ctx_save_restore_size + q->debug_memory_size)); + munmap(q->ctx_save_restore, q->total_mem_alloc_size); else if (q->ctx_save_restore) free_exec_aligned_memory(q->ctx_save_restore, q->total_mem_alloc_size, From 30da9a3cf9d5de8c0c6b8eb13866c71070216ef4 Mon Sep 17 00:00:00 2001 From: David Francis Date: Wed, 5 Oct 2022 14:28:08 -0400 Subject: [PATCH 1118/1247] kfdtest: Update shaders to compile on gfx940 gfx940 changed the semantics of the glc and slc coherency options on vector stores and loads. This means that shaders that use those bits no longer compile on gfx940. Add precompilation if statements to those shaders to use the new coherency bits. Also add gfx940 to ASMTest so that compilation is tested. Note: One of the tests enabled by this patch on gfx940, KFDEvictTest.QueueTest, does not pass on gfx940 emulators. Signed-off-by: David Francis Change-Id: I942f9d2536e9eb5510c4d5af30df6ff1a95c8cf7 --- tests/kfdtest/src/KFDASMTest.cpp | 1 + tests/kfdtest/src/ShaderStore.cpp | 43 ++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/src/KFDASMTest.cpp b/tests/kfdtest/src/KFDASMTest.cpp index 4b9f5d69c8..832b2ecff6 100644 --- a/tests/kfdtest/src/KFDASMTest.cpp +++ b/tests/kfdtest/src/KFDASMTest.cpp @@ -43,6 +43,7 @@ static const std::vector TargetList = { 0x090009, 0x09000a, 0x09000c, + 0x090400, 0x0a0100, 0x0a0101, 0x0a0102, diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index c676502a85..998282c142 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -95,9 +95,15 @@ const char *CopyDwordIsa = R"( v_mov_b32 v1, s1 v_mov_b32 v2, s2 v_mov_b32 v3, s3 - flat_load_dword v4, v[0:1] glc slc - s_waitcnt 0 - flat_store_dword v[2:3], v4 glc slc + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + flat_load_dword v4, v[0:1] nt sc1 sc0 + s_waitcnt 0 + flat_store_dword v[2:3], v4 nt sc1 sc0 + .else + flat_load_dword v4, v[0:1] glc slc + s_waitcnt 0 + flat_store_dword v[2:3], v4 glc slc + .endif s_endpgm )"; @@ -112,7 +118,10 @@ const char *AtomicIncIsa = R"( .text v_mov_b32 v0, s0 v_mov_b32 v1, s1 - .if (.amdgcn.gfx_generation_number >= 8) + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + v_mov_b32 v2, 1 + flat_atomic_add v3, v[0:1], v2 nt sc1 sc0 + .elseif (.amdgcn.gfx_generation_number >= 8) v_mov_b32 v2, 1 flat_atomic_add v3, v[0:1], v2 glc slc .else @@ -153,9 +162,15 @@ const char *ScratchCopyDwordIsa = R"( s_mov_b32 flat_scratch_hi, 0 .endif // Copy a dword between the passed addresses - flat_load_dword v4, v[0:1] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) - flat_store_dword v[2:3], v4 slc + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + flat_load_dword v4, v[0:1] nt sc1 sc0 + s_waitcnt vmcnt(0) & lgkmcnt(0) + flat_store_dword v[2:3], v4 nt sc1 sc0 + .else + flat_load_dword v4, v[0:1] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + flat_store_dword v[2:3], v4 slc + .endif s_endpgm )"; @@ -179,6 +194,8 @@ const char *PollMemoryIsa = R"( s_cbranch_scc0 LOOP .if (.amdgcn.gfx_generation_number >= 10) flat_store_dword v[0:1], v2 slc + .elseif (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + flat_store_dword v[0:1], v2 nt sc1 sc0 .else s_store_dword s18, s[2:3], 0x0 glc .endif @@ -524,7 +541,11 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( V_ADD_CO_CI_U32 v3, v3, 0 // v[2:3] = s[0:1] + v0 * 8 // Load 64bit local buffer address stored at v[2:3] to v[6:7] - flat_load_dwordx2 v[6:7], v[2:3] slc + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + flat_load_dwordx2 v[6:7], v[2:3] nt sc1 sc0 + .else + flat_load_dwordx2 v[6:7], v[2:3] slc + .endif s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish v_mov_b32 v8, 0x5678 s_movk_i32 s8, 0x5678 @@ -542,7 +563,11 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( v_mov_b32 v12, v6 v_mov_b32 v13, v7 L_LOOP_READ: - flat_load_dwordx2 v[14:15], v[12:13] slc + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + flat_load_dwordx2 v[14:15], v[12:13] nt sc1 sc0 + .else + flat_load_dwordx2 v[14:15], v[12:13] slc + .endif V_ADD_CO_U32 v9, v9, v10 V_ADD_CO_U32 v12, v12, v10 V_ADD_CO_CI_U32 v13, v13, 0 From c234f84245d8871fd876f7057563bd19df1d8f08 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Thu, 22 Sep 2022 15:54:06 -0400 Subject: [PATCH 1119/1247] kfdtest: Update blacklist for Aqua Vanjanran Signed-off-by: Ori Messinger Signed-off-by: Amber Lin Signed-off-by: Harish Kasiviswanathan Signed-off-by: David Francis Signed-off-by: Philip Yang Change-Id: I8f822bb71e8e5dbee6bdb62f77cbe5ea83faabb5 --- tests/kfdtest/scripts/kfdtest.exclude | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 672c27ec3a..6a0bfe1776 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -246,10 +246,15 @@ FILTER[aldebaran]=\ FILTER[aqua_vanjaram]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDExceptionTest.FaultStorm:"\ +"KFDQMTest.BasicCuMaskingEven:"\ +"KFDMemoryTest.LargestSysBufferTest:"\ +"KFDMemoryTest.BigSysBufferStressTest:"\ +"KFDMemoryTest.FlatScratchAccess:"\ +"KFDIPCTest.BasicTest:"\ +"KFDEvictTest.BasicTest:"\ +"KFDEvictTest.QueueTest:"\ "KFDEvictTest.BurstyTest:"\ -"KFDMemoryTest.PtraceAccess:"\ -"KFDMemoryTest.DeviceHdpFlush" +"KFDSVMEvictTest.*" FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From 2a01e5c33b063e28956c14f925a3cfb644998c13 Mon Sep 17 00:00:00 2001 From: David Francis Date: Thu, 16 Feb 2023 15:10:54 -0500 Subject: [PATCH 1120/1247] kfdtest: Use scalar path for PollMemoryIsa Shader on gfx940 A gfx940 code path was erroneously added to this shader. It's unneccesary; without this path, the shader uses the scalar store, which works just fine on gfx940 without changes. Remove it. Signed-off-by: David Francis Change-Id: I825cbbebbdb25c4a7c2f16e228c2bea6a6bcc30c --- tests/kfdtest/src/ShaderStore.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 998282c142..a5fbbc2d64 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -194,8 +194,6 @@ const char *PollMemoryIsa = R"( s_cbranch_scc0 LOOP .if (.amdgcn.gfx_generation_number >= 10) flat_store_dword v[0:1], v2 slc - .elseif (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) - flat_store_dword v[0:1], v2 nt sc1 sc0 .else s_store_dword s18, s[2:3], 0x0 glc .endif From 16c65303304b6026e2e04877cd9bc36044f80977 Mon Sep 17 00:00:00 2001 From: David Francis Date: Wed, 1 Feb 2023 10:05:47 -0500 Subject: [PATCH 1121/1247] kfdtest: Deprecate PollNCMemoryIsa Even with the restriction to only compile on gfx90a, this shader still fails CompileShaders test. There don't seem to be any systems that actually use it. Leave it in the shader store, but remove it otherwise Signed-off-by: David Francis Change-Id: I41bec6ba10363d42b163ac101c3a92edaad6d6df --- tests/kfdtest/src/KFDMemoryTest.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 3e0612aa05..30327f0c17 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -176,14 +176,7 @@ TEST_F(KFDMemoryTest, MapUnmapToNodes) { HsaMemoryBuffer srcBuffer(PAGE_SIZE, defaultGPUNode); HsaMemoryBuffer dstBuffer(PAGE_SIZE, defaultGPUNode); - const char *pReadMemory; - if (m_NodeInfo.IsNodeXGMItoCPU(defaultGPUNode) && (m_FamilyId != FAMILY_AL)) - /* On A+A system memory is mapped as NC */ - pReadMemory = PollNCMemoryIsa; - else - pReadMemory = PollMemoryIsa; - - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(pReadMemory, isaBuffer.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PollMemoryIsa, isaBuffer.As())); PM4Queue pm4Queue; ASSERT_SUCCESS(pm4Queue.Create(defaultGPUNode)); From e32278a6126bce8d8bfae71cd1b097cba21d92a5 Mon Sep 17 00:00:00 2001 From: David Francis Date: Wed, 1 Mar 2023 10:34:53 -0500 Subject: [PATCH 1122/1247] kfdtest: Restrict DriverHDPFlush to systems with PCIe Nodes with XGMI have no HDP, so DriverHDPFlush should skip. Signed-off-by: David Francis Change-Id: If5a87e660712e51d03e750d8e044786036b2e603 --- tests/kfdtest/src/KFDMemoryTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 30327f0c17..13f453f4ef 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1919,6 +1919,11 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { return; } + if (m_NodeInfo.IsNodeXGMItoCPU(nodes[0])) { + LOG() << "Skipping test: PCIe link to CPU is required." << std::endl; + return; + } + HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(nodes[0], pNodeProperties->NumMemoryBanks, memoryProperties)); From 598e3e8d86ac61793d331239ca230fd3d43ae79f Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 20 Mar 2023 13:54:50 -0400 Subject: [PATCH 1123/1247] kfdtest: KFDMemoryTest.DeviceHdpFlush requires large bar KFDMemoryTest.DeviceHdpFlush requires device node 0 is large bar to check VRAM content from CPU, run the test only if device 0 is large bar GPU. Change-Id: I874b153219550c50b724625e971e3ed3a84dc652 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDMemoryTest.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 13f453f4ef..e77b600b0e 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1924,6 +1924,11 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { return; } + if (!m_NodeInfo.IsGPUNodeLargeBar(nodes[0])) { + LOG() << "Skipping test: Test requires device 0 large bar GPU." << std::endl; + return; + } + HsaMemoryProperties *memoryProperties = new HsaMemoryProperties[pNodeProperties->NumMemoryBanks]; EXPECT_SUCCESS(hsaKmtGetNodeMemoryProperties(nodes[0], pNodeProperties->NumMemoryBanks, memoryProperties)); From 831d1ad352445088c07e5726a20847224e8a1960 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 3 Apr 2023 15:57:00 -0400 Subject: [PATCH 1124/1247] libhsakmt: Mask stepping version for GC 9.4.3 checks GC 9.4.3 to set gfx target version to 9.4.x dependent on revision and capabilities. Due to this, where applicable, mask off the gfx target stepping version and only check major/minor version (9.4). There are no collisions due to this change since GC 9.4.3 is the only ASIC that uses gfx target version 9.4.x. Signed-off-by: Graham Sider Change-Id: I72803e594c421f054d18ccfa7e92c507128fa5be --- src/queues.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/queues.c b/src/queues.c index 0f3b71c755..992290c791 100644 --- a/src/queues.c +++ b/src/queues.c @@ -86,23 +86,16 @@ static struct process_doorbells *doorbells; uint32_t get_vgpr_size_per_cu(uint32_t gfxv) { - uint32_t vgpr_size = 0; + uint32_t vgpr_size = 0x40000; - switch (gfxv) - { - case GFX_VERSION_ARCTURUS: - case GFX_VERSION_ALDEBARAN: - case GFX_VERSION_AQUA_VANJARAM: - vgpr_size = 0x80000; - break; - case GFX_VERSION_PLUM_BONITO: - case GFX_VERSION_WHEAT_NAS: - vgpr_size = 0x60000; - break; - default: - vgpr_size = 0x40000; - break; - } + if ((gfxv & ~(0xff)) == GFX_VERSION_AQUA_VANJARAM || + gfxv == GFX_VERSION_ALDEBARAN || + gfxv == GFX_VERSION_ARCTURUS) + vgpr_size = 0x80000; + + else if (gfxv == GFX_VERSION_PLUM_BONITO || + gfxv == GFX_VERSION_WHEAT_NAS) + vgpr_size = 0x60000; return vgpr_size; } @@ -616,7 +609,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, if (q->gfxv == GFX_VERSION_TONGA) q->eop_buffer_size = TONGA_PAGE_SIZE; - else if (q->gfxv == GFX_VERSION_AQUA_VANJARAM) + else if ((q->gfxv & ~(0xff)) == GFX_VERSION_AQUA_VANJARAM) q->eop_buffer_size = ((Type == HSA_QUEUE_COMPUTE) ? 4096 : 0); else if (q->gfxv >= 0x80000) q->eop_buffer_size = 4096; From 680c8ca5a9a355b67bb72e4276aca27f6805c462 Mon Sep 17 00:00:00 2001 From: David Francis Date: Tue, 17 Jan 2023 10:42:03 -0500 Subject: [PATCH 1125/1247] kfdtest: Make queue evict tests use constant number of wavefronts. Previously, KFDEvictTest.QueueTest and KFDSVMEvictTest.QueueTest would create a variable number of wavefronts, one for each 64MB of memory under test. This ran into limits on the buffers used by the wavefronts, and may at some point have exceeded the wavefront limit. Restrict the number of wavefronts to 512, and adjust the shader to accomodate a variable buffer size Signed-off-by: David Francis Change-Id: I2ec292e2900e2efa62a08313bca3d2f4bdabca8b --- tests/kfdtest/src/KFDEvictTest.cpp | 11 +++++++--- tests/kfdtest/src/KFDSVMEvictTest.cpp | 31 +++++++++++++++++++++++---- tests/kfdtest/src/KFDSVMEvictTest.hpp | 1 + tests/kfdtest/src/ShaderStore.cpp | 11 ++++++++-- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 1effa639b0..417123af37 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -33,6 +33,7 @@ #define N_PROCESSES (2) /* Number of processes running in parallel, must be at least 2 */ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +#define MAX_WAVEFRONTS (512) #define SDMA_NOP 0x0 @@ -382,7 +383,7 @@ TEST_F(KFDEvictTest, QueueTest) { HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + unsigned int count = MAX_WAVEFRONTS; const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); @@ -406,9 +407,10 @@ TEST_F(KFDEvictTest, QueueTest) { // Use 7/8 of VRAM between all processes HSAuint64 testSize = vramSize * 7 / 8; - HSAuint32 count = testSize / (vramBufSize * N_PROCESSES); + HSAuint32 vramBufSize = testSize / (count * N_PROCESSES); + vramBufSize = (vramBufSize / (1024 * 1024)) * (1024 * 1024); - if (count == 0) { + if (vramBufSize == 0) { LOG() << "Skipping test: Not enough system memory available." << std::endl; return; } @@ -455,6 +457,9 @@ TEST_F(KFDEvictTest, QueueTest) { for (i = 0; i < wavefront_num; i++) *(localBufAddr + i) = pBuffers[i]; + for (i = 0; i < wavefront_num; i++) + *(result + i) = vramBufSize; + dispatch0.SetArgs(localBufAddr, result); dispatch0.SetDim(wavefront_num, 1, 1); /* Submit the packet and start shader */ diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index cd61de3c3e..b97e6c7089 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -34,6 +34,7 @@ #define N_PROCESSES (4) /* number of processes running in parallel, at least 2 */ #define ALLOCATE_BUF_SIZE_MB (64) #define ALLOCATE_RETRY_TIMES (3) +#define MAX_WAVEFRONTS (512) void KFDSVMEvictTest::SetUp() { ROUTINE_START @@ -64,7 +65,7 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; /* use one third of total system memory for eviction buffer to test - * limit max allocate size to duoble of vramSize + * limit max allocate size to double of vramSize * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) */ size = sysMemSize / 3 + vramSize; @@ -81,6 +82,25 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf return count; } +HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count) { + HSAuint64 sysMemSize = GetSysMemSize(); + HSAuint64 size, sizeInPages; + HSAuint64 vramBufSizeInPages; + + LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; + + /* use one third of total system memory for eviction buffer to test + * limit max allocate size to duoble of vramSize + * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) + */ + size = sysMemSize / 3 + vramSize; + size = size > vramSize << 1 ? vramSize << 1 : size; + sizeInPages = size >> PAGE_SHIFT; + vramBufSizeInPages = sizeInPages / (count * N_PROCESSES); + + return vramBufSizeInPages << 20; +} + void KFDSVMEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, std::vector &pBuffers) { HSAuint64 totalMB; @@ -266,7 +286,7 @@ TEST_F(KFDSVMEvictTest, QueueTest) { HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + unsigned int count = MAX_WAVEFRONTS; const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); @@ -286,8 +306,8 @@ TEST_F(KFDSVMEvictTest, QueueTest) { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; } - HSAuint32 count = GetBufferCounter(vramSize, vramBufSize); - if (count == 0) { + HSAuint64 vramBufSize = GetBufferSize(vramSize, count); + if (vramBufSize == 0) { LOG() << "Not enough system memory, skipping the test" << std::endl; return; } @@ -315,6 +335,9 @@ TEST_F(KFDSVMEvictTest, QueueTest) { for (i = 0; i < wavefront_num; i++) *(localBufAddr + i) = pBuffers[i]; + for (i = 0; i < wavefront_num; i++) + *(result + i) = vramBufSize; + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadMemoryIsa, isaBuffer.As())); PM4Queue pm4Queue; diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/tests/kfdtest/src/KFDSVMEvictTest.hpp index 3f26287cc0..7e772105ae 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.hpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.hpp @@ -60,6 +60,7 @@ class KFDSVMEvictTest : public KFDLocalMemoryTest { void ForkChildProcesses(int nprocesses); void WaitChildProcesses(); HSAint32 GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize); + HSAint64 GetBufferSize(HSAuint64 vramSize, HSAuint32 count); protected: // members std::string m_psName; diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index a5fbbc2d64..d9b32c7e37 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -523,6 +523,7 @@ const char *PersistentIterateIsa = SHADER_MACROS R"( * v[2:3] - address of corresponding local buf address offset: s[0:1] + v0 * 8 * v[4:5] - corresponding output buf address: s[2:3] + v0 * 4 * v[6:7] - local buf address used for read test + * v11 - size of local buffer in MB */ const char *ReadMemoryIsa = SHADER_MACROS R"( // Compute address of corresponding output buffer @@ -538,6 +539,13 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v0 * 8 V_ADD_CO_CI_U32 v3, v3, 0 // v[2:3] = s[0:1] + v0 * 8 + //Load local buffer size from output buffer + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + flat_load_dword v11, v[4:5] nt sc1 sc0 + .else + flat_load_dword v11, v[4:5] slc + .endif + // Load 64bit local buffer address stored at v[2:3] to v[6:7] .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) flat_load_dwordx2 v[6:7], v[2:3] nt sc1 sc0 @@ -553,11 +561,10 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( s_cmp_eq_i32 s16, s8 s_cbranch_scc1 L_QUIT // if notified to quit by host - // Loop read 64M local buffer starting at v[6:7] + // Loop read local buffer starting at v[6:7] // every 4k page only read once v_mov_b32 v9, 0 v_mov_b32 v10, 0x1000 // 4k page - v_mov_b32 v11, 0x4000000 // 64M size v_mov_b32 v12, v6 v_mov_b32 v13, v7 L_LOOP_READ: From e2435d9e937a4622f64d531cfd52e7e215722f69 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 12 Apr 2023 11:54:24 -0400 Subject: [PATCH 1126/1247] kfdtest: Add flat compatability macros for gfx target 9.4.x For GC 9.4.0, modifications were made to various shaders since certain flat_ instructions no longer support glc/slc modifiers (replaced with nt/sc1/sc0). Instead of repeating conditionals inside various shader bodies, we can make use of LLVM AMDGCN macros. This patch modularizes the shader macros into seperated defines. Prior to the core raw-string literal, each shader now starts with the SHADER_START literal (".text\n") plus any number of SHADER_MACRO_* literals. This allows us to seperate the macro definitions logically and use the pre-processor to only include the required macro groups on a per-shader basis. Signed-off-by: Graham Sider Change-Id: I19eb3fd14252a0601bb7509249051b68e7fdb02a --- tests/kfdtest/src/ShaderStore.cpp | 179 ++++++++++++++++++------------ 1 file changed, 108 insertions(+), 71 deletions(-) diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index d9b32c7e37..610217b6bd 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -49,10 +49,12 @@ const std::vector ShaderList = { * Macros */ -/* Create macro for portable v_add_co_u32, v_add_co_ci_u32, - * and v_cmp_lt_u32 +#define SHADER_START ".text\n" + +/* Macros for portable v_add_co_u32, v_add_co_ci_u32, + * and v_cmp_lt_u32. */ -#define SHADER_MACROS \ +#define SHADER_MACROS_U32 \ " .text\n"\ " .macro V_ADD_CO_U32 vdst, src0, vsrc1\n"\ " .if (.amdgcn.gfx_generation_number >= 10)\n"\ @@ -80,50 +82,85 @@ const std::vector ShaderList = { " .endif\n"\ " .endm\n" +/* Macros for portable flat load/store/atomic instructions. + * + * gc943 (gfx94x) deprecates glc/slc in favour of nt/sc1/sc0. + * The below macros when used will always use the nt sc1 sc0 + * modifiers for gfx94x, but also take in arg0 arg1 to specify + * (for non-gfx94x): glc, slc, or glc slc. + */ +#define SHADER_MACROS_FLAT \ + " .macro FLAT_LOAD_DWORD_NSS vdst, vaddr arg0 arg1\n"\ + " .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4)\n"\ + " flat_load_dword \\vdst, \\vaddr nt sc1 sc0\n"\ + " .else\n"\ + " flat_load_dword \\vdst, \\vaddr \\arg0 \\arg1\n"\ + " .endif\n"\ + " .endm\n"\ + " .macro FLAT_LOAD_DWORDX2_NSS vdst, vaddr arg0 arg1\n"\ + " .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4)\n"\ + " flat_load_dwordx2 \\vdst, \\vaddr nt sc1 sc0\n"\ + " .else\n"\ + " flat_load_dwordx2 \\vdst, \\vaddr \\arg0 \\arg1\n"\ + " .endif\n"\ + " .endm\n"\ + " .macro FLAT_STORE_DWORD_NSS vaddr, vsrc arg0 arg1\n"\ + " .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4)\n"\ + " flat_store_dword \\vaddr, \\vsrc nt sc1 sc0\n"\ + " .else\n"\ + " flat_store_dword \\vaddr, \\vsrc \\arg0 \\arg1\n"\ + " .endif\n"\ + " .endm\n"\ + " .macro FLAT_ATOMIC_ADD_NSS vdst, vaddr, vsrc arg0 arg1\n"\ + " .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4)\n"\ + " flat_atomic_add \\vdst, \\vaddr, \\vsrc nt sc1 sc0\n"\ + " .else\n"\ + " flat_atomic_add \\vdst, \\vaddr, \\vsrc \\arg0 \\arg1\n"\ + " .endif\n"\ + " .endm\n" + /** * Common */ -const char *NoopIsa = R"( - .text +const char *NoopIsa = + SHADER_START + R"( s_endpgm )"; -const char *CopyDwordIsa = R"( - .text +const char *CopyDwordIsa = + SHADER_START + SHADER_MACROS_FLAT + R"( v_mov_b32 v0, s0 v_mov_b32 v1, s1 v_mov_b32 v2, s2 v_mov_b32 v3, s3 - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) - flat_load_dword v4, v[0:1] nt sc1 sc0 - s_waitcnt 0 - flat_store_dword v[2:3], v4 nt sc1 sc0 - .else - flat_load_dword v4, v[0:1] glc slc - s_waitcnt 0 - flat_store_dword v[2:3], v4 glc slc - .endif + FLAT_LOAD_DWORD_NSS v4, v[0:1] glc slc + s_waitcnt 0 + FLAT_STORE_DWORD_NSS v[2:3], v4 glc slc s_endpgm )"; -const char *InfiniteLoopIsa = R"( +const char *InfiniteLoopIsa = + SHADER_START + R"( .text LOOP: s_branch LOOP s_endpgm )"; -const char *AtomicIncIsa = R"( - .text +const char *AtomicIncIsa = + SHADER_START + SHADER_MACROS_FLAT + R"( v_mov_b32 v0, s0 v_mov_b32 v1, s1 - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) + .if (.amdgcn.gfx_generation_number >= 8) v_mov_b32 v2, 1 - flat_atomic_add v3, v[0:1], v2 nt sc1 sc0 - .elseif (.amdgcn.gfx_generation_number >= 8) - v_mov_b32 v2, 1 - flat_atomic_add v3, v[0:1], v2 glc slc + FLAT_ATOMIC_ADD_NSS v3, v[0:1], v2 glc slc .else v_mov_b32 v2, -1 flat_atomic_inc v3, v[0:1], v2 glc slc @@ -136,8 +173,10 @@ const char *AtomicIncIsa = R"( * KFDMemoryTest */ -const char *ScratchCopyDwordIsa = R"( - .text +const char *ScratchCopyDwordIsa = + SHADER_START + SHADER_MACROS_FLAT + R"( // Copy the parameters from scalar registers to vector registers .if (.amdgcn.gfx_generation_number >= 9) v_mov_b32 v0, s0 @@ -162,15 +201,9 @@ const char *ScratchCopyDwordIsa = R"( s_mov_b32 flat_scratch_hi, 0 .endif // Copy a dword between the passed addresses - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) - flat_load_dword v4, v[0:1] nt sc1 sc0 - s_waitcnt vmcnt(0) & lgkmcnt(0) - flat_store_dword v[2:3], v4 nt sc1 sc0 - .else - flat_load_dword v4, v[0:1] slc - s_waitcnt vmcnt(0) & lgkmcnt(0) - flat_store_dword v[2:3], v4 slc - .endif + FLAT_LOAD_DWORD_NSS v4, v[0:1] slc + s_waitcnt vmcnt(0) & lgkmcnt(0) + FLAT_STORE_DWORD_NSS v[2:3], v4 slc s_endpgm )"; @@ -179,8 +212,9 @@ const char *ScratchCopyDwordIsa = R"( * by host program), fill dst buffer with specific * value(0x5678) and quit */ -const char *PollMemoryIsa = R"( - .text +const char *PollMemoryIsa = + SHADER_START + R"( // Assume src address in s0, s1, and dst address in s2, s3 s_movk_i32 s18, 0x5678 .if (.amdgcn.gfx_generation_number >= 10) @@ -205,8 +239,9 @@ const char *PollMemoryIsa = R"( * cache coherence is not supported in scalar (smem) path. * Use vmem operations with scc */ -const char *PollNCMemoryIsa = R"( - .text +const char *PollNCMemoryIsa = + SHADER_START + R"( // Assume src address in s0, s1, and dst address in s2, s3 v_mov_b32 v6, 0x5678 v_mov_b32 v0, s0 @@ -230,8 +265,9 @@ const char *PollNCMemoryIsa = R"( * Once signal buffer is signaled, it copies input buffer * to output buffer */ -const char *CopyOnSignalIsa = R"( - .text +const char *CopyOnSignalIsa = + SHADER_START + R"( // Assume input buffer in s0, s1 .if (.amdgcn.gfx_generation_number >= 10) s_add_u32 s2, s0, 0x8 @@ -267,8 +303,9 @@ const char *CopyOnSignalIsa = R"( * Note: Only works on GFX9 (only used in * aldebaran tests) */ -const char *PollAndCopyIsa = R"( - .text +const char *PollAndCopyIsa = + SHADER_START + R"( // Assume src buffer in s[0:1] and dst buffer in s[2:3] .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_stepping == 10) // Path for Aldebaran @@ -309,8 +346,9 @@ const char *PollAndCopyIsa = R"( * * Note: Only works on Aldebaran */ -const char *WriteFlagAndValueIsa = R"( - .text +const char *WriteFlagAndValueIsa = + SHADER_START + R"( // Assume two inputs buffer in s[0:1] and s[2:3] .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_stepping == 10) v_mov_b32 v0, s0 @@ -332,8 +370,9 @@ const char *WriteFlagAndValueIsa = R"( * DW1: Write to this buffer for other device to read. * Input1: mmio base address */ -const char *WriteAndSignalIsa = R"( - .text +const char *WriteAndSignalIsa = + SHADER_START + R"( // Assume input buffer in s0, s1 .if (.amdgcn.gfx_generation_number >= 10) s_add_u32 s4, s0, 0x4 @@ -369,8 +408,9 @@ const char *WriteAndSignalIsa = R"( * s1 controls the number iterations of the loop * This shader can be used by GFX8, GFX9 and GFX10 */ -const char *LoopIsa = R"( - .text +const char *LoopIsa = + SHADER_START + R"( s_movk_i32 s0, 0x0008 s_movk_i32 s1, 0x00ff v_mov_b32 v0, 0 @@ -474,7 +514,10 @@ const char *LoopIsa = R"( * v6 - register storing known-value output for mangle testing * v7 - counter */ -const char *PersistentIterateIsa = SHADER_MACROS R"( +const char *PersistentIterateIsa = + SHADER_START + SHADER_MACROS_U32 + R"( // Compute address of output buffer v_mov_b32 v0, s4 // use workgroup id as index v_lshlrev_b32 v0, 2, v0 // v0 *= 4 @@ -525,7 +568,11 @@ const char *PersistentIterateIsa = SHADER_MACROS R"( * v[6:7] - local buf address used for read test * v11 - size of local buffer in MB */ -const char *ReadMemoryIsa = SHADER_MACROS R"( +const char *ReadMemoryIsa = + SHADER_START + SHADER_MACROS_U32 + SHADER_MACROS_FLAT + R"( // Compute address of corresponding output buffer v_mov_b32 v0, s4 // use workgroup id as index v_lshlrev_b32 v0, 2, v0 // v0 *= 4 @@ -539,19 +586,11 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( v_mov_b32 v3, s1 // v[2:3] = s[0:1] + v0 * 8 V_ADD_CO_CI_U32 v3, v3, 0 // v[2:3] = s[0:1] + v0 * 8 - //Load local buffer size from output buffer - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) - flat_load_dword v11, v[4:5] nt sc1 sc0 - .else - flat_load_dword v11, v[4:5] slc - .endif + // Load local buffer size from output buffer + FLAT_LOAD_DWORD_NSS v11, v[4:5] slc // Load 64bit local buffer address stored at v[2:3] to v[6:7] - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) - flat_load_dwordx2 v[6:7], v[2:3] nt sc1 sc0 - .else - flat_load_dwordx2 v[6:7], v[2:3] slc - .endif + FLAT_LOAD_DWORDX2_NSS v[6:7], v[2:3] slc s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish v_mov_b32 v8, 0x5678 s_movk_i32 s8, 0x5678 @@ -568,11 +607,7 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( v_mov_b32 v12, v6 v_mov_b32 v13, v7 L_LOOP_READ: - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4 && .amdgcn.gfx_generation_stepping == 0) - flat_load_dwordx2 v[14:15], v[12:13] nt sc1 sc0 - .else - flat_load_dwordx2 v[14:15], v[12:13] slc - .endif + FLAT_LOAD_DWORDX2_NSS v[14:15], v[12:13] slc V_ADD_CO_U32 v9, v9, v10 V_ADD_CO_U32 v12, v12, v10 V_ADD_CO_CI_U32 v13, v13, 0 @@ -590,8 +625,9 @@ const char *ReadMemoryIsa = SHADER_MACROS R"( */ /* Shader to initialize gws counter to 1 */ -const char *GwsInitIsa = R"( - .text +const char *GwsInitIsa = + SHADER_START + R"( s_mov_b32 m0, 0 s_nop 0 s_load_dword s16, s[0:1], 0x0 glc @@ -609,8 +645,9 @@ const char *GwsInitIsa = R"( * GWS semaphore is used to guarantee * the operation is atomic. */ -const char *GwsAtomicIncreaseIsa = R"( - .text +const char *GwsAtomicIncreaseIsa = + SHADER_START + R"( // Assume src address in s0, s1 .if (.amdgcn.gfx_generation_number >= 10) s_mov_b32 m0, 0 From 11a04fe1f55fddd11684cf4e7226e67ec7e71720 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 12 Apr 2023 12:08:38 -0400 Subject: [PATCH 1127/1247] kfdtest: Fix PersistentIterateShader for gfx target 9.4.x Replace 'flat_load_dword <...> glc' with appropriate macro. Signed-off-by: Graham Sider Change-Id: I9fdc7c916c685304457cd9698e741577f6c10c82 --- tests/kfdtest/src/ShaderStore.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 610217b6bd..705aa09711 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -517,6 +517,7 @@ const char *LoopIsa = const char *PersistentIterateIsa = SHADER_START SHADER_MACROS_U32 + SHADER_MACROS_FLAT R"( // Compute address of output buffer v_mov_b32 v0, s4 // use workgroup id as index @@ -526,7 +527,7 @@ const char *PersistentIterateIsa = V_ADD_CO_CI_U32 v5, v5, 0 // v[4:5] = s[2:3] + v0 * 4 // Store known-value output in register - flat_load_dword v6, v[4:5] glc + FLAT_LOAD_DWORD_NSS v6, v[4:5] glc s_waitcnt vmcnt(0) & lgkmcnt(0) // wait for memory reads to finish // Initialize counter From 4d54d6e70666a6aea049a03fc9c3a27a61cbe7ce Mon Sep 17 00:00:00 2001 From: Jesse Zhang Date: Thu, 13 Apr 2023 17:23:45 +0800 Subject: [PATCH 1128/1247] libhsakmt: Add compute core check for APU We should check compute core instead of cpu core, in order to exclude the case of APU. Signed-off-by: Jesse zhang Change-Id: I2ec2a6807f51f49f80e0e500f5d9af81c2efae37 --- src/topology.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/topology.c b/src/topology.c index ba5802da86..8b0b786b9a 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1098,7 +1098,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, read_size = fread(read_buf, 1, PAGE_SIZE, fd); if (read_size <= 0) { ret = HSAKMT_STATUS_ERROR; - goto err; + goto out; } /* Since we're using the buffer as a string, we make sure the string terminates */ @@ -1193,8 +1193,8 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, } /* Bail out early, if a CPU node */ - if (props->NumCPUCores) - goto err; + if (!props->NumFComputeCores) + goto out; if (props->NumArrays != 0) props->NumShaderBanks = simd_arrays_count/props->NumArrays; @@ -1214,7 +1214,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, pr_err("HSA_OVERRIDE_GFX_VERSION %s is invalid\n", envvar); ret = HSAKMT_STATUS_ERROR; - goto err; + goto out; } props->EngineId.ui32.Major = major & 0x3f; props->EngineId.ui32.Minor = minor & 0xff; @@ -1263,7 +1263,7 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, if (!props->NumXcc) props->NumXcc = 1; -err: +out: free(read_buf); fclose(fd); return ret; From e2df2c21afde3ef856954a94755f7a658df0c5e3 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 24 Mar 2023 11:14:40 -0400 Subject: [PATCH 1129/1247] kfdtest: Add helper to check if IsAppAPU system If there is connection between GPU and CPU with weight 13, KFD_CRAT_INTRA_SOCKET_WEIGHT, then this is AppAPU. This will be used to skip tests not suitable for AppAPU. Change-Id: If6fad81528b52afd4ac4cefa508d787b0f6637ca Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDTestUtil.cpp | 33 +++++++++++++++++++++++++++++++ tests/kfdtest/src/KFDTestUtil.hpp | 1 + 2 files changed, 34 insertions(+) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index b9d9d23d4a..98c90f7b19 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -672,6 +672,39 @@ const bool HsaNodeInfo::IsGPUNodeLargeBar(int node) const { return false; } +const bool HsaNodeInfo::IsAppAPU(int node) const { + const HsaNodeProperties *pNodeProperties = GetNodeProperties(node); + + /* CPU with compute cores is small APU, not AppAPU */ + if (pNodeProperties->NumCPUCores && pNodeProperties->NumFComputeCores) + return false; + + HsaIoLinkProperties *IolinkProperties = new HsaIoLinkProperties[pNodeProperties->NumIOLinks]; + if (hsaKmtGetNodeIoLinkProperties(node, pNodeProperties->NumIOLinks, IolinkProperties)) { + LOG() << "Unable to get Node IO Link Information for node " << node << std::endl; + delete [] IolinkProperties; + return false; + } + + /* Checking GPU-to-CPU connection weight */ + for (int linkId = 0; linkId < pNodeProperties->NumIOLinks; linkId++) { + HsaNodeProperties linkProps; + + if (hsaKmtGetNodeProperties(IolinkProperties[linkId].NodeTo, &linkProps)) { + LOG() << "Unable to get connected device's IO Link information" << std::endl; + break; + } + + /* If it's GPU-CPU link with connection weight KFD_CRAT_INTRA_SOCKET_WEIGHT 13 */ + if (linkProps.NumCPUCores && IolinkProperties[linkId].Weight == 13) { + delete [] IolinkProperties; + return true; + } + } + delete [] IolinkProperties; + return false; +} + const bool HsaNodeInfo::IsPeerAccessibleByNode(int peer, int node) const { const HsaNodeProperties *pNodeProperties; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 938ff8bf69..756b024e90 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -233,6 +233,7 @@ class HsaNodeInfo { void PrintNodeInfo() const; const bool IsGPUNodeLargeBar(int node) const; + const bool IsAppAPU(int node) const; const bool IsPeerAccessibleByNode(int peer, int node) const; // @brief Find the first available Large-BAR GPU node // @return: Node ID if successful or -1 From 21abaef3f864b542ad127153aaefe915fb1710b7 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 27 Mar 2023 11:28:22 -0400 Subject: [PATCH 1130/1247] kfdtest: AppAPU Skip KFDEvictTest, KDFSVMEvictTest, HMMProfilingEvent AppAPU VRAM is part of system memory managed by Linux kernel, no VRAM eviction and restore is needed between VRAM and system memory. Those Evict test failed on AppAPU now, skip those tests on AppAPU. No page migration between VRAM and system on AppAPU, HMMProfilingEvent depends on migration event, skip it on AppAPU. Change-Id: I4c809b97c947e809d136c1f88db2278cf74f5b47 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDEvictTest.cpp | 15 +++++++++++++++ tests/kfdtest/src/KFDSVMEvictTest.cpp | 10 ++++++++++ tests/kfdtest/src/KFDSVMRangeTest.cpp | 5 +++++ 3 files changed, 30 insertions(+) diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/tests/kfdtest/src/KFDEvictTest.cpp index 417123af37..e6cb080589 100644 --- a/tests/kfdtest/src/KFDEvictTest.cpp +++ b/tests/kfdtest/src/KFDEvictTest.cpp @@ -319,6 +319,11 @@ TEST_F(KFDEvictTest, BasicTest) { return; } + if (m_NodeInfo.IsAppAPU(defaultGPUNode)) { + LOG() << "Skipping test on AppAPU." << std::endl; + return; + } + LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB" << std::endl; LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; @@ -393,6 +398,11 @@ TEST_F(KFDEvictTest, QueueTest) { return; } + if (m_NodeInfo.IsAppAPU(defaultGPUNode)) { + LOG() << "Skipping test on AppAPU." << std::endl; + return; + } + HSAuint32 i; HSAuint64 vramSize = GetVramSize(defaultGPUNode); HSAuint64 sysMemSize = GetSysMemSize(); @@ -509,6 +519,11 @@ TEST_F(KFDEvictTest, BurstyTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + if (m_NodeInfo.IsAppAPU(defaultGPUNode)) { + LOG() << "Skipping test on AppAPU." << std::endl; + return; + } + HSAuint64 vramSize = GetVramSize(defaultGPUNode); HSAuint64 sysMemSize = GetSysMemSize(); diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index b97e6c7089..75eae28f82 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -228,6 +228,11 @@ TEST_F(KFDSVMEvictTest, BasicTest) { ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; HSAuint64 vramBufSize = ALLOCATE_BUF_SIZE_MB * 1024 * 1024; + if (m_NodeInfo.IsAppAPU(defaultGPUNode)) { + LOG() << "Skipping test on AppAPU." << std::endl; + return; + } + HSAuint64 vramSize = GetVramSize(defaultGPUNode); if (!vramSize) { @@ -296,6 +301,11 @@ TEST_F(KFDSVMEvictTest, QueueTest) { return; } + if (m_NodeInfo.IsAppAPU(defaultGPUNode)) { + LOG() << "Skipping test on AppAPU." << std::endl; + return; + } + HSAuint32 i; HSAuint64 vramSize = GetVramSize(defaultGPUNode); diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index f020529441..9e86e81b12 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1466,6 +1466,11 @@ TEST_F(KFDSVMRangeTest, HMMProfilingEvent) { return; } + if (m_NodeInfo.IsAppAPU(defaultGPUNode)) { + LOG() << "Skipping test on AppAPU." << std::endl; + return; + } + pthread_barrier_t barrier; ASSERT_SUCCESS(pthread_barrier_init(&barrier, NULL, 2)); From 0696f06c162fa69e94058a9b8312451c429dc35d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Tue, 24 Jan 2023 09:20:10 -0500 Subject: [PATCH 1131/1247] kfdtest: fix KFDSVMRangeTest.MultiGPU tests vector iterator For vector iterator loop access current node directly, don't need gpuNodesAll.at(i), which also causes out of range access. Change vector index loop to iterator loop to simplify the code. Change-Id: I2627ef8d13b5d2c9cd8c51cf4dacc3e8a97fcfb0 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 9e86e81b12..ba502da94c 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1001,12 +1001,12 @@ TEST_F(KFDSVMRangeTest, MultiGPUMigrationTest) { const std::vector gpuNodesAll = m_NodeInfo.GetNodesWithGPU(); std::vector gpuNodes; - for (int i : gpuNodesAll) { + for (auto node : gpuNodesAll) { const HsaNodeProperties *pNodeProperties; - pNodeProperties = m_NodeInfo.GetNodeProperties(gpuNodesAll.at(i)); + pNodeProperties = m_NodeInfo.GetNodeProperties(node); if (pNodeProperties->Capability.ui32.SVMAPISupported) - gpuNodes.push_back(gpuNodesAll.at(i)); + gpuNodes.push_back(node); } if (gpuNodes.size() < 2) { LOG() << "Skipping test: at least two SVM supported GPUs needed." << std::endl; @@ -1025,16 +1025,16 @@ TEST_F(KFDSVMRangeTest, MultiGPUMigrationTest) { for (HSAuint64 i = 0; i < BufferSize / 8; i++) pBuf[i] = i; - for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { - EXPECT_SUCCESS(SVMRangeMapToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); - EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); + for (auto node : gpuNodes) { + EXPECT_SUCCESS(SVMRangeMapToNode(pBuf, BufferSize, node)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, node)); - EXPECT_SUCCESS(SVMRangeMapToNode(pData, BufferSize, gpuNodes.at(gpuidx))); - EXPECT_SUCCESS(SVMRangePrefetchToNode(pData, BufferSize, gpuNodes.at(gpuidx))); + EXPECT_SUCCESS(SVMRangeMapToNode(pData, BufferSize, node)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pData, BufferSize, node)); } - for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { - ASSERT_SUCCESS(sdmaQueue.Create(gpuNodes.at(gpuidx))); + for (auto node : gpuNodes) { + ASSERT_SUCCESS(sdmaQueue.Create(node)); sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), pData, pBuf, BufferSize)); @@ -1085,12 +1085,12 @@ TEST_F(KFDSVMRangeTest, MultiGPUAccessInPlaceTest) { const std::vector gpuNodesAll = m_NodeInfo.GetNodesWithGPU(); std::vector gpuNodes; - for (int i : gpuNodesAll) { + for (auto node : gpuNodesAll) { const HsaNodeProperties *pNodeProperties; - pNodeProperties = m_NodeInfo.GetNodeProperties(gpuNodesAll.at(i)); + pNodeProperties = m_NodeInfo.GetNodeProperties(node); if (pNodeProperties->Capability.ui32.SVMAPISupported) - gpuNodes.push_back(gpuNodesAll.at(i)); + gpuNodes.push_back(node); } if (gpuNodes.size() < 2) { LOG() << "Skipping test: at least two SVM supported GPUs needed." << std::endl; @@ -1109,16 +1109,16 @@ TEST_F(KFDSVMRangeTest, MultiGPUAccessInPlaceTest) { for (HSAuint64 i = 0; i < BufferSize / 8; i++) pBuf[i] = i; - for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { - EXPECT_SUCCESS(SVMRangeMapInPlaceToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); - EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, gpuNodes.at(gpuidx))); + for (auto node : gpuNodes) { + EXPECT_SUCCESS(SVMRangeMapInPlaceToNode(pBuf, BufferSize, node)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pBuf, BufferSize, node)); - EXPECT_SUCCESS(SVMRangeMapInPlaceToNode(pData, BufferSize, gpuNodes.at(gpuidx))); - EXPECT_SUCCESS(SVMRangePrefetchToNode(pData, BufferSize, gpuNodes.at(gpuidx))); + EXPECT_SUCCESS(SVMRangeMapInPlaceToNode(pData, BufferSize, node)); + EXPECT_SUCCESS(SVMRangePrefetchToNode(pData, BufferSize, node)); } - for (HSAuint64 gpuidx = 0; gpuidx < gpuNodes.size(); gpuidx++) { - ASSERT_SUCCESS(sdmaQueue.Create(gpuNodes.at(gpuidx))); + for (auto node : gpuNodes) { + ASSERT_SUCCESS(sdmaQueue.Create(node)); sdmaQueue.PlaceAndSubmitPacket(SDMACopyDataPacket(sdmaQueue.GetFamilyId(), pData, pBuf, BufferSize)); From 30b1f23f7af05007dcdd3bb5c6c272adf9cae24e Mon Sep 17 00:00:00 2001 From: David Francis Date: Fri, 13 May 2022 14:33:44 -0400 Subject: [PATCH 1132/1247] kfdtest: Add coherency tests for Aqua Vanjaram Aqua Vanjaram is intended to have fine-grained coherency from anywhere to anywhere else using read-acquire and write-release primitives. Add a test that writes to memory covered by five different cache lines, then write-releases, while another thread read-acquires, then reads those five locations in memory. There are nine variations of the test to cover CPU-GPU, same-GPU and across-GPU, vector instructions and scalar instructions, and data local to the acquirer or receiver. Signed-off-by: David Francis Change-Id: I20d2db5c53bd280e971479aad7e61df6ed5d3623 --- tests/kfdtest/src/KFDMemoryTest.cpp | 330 ++++++++++++++++++++++++++++ tests/kfdtest/src/KFDMemoryTest.hpp | 4 + tests/kfdtest/src/ShaderStore.cpp | 209 ++++++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 5 + 4 files changed, 548 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index e77b600b0e..964e6dce12 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2394,6 +2394,336 @@ TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) { TEST_END } +void KFDMemoryTest::AcquireReleaseTestRunCPU(HSAuint32 acquireNode, bool scalar) { + + LOG() << "Testing coherency from CPU to node " << std::dec << acquireNode << std::endl; + + /* Allocate shared buffer - must be at least 64 * 6 bytes */ + HsaMemoryBuffer buffer(PAGE_SIZE, acquireNode, false/*zero*/, false/*local*/, false/*exec*/); + buffer.MapMemToNodes(&acquireNode, 1); + + /* Allocate output buffer and insert magic numbers */ + HsaMemoryBuffer outputBuffer(PAGE_SIZE, acquireNode, true, false, false); + outputBuffer.As()[0x40] = 99; + outputBuffer.As()[0x80] = 99; + outputBuffer.As()[0xc0] = 99; + outputBuffer.As()[0x100] = 99; + outputBuffer.As()[0x140] = 99; + + /* Flush results of previous tests from the buffer */ + /* This would be done with SDMA, but SDMA doesn't work on some Aqua Vanjaram emulators */ + PM4Queue flushQueue; + ASSERT_SUCCESS(flushQueue.Create(acquireNode)); + HsaMemoryBuffer flushBuffer(PAGE_SIZE, acquireNode, true/*zero*/, false/*local*/, true/*exec*/); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(FlushBufferForAcquireReleaseIsa, flushBuffer.As())); + Dispatch flushDispatch(flushBuffer); + flushDispatch.SetArgs(buffer.As(), NULL); + flushDispatch.SetDim(1, 1, 1); + flushDispatch.Submit(flushQueue); + flushDispatch.Sync(g_TestTimeOut); + + /* Start acquiring thread */ + PM4Queue acquireQueue; + ASSERT_SUCCESS(acquireQueue.Create(acquireNode)); + HsaMemoryBuffer acquireBuffer(PAGE_SIZE, acquireNode, true/*zero*/, false/*local*/, true/*exec*/); + if (!scalar) + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadAcquireVectorIsa, acquireBuffer.As())); + else + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadAcquireScalarIsa, acquireBuffer.As())); + Dispatch acquireDispatch(acquireBuffer); + acquireDispatch.SetArgs(buffer.As(), outputBuffer.As()); + acquireDispatch.SetDim(1, 1, 1); + acquireDispatch.Submit(acquireQueue); + + /* Delay 100ms to ensure acquirer is waiting */ + Delay(100); + + if (!scalar) { + buffer.As()[0x40] = 0x1; + buffer.As()[0x80] = 0x2; + buffer.As()[0xc0] = 0x3; + buffer.As()[0x100] = 0x4; + buffer.As()[0x140] = 0x5; + } else { + buffer.As()[0x40] = 0x6; + buffer.As()[0x80] = 0x7; + buffer.As()[0xc0] = 0x8; + buffer.As()[0x100] = 0x9; + buffer.As()[0x140] = 0xa; + } + buffer.As()[0x0] = 0x1; + + acquireDispatch.Sync(g_TestTimeOut); + + /* Check test result*/ + if (!scalar) { + EXPECT_EQ(0x1, outputBuffer.As()[0x40]); + EXPECT_EQ(0x2, outputBuffer.As()[0x80]); + EXPECT_EQ(0x3, outputBuffer.As()[0xc0]); + EXPECT_EQ(0x4, outputBuffer.As()[0x100]); + EXPECT_EQ(0x5, outputBuffer.As()[0x140]); + } else { + EXPECT_EQ(0x6, outputBuffer.As()[0x40]); + EXPECT_EQ(0x7, outputBuffer.As()[0x80]); + EXPECT_EQ(0x8, outputBuffer.As()[0xc0]); + EXPECT_EQ(0x9, outputBuffer.As()[0x100]); + EXPECT_EQ(0xa, outputBuffer.As()[0x140]); + } + + /* + * Guide to results: + * 0x99: acquiring shader did not write to output buffer at all + * 0x77: coherency error. Either releasing shader did not write or acquiring shader read stale value + * All five EXPECT_EQ fail: error occurs even when releasing shader bypasses cache + * Only first four EXPECT_EQ fail: error occurs only when releasing shader uses cache + */ + + /* Clean up */ + EXPECT_SUCCESS(acquireQueue.Destroy()); + EXPECT_SUCCESS(flushQueue.Destroy()); +} + +void KFDMemoryTest::AcquireReleaseTestRun(HSAuint32 acquireNode, HSAuint32 releaseNode, + bool localToRemote, bool scalar) { + + LOG() << "Testing coherency from node " << std::dec << releaseNode << " to node " << std::dec << acquireNode << std::endl; + + /* Allocate shared buffer - must be at least 64 * 6 bytes */ + HSAuint32 localNode; + if (!localToRemote) + localNode = acquireNode; + else + localNode = releaseNode; + HsaMemoryBuffer buffer(PAGE_SIZE, localNode, false/*zero*/, true/*local*/, false/*exec*/); + unsigned int nodes[2] = {acquireNode, releaseNode}; + buffer.MapMemToNodes(&nodes[0], 2); + + /* Allocate output buffer and insert magic numbers */ + HsaMemoryBuffer outputBuffer(PAGE_SIZE, acquireNode, true, false, false); + outputBuffer.As()[0x40] = 99; + outputBuffer.As()[0x80] = 99; + outputBuffer.As()[0xc0] = 99; + outputBuffer.As()[0x100] = 99; + outputBuffer.As()[0x140] = 99; + + /* Flush results of previous tests from the buffer */ + /* This would be done with SDMA, but SDMA doesn't work on some Aqua Vanjaram emulators */ + PM4Queue flushQueue; + ASSERT_SUCCESS(flushQueue.Create(acquireNode)); + HsaMemoryBuffer flushBuffer(PAGE_SIZE, acquireNode, true/*zero*/, false/*local*/, true/*exec*/); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(FlushBufferForAcquireReleaseIsa, flushBuffer.As())); + Dispatch flushDispatch(flushBuffer); + flushDispatch.SetArgs(buffer.As(), NULL); + flushDispatch.SetDim(1, 1, 1); + flushDispatch.Submit(flushQueue); + flushDispatch.Sync(g_TestTimeOut); + + /* Start acquiring thread */ + PM4Queue acquireQueue; + ASSERT_SUCCESS(acquireQueue.Create(acquireNode)); + HsaMemoryBuffer acquireBuffer(PAGE_SIZE, acquireNode, true/*zero*/, false/*local*/, true/*exec*/); + if (!scalar) + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadAcquireVectorIsa, acquireBuffer.As())); + else + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(ReadAcquireScalarIsa, acquireBuffer.As())); + Dispatch acquireDispatch(acquireBuffer); + acquireDispatch.SetArgs(buffer.As(), outputBuffer.As()); + acquireDispatch.SetDim(1, 1, 1); + acquireDispatch.Submit(acquireQueue); + + /* Delay 100ms to ensure acquirer is waiting */ + Delay(100); + + /* Start releasing thread */ + PM4Queue releaseQueue; + ASSERT_SUCCESS(releaseQueue.Create(releaseNode)); + HsaMemoryBuffer releaseBuffer(PAGE_SIZE, releaseNode, true/*zero*/, false/*local*/, true/*exec*/); + if (!scalar) + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteReleaseVectorIsa, releaseBuffer.As())); + else + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteReleaseScalarIsa, releaseBuffer.As())); + Dispatch releaseDispatch(releaseBuffer); + releaseDispatch.SetArgs(buffer.As(), NULL); + releaseDispatch.SetDim(1, 1, 1); + releaseDispatch.Submit(releaseQueue); + + /* Wait for threads to finish */ + releaseDispatch.Sync(g_TestTimeOut); + acquireDispatch.Sync(g_TestTimeOut); + + /* Check test result*/ + if (!scalar) { + EXPECT_EQ(0x1, outputBuffer.As()[0x40]); + EXPECT_EQ(0x2, outputBuffer.As()[0x80]); + EXPECT_EQ(0x3, outputBuffer.As()[0xc0]); + EXPECT_EQ(0x4, outputBuffer.As()[0x100]); + EXPECT_EQ(0x5, outputBuffer.As()[0x140]); + } else { + EXPECT_EQ(0x6, outputBuffer.As()[0x40]); + EXPECT_EQ(0x7, outputBuffer.As()[0x80]); + EXPECT_EQ(0x8, outputBuffer.As()[0xc0]); + EXPECT_EQ(0x9, outputBuffer.As()[0x100]); + EXPECT_EQ(0xa, outputBuffer.As()[0x140]); + } + + /* + * Guide to results: + * 0x99: acquiring shader did not write to output buffer at all + * 0x77: coherency error. Either releasing shader did not write or acquiring shader read stale value + * All five EXPECT_EQ fail: error occurs even when releasing shader bypasses cache + * Only first four EXPECT_EQ fail: error occurs only when releasing shader uses cache + */ + + /* Clean up */ + EXPECT_SUCCESS(acquireQueue.Destroy()); + EXPECT_SUCCESS(releaseQueue.Destroy()); + EXPECT_SUCCESS(flushQueue.Destroy()); +} + +/* A test of the memory coherence features on Aqua_Vanjaram. + * One shader stores values at 5 positions in memory, then performs + * a write-release. The other shader performs a read-acquire, then loads + * those 5 values, then stores them in a CPU-visible buffer + * + * withinGPU: When true, the two shaders will be loaded onto two nodes within + * the same GPU. When false, the two shaders will be loaded onto different + * GPUs. + * + * localToRemote: When true, the shared memory will be local to the releasing node. + * When false, the shared memory will be local to the acquiring node. + * + * scalar: When true, the shared data will be stored and loaded with scalar instructions. + * When false, the shared data will be stored and loaded with vector instructions. + */ +void KFDMemoryTest::AcquireReleaseTest(bool withinGPU, bool localToRemote, bool scalar) { + + if (m_FamilyId != FAMILY_AV) { + LOG() << "Skipping test: Test requires aqua vanjaram series asics." << std::endl; + return; + } + + /* Find second node - nodes with the same DrmRenderMinor are on the same GPU */ + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + HSAuint32 acquireNode; + HSAint32 acquireDRM; + bool foundSecondNode = false; + for (unsigned i = 0; i < gpuNodes.size(); i++) { + acquireNode = gpuNodes.at(i); + acquireDRM = m_NodeInfo.GetNodeProperties(acquireNode)->DrmRenderMinor; + for (unsigned j = 0; j < gpuNodes.size(); j++) { + if (!withinGPU) { + if (m_NodeInfo.GetNodeProperties(gpuNodes.at(j))->DrmRenderMinor != acquireDRM) { + foundSecondNode = true; + AcquireReleaseTestRun(acquireNode, gpuNodes.at(j), localToRemote, scalar); + } + } else { + if (m_NodeInfo.GetNodeProperties(gpuNodes.at(j))->DrmRenderMinor == acquireDRM && gpuNodes.at(j) != acquireNode) { + foundSecondNode = true; + AcquireReleaseTestRun(acquireNode, gpuNodes.at(j), localToRemote, scalar); + } + } + } + } + if (!foundSecondNode) { + if (!withinGPU) { + LOG() << "Skipping test: At least two GPUs are required." << std::endl; + } else { + LOG() << "Skipping test: At least two nodes on the same GPU are required." << std::endl; + } + + } +} + +TEST_F(KFDMemoryTest, AcquireReleaseCPU) { + if (m_FamilyId != FAMILY_AV) { + LOG() << "Skipping test: Test requires aqua vanjaram series asics." << std::endl; + return; + } + + /* Find second node - nodes with the same DrmRenderMinor are on the same GPU */ + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + HSAuint32 acquireNode; + for (unsigned i = 0; i < gpuNodes.size(); i++) { + acquireNode = gpuNodes.at(i); + AcquireReleaseTestRunCPU(acquireNode, true); + AcquireReleaseTestRunCPU(acquireNode, false); + } +} + + +TEST_F(KFDMemoryTest, AcquireReleaseFarLocalVector) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(false /* multi-GPU */, false /* acquirer is local */, false /* vector */); + + TEST_END +} + +TEST_F(KFDMemoryTest, AcquireReleaseFarLocalScalar) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(false /* multi-GPU */, false /* acquirer is local */, true /* scalar */); + + TEST_END +} + +TEST_F(KFDMemoryTest, AcquireReleaseFarRemoteVector) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(false /* multi-GPU */, true /* releaser is local */, false /* vector */); + + TEST_END +} + +TEST_F(KFDMemoryTest, AcquireReleaseFarRemoteScalar) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(false /* multi-GPU */, true /* releaser is local */, true /* scalar */); + + TEST_END +} + +TEST_F(KFDMemoryTest, AcquireReleaseCloseLocalVector) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(true /* within-GPU */, false /* acquirer is local */, false /* vector */); + + TEST_END +} + +TEST_F(KFDMemoryTest, AcquireReleaseCloseLocalScalar) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(true /* within-GPU */, false /* acquirer is local */, true /* scalar */); + + TEST_END +} + +TEST_F(KFDMemoryTest, AcquireReleaseCloseRemoteVector) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(true /* within-GPU */, true /* releaser is local */, false /* vector */); + + TEST_END +} + +TEST_F(KFDMemoryTest, AcquireReleaseCloseRemoteScalar) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + AcquireReleaseTest(true /* within-GPU */, true /* releaser is local */, true /* scalar */); + + TEST_END +} + + /* Application register same userptr to multiple GPUs using multiple threads * Test multiple threads register/deregister same userptr, to verify Thunk race handling */ diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp index 03149e5639..3417a4eb30 100644 --- a/tests/kfdtest/src/KFDMemoryTest.hpp +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -42,6 +42,10 @@ class KFDMemoryTest : public KFDBaseComponentTest { void BinarySearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, HSAuint64 highMB, int nodeToMap, HSAuint64 *lastSizeMB); + void AcquireReleaseTestRunCPU(HSAuint32 acquireNode, bool scalar); + void AcquireReleaseTestRun(HSAuint32 acquireNode, HSAuint32 releaseNode, + bool localToRemote, bool scalar); + void AcquireReleaseTest(bool withinGPU, bool localToRemote, bool scalar); }; #endif // __KFD_MEMORY_TEST__H__ diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 705aa09711..64726a68dd 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -400,6 +400,215 @@ const char *WriteAndSignalIsa = s_endpgm )"; +/* Input: + * s[0:1], A buffer of at least 64 * 6 bytes + * + * Store the value 0x77 at the 5 addresses 0x40, + * 0x80, ..., 0x140 in the buffer + * + * Aqua Vanjaram only + */ +const char *FlushBufferForAcquireReleaseIsa = + SHADER_START + R"( + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + s_mov_b32 s11, 0x77 + s_mov_b32 s12, 0x0 + // Store some data on 5 different cache lines + s_store_dword s12, s[0:1], 0x0 glc + s_store_dword s11, s[0:1], 0x40 glc + s_store_dword s11, s[0:1], 0x80 glc + s_store_dword s11, s[0:1], 0xc0 glc + s_store_dword s11, s[0:1], 0x100 glc + s_store_dword s11, s[0:1], 0x140 glc + s_waitcnt lgkmcnt(0) + .endif + s_endpgm +)"; + +/* Input: + * s[0:1], A buffer of at least 64 * 6 bytes, + * shared with the acquiring shader + * + * Store the values 1 - 5 at the 5 addresses 0x40, + * 0x80, ..., 0x140 in the buffer, then signal + * the flag at address 0x0 in the buffer. + * + * Uses vector stores + * + * Aqua Vanjaram only + */ +const char *WriteReleaseVectorIsa = + SHADER_START + R"( + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + v_mov_b32 v11, 0x1 + v_mov_b32 v12, 0x2 + v_mov_b32 v13, 0x3 + v_mov_b32 v14, 0x4 + v_mov_b32 v15, 0x5 + v_mov_b32 v21, 0x40 + v_mov_b32 v22, 0x80 + v_mov_b32 v23, 0xc0 + v_mov_b32 v24, 0x100 + v_mov_b32 v25, 0x140 + // Store some data on 5 different cache lines + global_store_dword v21, v11, s[0:1] + global_store_dword v22, v12, s[0:1] + global_store_dword v23, v13, s[0:1] + global_store_dword v24, v14, s[0:1] + global_store_dword v25, v15, s[0:1] nt sc1 sc0 + s_waitcnt vmcnt(0) + // Write-Release + s_mov_b32 s16, 0x1 + buffer_wbl2 sc1 sc0 + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s16, s[0:1], 0x0 glc + .endif + s_endpgm +)"; + +/* Input: + * s[0:1], A buffer of at least 64 * 6 bytes, + * shared with the acquiring shader + * + * Store the values 6 - 10 at the 5 addresses 0x40, + * 0x80, ..., 0x140 in the buffer, then signal + * the flag at address 0x0 in the buffer. + * + * Uses scalar stores + * + * Aqua Vanjaram only + */ +const char *WriteReleaseScalarIsa = + SHADER_START + R"( + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + s_mov_b32 s11, 0x6 + s_mov_b32 s12, 0x7 + s_mov_b32 s13, 0x8 + s_mov_b32 s14, 0x9 + s_mov_b32 s15, 0xa + // Store some data on 5 different cache lines + s_store_dword s11, s[0:1], 0x40 + s_store_dword s12, s[0:1], 0x80 + s_store_dword s13, s[0:1], 0xc0 + s_store_dword s14, s[0:1], 0x100 + s_store_dword s15, s[0:1], 0x140 glc + s_waitcnt lgkmcnt(0) + // Write-Release + s_dcache_wb // WB Scalar L1 cache + s_mov_b32 s16, 0x1 + buffer_wbl2 sc1 sc0 + s_waitcnt vmcnt(0) & lgkmcnt(0) + s_store_dword s16, s[0:1], 0x0 glc + s_waitcnt lgkmcnt(0) + .endif + s_endpgm +)"; + +/* Input: + * s[0:1], A buffer of at least 64 * 6 bytes, + * shared with the releasing shader + * s[2:3], A buffer of at least 64 * 6 bytes, + * accessible by the CPU, used for output + * + * Polls the flag at address 0x0 in the shared buffer. + * When the signal is received, read the values + * at the 5 addresses 0x40, 0x80, ... 0x140, + * and store them at the same locations in + * the output buffer + * + * Uses vector loads + * + * Aqua Vanjaram only + */ +const char *ReadAcquireVectorIsa = + SHADER_START + R"( + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + // Read-Acquire + s_mov_b32 s18, 0x1 + LOOP: + s_load_dword s17, s[0:1], 0x0 glc + s_waitcnt lgkmcnt(0) + s_cmp_eq_i32 s17, s18 + s_cbranch_scc0 LOOP + buffer_inv sc1 sc0 + // Load data + v_mov_b32 v21, 0x40 + v_mov_b32 v22, 0x80 + v_mov_b32 v23, 0xc0 + v_mov_b32 v24, 0x100 + v_mov_b32 v25, 0x140 + global_load_dword v11, v21, s[0:1] + global_load_dword v12, v22, s[0:1] + global_load_dword v13, v23, s[0:1] + global_load_dword v14, v24, s[0:1] + global_load_dword v15, v25, s[0:1] + s_waitcnt vmcnt(0) + // Store data for output + v_mov_b32 v21, 0x40 + v_mov_b32 v22, 0x80 + v_mov_b32 v23, 0xc0 + v_mov_b32 v24, 0x100 + v_mov_b32 v25, 0x140 + global_store_dword v21, v11, s[2:3] nt sc1 sc0 + global_store_dword v22, v12, s[2:3] nt sc1 sc0 + global_store_dword v23, v13, s[2:3] nt sc1 sc0 + global_store_dword v24, v14, s[2:3] nt sc1 sc0 + global_store_dword v25, v15, s[2:3] nt sc1 sc0 + s_waitcnt vmcnt(0) + .endif + s_endpgm +)"; + +/* Input: + * s[0:1], A buffer of at least 64 * 6 bytes, + * shared with the releasing shader + * s[2:3], A buffer of at least 64 * 6 bytes, + * accessible by the CPU, used for output + * + * Polls the flag at address 0x0 in the shared buffer. + * When the signal is received, read the values + * at the 5 addresses 0x40, 0x80, ... 0x140, + * and store them at the same locations in + * the output buffer + * + * Uses scalar loads + * + * Aqua Vanjaram only + */ +const char *ReadAcquireScalarIsa = + SHADER_START + R"( + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + // Read-Acquire + s_mov_b32 s18, 0x1 + LOOP: + s_load_dword s17, s[0:1], 0x0 glc + s_waitcnt lgkmcnt(0) + s_cmp_eq_i32 s17, s18 + s_cbranch_scc0 LOOP + buffer_inv sc1 sc0 + // Load data + s_load_dword s21, s[0:1], 0x40 + s_load_dword s22, s[0:1], 0x80 + s_load_dword s23, s[0:1], 0xc0 + s_load_dword s24, s[0:1], 0x100 + s_load_dword s25, s[0:1], 0x140 + s_waitcnt lgkmcnt(0) + // Store data for output + s_store_dword s21, s[2:3], 0x40 glc + s_store_dword s22, s[2:3], 0x80 glc + s_store_dword s23, s[2:3], 0xc0 glc + s_store_dword s24, s[2:3], 0x100 glc + s_store_dword s25, s[2:3], 0x140 glc + s_waitcnt lgkmcnt(0) + .endif + s_endpgm +)"; + /** * KFDQMTest */ diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index c655030f9b..3964a5b397 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -43,6 +43,11 @@ extern const char *CopyOnSignalIsa; extern const char *PollAndCopyIsa; extern const char *WriteFlagAndValueIsa; extern const char *WriteAndSignalIsa; +extern const char *WriteReleaseVectorIsa; +extern const char *WriteReleaseScalarIsa; +extern const char *ReadAcquireVectorIsa; +extern const char *ReadAcquireScalarIsa; +extern const char *FlushBufferForAcquireReleaseIsa; /* KFDQMTest */ extern const char *LoopIsa; From eed5518e4c2bb27f80b11ac9642b1a3a3933b135 Mon Sep 17 00:00:00 2001 From: David Francis Date: Thu, 5 Jan 2023 14:31:57 -0500 Subject: [PATCH 1133/1247] kfdtest: Enable gfx90a coherency tests on Aqua Vanjaram These tests should also pass on Aqua Vanjaram, so enable them Signed-off-by: David Francis Signed-off-by: Graham Sider Change-Id: Ibbb9cd43d653c63b08c39efd1d7326cfac1f8411 --- tests/kfdtest/src/KFDMemoryTest.cpp | 12 +++++------ tests/kfdtest/src/ShaderStore.cpp | 32 +++++++++++++++++++---------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 964e6dce12..c161431d02 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2191,8 +2191,8 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithRemoteGPU) { const int dwSource = 0x40 * sizeof(int); /* At 3rd cache line */ const int dwLocation = 0x80 * sizeof(int); /* At 5th cache line */ - if (m_FamilyId != FAMILY_AL) { - LOG() << "Skipping test: Test requires aldebaran series asics." << std::endl; + if (m_FamilyId != FAMILY_AL && m_FamilyId != FAMILY_AV) { + LOG() << "Skipping test: Test requires aldebaran or aqua vanjaram series asics." << std::endl; return; } @@ -2271,8 +2271,8 @@ TEST_F(KFDMemoryTest, VramCacheCoherenceWithCPU) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); - if (m_FamilyId != FAMILY_AL) { - LOG() << "Skipping test: Test requires aldebaran series asics." << std::endl; + if (m_FamilyId != FAMILY_AL && m_FamilyId != FAMILY_AV) { + LOG() << "Skipping test: Test requires aldebaran or aqua vanjaram series asics." << std::endl; return; } @@ -2335,8 +2335,8 @@ TEST_F(KFDMemoryTest, SramCacheCoherenceWithGPU) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); - if (m_FamilyId != FAMILY_AL) { - LOG() << "Skipping test: Test requires aldebaran series asics." << std::endl; + if (m_FamilyId != FAMILY_AL && m_FamilyId != FAMILY_AV) { + LOG() << "Skipping test: Test requires aldebaran or aqua vanjaram series asics." << std::endl; return; } diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 64726a68dd..604ebb8e24 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -305,19 +305,26 @@ const char *CopyOnSignalIsa = */ const char *PollAndCopyIsa = SHADER_START + SHADER_MACROS_FLAT R"( // Assume src buffer in s[0:1] and dst buffer in s[2:3] - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_stepping == 10) - // Path for Aldebaran + // Path for Aldebaran, Aqua Vanjaram + .if (.amdgcn.gfx_generation_number == 9 && + (.amdgcn.gfx_generation_minor == 4 || + .amdgcn.gfx_generation_stepping == 10)) v_mov_b32 v0, s0 v_mov_b32 v1, s1 v_mov_b32 v18, 0x1 - LOOP_ALDBRN: - flat_load_dword v16, v[0:1] glc + LOOP0: + FLAT_LOAD_DWORD_NSS v16, v[0:1] glc s_waitcnt vmcnt(0) & lgkmcnt(0) v_cmp_eq_i32 vcc, v16, v18 - s_cbranch_vccz LOOP_ALDBRN - buffer_invl2 + s_cbranch_vccz LOOP0 + .if (.amdgcn.gfx_generation_minor == 4) + buffer_invl2 sc1 sc0 + .else + buffer_invl2 + .endif s_load_dword s17, s[0:1], 0x4 glc s_waitcnt vmcnt(0) & lgkmcnt(0) s_store_dword s17, s[2:3], 0x0 glc @@ -325,10 +332,10 @@ const char *PollAndCopyIsa = buffer_wbl2 .elseif (.amdgcn.gfx_generation_number == 9) s_movk_i32 s18, 0x1 - LOOP: + LOOP1: s_load_dword s16, s[0:1], 0x0 glc s_cmp_eq_i32 s16, s18 - s_cbranch_scc0 LOOP + s_cbranch_scc0 LOOP1 s_load_dword s17, s[0:1], 0x4 glc s_waitcnt vmcnt(0) & lgkmcnt(0) s_store_dword s17, s[2:3], 0x0 glc @@ -344,13 +351,16 @@ const char *PollAndCopyIsa = * Input1: A buffer of at least 2 dwords. * DW0: used as the value to be written. * - * Note: Only works on Aldebaran + * Note: Only works on Aldebaran and Aqua Vanjaram */ const char *WriteFlagAndValueIsa = SHADER_START + SHADER_MACROS_FLAT R"( // Assume two inputs buffer in s[0:1] and s[2:3] - .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_stepping == 10) + .if (.amdgcn.gfx_generation_number == 9 && + (.amdgcn.gfx_generation_minor == 4 || + .amdgcn.gfx_generation_stepping == 10)) v_mov_b32 v0, s0 v_mov_b32 v1, s1 s_load_dword s18, s[2:3], 0x0 glc @@ -360,7 +370,7 @@ const char *WriteFlagAndValueIsa = buffer_wbl2 s_waitcnt vmcnt(0) & lgkmcnt(0) v_mov_b32 v16, 0x1 - flat_store_dword v[0:1], v16 glc + FLAT_STORE_DWORD_NSS v[0:1], v16 glc .endif s_endpgm )"; From 77761836aece988c60ee945e7b9309374e04a687 Mon Sep 17 00:00:00 2001 From: xinhui pan Date: Wed, 19 Apr 2023 10:57:37 +0800 Subject: [PATCH 1134/1247] thunk: Fix and optimise for pointer range search Previous code might fail to get the correct ln node. And trigger extra walk through of the tree. Fix it. While walking through the tree, better to search from right to left as the node->start likely close to *address*. Change-Id: If86ddf73e59a1eb88225d1ea90797818e8165488 Signed-off-by: xinhui pan --- src/fmm.c | 48 ++++++++++++++++++++++++++---------------------- src/rbtree.c | 28 ++++++++++++++++++++++++++++ src/rbtree.h | 2 ++ 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ff02fa9394..2f088d2e51 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -470,17 +470,34 @@ static vm_object_t *vm_find_object_by_address_userptr_range(manageable_aperture_ vm_object_t *cur = NULL; rbtree_t *tree = vm_object_tree(app, is_userptr); rbtree_key_t key = rbtree_key((unsigned long)address, 0); - rbtree_node_t *ln = rbtree_lookup_nearest(tree, &key, - LKP_ALL, LEFT); - rbtree_node_t *rn = rbtree_lookup_nearest(tree, &key, - LKP_ALL, RIGHT); + rbtree_node_t *rn = rbtree_lookup_nearest(tree, &key, LKP_ALL, RIGHT); + rbtree_node_t *ln; void *start; uint64_t size; - int bad = 0; -loop: - while (ln) { - cur = vm_object_entry(ln, is_userptr); + /* all nodes might sit on left side of *address*, in this case rn is NULL. + * So pick up the rightest one as rn. + */ + if (!rn) + rn = rbtree_min_max(tree, RIGHT); + + if (is_userptr) { + /* userptr might overlap. Need walk through the tree from right to left as only left nodes + * can obtain the *address* + */ + ln = rbtree_min_max(tree, LEFT); + } else { + /* if key->size is -1, it match the node with start <= address. + * if key->size is 0, it match the node with start < address. + */ + key = rbtree_key((unsigned long)address, -1); + ln = rbtree_lookup_nearest(tree, &key, LKP_ALL, LEFT); + } + if (!ln) + return NULL; + + while (rn) { + cur = vm_object_entry(rn, is_userptr); if (is_userptr == 0) { start = cur->start; size = cur->size; @@ -498,20 +515,7 @@ loop: if (ln == rn) break; - ln = rbtree_next(tree, ln); - } - - if (cur == NULL && bad == 0) { - /* As there is area overlap, say, (address, size) like - * (0x100, 32), (0x108, 8), and the key.address is 0x118 - * The lookup above only find (0x108, 8), but the correct node should - * be (0x100, 16). So try to walk though the tree to find the node. - */ - rn = ln; - key = rbtree_key(0, 0); - ln = rbtree_lookup_nearest(tree, &key, LKP_ALL, RIGHT); - bad = 1; - goto loop; + rn = rbtree_prev(tree, rn); } return cur; /* NULL if not found */ diff --git a/src/rbtree.c b/src/rbtree.c index 86e2d59f7b..21d0d70d5a 100644 --- a/src/rbtree.c +++ b/src/rbtree.c @@ -372,3 +372,31 @@ rbtree_next(rbtree_t *tree, rbtree_node_t *node) node = parent; } } + +rbtree_node_t * +rbtree_prev(rbtree_t *tree, rbtree_node_t *node) +{ + rbtree_node_t *root, *sentinel, *parent; + + sentinel = &tree->sentinel; + + if (node->left != sentinel) { + return rbtree_max(node->left, sentinel); + } + + root = tree->root; + + for ( ;; ) { + parent = node->parent; + + if (node == root) { + return NULL; + } + + if (node == parent->right) { + return parent; + } + + node = parent; + } +} diff --git a/src/rbtree.h b/src/rbtree.h index 965c8fe773..fa13959369 100644 --- a/src/rbtree.h +++ b/src/rbtree.h @@ -64,6 +64,8 @@ struct rbtree_s { void rbtree_insert(rbtree_t *tree, rbtree_node_t *node); void rbtree_delete(rbtree_t *tree, rbtree_node_t *node); +rbtree_node_t *rbtree_prev(rbtree_t *tree, + rbtree_node_t *node); rbtree_node_t *rbtree_next(rbtree_t *tree, rbtree_node_t *node); From 92f3d4a4580e96130233bc27959032058a8213c8 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 26 Apr 2023 11:53:40 -0400 Subject: [PATCH 1135/1247] kfdtest: Fix new shader directives LLVM MC does not seem to accept multi-line conditionals. This may be fixable in the future with macros. The Aqua Vanjaram shader spec states that while buffer_invl2 has been replaced by buffer_inv, the former may still be used for compatibility. However, this does not seem to be implemented. For now, fix conditional. Signed-off-by: Graham Sider Change-Id: I7f8b64c96055371d7e0090b758d2cfd2a37ecd3c --- tests/kfdtest/src/ShaderStore.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 604ebb8e24..58a2f6674e 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -309,9 +309,7 @@ const char *PollAndCopyIsa = R"( // Assume src buffer in s[0:1] and dst buffer in s[2:3] // Path for Aldebaran, Aqua Vanjaram - .if (.amdgcn.gfx_generation_number == 9 && - (.amdgcn.gfx_generation_minor == 4 || - .amdgcn.gfx_generation_stepping == 10)) + .if (.amdgcn.gfx_generation_number == 9 && (.amdgcn.gfx_generation_minor == 4 || .amdgcn.gfx_generation_stepping == 10)) v_mov_b32 v0, s0 v_mov_b32 v1, s1 v_mov_b32 v18, 0x1 @@ -321,7 +319,7 @@ const char *PollAndCopyIsa = v_cmp_eq_i32 vcc, v16, v18 s_cbranch_vccz LOOP0 .if (.amdgcn.gfx_generation_minor == 4) - buffer_invl2 sc1 sc0 + buffer_inv sc1 sc0 .else buffer_invl2 .endif @@ -358,9 +356,7 @@ const char *WriteFlagAndValueIsa = SHADER_MACROS_FLAT R"( // Assume two inputs buffer in s[0:1] and s[2:3] - .if (.amdgcn.gfx_generation_number == 9 && - (.amdgcn.gfx_generation_minor == 4 || - .amdgcn.gfx_generation_stepping == 10)) + .if (.amdgcn.gfx_generation_number == 9 && (.amdgcn.gfx_generation_minor == 4 || .amdgcn.gfx_generation_stepping == 10)) v_mov_b32 v0, s0 v_mov_b32 v1, s1 s_load_dword s18, s[2:3], 0x0 glc From 54136f60a094589a233d2984ca8405d1d53c7b67 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 18 Apr 2023 10:48:02 -0400 Subject: [PATCH 1136/1247] kfdtest: Add Assembler::RunAssembleBuf overload Overload Assembler::RunAssembleBuf to take in an extra Gfxv parameter. Using this overload will temporarily set the target ASIC to Gfxv before calling RunAssemble, and copy back the original MCPU literal upon completion. The copy to reset the original MCPU in this case is safe as the MCPU length is always known. This will be useful in multi-device test cases whereby the devices are not necessarily the same gfx version. The overload is explicitly for the RunAssembleBuf wrapper rather than RunAssemble to ensure the default MCPU is always reset independent of errors in RunAssemble. Signed-off-by: Graham Sider Change-Id: I7fe5a962876314b6df32e4b7160174949d98f9e3 --- tests/kfdtest/src/Assemble.cpp | 9 +++++++++ tests/kfdtest/src/Assemble.hpp | 2 ++ 2 files changed, 11 insertions(+) diff --git a/tests/kfdtest/src/Assemble.cpp b/tests/kfdtest/src/Assemble.cpp index cf4b9e7de0..2077375ea2 100644 --- a/tests/kfdtest/src/Assemble.cpp +++ b/tests/kfdtest/src/Assemble.cpp @@ -224,6 +224,7 @@ int Assembler::ExtractELFText(const char* RawData) { * @param AssemblySource Shader source represented as a raw C string * @param OutBuf Raw instruction stream output buffer * @param BufSize Size of OutBuf (defaults to PAGE_SIZE) + * @param Gfxv Optional overload to temporarily set target ASIC * @return Value of RunAssemble() (0 on success) */ int Assembler::RunAssembleBuf(const char* const AssemblySource, char* OutBuf, @@ -231,6 +232,14 @@ int Assembler::RunAssembleBuf(const char* const AssemblySource, char* OutBuf, int ret = RunAssemble(AssemblySource); return ret ? ret : CopyInstrStream(OutBuf, BufSize); } +int Assembler::RunAssembleBuf(const char* const AssemblySource, char* OutBuf, + const size_t BufSize, const uint32_t Gfxv) { + const char* defaultMCPU = GetTargetAsic(); + SetTargetAsic(Gfxv); + int ret = RunAssemble(AssemblySource); + strncpy(MCPU, defaultMCPU, ASM_MCPU_LEN); + return ret ? ret : CopyInstrStream(OutBuf, BufSize); +} /** * Assemble shader and fill member vars diff --git a/tests/kfdtest/src/Assemble.hpp b/tests/kfdtest/src/Assemble.hpp index 46fb946a84..a519129ed8 100644 --- a/tests/kfdtest/src/Assemble.hpp +++ b/tests/kfdtest/src/Assemble.hpp @@ -81,6 +81,8 @@ class Assembler { int RunAssemble(const char* const AssemblySource); int RunAssembleBuf(const char* const AssemblySource, char* OutBuf, const size_t BufSize = PAGE_SIZE); + int RunAssembleBuf(const char* const AssemblySource, char* OutBuf, + const size_t BufSize, const uint32_t Gfxv); }; #endif // _ASSEMBLE_H_ From 5d6f900353d16b5e0c0c5ed9b9ee90f6180e22f3 Mon Sep 17 00:00:00 2001 From: Shane Xiao Date: Tue, 9 May 2023 16:36:55 +0800 Subject: [PATCH 1137/1247] kfdtest: DeviceHdpFlush need set target ASIC with different Gfx versions If Dev0 and Dev1 are not the same gfx, we should temporarily set the target ASIC for compiling Shader code. Signed-off-by: Shane Xiao Signed-off-by: Shikai Guo Change-Id: I5836beb16ade519f5a148d3d2b9c2875554f0c35 --- tests/kfdtest/src/KFDMemoryTest.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index c161431d02..07f9803437 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -1908,13 +1908,19 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { } } + const HsaNodeProperties *pNodePropertiesDev1 = NULL; + unsigned int m_FamilyIdDev1 = 0; + pNodeProperties = m_NodeInfo.GetNodeProperties(nodes[0]); - if (!pNodeProperties) { + pNodePropertiesDev1 = m_NodeInfo.GetNodeProperties(nodes[1]); + if (!pNodeProperties || !pNodePropertiesDev1) { LOG() << "Failed to get gpu node properties." << std::endl; return; } - if (m_FamilyId < FAMILY_AI) { + m_FamilyIdDev1 = FamilyIdFromNode(pNodePropertiesDev1); + + if (m_FamilyId < FAMILY_AI || m_FamilyIdDev1 < FAMILY_AI) { LOG() << "Skipping test: Test requires gfx9 and later asics." << std::endl; return; } @@ -1969,7 +1975,9 @@ TEST_F(KFDMemoryTest, DeviceHdpFlush) { ASSERT_SUCCESS(queue0.Create(nodes[1])); HsaMemoryBuffer isaBuffer0(PAGE_SIZE, nodes[1], true/*zero*/, false/*local*/, true/*exec*/); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteAndSignalIsa, isaBuffer0.As())); + /* Temporarily set target ASIC for Dev1 */ + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WriteAndSignalIsa, isaBuffer0.As(), + PAGE_SIZE, GetGfxVersion(pNodePropertiesDev1))); Dispatch dispatch0(isaBuffer0); dispatch0.SetArgs(buffer, mmioBase); From b487f8736390c373690e5e469ef2fec29c805084 Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Mon, 8 May 2023 16:32:52 -0700 Subject: [PATCH 1138/1247] Set the default value of ROCM_HEADER_WRAPPER_WERROR to OFF Using wrapper header files will result in #warning message by default Change-Id: I8301e433d39f3e5d39384ede6f0e4464d0eb20a6 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e7a9a361be..c4b5906056 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,7 +223,7 @@ if(INCLUDE_PATH_COMPATIBILITY) set(ROCM_HEADER_WRAPPER_WERROR "$ENV{ROCM_HEADER_WRAPPER_WERROR}" CACHE STRING "Header wrapper warnings as errors.") else() - set(ROCM_HEADER_WRAPPER_WERROR "ON" CACHE STRING "Header wrapper warnings as errors.") + set(ROCM_HEADER_WRAPPER_WERROR "OFF" CACHE STRING "Header wrapper warnings as errors.") endif() endif() if(ROCM_HEADER_WRAPPER_WERROR) From 53ed978c3d52b62eacbd64867cf2719b0f6651a8 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 23 Feb 2023 13:51:51 +0800 Subject: [PATCH 1139/1247] kfdtest: add non paged wptr judging API. If MES is enabled, wptr has to be non paged memory, Add an API to check this condition. Change-Id: I53af1f6687d5332d102e7062c3d760e33b96e722 Signed-off-by: Yifan Zhang --- tests/kfdtest/src/KFDBaseComponentTest.cpp | 5 +++++ tests/kfdtest/src/KFDBaseComponentTest.hpp | 1 + 2 files changed, 6 insertions(+) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index bb510e040b..2c0625972e 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -164,6 +164,11 @@ unsigned int KFDBaseComponentTest::GetFamilyIdFromNodeId(unsigned int nodeId) return FamilyIdFromNode(m_NodeInfo.GetNodeProperties(nodeId)); } +bool KFDBaseComponentTest::NeedNonPagedWptr(unsigned int nodeId) +{ + return GetFamilyIdFromNodeId(nodeId) >= FAMILY_GFX11; +} + int KFDBaseComponentTest::FindDRMRenderNode(int gpuNode) { HsaNodeProperties *nodeProperties; _HSAKMT_STATUS status; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index cc87465ad5..8f714d54b3 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -58,6 +58,7 @@ class KFDBaseComponentTest : public testing::Test { // @return DRM Render Node if successful or -1 on failure int FindDRMRenderNode(int gpuNode); unsigned int GetFamilyIdFromNodeId(unsigned int nodeId); + bool NeedNonPagedWptr(unsigned int nodeId); unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; } // @brief Executed before the first test that uses KFDBaseComponentTest. From d319660838b3ef25235fc28e259b67dbe0f38730 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 23 Feb 2023 14:08:21 +0800 Subject: [PATCH 1140/1247] =?UTF-8?q?kfdtest:=20Using=20non-paged=20memory?= =?UTF-8?q?=20allocation=20for=20wptr=20on=20devices=20that=C2=A0have=20ME?= =?UTF-8?q?S=20scheduler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work on unmapped queues for usermode queue oversubscription (no aggregated doorbell) Change-Id: I10e30fdc2bec587cef9427faa4874957988c34b3 Signed-off-by: Yifan Zhang --- tests/kfdtest/src/KFDQMTest.cpp | 2 +- tests/kfdtest/src/KFDTestUtil.cpp | 4 ++-- tests/kfdtest/src/KFDTestUtil.hpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 2f156ad23b..ee01300e4c 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1174,7 +1174,7 @@ TEST_F(KFDQMTest, CreateAqlCpQueue) { int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - HsaMemoryBuffer pointers(PAGE_SIZE, defaultGPUNode); + HsaMemoryBuffer pointers(PAGE_SIZE, defaultGPUNode, /*zero*/true, /*local*/false, /*exec*/false, /*isScratch */false, /* isReadOnly */false, /* isUncached */false, /* NonPaged */g_baseTest->NeedNonPagedWptr(defaultGPUNode)); ASSERT_SUCCESS(queue.Create(defaultGPUNode, PAGE_SIZE, pointers.As())); diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 98c90f7b19..d02804eb09 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -255,7 +255,7 @@ HSAuint64 GetSystemTickCountInMicroSec() { const HsaMemoryBuffer HsaMemoryBuffer::Null; HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, bool isLocal, bool isExec, - bool isScratch, bool isReadOnly, bool isUncached) + bool isScratch, bool isReadOnly, bool isUncached, bool NonPaged) :m_Size(size), m_pUser(NULL), m_pBuf(NULL), @@ -279,7 +279,7 @@ HsaMemoryBuffer::HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero, b EXPECT_EQ(isUncached, 0) << "Uncached flag is relevant only for system or host memory"; } else { m_Flags.ui32.HostAccess = 1; - m_Flags.ui32.NonPaged = 0; + m_Flags.ui32.NonPaged = NonPaged ? 1 : 0; m_Flags.ui32.CoarseGrain = 0; m_Flags.ui32.NoNUMABind = 1; m_Flags.ui32.Uncached = isUncached; diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 756b024e90..534a74a456 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -68,7 +68,7 @@ class HsaMemoryBuffer { public: HsaMemoryBuffer(HSAuint64 size, unsigned int node, bool zero = true, bool isLocal = false, - bool isExec = false, bool isScratch = false, bool isReadOnly = false, bool isUncached = false); + bool isExec = false, bool isScratch = false, bool isReadOnly = false, bool isUncached = false, bool NonPaged = false); HsaMemoryBuffer(void *addr, HSAuint64 size); template RetType As() { From d96624378397227b3c1179e486e9607173378f70 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 2 May 2023 15:49:46 -0400 Subject: [PATCH 1141/1247] kfdtest: Add include directory for ROCr merge When we merge thunk into ROCr, kfdtest will be in a different folder structure. Add the new location to ensure that we can build now and in the future with no disruptions Signed-off-by: Kent Russell Change-Id: I6517e061cb0da7137d903abbc380bfc7126f40d4 --- tests/kfdtest/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 61c9b84f60..1ec430da20 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -159,6 +159,7 @@ llvm_map_components_to_libnames(llvm_libs AMDGPUAsmParser Core Support) include_directories(${PROJECT_SOURCE_DIR}/gtest-1.6.0) include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(${PROJECT_SOURCE_DIR}/../../include) +include_directories(${PROJECT_SOURCE_DIR}/../../libhsakmt/include) include_directories(${DRM_INCLUDE_DIRS}) From 1e6d728730b6159987cb8e70f424e6be9ca9252d Mon Sep 17 00:00:00 2001 From: Bing Ma Date: Mon, 1 May 2023 15:20:21 -0700 Subject: [PATCH 1142/1247] libhsakmt: Add support functions for ASAN Add support functions to remap the first page of device memory (GPU/GTT) to share host ASAN logic. Signed-off-by: David Yat Sin Change-Id: I4c27d5417ba80a172dccb0a079a597c5dc1c8f85 --- include/hsakmt.h | 24 ++++++++++++++ src/fmm.c | 83 +++++++++++++++++++++++++++++++++++++++++++++++- src/fmm.h | 4 +++ src/libhsakmt.h | 6 ++++ src/memory.c | 24 ++++++++++++++ 5 files changed, 140 insertions(+), 1 deletion(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 63ac299511..63e6312818 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -913,6 +913,30 @@ hsaKmtOpenSMI( int *fd // OUT: anonymous file handle ); +/** + If this is GPU Mapped memory, remap the first page at this address to be normal system memory + + This is used in ASAN mode to remap the first page of device memory to share host ASAN logic. + This function is only supported when libhsakmt is compiled in ASAN mode. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtReplaceAsanHeaderPage( + void *addr // IN: Start of othe virtual address page +); + +/** + If this is GPU Mapped memory, remap the first page back to the original GPU memory + + This is used in ASAN mode to remap the first page back to its original mapping. + This function is only supported when libhsakmt is compiled in ASAN mode. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtReturnAsanHeaderPage( + void *addr // IN: Start of othe virtual address page +); + #ifdef __cplusplus } //extern "C" #endif diff --git a/src/fmm.c b/src/fmm.c index 2f088d2e51..f6c2082373 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -23,9 +23,9 @@ * DEALINGS IN THE SOFTWARE. */ +#include "libhsakmt.h" #include "fmm.h" #include "linux/kfd_ioctl.h" -#include "libhsakmt.h" #include #include #include @@ -106,6 +106,11 @@ struct vm_object { void *user_data; /* Flag to indicate imported KFD buffer */ bool is_imported_kfd_bo; +#ifdef SANITIZER_AMDGPU + int mmap_flags; + int mmap_fd; + off_t mmap_offset; +#endif }; typedef struct vm_object vm_object_t; @@ -340,6 +345,9 @@ static vm_object_t *vm_create_and_init_object(void *start, uint64_t size, object->is_imported_kfd_bo = false; object->node.key = rbtree_key((unsigned long)start, size); object->user_node.key = rbtree_key(0, 0); +#ifdef SANITIZER_AMDGPU + object->mmap_fd = 0; +#endif } return object; @@ -1495,6 +1503,13 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, __fmm_release(vm_obj, aperture); return NULL; } +#ifdef SANITIZER_AMDGPU + if (vm_obj) { + vm_obj->mmap_flags = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; + vm_obj->mmap_fd = gpu_mem[gpu_mem_id].drm_render_fd; + vm_obj->mmap_offset = mmap_offset; + } +#endif } return mem; @@ -1738,6 +1753,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, return NULL; } } + +#ifdef SANITIZER_AMDGPU + if (mem && vm_obj) { + vm_obj->mmap_flags = mflags.ui32.HostAccess ? PROT_READ | PROT_WRITE : PROT_NONE; + vm_obj->mmap_fd = gpu_drm_fd; + vm_obj->mmap_offset = mmap_offset; + } +#endif } if (mem && vm_obj) { @@ -3846,6 +3869,64 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) return ret; } +#ifdef SANITIZER_AMDGPU +HSAKMT_STATUS fmm_replace_asan_header_page(void* address) +{ + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + manageable_aperture_t* aperture; + vm_object_t* vm_obj; + + vm_obj = vm_find_object(address, UINT64_MAX, &aperture); + if (!vm_obj) + return HSAKMT_STATUS_ERROR; + /* Successful vm_find_object returns with the aperture locked */ + + /* If this is a GPU-mapped memory, remap the first page to be normal system memory*/ + if (vm_obj->mmap_fd) { + void* p = mmap(address, + PAGE_SIZE, + PROT_WRITE | PROT_READ, MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED, + -1, + 0); + + if (p == MAP_FAILED) + ret = HSAKMT_STATUS_ERROR; + } + + pthread_mutex_unlock(&aperture->fmm_mutex); + return ret; +} + +HSAKMT_STATUS fmm_return_asan_header_page(void* address) +{ + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + manageable_aperture_t* aperture; + vm_object_t* vm_obj; + + vm_obj = vm_find_object(address, UINT64_MAX, &aperture); + if (!vm_obj) + return HSAKMT_STATUS_ERROR; + /* Successful vm_find_object returns with the aperture locked */ + + /* If this is a GPU-mapped memory, remap the first page back to the original GPU memory*/ + if (vm_obj->mmap_fd) { + off_t mmap_offset = vm_obj->mmap_offset + ((char*)address - (char*)vm_obj->start); + void* p = mmap(address, + PAGE_SIZE, + vm_obj->mmap_flags, + MAP_SHARED | MAP_FIXED, + vm_obj->mmap_fd, + mmap_offset); + + if (p == MAP_FAILED) + ret = HSAKMT_STATUS_ERROR; + } + + pthread_mutex_unlock(&aperture->fmm_mutex); + return ret; +} +#endif + HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data) { manageable_aperture_t *aperture; diff --git a/src/fmm.h b/src/fmm.h index 93aa55fb76..347b9fe27a 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -62,6 +62,10 @@ int fmm_unmap_from_gpu(void *address); bool fmm_get_handle(void *address, uint64_t *handle); HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info); HSAKMT_STATUS fmm_set_mem_user_data(const void *mem, void *usr_data); +#ifdef SANITIZER_AMDGPU +HSAKMT_STATUS fmm_replace_asan_header_page(void* address); +HSAKMT_STATUS fmm_return_asan_header_page(void* address); +#endif /* Topology interface*/ HSAKMT_STATUS fmm_node_added(HSAuint32 gpu_id); diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 85e69f6d07..2d8da2c9c8 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -43,6 +43,12 @@ extern HsaVersionInfo kfd_version_info; #undef HSAKMTAPI #define HSAKMTAPI __attribute__((visibility ("default"))) +#if defined(__clang__) +#if __has_feature(address_sanitizer) +#define SANITIZER_AMDGPU 1 +#endif +#endif + /*Avoid pointer-to-int-cast warning*/ #define PORT_VPTR_TO_UINT64(vptr) ((uint64_t)(unsigned long)(vptr)) diff --git a/src/memory.c b/src/memory.c index 4cb05057ad..4d4cd53ba2 100644 --- a/src/memory.c +++ b/src/memory.c @@ -588,3 +588,27 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryUserData(const void *Pointer, return fmm_set_mem_user_data(Pointer, UserData); } + +HSAKMT_STATUS HSAKMTAPI hsaKmtReplaceAsanHeaderPage(void *addr) +{ +#ifdef SANITIZER_AMDGPU + pr_debug("[%s] address %p\n", __func__, addr); + CHECK_KFD_OPEN(); + + return fmm_replace_asan_header_page(addr); +#else + return HSAKMT_STATUS_NOT_SUPPORTED; +#endif +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtReturnAsanHeaderPage(void *addr) +{ +#ifdef SANITIZER_AMDGPU + pr_debug("[%s] address %p\n", __func__, addr); + CHECK_KFD_OPEN(); + + return fmm_return_asan_header_page(addr); +#else + return HSAKMT_STATUS_NOT_SUPPORTED; +#endif +} \ No newline at end of file From 5df82e3d14be42938efdea865ab65768b77daa0b Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 27 Mar 2023 17:02:45 -0400 Subject: [PATCH 1143/1247] kfdtest: Enable KFDEvictTest and KFDSVMEvictTest on aqua_vanjaram For aqua_vanjaram APU mode, KFDEvictTest and KFDSVMEvictTest are skipped. Those tests passed on dGPU mode with memory reporting partition support on GFX 9.4.3. Change-Id: I56357843c6743b01b807359dbb37b32391fd9a25 Signed-off-by: Philip Yang --- tests/kfdtest/scripts/kfdtest.exclude | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 6a0bfe1776..a4c4ee1db1 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -250,11 +250,7 @@ FILTER[aqua_vanjaram]=\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDMemoryTest.BigSysBufferStressTest:"\ "KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest:"\ -"KFDEvictTest.BasicTest:"\ -"KFDEvictTest.QueueTest:"\ -"KFDEvictTest.BurstyTest:"\ -"KFDSVMEvictTest.*" +"KFDIPCTest.BasicTest" FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ From 478a68d49c106941990d6707cc3bad4abb2fb028 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 16 May 2023 09:13:00 -0400 Subject: [PATCH 1144/1247] kfdtest: Test XNACK on and off for SVM tests Add parameterization for KFDSVM tests so that we test with both XNACK enabled and XNACK disabled. This will be overridden by HSA_XNACK, if set Change-Id: Ie96eb61c03115f947e08cfa076ac459f7440f5d8 --- tests/kfdtest/src/KFDBaseComponentTest.hpp | 27 +++++++---- tests/kfdtest/src/KFDSVMEvictTest.cpp | 7 +-- tests/kfdtest/src/KFDSVMEvictTest.hpp | 3 +- tests/kfdtest/src/KFDSVMRangeTest.cpp | 52 +++++++++++----------- tests/kfdtest/src/KFDSVMRangeTest.hpp | 3 +- 5 files changed, 54 insertions(+), 38 deletions(-) diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 8f714d54b3..22d5bcd12d 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -92,27 +92,38 @@ class KFDBaseComponentTest : public testing::Test { return supported; } - void SVMSetXNACKMode() { + // Set xnack_override to -1 if parameter is not passed in, to avoid unnecessary code churn + void SVMSetXNACKMode(int xnack_override = -1) { if (!SVMAPISupported()) return; m_xnack = -1; - - char *hsa_xnack = getenv("HSA_XNACK"); - if (!hsa_xnack) - return; - HSAKMT_STATUS ret = hsaKmtGetXNACKMode(&m_xnack); if (ret != HSAKMT_STATUS_SUCCESS) { LOG() << "Failed " << ret << " to get XNACK mode" << std::endl; return; } - // XNACK OFF if defined HSA_XNACK=0 - HSAint32 xnack_on = strncmp(hsa_xnack, "0", 1); + HSAint32 xnack_on = -1; + char *hsa_xnack = getenv("HSA_XNACK"); + + // HSA_XNACK takes priority over kfdtest parameters + if (hsa_xnack) + xnack_on = strncmp(hsa_xnack, "0", 1); + else if (xnack_override > -1) + xnack_on = xnack_override; + else + return; + + // No need to set XNACK if it's already the current value + if (xnack_on == m_xnack) + return; + ret = hsaKmtSetXNACKMode(xnack_on); if (ret != HSAKMT_STATUS_SUCCESS) LOG() << "Failed " << ret << " to set XNACK mode " << xnack_on << std::endl; + else + LOG() << "Setting XNACK mode to " << xnack_on << std::endl; } void SVMRestoreXNACKMode() { diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index 75eae28f82..e9ae0ca867 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -41,7 +41,7 @@ void KFDSVMEvictTest::SetUp() { KFDLocalMemoryTest::SetUp(); - SVMSetXNACKMode(); + SVMSetXNACKMode(GetParam()); ROUTINE_END } @@ -217,7 +217,7 @@ void KFDSVMEvictTest::WaitChildProcesses() { * - Synchronization between the processes, so they know for sure when * they are done allocating memory */ -TEST_F(KFDSVMEvictTest, BasicTest) { +TEST_P(KFDSVMEvictTest, BasicTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -282,7 +282,7 @@ TEST_F(KFDSVMEvictTest, BasicTest) { * - notify shader to quit * - check result buffer with specific value to confirm all wavefronts quit normally */ -TEST_F(KFDSVMEvictTest, QueueTest) { +TEST_P(KFDSVMEvictTest, QueueTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL) @@ -383,3 +383,4 @@ TEST_F(KFDSVMEvictTest, QueueTest) { TEST_END } +INSTANTIATE_TEST_CASE_P(, KFDSVMEvictTest,::testing::Values(0, 1)); diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/tests/kfdtest/src/KFDSVMEvictTest.hpp index 7e772105ae..16b2a71f31 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.hpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.hpp @@ -31,7 +31,8 @@ // @class KFDEvictTest // Test eviction and restore procedure using two processes -class KFDSVMEvictTest : public KFDLocalMemoryTest { +class KFDSVMEvictTest : public KFDLocalMemoryTest, + public ::testing::WithParamInterface { public: KFDSVMEvictTest(void): m_ChildStatus(HSAKMT_STATUS_ERROR), m_IsParent(true) {} diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index ba502da94c..86ec70f0fb 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -35,7 +35,7 @@ void KFDSVMRangeTest::SetUp() { KFDBaseComponentTest::SetUp(); - SVMSetXNACKMode(); + SVMSetXNACKMode(GetParam()); ROUTINE_END } @@ -50,7 +50,7 @@ void KFDSVMRangeTest::TearDown() { ROUTINE_END } -TEST_F(KFDSVMRangeTest, BasicSystemMemTest) { +TEST_P(KFDSVMRangeTest, BasicSystemMemTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -93,7 +93,7 @@ TEST_F(KFDSVMRangeTest, BasicSystemMemTest) { TEST_END } -TEST_F(KFDSVMRangeTest, SetGetAttributesTest) { +TEST_P(KFDSVMRangeTest, SetGetAttributesTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL) @@ -167,7 +167,7 @@ TEST_F(KFDSVMRangeTest, SetGetAttributesTest) { TEST_END } -TEST_F(KFDSVMRangeTest, XNACKModeTest) { +TEST_P(KFDSVMRangeTest, XNACKModeTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -204,7 +204,7 @@ TEST_F(KFDSVMRangeTest, XNACKModeTest) { TEST_END } -TEST_F(KFDSVMRangeTest, InvalidRangeTest) { +TEST_P(KFDSVMRangeTest, InvalidRangeTest) { TEST_START(TESTPROFILE_RUNALL) if (!SVMAPISupported()) @@ -303,7 +303,7 @@ void KFDSVMRangeTest::SplitRangeTest(int defaultGPUNode, int prefetch_location) delete sysBuffer; } -TEST_F(KFDSVMRangeTest, SplitSystemRangeTest) { +TEST_P(KFDSVMRangeTest, SplitSystemRangeTest) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); TEST_START(TESTPROFILE_RUNALL) @@ -323,7 +323,7 @@ TEST_F(KFDSVMRangeTest, SplitSystemRangeTest) { TEST_END } -TEST_F(KFDSVMRangeTest, EvictSystemRangeTest) { +TEST_P(KFDSVMRangeTest, EvictSystemRangeTest) { const HsaNodeProperties *pNodeProperties = m_NodeInfo.HsaDefaultGPUNodeProperties(); TEST_START(TESTPROFILE_RUNALL) @@ -426,7 +426,7 @@ TEST_F(KFDSVMRangeTest, EvictSystemRangeTest) { TEST_END } -TEST_F(KFDSVMRangeTest, PartialUnmapSysMemTest) { +TEST_P(KFDSVMRangeTest, PartialUnmapSysMemTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -478,7 +478,7 @@ TEST_F(KFDSVMRangeTest, PartialUnmapSysMemTest) { TEST_END } -TEST_F(KFDSVMRangeTest, BasicVramTest) { +TEST_P(KFDSVMRangeTest, BasicVramTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -526,7 +526,7 @@ TEST_F(KFDSVMRangeTest, BasicVramTest) { TEST_END } -TEST_F(KFDSVMRangeTest, SplitVramRangeTest) { +TEST_P(KFDSVMRangeTest, SplitVramRangeTest) { TEST_START(TESTPROFILE_RUNALL) if (!SVMAPISupported()) @@ -544,7 +544,7 @@ TEST_F(KFDSVMRangeTest, SplitVramRangeTest) { TEST_END } -TEST_F(KFDSVMRangeTest, PrefetchTest) { +TEST_P(KFDSVMRangeTest, PrefetchTest) { TEST_START(TESTPROFILE_RUNALL); if (!SVMAPISupported()) @@ -582,7 +582,7 @@ TEST_F(KFDSVMRangeTest, PrefetchTest) { TEST_END } -TEST_F(KFDSVMRangeTest, MigrateTest) { +TEST_P(KFDSVMRangeTest, MigrateTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -671,7 +671,7 @@ TEST_F(KFDSVMRangeTest, MigrateTest) { * 4. Use GPU sdma to fill the range in system memory * 5. Check if data is correct in system memory */ -TEST_F(KFDSVMRangeTest, MigrateAccessInPlaceTest) { +TEST_P(KFDSVMRangeTest, MigrateAccessInPlaceTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -735,7 +735,7 @@ TEST_F(KFDSVMRangeTest, MigrateAccessInPlaceTest) { * [ 292.730010] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f1000 * [ 292.730931] amdgpu:svm_migrate_to_ram:744: CPU page fault address 0x7f22597f2000 */ -TEST_F(KFDSVMRangeTest, MigrateGranularityTest) { +TEST_P(KFDSVMRangeTest, MigrateGranularityTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -789,7 +789,7 @@ TEST_F(KFDSVMRangeTest, MigrateGranularityTest) { TEST_END } -TEST_F(KFDSVMRangeTest, MigrateLargeBufTest) { +TEST_P(KFDSVMRangeTest, MigrateLargeBufTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -877,7 +877,7 @@ TEST_F(KFDSVMRangeTest, MigrateLargeBufTest) { TEST_END } -TEST_F(KFDSVMRangeTest, MigratePolicyTest) { +TEST_P(KFDSVMRangeTest, MigratePolicyTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -983,7 +983,7 @@ TEST_F(KFDSVMRangeTest, MigratePolicyTest) { * * Test will skip if only one GPU found */ -TEST_F(KFDSVMRangeTest, MultiGPUMigrationTest) { +TEST_P(KFDSVMRangeTest, MultiGPUMigrationTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1067,7 +1067,7 @@ TEST_F(KFDSVMRangeTest, MultiGPUMigrationTest) { * * Test will skip if only one GPU found */ -TEST_F(KFDSVMRangeTest, MultiGPUAccessInPlaceTest) { +TEST_P(KFDSVMRangeTest, MultiGPUAccessInPlaceTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1167,7 +1167,7 @@ unsigned int GpuReadThread(void* p) { return 0; } -TEST_F(KFDSVMRangeTest, MultiThreadMigrationTest) { +TEST_P(KFDSVMRangeTest, MultiThreadMigrationTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1228,7 +1228,7 @@ TEST_F(KFDSVMRangeTest, MultiThreadMigrationTest) { * Use sdma to write data to memory, should write to file * Close file, and then check if file data is updated correctly */ -TEST_F(KFDSVMRangeTest, MigrateFileBackedRangeTest) { +TEST_P(KFDSVMRangeTest, MigrateFileBackedRangeTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1287,7 +1287,7 @@ TEST_F(KFDSVMRangeTest, MigrateFileBackedRangeTest) { * Map read only range to GPU, test sdma can read the range * write to range should trigger GPU vm fault for both xnack on and off */ -TEST_F(KFDSVMRangeTest, ReadOnlyRangeTest) { +TEST_P(KFDSVMRangeTest, ReadOnlyRangeTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1448,7 +1448,7 @@ unsigned int ReadSMIEventThread(void* p) { return 0; } -TEST_F(KFDSVMRangeTest, HMMProfilingEvent) { +TEST_P(KFDSVMRangeTest, HMMProfilingEvent) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1498,7 +1498,7 @@ TEST_F(KFDSVMRangeTest, HMMProfilingEvent) { * KFD should support VRAM overcommitment by evicting SVM ranges to system memory to alloc * VRAM for new ranges. */ -TEST_F(KFDSVMRangeTest, VramOvercommitTest) { +TEST_P(KFDSVMRangeTest, VramOvercommitTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1558,7 +1558,7 @@ TEST_F(KFDSVMRangeTest, VramOvercommitTest) { * by spliting giant range into smaller ranges, evicting SVM ranges to system memory to * alloc VRAM for overcommitment ranges. */ -TEST_F(KFDSVMRangeTest, VramOvercommitGiantRangeTest) { +TEST_P(KFDSVMRangeTest, VramOvercommitGiantRangeTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1608,7 +1608,7 @@ TEST_F(KFDSVMRangeTest, VramOvercommitGiantRangeTest) { * use sdma to memset the rest 2 pages, each page has different value 0x1, 0x2, 0x3, 0x4 * then check if all page have the specific value after migrating 4 pages to system memory. */ -TEST_F(KFDSVMRangeTest, PrefaultPartialRangeTest) { +TEST_P(KFDSVMRangeTest, PrefaultPartialRangeTest) { TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); TEST_START(TESTPROFILE_RUNALL); @@ -1653,3 +1653,5 @@ TEST_F(KFDSVMRangeTest, PrefaultPartialRangeTest) { munmap(pBuf, BufSize); TEST_END } + +INSTANTIATE_TEST_CASE_P(, KFDSVMRangeTest,::testing::Values(0, 1)); diff --git a/tests/kfdtest/src/KFDSVMRangeTest.hpp b/tests/kfdtest/src/KFDSVMRangeTest.hpp index 03a245dba3..0f0d8a0f4d 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.hpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.hpp @@ -28,7 +28,8 @@ #include "KFDBaseComponentTest.hpp" -class KFDSVMRangeTest : public KFDBaseComponentTest { +class KFDSVMRangeTest : public KFDBaseComponentTest, + public ::testing::WithParamInterface { public: KFDSVMRangeTest() {} ~KFDSVMRangeTest() {} From cc4fb2d1a997783a125c935c76797eed02b00faf Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Wed, 24 May 2023 15:58:14 -0500 Subject: [PATCH 1145/1247] libhsakmt: Revert "libhsakmt: Update FD creation logic" This reverts commit fd48f14ceb6d5a2fa0b175768988394e21d2b443. Current amdgpu exposes one render node for one gpu node/partition, revert to previous way to open render node at Thunk. Signed-off-by: Xiaogang Chen Change-Id: I436be74f8e872a7ab5c4a1420b4ea884f5a00e57 --- src/fmm.c | 38 +++++++++++++------------------------- src/fmm.h | 2 +- src/topology.c | 2 +- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index f6c2082373..c3512e2982 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1933,32 +1933,23 @@ static HSAKMT_STATUS get_process_apertures( */ #define DRM_FIRST_RENDER_NODE 128 #define DRM_LAST_RENDER_NODE 255 -#define DRM_MAX_PARTITIONS 8 -static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE][DRM_MAX_PARTITIONS]; -static uint32_t gpu_id_fd_map[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE][DRM_MAX_PARTITIONS]; +static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE]; -int open_drm_render_device(int minor, uint32_t gpu_id) +int open_drm_render_device(int minor) { char path[128]; - int fd, render_idx, gpu_id_idx; + int index, fd; if (minor < DRM_FIRST_RENDER_NODE || minor > DRM_LAST_RENDER_NODE) { pr_err("DRM render minor %d out of range [%d, %d]\n", minor, DRM_FIRST_RENDER_NODE, DRM_LAST_RENDER_NODE); return -EINVAL; } - render_idx = minor - DRM_FIRST_RENDER_NODE; + index = minor - DRM_FIRST_RENDER_NODE; /* If the render node was already opened, keep using the same FD */ - for (gpu_id_idx = 0; gpu_id_idx < DRM_MAX_PARTITIONS && - gpu_id_fd_map[render_idx][gpu_id_idx] != 0; gpu_id_idx++) - if (gpu_id_fd_map[render_idx][gpu_id_idx] == gpu_id) - return drm_render_fds[render_idx][gpu_id_idx]; - - if (gpu_id_idx >= DRM_MAX_PARTITIONS) { - pr_err("Requesting more FDs for same render node than max supported partitions!\n"); - return -EINVAL; - } + if (drm_render_fds[index]) + return drm_render_fds[index]; sprintf(path, "/dev/dri/renderD%d", minor); fd = open(path, O_RDWR | O_CLOEXEC); @@ -1970,8 +1961,7 @@ int open_drm_render_device(int minor, uint32_t gpu_id) } return -errno; } - gpu_id_fd_map[render_idx][gpu_id_idx] = gpu_id; - drm_render_fds[render_idx][gpu_id_idx] = fd; + drm_render_fds[index] = fd; return fd; } @@ -2362,7 +2352,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) /* Skip non-GPU nodes */ if (props.KFDGpuID) { - int fd = open_drm_render_device(props.DrmRenderMinor, props.KFDGpuID); + int fd = open_drm_render_device(props.DrmRenderMinor); if (fd <= 0) { ret = HSAKMT_STATUS_ERROR; goto gpu_mem_init_failed; @@ -3962,17 +3952,15 @@ static void fmm_clear_aperture(manageable_aperture_t *app) */ void fmm_clear_all_mem(void) { - uint32_t i, j; + uint32_t i; void *map_addr; /* Close render node FDs. The child process needs to open new ones */ - for (i = 0; i <= DRM_LAST_RENDER_NODE - DRM_FIRST_RENDER_NODE; i++) { - for (j = 0; j < DRM_MAX_PARTITIONS && drm_render_fds[i][j] != 0; j++) { - close(drm_render_fds[i][j]); - drm_render_fds[i][j] = 0; - gpu_id_fd_map[i][j] = 0; + for (i = 0; i <= DRM_LAST_RENDER_NODE - DRM_FIRST_RENDER_NODE; i++) + if (drm_render_fds[i]) { + close(drm_render_fds[i]); + drm_render_fds[i] = 0; } - } fmm_clear_aperture(&cpuvm_aperture); fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); diff --git a/src/fmm.h b/src/fmm.h index 347b9fe27a..ce4bf55ca7 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -97,7 +97,7 @@ HSAKMT_STATUS fmm_register_shared_memory(const HsaSharedMemoryHandle *SharedMemo HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, uint32_t *nodes_to_map, uint64_t num_of_nodes, uint64_t *gpuvm_address); -int open_drm_render_device(int minor, uint32_t gpu_id); +int open_drm_render_device(int minor); void *mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align, uint64_t guard_size, void *aper_base, void *aper_limit); diff --git a/src/topology.c b/src/topology.c index 8b0b786b9a..d2c81a98d9 100644 --- a/src/topology.c +++ b/src/topology.c @@ -694,7 +694,7 @@ static HSAKMT_STATUS topology_sysfs_check_node_supported(uint32_t sysfs_node_id, } /* Open DRM Render device */ - ret_value = open_drm_render_device(drm_render_minor, gpu_id); + ret_value = open_drm_render_device(drm_render_minor); if (ret_value > 0) *is_node_supported = true; else if (ret_value != -ENOENT && ret_value != -EPERM) From 0a2989083b2848d6daf625ef0fc8215a805ba3ca Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Mon, 31 Oct 2022 15:38:32 -0500 Subject: [PATCH 1146/1247] libhsakmt: add/init a new manageable_aperture_t from NON_CANONICAL space. This new manageable_aperture_t is used for VRAM allocation-only and VA allocation-only. Signed-off-by: Xiaogang Chen Change-Id: I3866ef9d35386d6aef7b6934ac8d4a89ef843b50 --- src/fmm.c | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index c3512e2982..e757166e57 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -76,6 +76,9 @@ #define vm_object_tree(app, is_userptr) \ ((is_userptr) ? &(app)->user_tree : &(app)->tree) +#define START_NON_CANONICAL_ADDR (1ULL << 47) +#define END_NON_CANONICAL_ADDR (~0UL - (1UL << 47)) + struct vm_object { void *start; void *userptr; @@ -257,6 +260,12 @@ static svm_t svm = { */ static manageable_aperture_t cpuvm_aperture = INIT_MANAGEABLE_APERTURE(0, 0); +/* mem_handle_aperture is used to generate memory handles + * for allocations that don't have a valid virtual address + * its size is 47bits. +*/ +static manageable_aperture_t mem_handle_aperture = INIT_MANAGEABLE_APERTURE(START_NON_CANONICAL_ADDR, (START_NON_CANONICAL_ADDR + (1ULL << 47))); + /* GPU node array for default mappings */ static uint32_t all_gpu_id_array_size; static uint32_t *all_gpu_id_array; @@ -267,7 +276,8 @@ typedef enum _HSA_APERTURE { HSA_APERTURE_DGPU, HSA_APERTURE_DGPU_ALT, HSA_APERTURE_GPUVM, - HSA_APERTURE_CPUVM + HSA_APERTURE_CPUVM, + HSA_APERTURE_MEMHANDLE } HSA_APERTURE; typedef struct _HsaApertureInfo { @@ -890,6 +900,8 @@ static manageable_aperture_t *fmm_get_aperture(HsaApertureInfo info) return &gpu_mem[info.idx].gpuvm_aperture; case HSA_APERTURE_CPUVM: return &cpuvm_aperture; + case HSA_APERTURE_MEMHANDLE: + return &mem_handle_aperture; default: return NULL; } @@ -918,7 +930,13 @@ static manageable_aperture_t *fmm_find_aperture(const void *address, uint32_t i; HsaApertureInfo _info = { .type = HSA_APERTURE_UNSUPPORTED, .idx = 0}; - if (is_dgpu) { + if ((address >= mem_handle_aperture.base) && + (address <= mem_handle_aperture.limit)){ + + aperture = &mem_handle_aperture; + _info.type = HSA_APERTURE_MEMHANDLE; + + } else if (is_dgpu) { if (address >= svm.dgpu_aperture->base && address <= svm.dgpu_aperture->limit) { @@ -1185,6 +1203,13 @@ static vm_object_t *vm_find_object(const void *addr, uint64_t size, break; } + if (!aper) { + if ((addr >= mem_handle_aperture.base) && + (addr <= mem_handle_aperture.limit)){ + aper = &mem_handle_aperture; + } + } + if (!aper) { if (!svm.dgpu_aperture) goto no_svm; @@ -2202,6 +2227,8 @@ static void fmm_init_rbtree(void) rbtree_init(&svm.apertures[SVM_COHERENT].user_tree); rbtree_init(&cpuvm_aperture.tree); rbtree_init(&cpuvm_aperture.user_tree); + rbtree_init(&mem_handle_aperture.tree); + rbtree_init(&mem_handle_aperture.user_tree); } while (i--) { @@ -2273,6 +2300,66 @@ static void release_mmio(void) } } +static bool two_apertures_overlap(void *start_1, void *limit_1, void *start_2, void *limit_2) +{ + return (start_1 >= start_2 && start_1 <= limit_2) || (start_2 >= start_1 && start_2 <= limit_1); +} + +static bool init_mem_handle_aperture(HSAuint32 align, HSAuint32 guard_pages) +{ + bool found; + uint32_t i; + + /* init mem_handle_aperture for buffer handler management*/ + mem_handle_aperture.align = align; + mem_handle_aperture.guard_pages = guard_pages; + mem_handle_aperture.is_cpu_accessible = false; + mem_handle_aperture.ops = &reserved_aperture_ops; + + while (PORT_VPTR_TO_UINT64(mem_handle_aperture.base) < END_NON_CANONICAL_ADDR - 1) { + + found = true; + for (i = 0; i < gpu_mem_count; i++) { + + if (gpu_mem[i/*gpu_mem_id*/].lds_aperture.base && + two_apertures_overlap(gpu_mem[i].lds_aperture.base, gpu_mem[i].lds_aperture.limit, + mem_handle_aperture.base, mem_handle_aperture.limit)) { + found = false; + break; + } + + if (gpu_mem[i].scratch_aperture.base && + two_apertures_overlap(gpu_mem[i].scratch_aperture.base, gpu_mem[i].scratch_aperture.limit, + mem_handle_aperture.base, mem_handle_aperture.limit)){ + found = false; + break; + } + + if (gpu_mem[i].gpuvm_aperture.base && + two_apertures_overlap(gpu_mem[i].gpuvm_aperture.base, gpu_mem[i].gpuvm_aperture.limit, + mem_handle_aperture.base, mem_handle_aperture.limit)){ + found = false; + break; + } + } + + if (found) { + pr_info("mem_handle_aperture start %p, mem_handle_aperture limit %p\n", mem_handle_aperture.base, mem_handle_aperture.limit); + return true; + } else { + /*increase base by 1UL<<47 to check next hole*/ + mem_handle_aperture.base = VOID_PTR_ADD(mem_handle_aperture.base, (1UL << 47)); + mem_handle_aperture.limit = VOID_PTR_ADD(mem_handle_aperture.base, (1ULL << 47)); + } + } + + /* set invalid aperture if fail locating a hole for it*/ + mem_handle_aperture.base = 0; + mem_handle_aperture.limit = 0; + + return false; +} + HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) { uint32_t i; @@ -2571,6 +2658,9 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) fmm_init_rbtree(); + if (!init_mem_handle_aperture(PAGE_SIZE, guardPages)) + pr_err("Failed to init mem_handle_aperture\n"); + for (gpu_mem_id = 0; (uint32_t)gpu_mem_id < gpu_mem_count; gpu_mem_id++) { if (!topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) continue; @@ -3962,6 +4052,7 @@ void fmm_clear_all_mem(void) drm_render_fds[i] = 0; } + fmm_clear_aperture(&mem_handle_aperture); fmm_clear_aperture(&cpuvm_aperture); fmm_clear_aperture(&svm.apertures[SVM_DEFAULT]); fmm_clear_aperture(&svm.apertures[SVM_COHERENT]); From 0138487aa4c6e714304c7558c05f2338fc365591 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Mon, 31 Oct 2022 16:33:15 -0500 Subject: [PATCH 1147/1247] libhsakmt: support vram-only and VA-only alloc/free. Signed-off-by: Xiaogang.Chen Change-Id: I47cf53642d2ea197c08b20e84d7cae04b2d431e0 --- include/hsakmttypes.h | 4 ++- src/fmm.c | 70 +++++++++++++++++++++++++++++++++++++++++-- src/memory.c | 7 +++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index e584fd75a9..ad5b565361 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -564,7 +564,9 @@ typedef struct _HsaMemFlags unsigned int FixedAddress : 1; // Allocate memory at specified virtual address. Fail if address is not free. unsigned int NoNUMABind: 1; // Don't bind system memory to a specific NUMA node unsigned int Uncached: 1; // Caching flag for fine-grained memory on A+A HW platform - unsigned int Reserved : 14; + unsigned int NoAddress: 1; // only do vram allocation, return a handle, not allocate virtual address. + unsigned int OnlyAddress: 1; // only do virtal address allocation without vram allocation. + unsigned int Reserved: 12; } ui32; HSAuint32 Value; diff --git a/src/fmm.c b/src/fmm.c index e757166e57..59639a2cc0 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1091,6 +1091,10 @@ static vm_object_t *fmm_allocate_memory_object(uint32_t gpu_id, void *mem, if (ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) args.mmap_offset = *mmap_offset; + /* if allocate vram-only, use an invalid VA */ + if (aperture == &mem_handle_aperture) + args.va_addr = 0; + if (kmtIoctl(kfd_fd, AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, &args)) return NULL; @@ -1471,6 +1475,35 @@ static void *fmm_map_to_cpu(void *mem, uint64_t size, bool host_access, return ret; } +static void *fmm_allocate_va(uint32_t gpu_id, void *address, uint64_t size, + manageable_aperture_t *aperture, HsaMemFlags mflags) +{ + void *mem = NULL; + vm_object_t *vm_obj = NULL; + + /* Check aperture is properly initialized/supported */ + if (!aperture_is_valid(aperture->base, aperture->limit)) + return NULL; + + /* Allocate address space */ + pthread_mutex_lock(&aperture->fmm_mutex); + mem = aperture_allocate_area(aperture, address, size); + /* assing handle 0 to vm_obj since no mem allocted */ + vm_obj = aperture_allocate_object(aperture, mem, 0, + size, mflags); + pthread_mutex_unlock(&aperture->fmm_mutex); + + if (!vm_obj) { + pthread_mutex_lock(&aperture->fmm_mutex); + aperture_release_area(aperture, mem, size); + pthread_mutex_unlock(&aperture->fmm_mutex); + + mem = NULL; + } + + return mem; +} + void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags mflags) { @@ -1501,6 +1534,14 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; } + /* special case for va allocation without vram alloc */ + if (mflags.ui32.OnlyAddress) + return fmm_allocate_va(gpu_id, address, size, aperture, mflags); + + /* special case for vram allocation without addr */ + if(mflags.ui32.NoAddress) + aperture = &mem_handle_aperture; + if (!mflags.ui32.CoarseGrain || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; @@ -1518,7 +1559,8 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, pthread_mutex_unlock(&aperture->fmm_mutex); } - if (mem) { + /* if alloc vram-only not mmap to cpu vm since no va */ + if (mem && !mflags.ui32.NoAddress) { void *ret = fmm_map_to_cpu(mem, MemorySizeInBytes, mflags.ui32.HostAccess, gpu_mem[gpu_mem_id].drm_render_fd, @@ -1838,7 +1880,7 @@ static int __fmm_release(vm_object_t *object, manageable_aperture_t *aperture) * free the BO before unmapping the pages. */ args.handle = object->handle; - if (kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) { + if (args.handle && kmtIoctl(kfd_fd, AMDKFD_IOC_FREE_MEMORY_OF_GPU, &args)) { pthread_mutex_unlock(&aperture->fmm_mutex); return -errno; } @@ -3040,6 +3082,18 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) } /* Successful vm_find_object returns with the aperture locked */ + /* allocate VA only */ + if (object && object->handle == 0) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + /* allocate buffer only, should be mapped by GEM API */ + if (aperture == &mem_handle_aperture) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + if (aperture == &cpuvm_aperture) { /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); @@ -3791,6 +3845,18 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_ERROR; /* Successful vm_find_object returns with aperture locked */ + /* allocates VA only */ + if (object && object->handle == 0) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + + /* allocates buffer only, should be mapped by GEM API */ + if (aperture == &mem_handle_aperture) { + pthread_mutex_unlock(&aperture->fmm_mutex); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + /* APU memory is not supported by this function */ if (aperture && (aperture == &cpuvm_aperture || !aperture->is_cpu_accessible)) { diff --git a/src/memory.c b/src/memory.c index 4d4cd53ba2..647bc44f3c 100644 --- a/src/memory.c +++ b/src/memory.c @@ -171,6 +171,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, } /* GPU allocated VRAM */ + /* sanity check cannot do OnlyAddress and NoAddress alloc at same time */ + if (MemFlags.ui32.OnlyAddress && MemFlags.ui32.NoAddress) { + pr_err("[%s] allocate addr-only and memory-only at same time\n", + __func__); + return HSAKMT_STATUS_INVALID_PARAMETER; + } + *MemoryAddress = fmm_allocate_device(gpu_id, PreferredNode, *MemoryAddress, SizeInBytes, MemFlags); From 108c0e5f92de6d945931381c43d87e901db117c1 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Mon, 31 Oct 2022 19:51:42 -0500 Subject: [PATCH 1148/1247] kfdtest: add kfdtest cases for VA-only, VRAM-only allocated VRAM. Alloc vram by kfd, then map by GEM api to GPU VM and map to CPU VM. Signed-off-by: Xiaogang Chen Change-Id: Ib5b2f35662cd5473f622f6ffc9b62925fe57ae42 --- tests/kfdtest/src/KFDMemoryTest.cpp | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 07f9803437..1c3b9daf11 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2889,3 +2889,51 @@ TEST_F(KFDMemoryTest, ExportDMABufTest) { TEST_END } + +TEST_F(KFDMemoryTest, VA_VRAM_Only_AllocTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (m_VersionInfo.KernelInterfaceMinorVersion < 12) { + LOG() << "Skipping test, requires KFD ioctl version 1.12 or newer" << std::endl; + return; + } + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + HsaMemFlags memFlags = m_MemoryFlags; + memFlags.ui32.NonPaged = 1; + memFlags.ui32.HostAccess = 0; + + HsaMemMapFlags mapFlags = {0}; + + HSAuint32 *buf; + + /*alloc va without vram alloc*/ + memFlags.ui32.OnlyAddress = 1; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, memFlags, + reinterpret_cast(&buf))); + + /*mapping VA allocated by kfd api would fail*/ + ASSERT_EQ(HSAKMT_STATUS_ERROR, hsaKmtMapMemoryToGPU(buf, PAGE_SIZE, NULL)); + ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtMapMemoryToGPUNodes(buf, PAGE_SIZE, NULL, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); + + ASSERT_SUCCESS(hsaKmtFreeMemory(buf, PAGE_SIZE)); + + /*alloc vram without va assigned*/ + memFlags.ui32.OnlyAddress = 0; + memFlags.ui32.NoAddress = 1; + ASSERT_SUCCESS(hsaKmtAllocMemory(defaultGPUNode, PAGE_SIZE, memFlags, + reinterpret_cast(&buf))); + + /*mapping handle allocated by kfd API would fail*/ + ASSERT_EQ(HSAKMT_STATUS_ERROR, hsaKmtMapMemoryToGPU(buf, PAGE_SIZE, NULL)); + ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtMapMemoryToGPUNodes(buf, PAGE_SIZE, NULL, + mapFlags, 1, reinterpret_cast(&defaultGPUNode))); + + ASSERT_SUCCESS(hsaKmtFreeMemory(buf, PAGE_SIZE)); + + TEST_END +} From 11ac57d2934a30a4855d54a13b98f9121e4eee0f Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Mon, 31 Oct 2022 22:06:28 -0500 Subject: [PATCH 1149/1247] libhsakmt: update HsaPointerInfo for address-only allocated VRAM. Signed-off-by: Xiaogang Chen Change-Id: Ib88b34dff772997d2b2e5f3c7e333cef3092ef56 --- include/hsakmttypes.h | 3 ++- src/events.c | 3 +++ src/fmm.c | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index ad5b565361..57f4a855de 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1303,8 +1303,9 @@ typedef enum _HSA_POINTER_TYPE { HSA_POINTER_ALLOCATED = 1, // Allocated with hsaKmtAllocMemory (except scratch) HSA_POINTER_REGISTERED_USER = 2, // Registered user pointer HSA_POINTER_REGISTERED_GRAPHICS = 3, // Registered graphics buffer - HSA_POINTER_REGISTERED_SHARED = 4 // Registered shared buffer (IPC) + HSA_POINTER_REGISTERED_SHARED = 4, // Registered shared buffer (IPC) // (hsaKmtRegisterGraphicsToNodes) + HSA_POINTER_RESERVED_ADDR = 5 // address-only reservation VA } HSA_POINTER_TYPE; typedef struct _HsaPointerInfo { diff --git a/src/events.c b/src/events.c index 1c5b03dcf2..512abc2be0 100644 --- a/src/events.c +++ b/src/events.c @@ -339,6 +339,9 @@ static void analysis_memory_exception(struct kfd_hsa_memory_exception_data * pr_err("Memory is allocated using hsaKmtAllocMemory\n"); pr_err("CPU address of the memory is %p\n", info.CPUAddress); break; + case HSA_POINTER_RESERVED_ADDR: + pr_err("Memory is allocated by OnlyAddress mode\n"); + break; default: pr_err("Invalid memory type %d\n", info.Type); break; diff --git a/src/fmm.c b/src/fmm.c index 59639a2cc0..4e8293fbcf 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3963,6 +3963,8 @@ HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info) info->Type = HSA_POINTER_REGISTERED_GRAPHICS; else if (vm_obj->userptr) info->Type = HSA_POINTER_REGISTERED_USER; + else if (vm_obj->handle == 0) + info->Type = HSA_POINTER_RESERVED_ADDR; else info->Type = HSA_POINTER_ALLOCATED; From d2a37894bb5adda283e66feb115da505996bdd6c Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Thu, 25 May 2023 13:38:39 -0500 Subject: [PATCH 1150/1247] libhsakmt: support allocating a fixed address at mmap_aperture. When HsaMemFlags.ui32.FixedAddress=1 allocate fixed address at mmap_aperture. Signed-off-by: Xiaogang Chen Change-Id: I1f3b532ec3c1a4fb0962126a0bd56441abaf6a9c --- src/fmm.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 4e8293fbcf..f552c21daf 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -785,14 +785,38 @@ static void *mmap_aperture_allocate_aligned(manageable_aperture_t *aper, uint64_t alignment_size = PAGE_SIZE << svm.alignment_order; uint64_t guard_size; - if (address) - return NULL; - if (!aper->is_cpu_accessible) { pr_err("MMap Aperture must be CPU accessible\n"); return NULL; } + if (address) { + void *addr; + +#ifdef MAP_FIXED_NOREPLACE + addr = mmap(address, size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED_NOREPLACE, + -1, 0); +#else + addr = mmap(address, size, PROT_NONE, + MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, + -1, 0); +#endif + if (addr == MAP_FAILED) { + pr_err("mmap failed: %s\n", strerror(errno)); + return NULL; + } + +#ifndef MAP_FIXED_NOREPLACE + if (address != addr) { + pr_err("mmap failed to return addr asked\n"); + munmap(addr, size); + return NULL; + } +#endif + return addr; + } + /* Align big buffers to the next power-of-2. By default, the max alignment * size is set to 2MB. This can be modified by the env variable * HSA_MAX_VA_ALIGN. This variable sets the order of the alignment size as From 989c6c617c3d1ad1ac9304b3ed82c60128cc47bd Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Mon, 31 Oct 2022 22:26:59 -0500 Subject: [PATCH 1151/1247] libhsakmt: add DMABuf import without address allocation. When gpu map info is not provided import DMABuf without VA assigned. Signed-off-by: Xiaogang Chen Change-Id: I996ab4eb46977af5064126529c28a8bf20a67292 --- src/fmm.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index f552c21daf..119e431f72 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3539,7 +3539,11 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, gpu_mem_id = gpu_mem_find_by_gpu_id(infoArgs.gpu_id); if (gpu_mem_id < 0) goto error_free_metadata; - if (topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) { + + /* import DMA buffer without VA assigned */ + if (!gpu_id_array && gpu_id_array_size == 0) { + aperture = &mem_handle_aperture; + } else if (topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) { aperture = svm.dgpu_aperture; aperture_base = NULL; } else { @@ -3556,7 +3560,11 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, goto error_free_metadata; /* Import DMA buffer */ - importArgs.va_addr = VOID_PTRS_SUB(mem, aperture_base); + if (aperture == &mem_handle_aperture) + importArgs.va_addr = 0; + else + importArgs.va_addr = VOID_PTRS_SUB(mem, aperture_base); + importArgs.gpu_id = infoArgs.gpu_id; importArgs.dmabuf_fd = GraphicsResourceHandle; r = kmtIoctl(kfd_fd, AMDKFD_IOC_IMPORT_DMABUF, (void *)&importArgs); From 9bebb276bee49a2a068e3dc676758a7ef65780f4 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Mon, 31 Oct 2022 22:43:36 -0500 Subject: [PATCH 1152/1247] libhsakmt: add NodeId at HsaGraphicsResourceInfo. Return GPU NodeId that exported the DMA buffer from amdgpu graphic driver at fmm_register_graphics_handle. Signed-off-by: Xiaogang Chen Change-Id: Iaeccce6e6d0b7e27f10b15ed89d1b5310d03d44b --- include/hsakmttypes.h | 2 +- src/fmm.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 57f4a855de..f84f634141 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -607,7 +607,7 @@ typedef struct _HsaGraphicsResourceInfo { HSAuint64 SizeInBytes; // Buffer size const void *Metadata; // Pointer to metadata owned by Thunk HSAuint32 MetadataSizeInBytes; // Size of metadata - HSAuint32 Reserved; // Reserved for future use, will be set to 0 + HSAuint32 NodeId; // GPU exported the buffer } HsaGraphicsResourceInfo; typedef enum _HSA_CACHING_TYPE diff --git a/src/fmm.c b/src/fmm.c index 119e431f72..023ba9e81a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3590,7 +3590,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, GraphicsResourceInfo->SizeInBytes = infoArgs.size; GraphicsResourceInfo->Metadata = (void *)(unsigned long)infoArgs.metadata_ptr; GraphicsResourceInfo->MetadataSizeInBytes = infoArgs.metadata_size; - GraphicsResourceInfo->Reserved = 0; + gpuid_to_nodeid(infoArgs.gpu_id, &GraphicsResourceInfo->NodeId); return HSAKMT_STATUS_SUCCESS; From ac1db60fc2f1e7c278573067192fd17c7bfd34b6 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Tue, 15 Nov 2022 12:18:27 -0600 Subject: [PATCH 1153/1247] libhsakmt: query/use render node fds that libdrm uses. Query render node fds that libdrm uses for current process and use them at Thunk if available. v2: avoid naming conflict with amdgpu_device_get_fd from amdgpu.h Signed-off-by: Xiaogang Chen Change-Id: Id7288c03730f4a4c9c3644e37ca4725fec71a471 --- include/hsakmt.h | 10 +++++++ include/hsakmttypes.h | 2 ++ src/fmm.c | 65 +++++++++++++++++++++++++++++++++++++------ src/fmm.h | 2 ++ src/memory.c | 10 ++++++- src/openclose.c | 15 ++++++++++ 6 files changed, 95 insertions(+), 9 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 63e6312818..582e907328 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -602,6 +602,16 @@ hsaKmtUnmapGraphicHandle( HSAuint64 SizeInBytes //IN ); +/** + * Get an AMDGPU device handle for a GPU node + */ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetAMDGPUDeviceHandle( + HSAuint32 NodeId, //IN + HsaAMDGPUDeviceHandle *DeviceHandle //OUT + ); + /** Allocate GWS resource for a queue */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index f84f634141..16e156d039 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1409,6 +1409,8 @@ typedef enum _HSA_SVM_UNMAP_TRIGGERS { #define HSA_SMI_EVENT_MASK_FROM_INDEX(i) (1ULL << ((i) - 1)) #define HSA_SMI_EVENT_MSG_SIZE 96 +typedef void *HsaAMDGPUDeviceHandle; + #pragma pack(pop, hsakmttypes_h) diff --git a/src/fmm.c b/src/fmm.c index 023ba9e81a..da970e0ba8 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -41,6 +41,8 @@ #include #include #include "rbtree.h" +#include + #ifndef MPOL_F_STATIC_NODES /* Bug in numaif.h, this should be defined in there. Definition copied * from linux/mempolicy.h. @@ -195,6 +197,7 @@ typedef struct { int drm_render_fd; uint32_t usable_peer_id_num; uint32_t *usable_peer_id_array; + int drm_render_minor; } gpu_mem_t; enum svm_aperture_type { @@ -2026,10 +2029,15 @@ static HSAKMT_STATUS get_process_apertures( #define DRM_LAST_RENDER_NODE 255 static int drm_render_fds[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE]; +/* amdgpu device handle for each gpu that libdrm uses */ +static struct amdgpu_device *amdgpu_handle[DRM_LAST_RENDER_NODE + 1 - DRM_FIRST_RENDER_NODE]; + int open_drm_render_device(int minor) { char path[128]; int index, fd; + uint32_t major_drm, minor_drm; + struct amdgpu_device **device_handle; if (minor < DRM_FIRST_RENDER_NODE || minor > DRM_LAST_RENDER_NODE) { pr_err("DRM render minor %d out of range [%d, %d]\n", minor, @@ -2054,6 +2062,23 @@ int open_drm_render_device(int minor) } drm_render_fds[index] = fd; + /* if amdgpu_device_get_fd availabe query render fd that libdrm uses, + * then close drm_render_fds above, replace it by fd libdrm uses. + */ + device_handle = &amdgpu_handle[index]; + if (fn_amdgpu_device_get_fd && + !amdgpu_device_initialize(fd, &major_drm, &minor_drm, device_handle)) { + fd = fn_amdgpu_device_get_fd(*device_handle); + if (fd > 0) { + close(drm_render_fds[index]); + drm_render_fds[index] = fd; + } else { + pr_err("amdgpu_device_get_fd failed: %d\n", fd); + amdgpu_device_deinitialize(*device_handle); + *device_handle = 0; + } + } + return fd; } @@ -2366,6 +2391,23 @@ static void release_mmio(void) } } +HSAKMT_STATUS fmm_get_amdgpu_device_handle(uint32_t node_id, + HsaAMDGPUDeviceHandle *DeviceHandle) +{ + int32_t i = gpu_mem_find_by_node_id(node_id); + int index; + + if (i < 0) + return HSAKMT_STATUS_INVALID_NODE_UNIT; + + index = gpu_mem[i].drm_render_minor - DRM_FIRST_RENDER_NODE; + if (!amdgpu_handle[index]) + return HSAKMT_STATUS_INVALID_HANDLE; + + *DeviceHandle = amdgpu_handle[index]; + return HSAKMT_STATUS_SUCCESS; +} + static bool two_apertures_overlap(void *start_1, void *limit_1, void *start_2, void *limit_2) { return (start_1 >= start_2 && start_1 <= limit_2) || (start_2 >= start_1 && start_2 <= limit_1); @@ -2376,7 +2418,7 @@ static bool init_mem_handle_aperture(HSAuint32 align, HSAuint32 guard_pages) bool found; uint32_t i; - /* init mem_handle_aperture for buffer handler management*/ + /* init mem_handle_aperture for buffer handler management */ mem_handle_aperture.align = align; mem_handle_aperture.guard_pages = guard_pages; mem_handle_aperture.is_cpu_accessible = false; @@ -2387,7 +2429,7 @@ static bool init_mem_handle_aperture(HSAuint32 align, HSAuint32 guard_pages) found = true; for (i = 0; i < gpu_mem_count; i++) { - if (gpu_mem[i/*gpu_mem_id*/].lds_aperture.base && + if (gpu_mem[i].lds_aperture.base && two_apertures_overlap(gpu_mem[i].lds_aperture.base, gpu_mem[i].lds_aperture.limit, mem_handle_aperture.base, mem_handle_aperture.limit)) { found = false; @@ -2410,16 +2452,17 @@ static bool init_mem_handle_aperture(HSAuint32 align, HSAuint32 guard_pages) } if (found) { - pr_info("mem_handle_aperture start %p, mem_handle_aperture limit %p\n", mem_handle_aperture.base, mem_handle_aperture.limit); + pr_info("mem_handle_aperture start %p, mem_handle_aperture limit %p\n", + mem_handle_aperture.base, mem_handle_aperture.limit); return true; } else { - /*increase base by 1UL<<47 to check next hole*/ + /* increase base by 1UL<<47 to check next hole */ mem_handle_aperture.base = VOID_PTR_ADD(mem_handle_aperture.base, (1UL << 47)); mem_handle_aperture.limit = VOID_PTR_ADD(mem_handle_aperture.base, (1ULL << 47)); } } - /* set invalid aperture if fail locating a hole for it*/ + /* set invalid aperture if fail locating a hole for it */ mem_handle_aperture.base = 0; mem_handle_aperture.limit = 0; @@ -2511,6 +2554,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) goto gpu_mem_init_failed; } + gpu_mem[gpu_mem_count].drm_render_minor = props.DrmRenderMinor; gpu_mem[gpu_mem_count].usable_peer_id_array = calloc(NumNodes, sizeof(uint32_t)); if (!gpu_mem[gpu_mem_count].usable_peer_id_array) { @@ -4146,11 +4190,16 @@ void fmm_clear_all_mem(void) void *map_addr; /* Close render node FDs. The child process needs to open new ones */ - for (i = 0; i <= DRM_LAST_RENDER_NODE - DRM_FIRST_RENDER_NODE; i++) - if (drm_render_fds[i]) { + for (i = 0; i <= DRM_LAST_RENDER_NODE - DRM_FIRST_RENDER_NODE; i++) { + + if (amdgpu_handle[i]) { + amdgpu_device_deinitialize(amdgpu_handle[i]); + amdgpu_handle[i] = NULL; + } else if (drm_render_fds[i]) { close(drm_render_fds[i]); - drm_render_fds[i] = 0; } + drm_render_fds[i] = 0; + } fmm_clear_aperture(&mem_handle_aperture); fmm_clear_aperture(&cpuvm_aperture); diff --git a/src/fmm.h b/src/fmm.h index ce4bf55ca7..cd6a14ad02 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -45,6 +45,7 @@ typedef struct { void *start_address; } aperture_properties_t; +HSAKMT_STATUS fmm_get_amdgpu_device_handle(uint32_t node_id, HsaAMDGPUDeviceHandle *DeviceHandle); HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes); void fmm_destroy_process_apertures(void); @@ -101,4 +102,5 @@ int open_drm_render_device(int minor); void *mmap_allocate_aligned(int prot, int flags, uint64_t size, uint64_t align, uint64_t guard_size, void *aper_base, void *aper_limit); +extern int (*fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle); #endif /* FMM_H_ */ diff --git a/src/memory.c b/src/memory.c index 647bc44f3c..c5ebfe99e8 100644 --- a/src/memory.c +++ b/src/memory.c @@ -618,4 +618,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtReturnAsanHeaderPage(void *addr) #else return HSAKMT_STATUS_NOT_SUPPORTED; #endif -} \ No newline at end of file +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetAMDGPUDeviceHandle( HSAuint32 NodeId, + HsaAMDGPUDeviceHandle *DeviceHandle) +{ + CHECK_KFD_OPEN(); + + return fmm_get_amdgpu_device_handle(NodeId, DeviceHandle); +} diff --git a/src/openclose.c b/src/openclose.c index df98e46219..4913349256 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -23,6 +23,11 @@ * DEALINGS IN THE SOFTWARE. */ +/* glibc macro that enables access some nonstandard GNU/Linux extensions + * such as RTLD_DEFAULT used by dlsym + */ +#define _GNU_SOURCE + #include "libhsakmt.h" #include @@ -34,6 +39,9 @@ #include #include #include "fmm.h" +#include + +int (*fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle); static const char kfd_device_name[] = "/dev/kfd"; static pid_t parent_pid = -1; @@ -143,6 +151,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) HSAKMT_STATUS result; int fd = -1; HsaSystemProperties sys_props; + char *error; pthread_mutex_lock(&hsakmt_mutex); @@ -156,6 +165,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (kfd_open_count == 0) { static bool atfork_installed = false; + fn_amdgpu_device_get_fd = dlsym(RTLD_DEFAULT, "amdgpu_device_get_fd"); + if ((error = dlerror()) != NULL) + pr_err("amdgpu_device_get_fd is not available: %s\n", error); + else + pr_info("amdgpu_device_get_fd is available %p\n", fn_amdgpu_device_get_fd); + result = init_vars_from_env(); if (result != HSAKMT_STATUS_SUCCESS) goto open_failed; From 7e4e57ae5fcf17c0e61ad368f5b9f16f950a6d6c Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Tue, 15 Nov 2022 14:12:53 -0600 Subject: [PATCH 1154/1247] libhsakmt: have hsaKmtMapMemoryToGPU return same value as fmm_map_to_gpu. Have hsaKmtMapMemoryToGPU return same value as fmm_map_to_gpu to keep consistency. Signed-off-by: Xiaogang Chen Change-Id: Ifabb72301e1d5a6c1310973bb1321714e12a1fa6 --- src/fmm.c | 52 +++++++++++++++-------------- src/fmm.h | 2 +- src/memory.c | 5 +-- tests/kfdtest/src/KFDMemoryTest.cpp | 4 +-- 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index da970e0ba8..efd424bb55 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2946,13 +2946,14 @@ static void add_device_ids_to_mapped_array(vm_object_t *obj, /* If nodes_to_map is not NULL, map the nodes specified; otherwise map all. */ -static int _fmm_map_to_gpu(manageable_aperture_t *aperture, +static HSAKMT_STATUS _fmm_map_to_gpu(manageable_aperture_t *aperture, void *address, uint64_t size, vm_object_t *obj, uint32_t *nodes_to_map, uint32_t nodes_array_size) { struct kfd_ioctl_map_memory_to_gpu_args args = {0}; vm_object_t *object; - int ret = 0; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; + int ret_ioctl; if (!obj) pthread_mutex_lock(&aperture->fmm_mutex); @@ -2962,7 +2963,7 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, /* Find the object to retrieve the handle */ object = vm_find_object_by_address(aperture, address, 0); if (!object) { - ret = -EINVAL; + ret = HSAKMT_STATUS_INVALID_HANDLE; goto err_object_not_found; } } @@ -3004,10 +3005,11 @@ static int _fmm_map_to_gpu(manageable_aperture_t *aperture, } args.n_success = 0; - ret = kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args); - if (ret) { + ret_ioctl = kmtIoctl(kfd_fd, AMDKFD_IOC_MAP_MEMORY_TO_GPU, &args); + if (ret_ioctl) { pr_err("GPU mapping failed (%d) for obj at %p, userptr %p, size %lu", - ret, object->start, object->userptr, object->size); + ret_ioctl, object->start, object->userptr, object->size); + ret = HSAKMT_STATUS_ERROR; goto err_map_failed; } @@ -3035,11 +3037,11 @@ err_map_failed: return ret; } -static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, +static HSAKMT_STATUS _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *aperture, void *address, uint64_t size) { int32_t gpu_mem_id; - int ret; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; bool is_debugger = 0; uint32_t flags; void *mmap_ret = NULL; @@ -3049,15 +3051,15 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert /* Retrieve gpu_mem id according to gpu_id */ gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); if (gpu_mem_id < 0) - return -1; + return HSAKMT_STATUS_INVALID_PARAMETER; if (!is_dgpu) - return 0; /* Nothing to do on APU */ + return HSAKMT_STATUS_SUCCESS; /* Nothing to do on APU */ /* sanity check the address */ if (address < aperture->base || VOID_PTR_ADD(address, size - 1) > aperture->limit) - return -1; + return HSAKMT_STATUS_INVALID_PARAMETER; is_debugger = debug_get_reg_status(gpu_mem[gpu_mem_id].node_id); flags = is_debugger ? KFD_IOC_ALLOC_MEM_FLAGS_GTT : @@ -3067,32 +3069,32 @@ static int _fmm_map_to_gpu_scratch(uint32_t gpu_id, manageable_aperture_t *apert obj = fmm_allocate_memory_object(gpu_id, address, size, aperture, &mmap_offset, flags); if (!obj) - return -1; + return HSAKMT_STATUS_INVALID_HANDLE; /* Create a CPU mapping for the debugger */ mmap_ret = fmm_map_to_cpu(address, size, is_debugger, gpu_mem[gpu_mem_id].drm_render_fd, mmap_offset); if (mmap_ret == MAP_FAILED) { __fmm_release(obj, aperture); - return -1; + return HSAKMT_STATUS_ERROR; } /* map to GPU */ ret = _fmm_map_to_gpu(aperture, address, size, NULL, &gpu_id, sizeof(uint32_t)); - if (ret != 0) + if (ret != HSAKMT_STATUS_SUCCESS) __fmm_release(obj, aperture); return ret; } -static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, - uint64_t *gpuvm_addr, vm_object_t *object, - uint32_t *nodes_to_map, uint32_t nodes_array_size) +static HSAKMT_STATUS _fmm_map_to_gpu_userptr(void *addr, uint64_t size, + uint64_t *gpuvm_addr, vm_object_t *object, + uint32_t *nodes_to_map, uint32_t nodes_array_size) { manageable_aperture_t *aperture; void *svm_addr; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); - int ret; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; aperture = svm.dgpu_aperture; @@ -3116,18 +3118,18 @@ static int _fmm_map_to_gpu_userptr(void *addr, uint64_t size, svm_addr = object->start; ret = _fmm_map_to_gpu(aperture, svm_addr, object->size, object, NULL, 0); } - if (ret == 0 && gpuvm_addr) + if (ret == HSAKMT_STATUS_SUCCESS && gpuvm_addr) *gpuvm_addr = (uint64_t)svm_addr + page_offset; return ret; } -int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) +HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) { manageable_aperture_t *aperture; vm_object_t *object; uint32_t i; - int ret; + HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; /* Special handling for scratch memory */ for (i = 0; i < gpu_mem_count; i++) @@ -3143,10 +3145,10 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if (!is_dgpu) { /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); - return 0; + return HSAKMT_STATUS_SUCCESS; } pr_err("Object not found at %p\n", address); - return -EINVAL; + return HSAKMT_STATUS_INVALID_PARAMETER; } /* Successful vm_find_object returns with the aperture locked */ @@ -3165,7 +3167,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) if (aperture == &cpuvm_aperture) { /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); - ret = 0; + ret = HSAKMT_STATUS_SUCCESS; } else if ((svm.is_svm_api_supported && !object) || object->userptr) { ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object, NULL, 0); } else { @@ -3173,7 +3175,7 @@ int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address) /* Update alternate GPUVM address only for * CPU-invisible apertures on old APUs */ - if (!ret && gpuvm_address && !aperture->is_cpu_accessible) + if (ret == HSAKMT_STATUS_SUCCESS && gpuvm_address && !aperture->is_cpu_accessible) *gpuvm_address = VOID_PTRS_SUB(object->start, aperture->base); } diff --git a/src/fmm.h b/src/fmm.h index cd6a14ad02..6cd7898cac 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -58,7 +58,7 @@ void *fmm_allocate_host(uint32_t node_id, void *address, uint64_t MemorySizeInBy HsaMemFlags flags); void fmm_print(uint32_t node); HSAKMT_STATUS fmm_release(void *address); -int fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); +HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_address); int fmm_unmap_from_gpu(void *address); bool fmm_get_handle(void *address, uint64_t *handle); HSAKMT_STATUS fmm_get_mem_info(const void *address, HsaPointerInfo *info); diff --git a/src/memory.c b/src/memory.c index c5ebfe99e8..d6e89ec565 100644 --- a/src/memory.c +++ b/src/memory.c @@ -457,10 +457,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPU(void *MemoryAddress, if (AlternateVAGPU) *AlternateVAGPU = 0; - if (!fmm_map_to_gpu(MemoryAddress, MemorySizeInBytes, AlternateVAGPU)) - return HSAKMT_STATUS_SUCCESS; - else - return HSAKMT_STATUS_ERROR; + return fmm_map_to_gpu(MemoryAddress, MemorySizeInBytes, AlternateVAGPU); } HSAKMT_STATUS HSAKMTAPI hsaKmtMapMemoryToGPUNodes(void *MemoryAddress, diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 1c3b9daf11..a00a4febd6 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -2916,7 +2916,7 @@ TEST_F(KFDMemoryTest, VA_VRAM_Only_AllocTest) { reinterpret_cast(&buf))); /*mapping VA allocated by kfd api would fail*/ - ASSERT_EQ(HSAKMT_STATUS_ERROR, hsaKmtMapMemoryToGPU(buf, PAGE_SIZE, NULL)); + ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtMapMemoryToGPU(buf, PAGE_SIZE, NULL)); ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtMapMemoryToGPUNodes(buf, PAGE_SIZE, NULL, mapFlags, 1, reinterpret_cast(&defaultGPUNode))); @@ -2929,7 +2929,7 @@ TEST_F(KFDMemoryTest, VA_VRAM_Only_AllocTest) { reinterpret_cast(&buf))); /*mapping handle allocated by kfd API would fail*/ - ASSERT_EQ(HSAKMT_STATUS_ERROR, hsaKmtMapMemoryToGPU(buf, PAGE_SIZE, NULL)); + ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtMapMemoryToGPU(buf, PAGE_SIZE, NULL)); ASSERT_EQ(HSAKMT_STATUS_INVALID_PARAMETER, hsaKmtMapMemoryToGPUNodes(buf, PAGE_SIZE, NULL, mapFlags, 1, reinterpret_cast(&defaultGPUNode))); From f6183f937e8c7095192ae61b443ff46a15a7f30d Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Wed, 24 May 2023 18:16:56 -0500 Subject: [PATCH 1155/1247] libhsakmt: allow gpu nodeid arrary is null and number of gpu is zero. Allow hsaKmtRegisterGraphicsHandleToNodes parameters NodeArray be null and NumberOfNodes be zero at same time. It is the case we want the imported buffer not be registered by kfd. Set gpu_id_array = NULL explicitly to avoid free uninitialized gpuid array. Report: Yat Sin, David Signed-off-by: Xiaogang Chen Change-Id: I3babc1160c9573e38dd11d81965c8de2b70cae2e --- src/memory.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/memory.c b/src/memory.c index d6e89ec565..6b3eaf496b 100644 --- a/src/memory.c +++ b/src/memory.c @@ -306,13 +306,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterGraphicsHandleToNodes(HSAuint64 GraphicsRe HSAuint32 *NodeArray) { CHECK_KFD_OPEN(); - uint32_t *gpu_id_array; + uint32_t *gpu_id_array = NULL; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; pr_debug("[%s] number of nodes %lu\n", __func__, NumberOfNodes); - ret = validate_nodeid_array(&gpu_id_array, - NumberOfNodes, NodeArray); + if (NodeArray != NULL || NumberOfNodes != 0) { + ret = validate_nodeid_array(&gpu_id_array, + NumberOfNodes, NodeArray); + } if (ret == HSAKMT_STATUS_SUCCESS) { ret = fmm_register_graphics_handle( From 718d95de7728848b1c476eee0e3ceaeef9593bc8 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Tue, 30 May 2023 15:43:36 -0400 Subject: [PATCH 1156/1247] fmm.c: Fix possibly initialized variable usage If we end up in the first if clause, aperture_base is not set, unlike the other 2 clauses. Initialize it to NULL at declaration time, and only change its value in the final else clause, where we set it to aperture->base Change-Id: I2bf44dc93cae8a03e66f41cedd85d57be2115bba Signed-off-by: Kent Russell --- src/fmm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index efd424bb55..bf9e5191f6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -3552,7 +3552,7 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, HsaMemFlags mflags; vm_object_t *obj; void *metadata; - void *mem, *aperture_base; + void *mem, *aperture_base = NULL; int32_t gpu_mem_id; int r; HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; @@ -3591,7 +3591,6 @@ HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, aperture = &mem_handle_aperture; } else if (topology_is_svm_needed(gpu_mem[gpu_mem_id].EngineId)) { aperture = svm.dgpu_aperture; - aperture_base = NULL; } else { aperture = &gpu_mem[gpu_mem_id].gpuvm_aperture; aperture_base = aperture->base; From 728162c2c8715d17fada03d76985e82eb2a60315 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Mon, 23 Jan 2023 13:38:07 -0600 Subject: [PATCH 1157/1247] libhsakmt: include changes for upstream debugger API Signed-off-by: Alex Sierra Change-Id: Id296e13dff431c7a151c5aae0b93412b1e116467 --- include/hsakmt.h | 10 - include/hsakmttypes.h | 34 ++ include/linux/kfd_ioctl.h | 703 +++++++++++++++++++++++++++----------- src/debug.c | 60 +--- src/libhsakmt.ver | 1 - 5 files changed, 559 insertions(+), 249 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 582e907328..ff71d0c068 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -685,16 +685,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void); -/** - Get the major and minor version of the kernel debugger support. -*/ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetKernelDebugTrapVersionInfo( - HSAuint32 *Major, //Out - HSAuint32 *Minor //Out - ); - /** Gets GPU and CPU clock counters for particular Node */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 16e156d039..2e086ca9b6 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -836,6 +836,40 @@ typedef enum _HSA_DBG_TRAP_MASK HSA_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION = 256 // Memory violation } HSA_DBG_TRAP_MASK; +typedef enum _HSA_DBG_TRAP_EXCEPTION_CODE { + HSA_DBG_EC_NONE = 0, + /* per queue */ + HSA_DBG_EC_QUEUE_WAVE_ABORT = 1, + HSA_DBG_EC_QUEUE_WAVE_TRAP = 2, + HSA_DBG_EC_QUEUE_WAVE_MATH_ERROR = 3, + HSA_DBG_EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION = 4, + HSA_DBG_EC_QUEUE_WAVE_MEMORY_VIOLATION = 5, + HSA_DBG_EC_QUEUE_WAVE_APERTURE_VIOLATION = 6, + HSA_DBG_EC_QUEUE_PACKET_DISPATCH_DIM_INVALID = 16, + HSA_DBG_EC_QUEUE_PACKET_DISPATCH_GROUP_SEGMENT_SIZE_INVALID = 17, + HSA_DBG_EC_QUEUE_PACKET_DISPATCH_CODE_INVALID = 18, + HSA_DBG_EC_QUEUE_PACKET_RESERVED = 19, + HSA_DBG_EC_QUEUE_PACKET_UNSUPPORTED = 20, + HSA_DBG_EC_QUEUE_PACKET_DISPATCH_WORK_GROUP_SIZE_INVALID = 21, + HSA_DBG_EC_QUEUE_PACKET_DISPATCH_REGISTER_INVALID = 22, + HSA_DBG_EC_QUEUE_PACKET_VENDOR_UNSUPPORTED = 23, + HSA_DBG_EC_QUEUE_PREEMPTION_ERROR = 30, + HSA_DBG_EC_QUEUE_NEW = 31, + /* per device */ + HSA_DBG_EC_DEVICE_QUEUE_DELETE = 32, + HSA_DBG_EC_DEVICE_MEMORY_VIOLATION = 33, + HSA_DBG_EC_DEVICE_RAS_ERROR = 34, + HSA_DBG_EC_DEVICE_FATAL_HALT = 35, + HSA_DBG_EC_DEVICE_NEW = 36, + /* per process */ + HSA_DBG_EC_PROCESS_RUNTIME = 48, + HSA_DBG_EC_PROCESS_DEVICE_REMOVE = 49, + HSA_DBG_EC_MAX +} HSA_DBG_TRAP_EXCEPTION_CODE; + +/* Mask generated by ecode defined in enum above. */ +#define HSA_EC_MASK(ecode) (1ULL << (ecode - 1)) + typedef enum _HSA_DBG_WAVE_LAUNCH_MODE { HSA_DBG_WAVE_LAUNCH_MODE_NORMAL = 0, // Wavefront launched normally. diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 8dc670a4d6..6c030a065e 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -38,9 +38,10 @@ * - 1.10 - Add SMI profiler event log * - 1.11 - Add unified memory for ctx save/restore area * - 1.12 - Add DMA buf export ioctl + * - 1.13 - Add debugger API */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 12 +#define KFD_IOCTL_MINOR_VERSION 13 /* * Debug revision change log @@ -152,7 +153,7 @@ struct kfd_queue_snapshot_entry { __u32 ring_size; __u32 queue_type; __u32 ctx_save_restore_area_size; - __u32 reserved[17]; + __u32 reserved; }; struct kfd_dbg_device_info_entry { @@ -164,7 +165,21 @@ struct kfd_dbg_device_info_entry { __u64 gpuvm_base; __u64 gpuvm_limit; __u32 gpu_id; - __u32 pad; + __u32 location_id; + __u32 vendor_id; + __u32 device_id; + __u32 revision_id; + __u32 subsystem_vendor_id; + __u32 subsystem_device_id; + __u32 fw_version; + __u32 gfx_target_version; + __u32 simd_count; + __u32 max_waves_per_simd; + __u32 array_count; + __u32 simd_arrays_per_engine; + __u32 num_xcc; + __u32 capability; + __u32 debug_prop; }; /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */ @@ -287,7 +302,29 @@ enum kfd_dbg_trap_mask { KFD_DBG_TRAP_MASK_FP_INEXACT = 32, KFD_DBG_TRAP_MASK_INT_DIVIDE_BY_ZERO = 64, KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH = 128, - KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION = 256 + KFD_DBG_TRAP_MASK_DBG_MEMORY_VIOLATION = 256, + KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START = (1 << 30), + KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END = (1 << 31) +}; + +/* Wave launch modes */ +enum kfd_dbg_trap_wave_launch_mode { + KFD_DBG_TRAP_WAVE_LAUNCH_MODE_NORMAL = 0, + KFD_DBG_TRAP_WAVE_LAUNCH_MODE_HALT = 1, + KFD_DBG_TRAP_WAVE_LAUNCH_MODE_DEBUG = 3 +}; + +/* Address watch modes */ +enum kfd_dbg_trap_address_watch_mode { + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_READ = 0, + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_NONREAD = 1, + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_ATOMIC = 2, + KFD_DBG_TRAP_ADDRESS_WATCH_MODE_ALL = 3 +}; + +/* Additional wave settings */ +enum kfd_dbg_trap_flags { + KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP = 1, }; enum kfd_dbg_trap_exception_code { @@ -369,217 +406,492 @@ struct kfd_runtime_info { __u32 ttmp_setup; }; -/* KFD_IOC_DBG_TRAP_ENABLE: - * exception_mask: exceptions to be reported to the debugger - * ptr: runtime info buffer to copy to (IN) - * data1: 0=disable, 1=enable - * data2: return value for fd - * data3: runtime info size - * data4: unused +/* Enable modes for runtime enable */ +#define KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK 1 +#define KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK 2 +#define KFD_RUNTIME_ENABLE_CAPS_SUPPORTS_CORE_DUMP_MASK 0x80000000 + +/** + * kfd_ioctl_runtime_enable_args - Arguments for runtime enable + * + * Coordinates debug exception signalling and debug device enablement with runtime. + * + * @r_debug - pointer to user struct for sharing information between ROCr and the debuggger + * @mode_mask - mask to set mode + * KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK - enable runtime for debugging, otherwise disable + * KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK - enable trap temporary setup (ignore on disable) + * + * Return - 0 on SUCCESS. + * - EBUSY if runtime enable call already pending. + * - EEXIST if user queues already active prior to call. + * If process is debug enabled, runtime enable will enable debug devices and + * wait for debugger process to send runtime exception EC_PROCESS_RUNTIME + * to unblock - see kfd_ioctl_dbg_trap_args. * */ -#define KFD_IOC_DBG_TRAP_ENABLE 0 +struct kfd_ioctl_runtime_enable_args { + __u64 r_debug; + __u32 mode_mask; + __u32 capabilities_mask; +}; -/* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE: - * exception_mask: unused - * ptr: unused - * data1: override mode (see enum kfd_dbg_trap_override_mode) - * data2: [in/out] trap mask (see enum kfd_dbg_trap_mask) - * data3: [in] requested mask, [out] supported mask - * data4: unused +/* Context save area header information */ +struct kfd_context_save_area_header { + struct { + __u32 control_stack_offset; + __u32 control_stack_size; + __u32 wave_state_offset; + __u32 wave_state_size; + } wave_state; + __u32 debug_offset; + __u32 debug_size; + __u64 err_payload_addr; + __u32 err_event_id; + __u32 reserved1; +}; + +/* + * Debug operations * - * May fail with -EPERM if the requested mode is not supported. + * For specifics on usage and return values, see documentation per operation + * below. Otherwise, generic error returns apply: + * - ESRCH if the process to debug does not exist. * - * May fail with -EACCES if requested trap mask bits are not supported. - * In that case the supported trap mask bits are returned in data3. + * - EINVAL (with KFD_IOC_DBG_TRAP_ENABLE exempt) if operation + * KFD_IOC_DBG_TRAP_ENABLE has not succeeded prior. + * Also returns this error if GPU hardware scheduling is not supported. * - * If successful, output parameters return the previous trap mask - * value and the hardware-dependent mask of supported trap mask bits. - */ -#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE 1 - -/* KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE: - * exception_mask: unused - * ptr: unused - * data1: 0=normal, 1=halt, 2=kill, 3=singlestep, 4=disable - * data2: unused - * data3: unused - * data4: unused - */ -#define KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE 2 - -/* KFD_IOC_DBG_TRAP_NODE_SUSPEND: - * exception_mask: exceptions to clear on suspend - * ptr: pointer to an array of Queues IDs (IN/OUT) - * data1: number of queues (IN) - * data2: grace period (IN) - * data3: unused - * data4: unused + * - EPERM (with KFD_IOC_DBG_TRAP_DISABLE exempt) if target process is not + * PTRACE_ATTACHED. KFD_IOC_DBG_TRAP_DISABLE is exempt to allow + * clean up of debug mode as long as process is debug enabled. * - * Returns the number of queues suspended from array of Queue IDs (ptr). - * Requested queues that fail to suspend are masked in the array: + * - EACCES if any DBG_HW_OP (debug hardware operation) is requested when + * AMDKFD_IOC_RUNTIME_ENABLE has not succeeded prior. * - * KFD_DBG_QUEUE_INVALID_MASK - requested queue does not exist or cannot be - * suspended (new or being destroyed). + * - ENODEV if any GPU does not support debugging on a DBG_HW_OP call. * - * KFD_DBG_QUEUE_ERROR_MASK - bad internal operation occurred on requested - * queue. + * - Other errors may be returned when a DBG_HW_OP occurs while the GPU + * is in a fatal state. * - * NOTE! All queue destroy requests will be blocked on a suspended queue. - * Queue resume will unblock. + */ +enum kfd_dbg_trap_operations { + KFD_IOC_DBG_TRAP_ENABLE = 0, + KFD_IOC_DBG_TRAP_DISABLE = 1, + KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT = 2, + KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED = 3, + KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE = 4, /* DBG_HW_OP */ + KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE = 5, /* DBG_HW_OP */ + KFD_IOC_DBG_TRAP_SUSPEND_QUEUES = 6, /* DBG_HW_OP */ + KFD_IOC_DBG_TRAP_RESUME_QUEUES = 7, /* DBG_HW_OP */ + KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH = 8, /* DBG_HW_OP */ + KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH = 9, /* DBG_HW_OP */ + KFD_IOC_DBG_TRAP_SET_FLAGS = 10, + KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT = 11, + KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO = 12, + KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT = 13, + KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT = 14 +}; + +/** + * kfd_ioctl_dbg_trap_enable_args * - * Grace period (data2) is time allowed for waves to complete before CWSR. - * 0 can be entered for immediate preemption. - */ -#define KFD_IOC_DBG_TRAP_NODE_SUSPEND 3 - -/* KFD_IOC_DBG_TRAP_NODE_RESUME: - * exception_mask: unused - * ptr: pointer to an array of Queues IDs (IN/OUT) - * data1: number of queues (IN) - * data2: unused - * data3: unused - * data4: unused + * Arguments for KFD_IOC_DBG_TRAP_ENABLE. * - * Returns the number of queues resumed from array of Queue IDs (ptr). - * Requested queues that fail to resume are masked in the array: + * Enables debug session for target process. Call @op KFD_IOC_DBG_TRAP_DISABLE in + * kfd_ioctl_dbg_trap_args to disable debug session. * - * KFD_DBG_QUEUE_INVALID_MASK - requested queue does not exist. + * @exception_mask (IN) - exceptions to raise to the debugger + * @rinfo_ptr (IN) - pointer to runtime info buffer (see kfd_runtime_info) + * @rinfo_size (IN/OUT) - size of runtime info buffer in bytes + * @dbg_fd (IN) - fd the KFD will nofify the debugger with of raised + * exceptions set in exception_mask. * - * KFD_DBG_QUEUE_ERROR_MASK - bad internal operation occurred on requested - * queue. - */ -#define KFD_IOC_DBG_TRAP_NODE_RESUME 4 - -/* KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT: - * exception_mask: exception to clear (IN) on query and report (OUT) - * ptr: unused - * data1: queue id (OUT) - * data2: gpu id (OUT) - * data3: unused - * data4: unused + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * Copies KFD saved kfd_runtime_info to @rinfo_ptr on enable. + * Size of kfd_runtime saved by the KFD returned to @rinfo_size. + * - EBADF if KFD cannot get a reference to dbg_fd. + * - EFAULT if KFD cannot copy runtime info to rinfo_ptr. + * - EINVAL if target process is already debug enabled. * - * Source ID (data 1 - OUT) returns the queue_id for a queue event, the gpu_id - * for a device event or 0 for a process event. + */ +struct kfd_ioctl_dbg_trap_enable_args { + __u64 exception_mask; + __u64 rinfo_ptr; + __u32 rinfo_size; + __u32 dbg_fd; +}; + +/** + * kfd_ioctl_dbg_trap_send_runtime_event_args * - * Returns EAGAIN if no event is found and 0 otherwise. - */ -#define KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT 5 - -/* KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT: - * exception_mask: exception to clear on snapshot - * ptr: user buffer (IN) - * data1: number of queue snapshots (IN/OUT) - 0 for IN ignores buffer writes - * data2: unused - * data3: unused - * data4: unused - */ -#define KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT 6 - -/* KFD_IOC_DBG_TRAP_GET_VERSION: - * exception_mask: unused - * ptr: unsused - * data1: major version (OUT) - * data2: minor version (OUT) - * data3: unused - * data4: unused - */ -#define KFD_IOC_DBG_TRAP_GET_VERSION 7 - -/* KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH: - * exception_mask: unused - * ptr: unused - * data1: watch ID - * data2: unused - * data3: unused - * data4: unused - */ -#define KFD_IOC_DBG_TRAP_CLEAR_ADDRESS_WATCH 8 - -/* KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH: - * exception_mask: unused - * ptr: Watch address - * data1: Watch ID (OUT) - * data2: watch_mode: 0=read, 1=nonread, 2=atomic, 3=all - * data3: watch address mask - * data4: unused - */ -#define KFD_IOC_DBG_TRAP_SET_ADDRESS_WATCH 9 - -/* KFD_IOC_DBG_TRAP_SET_PRECISE_MEM_OPS - * exception_mask: unused - * ptr: unused - * data1: 0=disable, 1=enable (IN) - * data2: unused - * data3: unused - * data4: unused - */ -#define KFD_IOC_DBG_TRAP_SET_PRECISE_MEM_OPS 10 - -/* KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO - * exception_mask: unused - * ptr: exception info pointer to copy to - * data1: source_id - * data2: exception_code - * data3: clear_exception (1 == true, 0 == false) - * data4: exception info data size * - * NOTE: If data2 == EC_PROCESS_RUNTIME, the saved runtime info will be copied - * to the exception info pointer. - */ -#define KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO 11 - -/* KFD_IOC_DBG_TRAP_DEVICE_SNAPSHOT - * exception_mask: exception to clear on snapshot - * ptr: user buffer for 'struct kfd_dbg_device_info_entry' entries (IN) - * data1: number of devices in snapshot (IN/OUT) - * data2: unused - * data3: unused - * data4: unused - */ -#define KFD_IOC_DBG_TRAP_DEVICE_SNAPSHOT 12 - -/* KFD_IOC_DBG_TRAP_RUNTIME_ENABLE - * exception_mask: unused - * ptr: r_debug info to save - * data1: enable (0=disable, 1=enable) - * data2: enable ttmp save (0=disable, 1=enable) - * data3: unused - * data4: unused + * Arguments for KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT. + * Raises exceptions to runtime. * - * FIXME: This option is temporary. Future upstream will use a separate IOCTL. + * @exception_mask (IN) - exceptions to raise to runtime + * @gpu_id (IN) - target device id + * @queue_id (IN) - target queue id + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * - ENODEV if gpu_id not found. + * If exception_mask contains EC_PROCESS_RUNTIME, unblocks pending + * AMDKFD_IOC_RUNTIME_ENABLE call - see kfd_ioctl_runtime_enable_args. + * All other exceptions are raised to runtime through err_payload_addr. + * See kfd_context_save_area_header. */ -#define KFD_IOC_DBG_TRAP_RUNTIME_ENABLE 13 +struct kfd_ioctl_dbg_trap_send_runtime_event_args { + __u64 exception_mask; + __u32 gpu_id; + __u32 queue_id; +}; -/* KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT - * exception_mask: exception to send - * ptr: unused - * data1: destination device id - * data2: destination queue id - * data3: usused - * data4: unused +/** + * kfd_ioctl_dbg_trap_set_exceptions_enabled_args + * + * Arguments for KFD_IOC_SET_EXCEPTIONS_ENABLED + * Set new exceptions to be raised to the debugger. + * + * @exception_mask (IN) - new exceptions to raise the debugger + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. */ -#define KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT 14 +struct kfd_ioctl_dbg_trap_set_exceptions_enabled_args { + __u64 exception_mask; +}; -/* KFD_IOC_DBG_TRAP_SET_EXCEPTION_ENABLED - * exception_mask: exception to set - * ptr: unused - * data1: unused - * data2: unused - * data3: usused - * data4: unused +/** + * kfd_ioctl_dbg_trap_set_wave_launch_override_args + * + * Arguments for KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE + * Enable HW exceptions to raise trap. + * + * @override_mode (IN) - see kfd_dbg_trap_override_mode + * @enable_mask (IN/OUT) - reference kfd_dbg_trap_mask. + * IN is the override modes requested to be enabled. + * OUT is referenced in Return below. + * @support_request_mask (IN/OUT) - reference kfd_dbg_trap_mask. + * IN is the override modes requested for support check. + * OUT is referenced in Return below. + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * Previous enablement is returned in @enable_mask. + * Actual override support is returned in @support_request_mask. + * - EINVAL if override mode is not supported. + * - EACCES if trap support requested is not actually supported. + * i.e. enable_mask (IN) is not a subset of support_request_mask (OUT). + * Otherwise it is considered a generic error (see kfd_dbg_trap_operations). */ -#define KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED 15 +struct kfd_ioctl_dbg_trap_set_wave_launch_override_args { + __u32 override_mode; + __u32 enable_mask; + __u32 support_request_mask; + __u32 pad; +}; +/** + * kfd_ioctl_dbg_trap_set_wave_launch_mode_args + * + * Arguments for KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_MODE + * Set wave launch mode. + * + * @mode (IN) - see kfd_dbg_trap_wave_launch_mode + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + */ +struct kfd_ioctl_dbg_trap_set_wave_launch_mode_args { + __u32 launch_mode; + __u32 pad; +}; + +/** + * kfd_ioctl_dbg_trap_suspend_queues_ags + * + * Arguments for KFD_IOC_DBG_TRAP_SUSPEND_QUEUES + * Suspend queues. + * + * @exception_mask (IN) - raised exceptions to clear + * @queue_array_ptr (IN) - pointer to array of queue ids (u32 per queue id) + * to suspend + * @num_queues (IN) - number of queues to suspend in @queue_array_ptr + * @grace_period (IN) - wave time allowance before preemption + * per 1K GPU clock cycle unit + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Destruction of a suspended queue is blocked until the queue is + * resumed. This allows the debugger to access queue information and + * the its context save area without running into a race condition on + * queue destruction. + * Automatically copies per queue context save area header information + * into the save area base + * (see kfd_queue_snapshot_entry and kfd_context_save_area_header). + * + * Return - Number of queues suspended on SUCCESS. + * . KFD_DBG_QUEUE_ERROR_MASK and KFD_DBG_QUEUE_INVALID_MASK masked + * for each queue id in @queue_array_ptr array reports unsuccessful + * suspend reason. + * KFD_DBG_QUEUE_ERROR_MASK = HW failure. + * KFD_DBG_QUEUE_INVALID_MASK = queue does not exist, is new or + * is being destroyed. + */ +struct kfd_ioctl_dbg_trap_suspend_queues_args { + __u64 exception_mask; + __u64 queue_array_ptr; + __u32 num_queues; + __u32 grace_period; +}; + +/** + * kfd_ioctl_dbg_trap_resume_queues_args + * + * Arguments for KFD_IOC_DBG_TRAP_RESUME_QUEUES + * Resume queues. + * + * @queue_array_ptr (IN) - pointer to array of queue ids (u32 per queue id) + * to resume + * @num_queues (IN) - number of queues to resume in @queue_array_ptr + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - Number of queues resumed on SUCCESS. + * KFD_DBG_QUEUE_ERROR_MASK and KFD_DBG_QUEUE_INVALID_MASK mask + * for each queue id in @queue_array_ptr array reports unsuccessful + * resume reason. + * KFD_DBG_QUEUE_ERROR_MASK = HW failure. + * KFD_DBG_QUEUE_INVALID_MASK = queue does not exist. + */ +struct kfd_ioctl_dbg_trap_resume_queues_args { + __u64 queue_array_ptr; + __u32 num_queues; + __u32 pad; +}; + +/** + * kfd_ioctl_dbg_trap_set_node_address_watch_args + * + * Arguments for KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH + * Sets address watch for device. + * + * @address (IN) - watch address to set + * @mode (IN) - see kfd_dbg_trap_address_watch_mode + * @mask (IN) - watch address mask + * @gpu_id (IN) - target gpu to set watch point + * @id (OUT) - watch id allocated + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * Allocated watch ID returned to @id. + * - ENODEV if gpu_id not found. + * - ENOMEM if watch IDs can be allocated + */ +struct kfd_ioctl_dbg_trap_set_node_address_watch_args { + __u64 address; + __u32 mode; + __u32 mask; + __u32 gpu_id; + __u32 id; +}; + +/** + * kfd_ioctl_dbg_trap_clear_node_address_watch_args + * + * Arguments for KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH + * Clear address watch for device. + * + * @gpu_id (IN) - target device to clear watch point + * @id (IN) - allocated watch id to clear + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * - ENODEV if gpu_id not found. + * - EINVAL if watch ID has not been allocated. + */ +struct kfd_ioctl_dbg_trap_clear_node_address_watch_args { + __u32 gpu_id; + __u32 id; +}; + +/** + * kfd_ioctl_dbg_trap_set_flags_args + * + * Arguments for KFD_IOC_DBG_TRAP_SET_FLAGS + * Sets flags for wave behaviour. + * + * @flags (IN/OUT) - IN = flags to enable, OUT = flags previously enabled + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * - EACCESS if any debug device does not allow flag options. + */ +struct kfd_ioctl_dbg_trap_set_flags_args { + __u32 flags; + __u32 pad; +}; + +/** + * kfd_ioctl_dbg_trap_query_debug_event_args + * + * Arguments for KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT + * + * Find one or more raised exceptions. This function can return multiple + * exceptions from a single queue or a single device with one call. To find + * all raised exceptions, this function must be called repeatedly until it + * returns -EAGAIN. Returned exceptions can optionally be cleared by + * setting the corresponding bit in the @exception_mask input parameter. + * However, clearing an exception prevents retrieving further information + * about it with KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO. + * + * @exception_mask (IN/OUT) - exception to clear (IN) and raised (OUT) + * @gpu_id (OUT) - gpu id of exceptions raised + * @queue_id (OUT) - queue id of exceptions raised + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on raised exception found + * Raised exceptions found are returned in @exception mask + * with reported source id returned in @gpu_id or @queue_id. + * - EAGAIN if no raised exception has been found + */ +struct kfd_ioctl_dbg_trap_query_debug_event_args { + __u64 exception_mask; + __u32 gpu_id; + __u32 queue_id; +}; + +/** + * kfd_ioctl_dbg_trap_query_exception_info_args + * + * Arguments KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO + * Get additional info on raised exception. + * + * @info_ptr (IN) - pointer to exception info buffer to copy to + * @info_size (IN/OUT) - exception info buffer size (bytes) + * @source_id (IN) - target gpu or queue id + * @exception_code (IN) - target exception + * @clear_exception (IN) - clear raised @exception_code exception + * (0 = false, 1 = true) + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * If @exception_code is EC_DEVICE_MEMORY_VIOLATION, copy @info_size(OUT) + * bytes of memory exception data to @info_ptr. + * If @exception_code is EC_PROCESS_RUNTIME, copy saved + * kfd_runtime_info to @info_ptr. + * Actual required @info_ptr size (bytes) is returned in @info_size. + */ +struct kfd_ioctl_dbg_trap_query_exception_info_args { + __u64 info_ptr; + __u32 info_size; + __u32 source_id; + __u32 exception_code; + __u32 clear_exception; +}; + +/** + * kfd_ioctl_dbg_trap_get_queue_snapshot_args + * + * Arguments KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT + * Get queue information. + * + * @exception_mask (IN) - exceptions raised to clear + * @snapshot_buf_ptr (IN) - queue snapshot entry buffer (see kfd_queue_snapshot_entry) + * @num_queues (IN/OUT) - number of queue snapshot entries + * The debugger specifies the size of the array allocated in @num_queues. + * KFD returns the number of queues that actually existed. If this is + * larger than the size specified by the debugger, KFD will not overflow + * the array allocated by the debugger. + * + * @entry_size (IN/OUT) - size per entry in bytes + * The debugger specifies sizeof(struct kfd_queue_snapshot_entry) in + * @entry_size. KFD returns the number of bytes actually populated per + * entry. The debugger should use the KFD_IOCTL_MINOR_VERSION to determine, + * which fields in struct kfd_queue_snapshot_entry are valid. This allows + * growing the ABI in a backwards compatible manner. + * Note that entry_size(IN) should still be used to stride the snapshot buffer in the + * event that it's larger than actual kfd_queue_snapshot_entry. + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * Copies @num_queues(IN) queue snapshot entries of size @entry_size(IN) + * into @snapshot_buf_ptr if @num_queues(IN) > 0. + * Otherwise return @num_queues(OUT) queue snapshot entries that exist. + */ +struct kfd_ioctl_dbg_trap_queue_snapshot_args { + __u64 exception_mask; + __u64 snapshot_buf_ptr; + __u32 num_queues; + __u32 entry_size; +}; + +/** + * kfd_ioctl_dbg_trap_get_device_snapshot_args + * + * Arguments for KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT + * Get device information. + * + * @exception_mask (IN) - exceptions raised to clear + * @snapshot_buf_ptr (IN) - pointer to snapshot buffer (see kfd_dbg_device_info_entry) + * @num_devices (IN/OUT) - number of debug devices to snapshot + * The debugger specifies the size of the array allocated in @num_devices. + * KFD returns the number of devices that actually existed. If this is + * larger than the size specified by the debugger, KFD will not overflow + * the array allocated by the debugger. + * + * @entry_size (IN/OUT) - size per entry in bytes + * The debugger specifies sizeof(struct kfd_dbg_device_info_entry) in + * @entry_size. KFD returns the number of bytes actually populated. The + * debugger should use KFD_IOCTL_MINOR_VERSION to determine, which fields + * in struct kfd_dbg_device_info_entry are valid. This allows growing the + * ABI in a backwards compatible manner. + * Note that entry_size(IN) should still be used to stride the snapshot buffer in the + * event that it's larger than actual kfd_dbg_device_info_entry. + * + * Generic errors apply (see kfd_dbg_trap_operations). + * Return - 0 on SUCCESS. + * Copies @num_devices(IN) device snapshot entries of size @entry_size(IN) + * into @snapshot_buf_ptr if @num_devices(IN) > 0. + * Otherwise return @num_devices(OUT) queue snapshot entries that exist. + */ +struct kfd_ioctl_dbg_trap_device_snapshot_args { + __u64 exception_mask; + __u64 snapshot_buf_ptr; + __u32 num_devices; + __u32 entry_size; +}; + +/** + * kfd_ioctl_dbg_trap_args + * + * Arguments to debug target process. + * + * @pid - target process to debug + * @op - debug operation (see kfd_dbg_trap_operations) + * + * @op determines which union struct args to use. + * Refer to kern docs for each kfd_ioctl_dbg_trap_*_args struct. + */ struct kfd_ioctl_dbg_trap_args { - __u64 exception_mask; /* to KFD */ - __u64 ptr; /* to KFD */ - __u32 pid; /* to KFD */ - __u32 op; /* to KFD */ - __u32 data1; /* to KFD */ - __u32 data2; /* to KFD */ - __u32 data3; /* to KFD */ - __u32 data4; /* to KFD */ + __u32 pid; + __u32 op; + + union { + struct kfd_ioctl_dbg_trap_enable_args enable; + struct kfd_ioctl_dbg_trap_send_runtime_event_args send_runtime_event; + struct kfd_ioctl_dbg_trap_set_exceptions_enabled_args set_exceptions_enabled; + struct kfd_ioctl_dbg_trap_set_wave_launch_override_args launch_override; + struct kfd_ioctl_dbg_trap_set_wave_launch_mode_args launch_mode; + struct kfd_ioctl_dbg_trap_suspend_queues_args suspend_queues; + struct kfd_ioctl_dbg_trap_resume_queues_args resume_queues; + struct kfd_ioctl_dbg_trap_set_node_address_watch_args set_node_address_watch; + struct kfd_ioctl_dbg_trap_clear_node_address_watch_args clear_node_address_watch; + struct kfd_ioctl_dbg_trap_set_flags_args set_flags; + struct kfd_ioctl_dbg_trap_query_debug_event_args query_debug_event; + struct kfd_ioctl_dbg_trap_query_exception_info_args query_exception_info; + struct kfd_ioctl_dbg_trap_queue_snapshot_args queue_snapshot; + struct kfd_ioctl_dbg_trap_device_snapshot_args device_snapshot; + }; }; /* Matching HSA_EVENTTYPE */ @@ -1355,8 +1667,14 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_EXPORT_DMABUF \ AMDKFD_IOWR(0x24, struct kfd_ioctl_export_dmabuf_args) +#define AMDKFD_IOC_RUNTIME_ENABLE \ + AMDKFD_IOWR(0x25, struct kfd_ioctl_runtime_enable_args) + +#define AMDKFD_IOC_DBG_TRAP \ + AMDKFD_IOWR(0x26, struct kfd_ioctl_dbg_trap_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x25 +#define AMDKFD_COMMAND_END 0x27 /* non-upstream ioctls */ #define AMDKFD_IOC_IPC_IMPORT_HANDLE \ @@ -1365,9 +1683,6 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_IPC_EXPORT_HANDLE \ AMDKFD_IOWR(0x81, struct kfd_ioctl_ipc_export_handle_args) -#define AMDKFD_IOC_DBG_TRAP \ - AMDKFD_IOWR(0x82, struct kfd_ioctl_dbg_trap_args) - #define AMDKFD_IOC_CROSS_MEMORY_COPY \ AMDKFD_IOWR(0x83, struct kfd_ioctl_cross_memory_copy_args) diff --git a/src/debug.c b/src/debug.c index d225e0cc9a..fcc10b1933 100644 --- a/src/debug.c +++ b/src/debug.c @@ -265,38 +265,13 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(HSAuint32 NodeId, return HSAKMT_STATUS_SUCCESS; } -/* Get the major and minor version of the kernel debugger support. */ -HSAKMT_STATUS -HSAKMTAPI -hsaKmtGetKernelDebugTrapVersionInfo( - HSAuint32 *Major, //Out - HSAuint32 *Minor //Out -) -{ - struct kfd_ioctl_dbg_trap_args args = {0}; - - memset(&args, 0x00, sizeof(args)); - args.op = KFD_IOC_DBG_TRAP_GET_VERSION; - args.pid = getpid(); - - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); - - if (err) - return HSAKMT_STATUS_ERROR; - - *Major = args.data1; - *Minor = args.data2; - return HSAKMT_STATUS_SUCCESS; -} - -#define HSA_RUNTIME_ENABLE_MIN_MAJOR 10 -#define HSA_RUNTIME_ENABLE_MAX_MAJOR 13 -#define HSA_RUNTIME_ENABLE_MIN_MINOR 0 +#define HSA_RUNTIME_ENABLE_MAX_MAJOR 1 +#define HSA_RUNTIME_ENABLE_MIN_MINOR 13 static HSAKMT_STATUS checkRuntimeDebugSupport(void) { - HSAuint32 kMajor, kMinor; HsaNodeProperties node = {0}; HsaSystemProperties props = {0}; + HsaVersionInfo versionInfo = {0}; memset(&node, 0x00, sizeof(node)); memset(&props, 0x00, sizeof(props)); @@ -315,12 +290,13 @@ static HSAKMT_STATUS checkRuntimeDebugSupport(void) { return HSAKMT_STATUS_NOT_SUPPORTED; } - if (hsaKmtGetKernelDebugTrapVersionInfo(&kMajor, &kMinor)) + if (hsaKmtGetVersion(&versionInfo)) return HSAKMT_STATUS_NOT_SUPPORTED; - if (kMajor < HSA_RUNTIME_ENABLE_MIN_MAJOR || kMajor > HSA_RUNTIME_ENABLE_MAX_MAJOR || - (kMajor == HSA_RUNTIME_ENABLE_MIN_MAJOR && - (int)kMinor < HSA_RUNTIME_ENABLE_MIN_MINOR)) + if (versionInfo.KernelInterfaceMajorVersion < HSA_RUNTIME_ENABLE_MAX_MAJOR || + (versionInfo.KernelInterfaceMajorVersion == + HSA_RUNTIME_ENABLE_MAX_MAJOR && + (int)versionInfo.KernelInterfaceMinorVersion < HSA_RUNTIME_ENABLE_MIN_MINOR)) return HSAKMT_STATUS_NOT_SUPPORTED; return HSAKMT_STATUS_SUCCESS; @@ -329,20 +305,18 @@ static HSAKMT_STATUS checkRuntimeDebugSupport(void) { HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, bool setupTtmp) { - struct kfd_ioctl_dbg_trap_args args = {0}; + struct kfd_ioctl_runtime_enable_args args = {0}; HSAKMT_STATUS result = checkRuntimeDebugSupport(); if (result) return result; memset(&args, 0x00, sizeof(args)); - args.op = KFD_IOC_DBG_TRAP_RUNTIME_ENABLE; - args.pid = getpid(); - args.data1 = 1; //enable - args.data2 = setupTtmp; - args.ptr = (HSAuint64)rDebug; + args.mode_mask = KFD_RUNTIME_ENABLE_MODE_ENABLE_MASK | + ((setupTtmp) ? KFD_RUNTIME_ENABLE_MODE_TTMP_SAVE_MASK : 0); + args.r_debug = (HSAuint64)rDebug; - long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_RUNTIME_ENABLE, &args); if (err) { if (errno == EBUSY) @@ -356,18 +330,16 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) { - struct kfd_ioctl_dbg_trap_args args = {0}; + struct kfd_ioctl_runtime_enable_args args = {0}; HSAKMT_STATUS result = checkRuntimeDebugSupport(); if (result) return result; memset(&args, 0x00, sizeof(args)); - args.op = KFD_IOC_DBG_TRAP_RUNTIME_ENABLE; - args.pid = getpid(); - args.data1 = 0; //disable + args.mode_mask = 0; //Disable - if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + if (kmtIoctl(kfd_fd, AMDKFD_IOC_RUNTIME_ENABLE, &args)) return HSAKMT_STATUS_ERROR; return HSAKMT_STATUS_SUCCESS; diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 50564de31b..fe34241b22 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -59,7 +59,6 @@ hsaKmtQueryPointerInfo; hsaKmtSetMemoryUserData; hsaKmtGetQueueInfo; hsaKmtAllocQueueGWS; -hsaKmtGetKernelDebugTrapVersionInfo; hsaKmtRuntimeEnable; hsaKmtRuntimeDisable; hsaKmtSPMAcquire; From 1428a7538ef4e9f10ae035f8594bc51d4fb9921d Mon Sep 17 00:00:00 2001 From: Sreekant Somasekharan Date: Mon, 5 Jun 2023 11:29:30 -0400 Subject: [PATCH 1158/1247] kfdtest: RoundToPowerOf2 function modified for compiler compliant bit shift values Compiler behavior is undefined if the right operand is negative, or greater than or equal to the width of the promoted left operand. For release builds with address sanitizer enabled, this compiler optimization behavior leads to unsupported queue size value since current method shifts till 128 bits on a 64 bit value. Signed-off-by: Sreekant Somasekharan Change-Id: Iafdc82d0dfb7f79e3012fb7bb70eda80e4b7a7a6 --- tests/kfdtest/src/KFDTestUtil.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index d02804eb09..6659bc036b 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -65,17 +65,18 @@ HSAKMT_STATUS fscanf_dec(const char *file, uint32_t *num) } uint64_t RoundToPowerOf2(uint64_t val) { - int bytes = sizeof(uint64_t); + val--; - val--; + /* Shift with amount larger than the bit width can result in + * undefined behavior by compiler for release builds. + * Shift till 32 bit only which is less than bit width of val. + */ + for (int i = 1; i <= 32; i *= 2) + val |= val >> i; - for (int i = 0; i < bytes; i++) { - val |= val >> (1 << i); - } + val++; - val++; - - return val; + return val; } bool WaitOnValue(const volatile unsigned int *buf, unsigned int value, unsigned int timeOut) { From d1a095123d7ee9ce7c3c38bedab30a821b7cc156 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Mon, 5 Jun 2023 12:51:57 -0400 Subject: [PATCH 1159/1247] kfdtest: Update GFX11 blacklist KFDQMTest.MultipleCpQueuesStressDispatch is fixed as of MES SCHQ version 0x3c (). Signed-off-by: Graham Sider Change-Id: I437f3eb5f12dc159339a9b7c7cff2e2b8214ad7c --- tests/kfdtest/scripts/kfdtest.exclude | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index a4c4ee1db1..e2d0b93e2b 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -117,11 +117,9 @@ BLACKLIST_GFX10_NV2X=\ "KFDPerfCountersTest.*" # KFDMemoryTest.FlatScratchAccess - SWDEV-329877 -# KFDQMTest.MultipleCpQueuesStressDispatch - SWDEV-340965 BLACKLIST_GFX11=\ "KFDQMTest.CreateAqlCpQueue:"\ "KFDQMTest.BasicCuMaskingEven:"\ -"KFDQMTest.MultipleCpQueuesStressDispatch:"\ "KFDCWSRTest.InterruptRestore:"\ "KFDPerfCountersTest.*:"\ "KFDMemoryTest.FlatScratchAccess" From 4675492852cc9d289a564320f25a2245faaaad9b Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Wed, 7 Jun 2023 16:01:24 -0400 Subject: [PATCH 1160/1247] kfdtest: Fix minor typo The purpose of this patch is to fix a minor typo in KFDSVMRangeTest. Before: "Skipping test: no enough system memory." After: "Skipping test: Not enough system memory." Signed-off-by: Ori Messinger Change-Id: I247cb558a177a1d25c393bf16c7386f4d79d0fba --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index 86ec70f0fb..a84ab5003c 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -1523,7 +1523,7 @@ TEST_P(KFDSVMRangeTest, VramOvercommitTest) { /* With XNACK off, KFD checks that all SVM memory will fit into system memory */ if (vramSize + overCommitSize > GetSysMemSize() / 2) { - LOG() << "Skipping test: no enough system memory." << std::endl; + LOG() << "Skipping test: Not enough system memory." << std::endl; return; } From a0cbf90b90cd7498c36e7c181fd675217f08dd87 Mon Sep 17 00:00:00 2001 From: James Zhu Date: Wed, 17 May 2023 16:30:30 -0400 Subject: [PATCH 1161/1247] libhsakmt: add event age tracking Keeping last signaled event age to avoid race conditions for HSA_EVENTTYPE_SIGNAL when event age init value is non-zero. Change-Id: Ifb9a11a6868e5762a9f92f579e45a0a2c8fa1017 Signed-off-by: James Zhu --- include/hsakmt.h | 43 +++++++++++++++++++++++++++++++++++++++ include/linux/kfd_ioctl.h | 13 ++++++++++-- src/events.c | 25 ++++++++++++++++++++++- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index ff71d0c068..5a7e2f4c37 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -243,6 +243,25 @@ hsaKmtWaitOnEvent( HSAuint32 Milliseconds //IN ); +/** + Checks the current state of the event object. If the object's state is + nonsignaled, the calling thread enters the wait state. event_age can + help avoiding race conditions. + + The function returns when one of the following occurs: +- The specified event object is in the signaled state. +- The time-out interval elapses. +- Tracking event age +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtWaitOnEvent_Ext( + HsaEvent* Event, //IN + HSAuint32 Milliseconds, //IN + uint64_t *event_age //IN/OUT + ); + /** Checks the current state of multiple event objects. @@ -264,6 +283,30 @@ hsaKmtWaitOnMultipleEvents( HSAuint32 Milliseconds //IN ); +/** + Checks the current state of multiple event objects. + event_age can help avoiding race conditions. + + The function returns when one of the following occurs: +- Either any one or all of the specified objects are in the signaled state + - if "WaitOnAll" is "true" the function returns when the state of all + objects in array is signaled + - if "WaitOnAll" is "false" the function returns when the state of any + one of the objects is set to signaled +- The time-out interval elapses. +- Tracking event age +*/ + +HSAKMT_STATUS +HSAKMTAPI +hsaKmtWaitOnMultipleEvents_Ext( + HsaEvent* Events[], //IN + HSAuint32 NumEvents, //IN + bool WaitOnAll, //IN + HSAuint32 Milliseconds, //IN + uint64_t *event_age //IN/OUT + ); + /** new TEMPORARY function definition - to be used only on "Triniti + Southern Islands" platform If used on other platforms the function will return HSAKMT_STATUS_ERROR diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 6c030a065e..9080505614 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -39,9 +39,10 @@ * - 1.11 - Add unified memory for ctx save/restore area * - 1.12 - Add DMA buf export ioctl * - 1.13 - Add debugger API + * - 1.14 - Update kfd_event_data */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 13 +#define KFD_IOCTL_MINOR_VERSION 14 /* * Debug revision change log @@ -979,12 +980,20 @@ struct kfd_hsa_hw_exception_data { __u32 gpu_id; }; +/* hsa signal event data */ +struct kfd_hsa_signal_event_data { + __u64 last_event_age; /* to and from KFD */ +}; + /* Event data */ struct kfd_event_data { union { + /* From KFD */ struct kfd_hsa_memory_exception_data memory_exception_data; struct kfd_hsa_hw_exception_data hw_exception_data; - }; /* From KFD */ + /* To and From KFD */ + struct kfd_hsa_signal_event_data signal_event_data; + }; __u64 kfd_event_data_ext; /* pointer to an extension structure for future exception types */ __u32 event_id; /* to KFD */ diff --git a/src/events.c b/src/events.c index 512abc2be0..6d3cd70805 100644 --- a/src/events.c +++ b/src/events.c @@ -213,11 +213,17 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtQueryEventState(HsaEvent *Event) HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent(HsaEvent *Event, HSAuint32 Milliseconds) +{ + return hsaKmtWaitOnEvent_Ext(Event, Milliseconds, NULL); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnEvent_Ext(HsaEvent *Event, + HSAuint32 Milliseconds, uint64_t *event_age) { if (!Event) return HSAKMT_STATUS_INVALID_HANDLE; - return hsaKmtWaitOnMultipleEvents(&Event, 1, true, Milliseconds); + return hsaKmtWaitOnMultipleEvents_Ext(&Event, 1, true, Milliseconds, event_age); } static HSAKMT_STATUS get_mem_info_svm_api(uint64_t address, uint32_t gpu_id) @@ -365,6 +371,15 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], HSAuint32 NumEvents, bool WaitOnAll, HSAuint32 Milliseconds) +{ + return hsaKmtWaitOnMultipleEvents_Ext(Events, NumEvents, WaitOnAll, Milliseconds, NULL); +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents_Ext(HsaEvent *Events[], + HSAuint32 NumEvents, + bool WaitOnAll, + HSAuint32 Milliseconds, + uint64_t *event_age) { CHECK_KFD_OPEN(); @@ -376,6 +391,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], for (HSAuint32 i = 0; i < NumEvents; i++) { event_data[i].event_id = Events[i]->EventId; event_data[i].kfd_event_data_ext = (uint64_t)(uintptr_t)NULL; + if (event_age && Events[i]->EventData.EventType == HSA_EVENTTYPE_SIGNAL) + event_data[i].signal_event_data.last_event_age = event_age[i]; } struct kfd_ioctl_wait_events_args args = {0}; @@ -413,6 +430,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents(HsaEvent *Events[], } } out: + + for (HSAuint32 i = 0; i < NumEvents; i++) { + if (event_age && Events[i]->EventData.EventType == HSA_EVENTTYPE_SIGNAL) + event_age[i] = event_data[i].signal_event_data.last_event_age; + } + free(event_data); return result; From 4ba8f1fe775b6ba74ced570738a863f578b29652 Mon Sep 17 00:00:00 2001 From: James Zhu Date: Thu, 1 Jun 2023 13:21:19 -0400 Subject: [PATCH 1162/1247] kfdtest: Add test for event wait with event age tracking enable Add 5 different test scenario to cover new event age tracking features. Change-Id: Icab43240fd127208b18abbd7542d6444127ef0c7 Signed-off-by: James Zhu --- tests/kfdtest/src/KFDEventTest.cpp | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/tests/kfdtest/src/KFDEventTest.cpp index a1ae0e6182..65c7585953 100644 --- a/tests/kfdtest/src/KFDEventTest.cpp +++ b/tests/kfdtest/src/KFDEventTest.cpp @@ -117,6 +117,71 @@ TEST_F(KFDEventTest, SignalEvent) { TEST_END; } +/* test event signaling with event age enabled wait */ +TEST_F(KFDEventTest, SignalEventExt) { + TEST_START(TESTPROFILE_RUNALL); + + PM4Queue queue; + HsaEvent *tmp_event; + uint64_t event_age; + + if (m_VersionInfo.KernelInterfaceMajorVersion == 1 && + m_VersionInfo.KernelInterfaceMinorVersion < 14) { + LOG() << "event age tracking isn't supported in KFD. Exiting." << std::endl; + return; + } + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &tmp_event)); + + /* Intentionally let event id for m_pHsaEvent be non zero */ + ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &m_pHsaEvent)); + ASSERT_NE(0, m_pHsaEvent->EventData.HWData2); + + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + /* 1. event_age gets incremented every time when the event signals */ + event_age = 1; + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, + m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent_Ext(m_pHsaEvent, g_TestTimeOut, &event_age)); + ASSERT_EQ(event_age, 2); + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, + m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent_Ext(m_pHsaEvent, g_TestTimeOut, &event_age)); + ASSERT_EQ(event_age, 3); + + /* 2. event wait return without sleep after the event signals */ + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, + m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); + sleep(1); /* wait for event signaling */ + EXPECT_SUCCESS(hsaKmtWaitOnEvent_Ext(m_pHsaEvent, g_TestTimeOut, &event_age)); + ASSERT_EQ(event_age, 4); + + /* 3. signaling from CPU */ + hsaKmtSetEvent(m_pHsaEvent); + EXPECT_SUCCESS(hsaKmtWaitOnEvent_Ext(m_pHsaEvent, g_TestTimeOut, &event_age)); + ASSERT_EQ(event_age, 5); + + /* 4. when event_age is 0, hsaKmtWaitOnEvent_Ext always sleeps */ + event_age = 0; + ASSERT_EQ(HSAKMT_STATUS_WAIT_TIMEOUT, hsaKmtWaitOnEvent_Ext(m_pHsaEvent, g_TestTimeOut, &event_age)); + + /* 5. when event_age is 0, it always stays 0 after the event signals */ + queue.PlaceAndSubmitPacket(PM4ReleaseMemoryPacket(m_FamilyId, false, + m_pHsaEvent->EventData.HWData2, m_pHsaEvent->EventId)); + EXPECT_SUCCESS(hsaKmtWaitOnEvent_Ext(m_pHsaEvent, g_TestTimeOut, &event_age)); + ASSERT_EQ(event_age, 0); + + EXPECT_SUCCESS(hsaKmtDestroyEvent(tmp_event)); + + EXPECT_SUCCESS(queue.Destroy()); + + TEST_END; +} + static uint64_t gettime() { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); From 29b04c25349a409aa153c550eed08a658efd23c3 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 12 Jun 2023 12:39:31 -0400 Subject: [PATCH 1163/1247] kfdtest: Fix KFDSVMEvictTest.QueueTest OOM Typo to calculate bufferSize from vramBufSizeInPages. The OOM shows up only with HSA_XNACK=1 because HSA_XNACK=0 doesn't support VRAM oversubscription. We changed to run SVM tests with both XNACK off and on. Change-Id: I3949959288fd92f4e7f4a87115a5f1547e225042 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index e9ae0ca867..fb7a2e9c7e 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -98,7 +98,7 @@ HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count) { sizeInPages = size >> PAGE_SHIFT; vramBufSizeInPages = sizeInPages / (count * N_PROCESSES); - return vramBufSizeInPages << 20; + return vramBufSizeInPages << PAGE_SHIFT; } void KFDSVMEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, From 9bf1cbe4ed7d3b58d9513de130dd08b56e0e312d Mon Sep 17 00:00:00 2001 From: Ruili Ji Date: Thu, 8 Jun 2023 15:05:19 +0800 Subject: [PATCH 1164/1247] kfdtest: Update COMPUTE_PGM_RSRC1 for software trap If asics don't need software traps within GFX11 domain, test with COMPUTE_PGM_RSRC1.PRIV = 1 will make system hang. Change-Id: I00cf8eb6d6b07856885c77bd343ca3c41cc3cad5 Signed-off-by: Ruili Ji Signed-off-by: Aaron Liu --- tests/kfdtest/src/Dispatch.cpp | 7 ++++++- tests/kfdtest/src/Dispatch.hpp | 1 + tests/kfdtest/src/KFDBaseComponentTest.cpp | 19 +++++++++++++++++++ tests/kfdtest/src/KFDBaseComponentTest.hpp | 1 + 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index ce3a8f4626..6209d8cb59 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -45,6 +45,7 @@ Dispatch::Dispatch(const HsaMemoryBuffer& isaBuf, const bool eventAutoReset) hsaKmtCreateEvent(&eventDesc, !eventAutoReset, false, &m_pEop); m_FamilyId = g_baseTest->GetFamilyIdFromNodeId(isaBuf.Node()); + m_NeedCwsrWA = g_baseTest->NeedCwsrWA(isaBuf.Node()); } Dispatch::~Dispatch() { @@ -138,7 +139,11 @@ void Dispatch::BuildIb() { pgmRsrc2 |= (1 << COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT) & COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK; - const bool priv = (m_FamilyId == FAMILY_GFX11); + /* + * For some special asics in the list of DEGFX11_12113 + * COMPUTE_PGM_RSRC needs priv=1 to prevent hardware traps + */ + const bool priv = m_NeedCwsrWA; const unsigned int COMPUTE_PGM_RSRC[] = { // PGM_RSRC1 = { VGPRS: 16 SGPRS: 16 PRIORITY: m_SpiPriority FLOAT_MODE: c0 // PRIV: 0 (1 for GFX11) DX10_CLAMP: 0 DEBUG_MODE: 0 IEEE_MODE: 0 BULKY: 0 CDBG_USER: 0 } diff --git a/tests/kfdtest/src/Dispatch.hpp b/tests/kfdtest/src/Dispatch.hpp index ce8ffba21b..b43becf01b 100644 --- a/tests/kfdtest/src/Dispatch.hpp +++ b/tests/kfdtest/src/Dispatch.hpp @@ -71,6 +71,7 @@ class Dispatch { HSAuint64 m_scratch_base; unsigned int m_SpiPriority; unsigned int m_FamilyId; + bool m_NeedCwsrWA; }; #endif // __KFD_DISPATCH__H__ diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/tests/kfdtest/src/KFDBaseComponentTest.cpp index 2c0625972e..c257add6a4 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.cpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.cpp @@ -164,6 +164,25 @@ unsigned int KFDBaseComponentTest::GetFamilyIdFromNodeId(unsigned int nodeId) return FamilyIdFromNode(m_NodeInfo.GetNodeProperties(nodeId)); } +/* + * Some asics need CWSR workround for DEGFX11_12113 + */ +bool KFDBaseComponentTest::NeedCwsrWA(unsigned int nodeId) +{ + bool needCwsrWA = false; + const HsaNodeProperties *props = m_NodeInfo.GetNodeProperties(nodeId); + + needCwsrWA = props->EngineId.ui32.Major == 11 && + props->EngineId.ui32.Minor == 0 && + (props->EngineId.ui32.Stepping == 0 || + props->EngineId.ui32.Stepping == 1 || + props->EngineId.ui32.Stepping == 2 || + props->EngineId.ui32.Stepping == 5 || + (props->EngineId.ui32.Stepping == 3 && props->NumArrays > 1)); + + return needCwsrWA; +} + bool KFDBaseComponentTest::NeedNonPagedWptr(unsigned int nodeId) { return GetFamilyIdFromNodeId(nodeId) >= FAMILY_GFX11; diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 22d5bcd12d..7fddd23844 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -58,6 +58,7 @@ class KFDBaseComponentTest : public testing::Test { // @return DRM Render Node if successful or -1 on failure int FindDRMRenderNode(int gpuNode); unsigned int GetFamilyIdFromNodeId(unsigned int nodeId); + bool NeedCwsrWA(unsigned int nodeId); bool NeedNonPagedWptr(unsigned int nodeId); unsigned int GetFamilyIdFromDefaultNode(){ return m_FamilyId; } From 9a22bade89f0eed21604b928c0eb4b8cc79eb7e7 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Fri, 9 Jun 2023 09:30:56 -0400 Subject: [PATCH 1165/1247] kfdtest.exclude: Blacklist CuMaskingEven on all ASICs This has slowly become less and less reliable on more and more ASICs, so just blacklist it altogether. Using wall clock for performance is not a reliable method for testing performance, so skip it to avoid more failure reports on various systems. Change-Id: I1a5744604e4620bc7675a629d146ba4ffba669d2 --- tests/kfdtest/scripts/kfdtest.exclude | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index e2d0b93e2b..b1eb8c8b78 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -41,6 +41,10 @@ FILTER[core]=\ # CU Masking Linear are not working correctly due to how the HW distributes work over CUs. # They are available for testing but are not currently expected to pass on CI/VI/AI. # +# CU Masking Even is added here due to some non-obvious baseline measurements. Though +# using wallclock to measure performance is always risky, there are just too many ASICs +# where this test is failing. Ideally we'll get better CU Masking coverage via rocrtst +# # The CheckZeroInitializationVram test is no longer expected to pass as KFD no longer # clears memory at allocation time. PERMANENT_BLACKLIST_ALL_ASICS=\ @@ -48,6 +52,7 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ "${FILTER[pm]}:"\ "KFDLocalMemoryTest.Fragmentation:"\ "KFDQMTest.BasicCuMaskingLinear:"\ +"KFDQMTest.BasicCuMaskingEven:"\ "RDMATest.GPUDirect:"\ "KFDRASTest.*:"\ "KFDLocalMemoryTest.CheckZeroInitializationVram" @@ -108,7 +113,6 @@ KV_QUEUE_BLACKLIST=\ # KFDCWSRTest.BasicTest*: SWDEV-353206 BLACKLIST_GFX10=\ "KFDMemoryTest.DeviceHdpFlush:"\ -"KFDQMTest.BasicCuMaskingEven:"\ "KFDSVMEvictTest.*:"\ "KFDCWSRTest.BasicTest*" @@ -119,7 +123,6 @@ BLACKLIST_GFX10_NV2X=\ # KFDMemoryTest.FlatScratchAccess - SWDEV-329877 BLACKLIST_GFX11=\ "KFDQMTest.CreateAqlCpQueue:"\ -"KFDQMTest.BasicCuMaskingEven:"\ "KFDCWSRTest.InterruptRestore:"\ "KFDPerfCountersTest.*:"\ "KFDMemoryTest.FlatScratchAccess" @@ -226,15 +229,12 @@ FILTER[renoir]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDEvictTest.*:"\ "KFDMemoryTest.LargestSysBufferTest:"\ -"KFDQMTest.BasicCuMaskingEven:"\ "KFDMemoryTest.SignalHandling" # KFDExceptionTest.* (KFD-435) -# KFQMTest.BasicCuMaskingEven (Issues with baseline measurement) FILTER[arcturus]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDExceptionTest.FaultStorm:"\ -"KFDQMTest.BasicCuMaskingEven"\ +"KFDExceptionTest.FaultStorm"\ FILTER[aldebaran]=\ "$BLACKLIST_ALL_ASICS:"\ @@ -244,7 +244,6 @@ FILTER[aldebaran]=\ FILTER[aqua_vanjaram]=\ "$BLACKLIST_ALL_ASICS:"\ -"KFDQMTest.BasicCuMaskingEven:"\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDMemoryTest.BigSysBufferStressTest:"\ "KFDMemoryTest.FlatScratchAccess:"\ From 3447b795df82d53c01ce22d71909a48525189a47 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Wed, 21 Jun 2023 14:28:48 -0400 Subject: [PATCH 1166/1247] kfdtest: Fix gfx_target_version Parsing Issue The purpose of this patch is to fix an issue in the run_kfdtest.sh script's gfx_target_version parsing. When the character length of the "gfx_target_version" value is equal to 5 instead of 6, it will now be zero padded on the left to allow each Major/Minor/Stepping value to be parsed correctly. Also, kfdtest.exclude file now replaces the default filter for aqua_vanjaram with the following 3 gfx filters: gfx940, gfx941, & gfx942 Signed-off-by: Ori Messinger Change-Id: I1f0264d3705803f24ad3c458e6bd367fbbec62be --- tests/kfdtest/scripts/kfdtest.exclude | 28 ++++++++++++++++++++------- tests/kfdtest/scripts/run_kfdtest.sh | 3 +++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index b1eb8c8b78..6a442954c9 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -242,13 +242,6 @@ FILTER[aldebaran]=\ "KFDMemoryTest.PtraceAccess:"\ "KFDMemoryTest.DeviceHdpFlush" -FILTER[aqua_vanjaram]=\ -"$BLACKLIST_ALL_ASICS:"\ -"KFDMemoryTest.LargestSysBufferTest:"\ -"KFDMemoryTest.BigSysBufferStressTest:"\ -"KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest" - FILTER[navi10]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10:"\ @@ -310,3 +303,24 @@ FILTER[gfx1103]=\ FILTER[gfx1036]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X" + +FILTER[gfx940]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDMemoryTest.LargestSysBufferTest:"\ +"KFDMemoryTest.BigSysBufferStressTest:"\ +"KFDMemoryTest.FlatScratchAccess:"\ +"KFDIPCTest.BasicTest" + +FILTER[gfx941]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDMemoryTest.LargestSysBufferTest:"\ +"KFDMemoryTest.BigSysBufferStressTest:"\ +"KFDMemoryTest.FlatScratchAccess:"\ +"KFDIPCTest.BasicTest" + +FILTER[gfx942]=\ +"$BLACKLIST_ALL_ASICS:"\ +"KFDMemoryTest.LargestSysBufferTest:"\ +"KFDMemoryTest.BigSysBufferStressTest:"\ +"KFDMemoryTest.FlatScratchAccess:"\ +"KFDIPCTest.BasicTest" \ No newline at end of file diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index f84ed0e760..fe60fdb8c2 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -161,6 +161,9 @@ getNodeName() { gpuName="gfx$(echo "$HSA_OVERRIDE_GFX_VERSION" | awk 'BEGIN {FS="."; RS=""} {printf "%d%x%x", $1, $2, $3 }')" else local GfxVersionDec=$(cat $TOPOLOGY_SYSFS_DIR/$nodeId/properties | grep gfx_target_version | awk '{print $2}') + if [[ ${#GfxVersionDec} = 5 ]]; then + GfxVersionDec="0${GfxVersionDec}" + fi gpuName="gfx$(printf "$GfxVersionDec" | fold -w2 | awk 'BEGIN {FS="\n"; RS=""} {printf "%d%x%x", $1, $2, $3}')" fi fi From 5e0a32d7b3781bbe9642b00137bee980df79d230 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Mon, 27 Mar 2023 15:47:55 -0500 Subject: [PATCH 1167/1247] add hsaKmtGetRuntimeCapabilities API Queries for runtime capabilities after its being enabled Signed-off-by: Alex Sierra Change-Id: I098c0e9862c0c1d5e304b111cdc281c0ccd09691 --- include/hsakmt.h | 6 ++++++ include/hsakmttypes.h | 1 + src/debug.c | 9 ++++++++- src/libhsakmt.ver | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 5a7e2f4c37..dae3f5a980 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -728,6 +728,12 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void); +HSAKMT_STATUS +HSAKMTAPI +hsaKmtGetRuntimeCapabilities( + HSAuint32 *caps_mask // OUT + ); + /** Gets GPU and CPU clock counters for particular Node */ diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 2e086ca9b6..ef0a4a23a4 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -893,6 +893,7 @@ typedef enum HSA_DBG_NODE_CONTROL { HSA_DBG_NODE_CONTROL_FLAG_MAX = 0x01 } HSA_DBG_NODE_CONTROL; +#define HSA_RUNTIME_ENABLE_CAPS_SUPPORTS_CORE_DUMP_MASK 0x80000000 //This structure is hardware specific and may change in the future typedef struct _HsaDbgWaveMsgAMDGen2 diff --git a/src/debug.c b/src/debug.c index fcc10b1933..51022a2ba3 100644 --- a/src/debug.c +++ b/src/debug.c @@ -31,7 +31,7 @@ #include static bool *is_device_debugged; - +static uint32_t runtime_capabilities_mask = 0; HSAKMT_STATUS init_device_debugging_memory(unsigned int NumNodes) { @@ -324,6 +324,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, else return HSAKMT_STATUS_ERROR; } + runtime_capabilities_mask= args.capabilities_mask; return HSAKMT_STATUS_SUCCESS; } @@ -344,3 +345,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) return HSAKMT_STATUS_SUCCESS; } + +HSAKMT_STATUS HSAKMTAPI hsaKmtGetRuntimeCapabilities(HSAuint32 *caps_mask) +{ + *caps_mask = runtime_capabilities_mask; + return HSAKMT_STATUS_SUCCESS; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index fe34241b22..80aedc224f 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -61,6 +61,7 @@ hsaKmtGetQueueInfo; hsaKmtAllocQueueGWS; hsaKmtRuntimeEnable; hsaKmtRuntimeDisable; +hsaKmtGetRuntimeCapabilities; hsaKmtSPMAcquire; hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; From 0cbf26c148c87bd69af9be2949132f17c0b6c1a2 Mon Sep 17 00:00:00 2001 From: Alex Sierra Date: Thu, 3 Nov 2022 21:13:33 +0000 Subject: [PATCH 1168/1247] src: add debug API to support GPU core dump Functions to API added to extract the following information from KFD Runtime information, device info and queues snapshot. Signed-off-by: Alex Sierra Change-Id: If995ecc54497ab61189bb0f209c64af0bbb0f56f --- include/hsakmt.h | 40 ++++++++++++ src/debug.c | 161 ++++++++++++++++++++++++++++++++++++++++++++++ src/libhsakmt.ver | 4 ++ 3 files changed, 205 insertions(+) diff --git a/include/hsakmt.h b/include/hsakmt.h index dae3f5a980..c95afb8a07 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -734,6 +734,46 @@ hsaKmtGetRuntimeCapabilities( HSAuint32 *caps_mask // OUT ); +/** + Enable debug trap. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgEnable( + void **runtime_info, //Out + HSAuint32 *data_size //Out + ); + +/** + Disable debug trap. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgDisable(void); + +/** + Get device snapshot. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgGetDeviceData( + void **data, //Out + HSAuint32 *n_entries, //Out + HSAuint32 *entry_size //Out + ); + +/** + Get queues snapshot. +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtDbgGetQueueData( + void **data, //Out + HSAuint32 *n_entries, //Out + HSAuint32 *entry_size, //Out + bool suspend_queues //In + ); + /** Gets GPU and CPU clock counters for particular Node */ diff --git a/src/debug.c b/src/debug.c index 51022a2ba3..e97b809844 100644 --- a/src/debug.c +++ b/src/debug.c @@ -351,3 +351,164 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtGetRuntimeCapabilities(HSAuint32 *caps_mask) *caps_mask = runtime_capabilities_mask; return HSAKMT_STATUS_SUCCESS; } + +static HSAKMT_STATUS dbg_trap_get_device_data(void *data, + uint32_t *n_entries, + uint32_t entry_size) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + args.device_snapshot.snapshot_buf_ptr = (uint64_t) data; + args.device_snapshot.num_devices = *n_entries; + args.device_snapshot.entry_size = entry_size; + args.op = KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT; + args.pid = getpid(); + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + return HSAKMT_STATUS_ERROR; + *n_entries = args.device_snapshot.num_devices; + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS dbg_trap_get_queue_data(void *data, + uint32_t *n_entries, + uint32_t entry_size, + uint32_t *queue_ids) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + args.queue_snapshot.num_queues = *n_entries; + args.queue_snapshot.entry_size = entry_size; + args.queue_snapshot.exception_mask = KFD_EC_MASK(EC_QUEUE_NEW); + args.op = KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT; + args.queue_snapshot.snapshot_buf_ptr = (uint64_t) data; + args.pid = getpid(); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + return HSAKMT_STATUS_ERROR; + + *n_entries = args.queue_snapshot.num_queues; + if (queue_ids && *n_entries) { + struct kfd_queue_snapshot_entry *queue_entry = + (struct kfd_queue_snapshot_entry *) data; + for (uint32_t i = 0; i < *n_entries; i++) + queue_ids[i] = queue_entry[i].queue_id; + } + + return HSAKMT_STATUS_SUCCESS; +} + +static HSAKMT_STATUS dbg_trap_suspend_queues(uint32_t *queue_ids, + uint32_t num_queues) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + int r; + + args.suspend_queues.queue_array_ptr = (uint64_t) queue_ids; + args.suspend_queues.num_queues = num_queues; + args.suspend_queues.exception_mask = KFD_EC_MASK(EC_QUEUE_NEW); + args.op = KFD_IOC_DBG_TRAP_SUSPEND_QUEUES; + args.pid = getpid(); + + r = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args); + if (r < 0) + return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgEnable(void **runtime_info, + HSAuint32 *data_size) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + *data_size = sizeof(struct kfd_runtime_info); + args.enable.rinfo_size = *data_size; + args.enable.dbg_fd = kfd_fd; + *runtime_info = malloc(args.enable.rinfo_size); + if (!*runtime_info) + return HSAKMT_STATUS_NO_MEMORY; + args.enable.rinfo_ptr = (uint64_t) *runtime_info; + args.op = KFD_IOC_DBG_TRAP_ENABLE; + args.pid = getpid(); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) { + free(*runtime_info); + return HSAKMT_STATUS_ERROR; + } + + return HSAKMT_STATUS_SUCCESS; +} +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgDisable(void) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + args.enable.dbg_fd = kfd_fd; + args.op = KFD_IOC_DBG_TRAP_DISABLE; + args.pid = getpid(); + + if (kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, &args)) + return HSAKMT_STATUS_ERROR; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetDeviceData(void **data, + HSAuint32 *n_entries, + HSAuint32 *entry_size) +{ + HSAKMT_STATUS ret = HSAKMT_STATUS_NO_MEMORY; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + *n_entries = UINT32_MAX; + *entry_size = sizeof(struct kfd_dbg_device_info_entry); + *data = malloc(*entry_size * *n_entries); + if (!*data) + return ret; + ret = dbg_trap_get_device_data(*data, n_entries, *entry_size); + if (ret) + free(*data); + + return ret; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtDbgGetQueueData(void **data, + HSAuint32 *n_entries, + HSAuint32 *entry_size, + bool suspend_queues) +{ + uint32_t *queue_ids = NULL; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(KFD_IOCTL_MINOR_VERSION); + *entry_size = sizeof(struct kfd_queue_snapshot_entry); + *n_entries = 0; + if (dbg_trap_get_queue_data(NULL, n_entries, *entry_size, NULL)) + return HSAKMT_STATUS_ERROR; + *data = malloc(*n_entries * *entry_size); + if (!*data) + return HSAKMT_STATUS_NO_MEMORY; + if (suspend_queues && *n_entries) + queue_ids = (uint32_t *)malloc(sizeof(uint32_t) * *n_entries); + if (!queue_ids || + dbg_trap_get_queue_data(*data, n_entries, *entry_size, queue_ids)) + goto free_data; + if (queue_ids) { + if (dbg_trap_suspend_queues(queue_ids, *n_entries) || + dbg_trap_get_queue_data(*data, n_entries, *entry_size, NULL)) + goto free_data; + free(queue_ids); + } + return HSAKMT_STATUS_SUCCESS; +free_data: + free(*data); + if (queue_ids) + free(queue_ids); + + return HSAKMT_STATUS_ERROR; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 80aedc224f..dc17215263 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -42,6 +42,10 @@ hsaKmtDbgRegister; hsaKmtDbgUnregister; hsaKmtDbgWavefrontControl; hsaKmtDbgAddressWatch; +hsaKmtDbgEnable; +hsaKmtDbgDisable; +hsaKmtDbgGetDeviceData; +hsaKmtDbgGetQueueData; hsaKmtGetClockCounters; hsaKmtPmcGetCounterProperties; hsaKmtPmcRegisterTrace; From 473a66d11536021a17360d7f1a3102871329b670 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Thu, 29 Jun 2023 09:34:18 -0400 Subject: [PATCH 1169/1247] Don't install asan license if disabled Change-Id: I8bffe5ec8496ff11e6d66995dd470cddb13f3c0d Signed-off-by: Jeremy Newton --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c4b5906056..b941fd174a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -307,7 +307,9 @@ set ( ENABLE_LDCONFIG ON CACHE BOOL "Set library links and caches using ldconfig # Install License file install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel) -install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR}-asan COMPONENT asan) +if ( ENABLE_ASAN_PACKAGING ) + install ( FILES ${CPACK_RESOURCE_FILE_LICENSE} DESTINATION ${CMAKE_INSTALL_DOCDIR}-asan COMPONENT asan) +endif() # Make proper version for appending # Default Value is 99999, setting it first From 7cc3ffc115cd23359a42e554049f580f7b3ef805 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Thu, 29 Jun 2023 12:11:44 -0400 Subject: [PATCH 1170/1247] kfdtest: Fix kfdtest.exclude "ReadOnlyRangeTest" Issue The purpose of this patch is to fix an issue in kfdtest.exclude's blacklist for KFDSVMRangeTest.ReadOnlyRangeTest. Excluding "KFDSVMRangeTest.ReadOnlyRangeTest" without adding a "*" to the end causes the test to still run, since after a recent patch the test actually runs these two variants instead: -"KFDSVMRangeTest.ReadOnlyRangeTest/0" -"KFDSVMRangeTest.ReadOnlyRangeTest/1" (For XNACK OFF/ON) Now, the test is excluded as "KFDSVMRangeTest.ReadOnlyRangeTest*" to cover those two XNACK ON/OFF variants. Signed-off-by: Ori Messinger Change-Id: I067c4c99fe839ce6cec5d134bd605e8cb41b8291 --- tests/kfdtest/scripts/kfdtest.exclude | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 6a442954c9..388f71b182 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -76,7 +76,7 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDMemoryTest.CacheInvalidateOnRemoteWrite:"\ "KFDEvictTest.BurstyTest:"\ "KFDHWSTest.*:"\ -"KFDSVMRangeTest.ReadOnlyRangeTest" +"KFDSVMRangeTest.ReadOnlyRangeTest*" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From 2d3a09cbd6cd961a45283621ff716e6395b213e5 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 20 Jul 2023 11:22:56 -0400 Subject: [PATCH 1171/1247] kfdtest: disable gws tests for gfx11 GFX11 will no longer use GWS for cooperative launch so disable the test. Signed-off-by: Jonathan Kim Change-Id: I8611c8158e1654782150ad10f1f65edb578e6435 --- tests/kfdtest/scripts/kfdtest.exclude | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 388f71b182..60ed132eb1 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -121,11 +121,13 @@ BLACKLIST_GFX10_NV2X=\ "KFDPerfCountersTest.*" # KFDMemoryTest.FlatScratchAccess - SWDEV-329877 +# KFDGWSTest.*: GFX11 will no longer use global wave sync BLACKLIST_GFX11=\ "KFDQMTest.CreateAqlCpQueue:"\ "KFDCWSRTest.InterruptRestore:"\ "KFDPerfCountersTest.*:"\ -"KFDMemoryTest.FlatScratchAccess" +"KFDMemoryTest.FlatScratchAccess:"\ +"KFDGWSTest.*" # KFDQMTest.CpuWriteCoherence fails. 0 dwordsAvailable (KFD-338) # KFDMemoryTest.MemoryRegister fails on SDMA queue creation (KFD-337) @@ -323,4 +325,4 @@ FILTER[gfx942]=\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDMemoryTest.BigSysBufferStressTest:"\ "KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest" \ No newline at end of file +"KFDIPCTest.BasicTest" From 1958224379a9939d1edbd577ab4206d29539834d Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 6 Jul 2023 13:50:58 -0400 Subject: [PATCH 1172/1247] run_kfdtest.sh: Clarify parameters taking arguments For --node and --exclude, these flags take arguments, but usage was unclear. This led to attempts like --node=1 , which will not work appropriately. Add examples for flags that take parameters, as well as the requirements for those parameters. Also change --exclude parsing to match --node parsing, for consistency Change-Id: I563ba9b370a24d9a84b9c39093f3cb1a5d723cef --- tests/kfdtest/scripts/run_kfdtest.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/tests/kfdtest/scripts/run_kfdtest.sh index fe60fdb8c2..a850a027c9 100755 --- a/tests/kfdtest/scripts/run_kfdtest.sh +++ b/tests/kfdtest/scripts/run_kfdtest.sh @@ -91,12 +91,16 @@ printUsage() { "pass on the specified platform. Usually you"\ "don't need this option" echo " -g , --gdb Run in debugger" - echo " -n , --node NodeId to test. If"\ - "not specified test will be run on all nodes" + echo " -n , --node NodeId(s) to test. Takes a single integer, or a"\ + "quoted, space-separated string as an argument"\ + "(e.g. -n 1 OR -n \"1 2 3\")"\ + "NOTE: Node numbers come from /sys/class/kfd/kfd/topology/nodes/#" echo " -l , --list List available nodes" echo " --high Force clocks to high for test execution" echo " -d , --docker Run in docker container" - echo " -e , --exclude Additional tests to exclude, in addition to kfdtest.exclude (colon-separated, single quoted string as an argument)" + echo " -e , --exclude Additional tests to exclude, in addition to kfdtest.exclude."\ + "Takes a colon-separated string as an argument"\ + "(e.g. -e KFDEvictTest.*:KFDSVMEvictTest.*)" echo " -h , --help Prints this help" echo echo "Gtest arguments will be forwarded to the app" @@ -264,7 +268,7 @@ while [ "$1" != "" ]; do -d | --docker ) RUN_IN_DOCKER="true" ;; -e | --exclude ) - ADDITIONAL_EXCLUDE="$2" ; shift ;; + shift 1; ADDITIONAL_EXCLUDE="$1" ;; -h | --help ) printUsage; exit 0 ;; *) From a395dd73066101ad0703012828c78d2dd117671b Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Fri, 7 Jul 2023 15:46:02 -0400 Subject: [PATCH 1173/1247] kfdtest: KFDSVMEvictTest support large VRAM or small system memory For xnack off, skip SVM evict tests if memory allocation size is larger than 15/16 total system memory, because the test may fail to allocate CWSR svm range to create queue after allocating test memory. Limit eviction size from total VRAM size to 1/2 total VRAM size, because for 192GB VRAM, evict 192GB may takes more than 120 seconds and cause test timeout failed. Change-Id: Ib1483b9aab580a8539187b2943cadea0fd5a7c71 Signed-off-by: Philip Yang --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 31 ++++++++++++++++++--------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index fb7a2e9c7e..fae3b09aae 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -66,14 +66,17 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf /* use one third of total system memory for eviction buffer to test * limit max allocate size to double of vramSize - * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) + * count is zero if not enough memory for XNACK off case */ - size = sysMemSize / 3 + vramSize; - size = size > vramSize << 1 ? vramSize << 1 : size; - /* Check if there is enough system memory to pass test, + size = MIN(sysMemSize / 3, vramSize / 2); + size += vramSize; + + /* Check if there is enough system memory to pass test for XNACK off * KFD system memory limit is 15/16. */ - if (size > (sysMemSize - (sysMemSize >> 4))) + HSAint32 xnack_enable = 0; + EXPECT_SUCCESS(hsaKmtGetXNACKMode(&xnack_enable)); + if (!xnack_enable && size > (sysMemSize - (sysMemSize >> 4))) return 0; sizeInPages = size >> PAGE_SHIFT; @@ -89,12 +92,20 @@ HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count) { LOG() << "Found System RAM of " << std::dec << (sysMemSize >> 20) << "MB" << std::endl; - /* use one third of total system memory for eviction buffer to test - * limit max allocate size to duoble of vramSize - * count is zero if not enough memory (sysMemSize/3 + vramSize) < (vramBufSize * N_PROCESSES) + /* use up to one third of total system memory for eviction buffer to test + * limit max eviction size to 1/2 of vramSize. */ - size = sysMemSize / 3 + vramSize; - size = size > vramSize << 1 ? vramSize << 1 : size; + size = MIN(sysMemSize / 3, vramSize / 2); + size += vramSize; + + /* Check if there is enough system memory to pass test for XNACK off + * KFD system memory limit is 15/16. + */ + HSAint32 xnack_enable = 0; + EXPECT_SUCCESS(hsaKmtGetXNACKMode(&xnack_enable)); + if (!xnack_enable && size > (sysMemSize - (sysMemSize >> 4))) + return 0; + sizeInPages = size >> PAGE_SHIFT; vramBufSizeInPages = sizeInPages / (count * N_PROCESSES); From 8471f80bacd85e75b7e3fca2b51af99903712fe0 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 26 Jul 2023 09:41:38 -0400 Subject: [PATCH 1174/1247] libhsakmt: remove old debugger versioning Current debugger uses KFD version directly. Signed-off-by: Jonathan Kim Change-Id: I212a53560a94dd24c599addce72f59c527c8af25 --- include/linux/kfd_ioctl.h | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index 9080505614..c56347a7d6 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -44,43 +44,6 @@ #define KFD_IOCTL_MAJOR_VERSION 1 #define KFD_IOCTL_MINOR_VERSION 14 -/* - * Debug revision change log - * - * 0.1 - Initial revision - * 0.2 - Fix to include querying pending event that is both trap and vmfault - * 1.0 - Removed function to set debug data (renumbering functions broke ABI) - * 1.1 - Allow attaching to processes that have not opened /dev/kfd yet - * 1.2 - Allow flag option to clear queue status on queue suspend - * 1.3 - Fix race condition between clear on suspend and trap event handling - * 1.4 - Fix bad kfifo free - * 1.5 - Fix ABA issue between queue snapshot and suspend - * 2.0 - Return number of queues suspended/resumed and mask invalid/error - * array slots - * 2.1 - Add Set Address Watch, and Clear Address Watch support. - * 3.0 - Overhaul set wave launch override API - * 3.1 - Add support for GFX10 - * 3.2 - Add support for GFX10.3 - * 3.3 - Add precise memory operations enable - * 4.0 - Remove gpu_id from api - * 5.0 - Report exception codes to the debugger - * 6.0 - Pass event file descriptor from userspace. - * 6.1 - Add KFD_IOC_DBG_TRAP_QUERY_EXCEPTION_INFO. - * 6.2 - Add KFD_IOC_DBG_TRAP_DEVICE_SNAPSHOT. - * 7.0 - Redefine exception codes - * 7.1 - Add KFD_IOC_DBG_TRAP_RUNTIME_ENABLE - * 7.2 - Add KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT - * 8.0 - Expand runtime information given to the debugger - * 8.1 - Allow the debugger to set the exception mask - * 9.0 - Handle multiple exceptions from single trap signal - * 10.0 - Query debug event returns both queue_id and gpu_id - * 10.1 - Add additional debug capability information - * 10.2 - Reserved - * 10.3 - Pass context_save_restore_area size to user-space - */ -#define KFD_IOCTL_DBG_MAJOR_VERSION 10 -#define KFD_IOCTL_DBG_MINOR_VERSION 3 - struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ __u32 minor_version; /* from KFD */ From 98c6784cc1df225b55d607f5b7f3eeb93725daee Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 21 Mar 2022 12:23:37 -0400 Subject: [PATCH 1175/1247] kfdtest: remove deprecated debug references Remove all unused material from KFDDBGTest. Signed-off-by: Jonathan Kim Change-Id: I13ed68656efadef7bbaf8bb737ce5a04829eca9b --- tests/kfdtest/src/KFDDBGTest.cpp | 771 ------------------------------- 1 file changed, 771 deletions(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index f256d8a135..224c43e46e 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -31,146 +31,6 @@ #include "Dispatch.hpp" #include -#if 0 -static const char* loop_inc_isa = \ -"\ -shader loop_inc_isa\n\ -asic(VI)\n\ -type(CS)\n\ -vgpr_count(16)\n\ -trap_present(1)\n\ -/* TODO Enable here Address Watch Exception: */ \n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ - v_mov_b32 v5, 0 \n\ -/*Iteration 1*/\n\ - v_mov_b32 v4, -1\n\ - flat_atomic_inc v5, v[0:1], v4 glc \n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - flat_load_dword v8, v[2:3] \n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - v_cmp_gt_u32 vcc, 2, v5 \n\ -/*Iteration 2*/\n\ - v_mov_b32 v4, -1\n\ - flat_atomic_inc v5, v[0:1], v4 glc \n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - flat_load_dword v8, v[2:3] \n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - v_cmp_gt_u32 vcc, 2, v5 \n\ -/*Epilogue*/\n\ - v_mov_b32 v5, 0 \n\ - s_endpgm\n\ - \n\ -end\n\ -"; -#endif - -static const char* iterate_isa_gfx = \ -"\ -shader iterate_isa\n\ -wave_size(32) \n\ -type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ - flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - v_mov_b32 v5, 0\n\ -LOOP:\n\ - v_add_co_u32 v5, vcc, 1, v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - /*compare the result value (v5) to iteration value (v4),*/\n\ - /*and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ - v_cmp_lt_u32 vcc, v5, v4\n\ - s_cbranch_vccnz LOOP\n\ - flat_store_dword v[2,3], v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; - -static const char* jump_to_trap_gfx = \ -"\ -shader jump_to_trap\n\ -wave_size(32) \n\ -type(CS)\n\ -/*copy the parameters from scalar registers to vector registers*/\n\ - s_trap 1\n\ - v_mov_b32 v0, s0\n\ - v_mov_b32 v1, s1\n\ - v_mov_b32 v2, s2\n\ - v_mov_b32 v3, s3\n\ - flat_load_dword v4, v[0:1] slc /*load target iteration value*/\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - v_mov_b32 v5, 0\n\ -LOOP:\n\ - v_add_co_u32 v5, vcc, 1, v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - /*compare the result value (v5) to iteration value (v4),*/\n\ - /*and jump if equal (i.e. if VCC is not zero after the comparison)*/\n\ - v_cmp_lt_u32 vcc, v5, v4\n\ - s_cbranch_vccnz LOOP\n\ - flat_store_dword v[2,3], v5\n\ - s_waitcnt vmcnt(0)&lgkmcnt(0)\n\ - s_endpgm\n\ - end\n\ -"; - -static const char* trap_handler_gfx = \ -"\ -shader trap_handler\n\ -wave_size(32) \n\ -type(CS)\n\ -CHECK_VMFAULT:\n\ - /*if trap jumped to by vmfault, restore skip m0 signalling*/\n\ - s_getreg_b32 ttmp14, hwreg(HW_REG_TRAPSTS)\n\ - s_and_b32 ttmp2, ttmp14, 0x800\n\ - s_cbranch_scc1 RESTORE_AND_EXIT\n\ -GET_DOORBELL:\n\ - s_mov_b32 ttmp2, exec_lo\n\ - s_mov_b32 ttmp3, exec_hi\n\ - s_mov_b32 exec_lo, 0x80000000\n\ - s_sendmsg 10\n\ -WAIT_SENDMSG:\n\ - /*wait until msb is cleared (i.e. doorbell fetched)*/\n\ - s_nop 7\n\ - s_bitcmp0_b32 exec_lo, 0x1F\n\ - s_cbranch_scc0 WAIT_SENDMSG\n\ -SEND_INTERRUPT:\n\ - /* set context bit and doorbell and restore exec*/\n\ - s_mov_b32 exec_hi, ttmp3\n\ - s_and_b32 exec_lo, exec_lo, 0xfff\n\ - s_mov_b32 ttmp3, exec_lo\n\ - s_bitset1_b32 ttmp3, 23\n\ - s_mov_b32 exec_lo, ttmp2\n\ - s_mov_b32 ttmp2, m0\n\ - /* set m0, send interrupt and restore m0 and exit trap*/\n\ - s_mov_b32 m0, ttmp3\n\ - s_nop 0x0\n\ - s_sendmsg sendmsg(MSG_INTERRUPT)\n\ - s_mov_b32 m0, ttmp2\n\ -RESTORE_AND_EXIT:\n\ - /* restore and increment program counter to skip shader trap jump*/\n\ - s_add_u32 ttmp0, ttmp0, 4\n\ - s_addc_u32 ttmp1, ttmp1, 0\n\ - s_and_b32 ttmp1, ttmp1, 0xffff\n\ - /* restore SQ_WAVE_IB_STS */\n\ - s_lshr_b32 ttmp2, ttmp11, (26 - 15)\n\ - s_and_b32 ttmp2, ttmp2, (0x8000 | 0x1F0000)\n\ - s_setreg_b32 hwreg(HW_REG_IB_STS), ttmp2\n\ - /* restore SQ_WAVE_STATUS */\n\ - s_and_b64 exec, exec, exec\n\ - s_and_b64 vcc, vcc, vcc\n\ - s_setreg_b32 hwreg(HW_REG_STATUS), ttmp12\n\ - s_rfe_b64 [ttmp0, ttmp1]\n\ - end\n\ -"; - void KFDDBGTest::SetUp() { ROUTINE_START @@ -190,634 +50,3 @@ void KFDDBGTest::TearDown() { ROUTINE_END } -#if 0 -// Functionality is deprecated now, keeping code for reference -TEST_F(KFDDBGTest, BasicAddressWatch) { - TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_VI) { - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - static const unsigned int TMA_TRAP_COUNT_OFFSET = 3; - static const unsigned int TMA_TRAP_COUNT_VALUE = 3; - - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - // Allocate Buffers on System Memory - HsaMemoryBuffer trapHandler(PAGE_SIZE*2, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - HsaMemoryBuffer dstBuf(PAGE_SIZE*4, defaultGPUNode, true, false, false); - HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); - - // Get Address Watch TrapHandler - m_pIsaGen->GetAwTrapHandler(trapHandler); - - ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, - trapHandler.As(), - 0x1000, - tmaBuf.As(), - 0x1000)); - - m_pIsaGen->CompileShader(loop_inc_isa, "loop_inc_isa", isaBuf); - PM4Queue queue, queue_flush; - - ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - ASSERT_SUCCESS(queue_flush.Create(defaultGPUNode)); - - // Set Address Watch Params - // TODO: Set WatchMode[1] to Atomic in case we want to test this mode. - - HSA_DBG_WATCH_MODE WatchMode[2]; - HSAuint64 WatchAddress[2]; - HSAuint64 WatchMask[2]; - HSAKMT_STATUS AddressWatchSuccess; - HSAuint64 AddressMask64 = 0x0; - unsigned char *secDstBuf = (unsigned char *)dstBuf.As() + 8192; - - WatchMode[0] = HSA_DBG_WATCH_ALL; - WatchAddress[0] = (HSAuint64) dstBuf.As() & (~AddressMask64); - WatchMask[0] = ~AddressMask64; - WatchMode[1] = HSA_DBG_WATCH_ALL; - WatchAddress[1] = (HSAuint64)secDstBuf & (~AddressMask64); - WatchMask[1] = ~AddressMask64; - - queue_flush.PlaceAndSubmitPacket(PM4WriteDataPacket(dstBuf.As(), 0x0, 0x0)); - queue_flush.PlaceAndSubmitPacket(PM4WriteDataPacket((unsigned int *)secDstBuf, 0x0, 0x0)); - Delay(50); - ASSERT_SUCCESS(hsaKmtDbgRegister(defaultGPUNode)); - - AddressWatchSuccess = hsaKmtDbgAddressWatch( - defaultGPUNode, // IN - 2, // # watch points - &WatchMode[0], // IN - reinterpret_cast(&WatchAddress[0]), // IN - &WatchMask[0], // IN, optional - NULL); // IN, optional - - EXPECT_EQ(AddressWatchSuccess, HSAKMT_STATUS_SUCCESS); - - Dispatch dispatch(isaBuf); - dispatch.SetArgs(dstBuf.As(), reinterpret_cast(secDstBuf)); - dispatch.SetDim(1, 1, 1); - - /* TODO: Use Memory ordering rules w/ atomics for host-GPU memory syncs. - * Set to std::memory_order_seq_cst - */ - - dispatch.Submit(queue); - - Delay(50); - dispatch.Sync(g_TestTimeOut); - - // Check that we got trap handler calls due to add watch triggers - EXPECT_GE(*(tmaBuf.As()+ TMA_TRAP_COUNT_OFFSET), TMA_TRAP_COUNT_VALUE); - - EXPECT_SUCCESS(hsaKmtDbgUnregister(defaultGPUNode)); - EXPECT_SUCCESS(queue.Destroy()); - EXPECT_SUCCESS(queue_flush.Destroy()); - } else { - LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; - } - TEST_END -} -#endif - -#if 0 -// Deprecated - Reference Use Only -/** - * checkDebugVersion: - * Inputs: - * HSAuint32 requiredMajor - * -- Required major version number - * HSAuint32 requiredMinor - * -- Required minor version number - * Output: - * bool: - * i) false if major version of thunk and kernel are not the same - * ii) false if kernel minor version is less than the required - * version if the major version is the required version. - * iii) false if thunk minor version is less than the required - * version if the major version is the required version. - * iv) false if hsaKmtGetKernelDebugTrapVersionInfo() call fails. - * v) true otherwise. - * -*/ -static bool checkDebugVersion(HSAuint32 requiredMajor, HSAuint32 requiredMinor) -{ - HSAuint32 kernelMajorNumber = 0; - HSAuint32 kernelMinorNumber = 0; - HSAuint32 thunkMajorNumber = 0; - HSAuint32 thunkMinorNumber = 0; - - hsaKmtGetThunkDebugTrapVersionInfo(&thunkMajorNumber, &thunkMinorNumber); - - if (hsaKmtGetKernelDebugTrapVersionInfo(&kernelMajorNumber, - &kernelMinorNumber)) { - LOG() << "Failed to get kernel debugger version!" << std::endl; - return false; - } - - if (kernelMajorNumber != thunkMajorNumber) - return false; - - if (kernelMajorNumber < requiredMajor || - (kernelMajorNumber == requiredMajor && - kernelMinorNumber < requiredMinor)) - return false; - - if (thunkMajorNumber < requiredMajor || - (thunkMajorNumber == requiredMajor && - thunkMinorNumber < requiredMinor)) - return false; - - return true; -} -#endif - -#if 0 -// Deprecated - Reference Use Only -TEST_F(KFDDBGTest, BasicDebuggerSuspendResume) { - TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI) { - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - HSAuint32 Flags = 0; - - if(!checkDebugVersion(0, 2)) { - LOG() << "Test disabled due to debug API version mismatch"; - goto exit; - } - - HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true/*zero*/, false/*local*/, true/*exec*/); - HsaMemoryBuffer iterateBuf(PAGE_SIZE, defaultGPUNode, true, false, false); - HsaMemoryBuffer resultBuf(PAGE_SIZE, defaultGPUNode, true, false, false); - - unsigned int* iter = iterateBuf.As(); - unsigned int* result = resultBuf.As(); - - int suspendTimeout = 500; - int syncStatus; - - m_pIsaGen->CompileShader(iterate_isa_gfx, - "iterate_isa", - isaBuffer); - - PM4Queue queue1; - HsaQueueResource *qResources; - HSA_QUEUEID queue_ids[2]; - - ASSERT_SUCCESS(queue1.Create(defaultGPUNode)); - - Dispatch *dispatch1; - - dispatch1 = new Dispatch(isaBuffer); - - dispatch1->SetArgs(&iter[0], &result[0]); - dispatch1->SetDim(1, 1, 1); - - // Need a loop large enough so we don't finish before we call Suspend. - // 150000000 takes between 5 and 6 seconds, which is long enough - // to test the suspend/resume. - iter[0] = 150000000; - - ASSERT_EQ(ptrace(PTRACE_TRACEME, 0, 0, 0), 0); - ASSERT_SUCCESS(hsaKmtEnableDebugTrap(defaultGPUNode, INVALID_QUEUEID)); - - // Submit the shader, queue1 - dispatch1->Submit(queue1); - qResources = queue1.GetResource(); - queue_ids[0] = qResources->QueueId; - - ASSERT_SUCCESS(hsaKmtQueueSuspend( - INVALID_PID, - 1, // one queue - queue_ids, - 10, // grace period - Flags)); - - syncStatus = dispatch1->SyncWithStatus(suspendTimeout); - ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); - - ASSERT_NE(iter[0], result[0]); - - // The shader hasn't finished, we will wait for 20 seconds, - // and then check if it has finished. If it was suspended, - // it should not have finished. - Delay(20000); - - // Check that the shader has not finished yet. - syncStatus = dispatch1->SyncWithStatus(suspendTimeout); - ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); - - ASSERT_NE(iter[0], result[0]); - - ASSERT_SUCCESS(hsaKmtQueueResume( - INVALID_PID, - 1, // Num queues - queue_ids, - Flags)); - - dispatch1->Sync(); - ASSERT_EQ(iter[0], result[0]); - - EXPECT_SUCCESS(queue1.Destroy()); - - ASSERT_SUCCESS(hsaKmtDisableDebugTrap(defaultGPUNode)); - - } else { - LOG() << "Skipping test: Test not supported on family ID 0x" << m_FamilyId << "." << std::endl; - } -exit: - LOG() << std::endl; - TEST_END -} -#endif - -#if 0 -// Deprecated - Reference Use Only -TEST_F(KFDDBGTest, BasicDebuggerQueryQueueStatus) { - TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI) { - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - HSAint32 PollFd; - - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - if (!checkDebugVersion(0, 2)) { - LOG() << "Test disabled due to debug API version mismatch"; - goto exit; - } - - // enable debug trap and check poll fd creation - ASSERT_SUCCESS(hsaKmtEnableDebugTrapWithPollFd(defaultGPUNode, - INVALID_QUEUEID, - &PollFd)); - ASSERT_GT(PollFd, 0) << "failed to create polling file descriptor"; - - // create shader and trap bufs then enable 2nd level trap - HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, true); - HsaMemoryBuffer iterBuf(PAGE_SIZE, defaultGPUNode, true, false, false); - HsaMemoryBuffer resBuf(PAGE_SIZE, defaultGPUNode, true, false, false); - - HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, true); - HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); - - ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, - trap.As(), - 0x1000, - tmaBuf.As(), - 0x1000)); - - // compile and dispatch shader - m_pIsaGen->CompileShader(jump_to_trap_gfx, "jump_to_trap", isaBuf); - m_pIsaGen->CompileShader(trap_handler_gfx, "trap_handler", trap); - - PM4Queue queue; - HsaQueueResource *qResources; - ASSERT_SUCCESS(queue.Create(defaultGPUNode)); - - unsigned int* iter = iterBuf.As(); - unsigned int* result = resBuf.As(); - int suspendTimeout = 500; - int syncStatus; - iter[0] = 150000000; - Dispatch *dispatch; - dispatch = new Dispatch(isaBuf); - dispatch->SetArgs(&iter[0], &result[0]); - dispatch->SetDim(1, 1, 1); - - dispatch->Submit(queue); - qResources = queue.GetResource(); - - // poll, read and query for pending trap event - struct pollfd fds[1]; - fds[0].fd = PollFd; - fds[0].events = POLLIN | POLLRDNORM; - ASSERT_GT(poll(fds, 1, 5000), 0); - - char trapChar; - ASSERT_GT(read(PollFd, &trapChar, 1), 0); - ASSERT_EQ('t', trapChar); - - HSAuint32 invalidQid = 0xffffffff; - HSAuint32 qid = invalidQid; - HSA_QUEUEID queueIds[1] = { qResources->QueueId}; - HSA_DEBUG_EVENT_TYPE EventReceived; - bool IsSuspended = false; - bool IsNew = false; - - ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, - false, &EventReceived, - &IsSuspended, &IsNew)); - ASSERT_NE(qid, invalidQid); - ASSERT_EQ(IsSuspended, false); - ASSERT_EQ(IsNew, true); - ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_TRAP); - - // suspend queue, get snapshot, query suspended queue - // and clear pending event - ASSERT_SUCCESS(hsaKmtQueueSuspend(INVALID_PID, 1, queueIds, 10, 0)); - - syncStatus = dispatch->SyncWithStatus(suspendTimeout); - ASSERT_NE(syncStatus, HSAKMT_STATUS_SUCCESS); - ASSERT_NE(iter[0], result[0]); - - struct kfd_queue_snapshot_entry qssBuf[1] = {}; - HSAuint32 QssEntries = 0; - - // get only number of queues and don't update the snapshot buffer - ASSERT_SUCCESS(hsaKmtGetQueueSnapshot(INVALID_NODEID, INVALID_PID, - false, - reinterpret_cast(qssBuf), - &QssEntries)); - - ASSERT_EQ(QssEntries, 1); - ASSERT_EQ(qssBuf[0].ctx_save_restore_address, 0); - ASSERT_EQ(qssBuf[0].ring_base_address, 0); - ASSERT_EQ(qssBuf[0].ring_size, 0); - - // update the snapshot buffer - QssEntries = 1; - ASSERT_SUCCESS(hsaKmtGetQueueSnapshot(INVALID_NODEID, INVALID_PID, - false, - reinterpret_cast(qssBuf), - &QssEntries)); - - ASSERT_EQ(QssEntries, 1); - ASSERT_NE(qssBuf[0].ctx_save_restore_address, 0); - ASSERT_NE(qssBuf[0].ring_base_address, 0); - ASSERT_NE(qssBuf[0].ring_size, 0); - - ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, - &qid, true, &EventReceived, - &IsSuspended, &IsNew)); - ASSERT_EQ(IsSuspended, true); - - ASSERT_SUCCESS(hsaKmtQueueResume(INVALID_PID, 1, queueIds, 0)); - - ASSERT_SUCCESS(hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid, - false, &EventReceived, - &IsSuspended, &IsNew)); - - ASSERT_EQ(IsSuspended, false); - ASSERT_EQ(IsNew, false); - ASSERT_EQ(EventReceived, HSA_DEBUG_EVENT_TYPE_NONE); - - dispatch->Sync(); - ASSERT_EQ(iter[0], result[0]); - EXPECT_SUCCESS(queue.Destroy()); - ASSERT_SUCCESS(hsaKmtDisableDebugTrap(defaultGPUNode)); - ASSERT_EQ(close(PollFd), 0); - - } else { - LOG() << "Skipping test: Test not supported on family ID 0x" - << m_FamilyId << "." << std::endl; - } -exit: - LOG() << std::endl; - TEST_END -} -#endif - -#if 0 -// clean up routine -// Deprecated - Reference Use Only -static void ExitVMFaultQueryChild(std::string errMsg, - int exitStatus, - HSAint32 pollFd, - PM4Queue *queue1, - PM4Queue *queue2, - HsaEvent *event, - int gpuNode) { - if (queue1) - queue1->Destroy(); - - if (queue2) - queue2->Destroy(); - - if (event) { - int ret = hsaKmtDestroyEvent(event); - if (ret) { - exitStatus = 1; - errMsg = "event failed to be destroyed"; - } - } - - if (pollFd >= 0) - close(pollFd); - - if (gpuNode >= 0) { - int ret = hsaKmtDisableDebugTrap(gpuNode); - if (ret) { - exitStatus = 1; - errMsg = "debug trap failed to disable"; - } - } - - if (!errMsg.empty()) - WARN() << errMsg << std::endl; - - exit(exitStatus); -} -#endif - -#if 0 -// Deprecated - Reference Use Only -TEST_F(KFDDBGTest, BasicDebuggerQueryVMFaultQueueStatus) { - TEST_START(TESTPROFILE_RUNALL) - if (m_FamilyId >= FAMILY_AI) { - int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); - - ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; - - if (!checkDebugVersion(0, 2)) { - LOG() << "Test disabled due to debug API version mismatch"; - goto exit; - } - - pid_t childPid = fork(); - ASSERT_GE(childPid, 0); - - // fork to child since vm faults halt all queues - if (childPid == 0) { - HSAint32 PollFd; - HSAKMT_STATUS ret; - bool childStatus; - - ret = hsaKmtOpenKFD(); - if (ret != HSAKMT_STATUS_SUCCESS) - ExitVMFaultQueryChild("KFD open failed", - 1, -1, NULL, NULL, NULL, -1); - - // enable debug trap and check poll fd creation - ret = hsaKmtEnableDebugTrapWithPollFd(defaultGPUNode, - INVALID_QUEUEID, - &PollFd); - - if (ret != HSAKMT_STATUS_SUCCESS || PollFd <= 0) - ExitVMFaultQueryChild("enable debug trap with poll fd failed", - 1, -1, NULL, NULL, NULL, defaultGPUNode); - - // create shader, vmfault and trap bufs then enable 2nd level trap - HsaMemoryBuffer vmFaultBuf(PAGE_SIZE, defaultGPUNode, true, false, - true); - HsaMemoryBuffer srcBuf(PAGE_SIZE, defaultGPUNode, false); - srcBuf.Fill(0xABCDABCD); - - HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, - true); - HsaMemoryBuffer iterBuf(PAGE_SIZE, defaultGPUNode, true, false, - false); - HsaMemoryBuffer resBuf(PAGE_SIZE, defaultGPUNode, true, false, - false); - - HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, - true); - HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, - false); - - ret = hsaKmtSetTrapHandler(defaultGPUNode, - trap.As(), 0x1000, - tmaBuf.As(), 0x1000); - - if (ret != HSAKMT_STATUS_SUCCESS) - ExitVMFaultQueryChild("setting trap handler failed", - 1, PollFd, NULL, NULL, NULL, - defaultGPUNode); - - // compile and dispatch shader - m_pIsaGen->CompileShader(jump_to_trap_gfx, "jump_to_trap", - isaBuf); - m_pIsaGen->CompileShader(trap_handler_gfx, "trap_handler", trap); - - PM4Queue queue1, queue2; - HSAuint32 qid1; - if (queue1.Create(defaultGPUNode) != HSAKMT_STATUS_SUCCESS) - ExitVMFaultQueryChild("queue 1 creation failed", - 1, PollFd, NULL, NULL, NULL, - defaultGPUNode); - - if (queue2.Create(defaultGPUNode) != HSAKMT_STATUS_SUCCESS) - ExitVMFaultQueryChild("queue 2 creation failed", - 1, PollFd, &queue1, NULL, NULL, - defaultGPUNode); - - unsigned int* iter = iterBuf.As(); - unsigned int* result = resBuf.As(); - int suspendTimeout = 500; - iter[0] = 150000000; - Dispatch *dispatch1; - dispatch1 = new Dispatch(isaBuf); - dispatch1->SetArgs(&iter[0], &result[0]); - dispatch1->SetDim(1, 1, 1); - dispatch1->Submit(queue1); - - // poll, read and query pending trap event - struct pollfd fds[1]; - fds[0].fd = PollFd; - fds[0].events = POLLIN | POLLRDNORM; - if (poll(fds, 1, 5000) <= 0) - ExitVMFaultQueryChild("poll wake on pending trap event failed", - 1, PollFd, &queue1, &queue2, NULL, - defaultGPUNode); - - int kMaxSize = 4096; - char fifoBuf[kMaxSize]; - childStatus = read(PollFd, fifoBuf, 1) == -1\ - || strchr(fifoBuf, 't') == NULL; - if (childStatus) - ExitVMFaultQueryChild("read on pending trap event failed", - 1, PollFd, &queue1, &queue2, NULL, - defaultGPUNode); - - memset(fifoBuf, 0, sizeof(fifoBuf)); - - HSA_DEBUG_EVENT_TYPE EventReceived; - bool IsSuspended; - bool IsNew; - HSAuint32 invalidQid = 0xffffffff; - qid1 = invalidQid; - - ret = hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, &qid1, - false, &EventReceived, &IsSuspended, - &IsNew); - - childStatus = ret != HSAKMT_STATUS_SUCCESS - || EventReceived != HSA_DEBUG_EVENT_TYPE_TRAP; - if (childStatus) - ExitVMFaultQueryChild("query on pending trap event failed", - 1, PollFd, &queue1, &queue2, NULL, - defaultGPUNode); - - // create and wait on pending vmfault event - HsaEvent *vmFaultEvent; - HsaEventDescriptor eventDesc; - eventDesc.EventType = HSA_EVENTTYPE_MEMORY; - eventDesc.NodeId = defaultGPUNode; - eventDesc.SyncVar.SyncVar.UserData = NULL; - eventDesc.SyncVar.SyncVarSize = 0; - ret = hsaKmtCreateEvent(&eventDesc, true, false, - &vmFaultEvent); - if (ret != HSAKMT_STATUS_SUCCESS) - ExitVMFaultQueryChild("create vmfault event failed", - 1, PollFd, &queue1, &queue2, NULL, - defaultGPUNode); - - Dispatch dispatch2(vmFaultBuf, false); - dispatch2.SetArgs( - reinterpret_cast(srcBuf.As()), - reinterpret_cast(0xABBAABBAULL)); - dispatch2.SetDim(1, 1, 1); - dispatch2.Submit(queue2); - - ret = hsaKmtWaitOnEvent(vmFaultEvent, g_TestTimeOut); - if (ret != HSAKMT_STATUS_SUCCESS) - ExitVMFaultQueryChild("wait on vmfault event failed", - 1, PollFd, &queue1, &queue2, - vmFaultEvent, defaultGPUNode); - - // poll, read and query on pending vmfault event - if (poll(fds, 1, 5000) <= 0) - ExitVMFaultQueryChild("poll wake on vmfault event failed", - 1, PollFd, &queue1, &queue2, - vmFaultEvent, defaultGPUNode); - - childStatus = read(PollFd, fifoBuf, kMaxSize) == -1 - || strchr(fifoBuf, 'v') == NULL - || strchr(fifoBuf, 't'); - - if (childStatus) - ExitVMFaultQueryChild("read on vmfault event failed", - 1, PollFd, &queue1, &queue2, - vmFaultEvent, defaultGPUNode); - - ret = hsaKmtQueryDebugEvent(defaultGPUNode, INVALID_PID, - &qid1, true, &EventReceived, - &IsSuspended, &IsNew); - - childStatus = ret != HSAKMT_STATUS_SUCCESS - || EventReceived != - HSA_DEBUG_EVENT_TYPE_TRAP_VMFAULT; - if (childStatus) - ExitVMFaultQueryChild("query on vmfault event failed", - 1, PollFd, &queue1, &queue2, - vmFaultEvent, defaultGPUNode); - - ExitVMFaultQueryChild("", 0, PollFd, &queue1, &queue2, - vmFaultEvent, defaultGPUNode); - - } else { - int childStatus; - ASSERT_EQ(childPid, waitpid(childPid, &childStatus, 0)); - ASSERT_NE(0, WIFEXITED(childStatus)); - ASSERT_EQ(0, WEXITSTATUS(childStatus)); - } - } else { - LOG() << "Skipping test: Test not supported on family ID 0x" - << m_FamilyId << "." << std::endl; - } -exit: - LOG() << std::endl; - TEST_END -} -#endif From aaab019960d3eb2721b87ce2d8a265bf269a1b47 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 21 Mar 2022 12:46:01 -0400 Subject: [PATCH 1176/1247] libhsakmt: add debug trap thunk call for testing Add generic thunk call for debug testing that assumes caller populations trap arguments correctly. Signed-off-by: Jonathan Kim Change-Id: I33a0bc66ca77e29f5b663d4bfe73f8684df8bfb6 --- include/hsakmt.h | 9 ++++++++- src/debug.c | 35 +++++++++++++++++++++++++++++++++++ src/libhsakmt.ver | 1 + 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index c95afb8a07..1c3cd84a78 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -774,6 +774,14 @@ hsaKmtDbgGetQueueData( bool suspend_queues //In ); +/** + Debug ops call primarily used for KFD testing + */ +HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl( + struct kfd_ioctl_dbg_trap_args *arg, + HSA_QUEUEID *Queues + ); + /** Gets GPU and CPU clock counters for particular Node */ @@ -1031,4 +1039,3 @@ hsaKmtReturnAsanHeaderPage( #endif #endif //_HSAKMT_H_ - diff --git a/src/debug.c b/src/debug.c index e97b809844..d12dc4a260 100644 --- a/src/debug.c +++ b/src/debug.c @@ -512,3 +512,38 @@ free_data: return HSAKMT_STATUS_ERROR; } + +HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *args, + HSA_QUEUEID *Queues) +{ + HSAKMT_STATUS result; + + CHECK_KFD_OPEN(); + + if (Queues) { + int num_queues = args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES ? + args->suspend_queues.num_queues : + args->resume_queues.num_queues; + void *queue_ptr = args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES ? + (void *)args->suspend_queues.queue_array_ptr : + (void *)args->resume_queues.queue_array_ptr; + + memcpy(queue_ptr, convert_queue_ids(num_queues, Queues), + num_queues * sizeof(uint32_t)); + } + + long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, args); + + if (args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES && + err >= 0 && err <= args->suspend_queues.num_queues) + result = HSAKMT_STATUS_SUCCESS; + else if (args->op == KFD_IOC_DBG_TRAP_RESUME_QUEUES && + err >= 0 && err <= args->resume_queues.num_queues) + result = HSAKMT_STATUS_SUCCESS; + else if (err == 0) + result = HSAKMT_STATUS_SUCCESS; + else + result = HSAKMT_STATUS_ERROR; + + return result; +} diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index dc17215263..c720476beb 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -66,6 +66,7 @@ hsaKmtAllocQueueGWS; hsaKmtRuntimeEnable; hsaKmtRuntimeDisable; hsaKmtGetRuntimeCapabilities; +hsaKmtDebugTrapIoctl; hsaKmtSPMAcquire; hsaKmtSPMRelease; hsaKmtSPMSetDestBuffer; From 66b66e42cd1eaf4119d0b0680e862759b6e37b97 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Mon, 31 Jul 2023 19:09:32 +0000 Subject: [PATCH 1177/1247] Keep libdrm device_handle on older libdrm Even if the version of libdrm older and does not support the amdgpu_device_get_fd function, the device_handle stored in amdgpu_handle[] is still valid and can be returned via hsaKmtGetAMDGPUDeviceHandle. Change-Id: I024a3e82e6cfebac5577aefe359b067746c4023e --- src/fmm.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index bf9e5191f6..d60a9ad647 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -2062,20 +2062,21 @@ int open_drm_render_device(int minor) } drm_render_fds[index] = fd; - /* if amdgpu_device_get_fd availabe query render fd that libdrm uses, - * then close drm_render_fds above, replace it by fd libdrm uses. - */ device_handle = &amdgpu_handle[index]; - if (fn_amdgpu_device_get_fd && - !amdgpu_device_initialize(fd, &major_drm, &minor_drm, device_handle)) { - fd = fn_amdgpu_device_get_fd(*device_handle); - if (fd > 0) { - close(drm_render_fds[index]); - drm_render_fds[index] = fd; - } else { - pr_err("amdgpu_device_get_fd failed: %d\n", fd); - amdgpu_device_deinitialize(*device_handle); - *device_handle = 0; + if (!amdgpu_device_initialize(fd, &major_drm, &minor_drm, device_handle)) { + /* if amdgpu_device_get_fd available query render fd that libdrm uses, + * then close drm_render_fds above, replace it by fd libdrm uses. + */ + if (fn_amdgpu_device_get_fd) { + fd = fn_amdgpu_device_get_fd(*device_handle); + if (fd > 0) { + close(drm_render_fds[index]); + drm_render_fds[index] = fd; + } else { + pr_err("amdgpu_device_get_fd failed: %d\n", fd); + amdgpu_device_deinitialize(*device_handle); + *device_handle = 0; + } } } From dd56b38c2f1451f14a3022886d8f092717591a39 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 21 Mar 2022 12:35:01 -0400 Subject: [PATCH 1178/1247] kfdtest: add base debug class and debug attach/detach operation Add base debug class and attach/detach operations. Signed-off-by: Jonathan Kim Change-Id: I60f3c166646f05838fec208ac2f59bba998c63f8 --- include/hsakmt.h | 9 +++ src/debug.c | 6 +- src/libhsakmt.ver | 1 + tests/kfdtest/CMakeLists.txt | 1 + tests/kfdtest/src/BaseDebug.cpp | 98 +++++++++++++++++++++++++++++++++ tests/kfdtest/src/BaseDebug.hpp | 50 +++++++++++++++++ 6 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 tests/kfdtest/src/BaseDebug.cpp create mode 100644 tests/kfdtest/src/BaseDebug.hpp diff --git a/include/hsakmt.h b/include/hsakmt.h index 1c3cd84a78..9d5ef5897f 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -774,6 +774,15 @@ hsaKmtDbgGetQueueData( bool suspend_queues //In ); +/** + Check whether gpu firmware and kernel support debugging +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtCheckRuntimeDebugSupport( + void + ); + /** Debug ops call primarily used for KFD testing */ diff --git a/src/debug.c b/src/debug.c index d12dc4a260..b3e312cbc1 100644 --- a/src/debug.c +++ b/src/debug.c @@ -268,7 +268,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDbgAddressWatch(HSAuint32 NodeId, #define HSA_RUNTIME_ENABLE_MAX_MAJOR 1 #define HSA_RUNTIME_ENABLE_MIN_MINOR 13 -static HSAKMT_STATUS checkRuntimeDebugSupport(void) { +HSAKMT_STATUS hsaKmtCheckRuntimeDebugSupport(void) { HsaNodeProperties node = {0}; HsaSystemProperties props = {0}; HsaVersionInfo versionInfo = {0}; @@ -306,7 +306,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, bool setupTtmp) { struct kfd_ioctl_runtime_enable_args args = {0}; - HSAKMT_STATUS result = checkRuntimeDebugSupport(); + HSAKMT_STATUS result = hsaKmtCheckRuntimeDebugSupport(); if (result) return result; @@ -332,7 +332,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeEnable(void *rDebug, HSAKMT_STATUS HSAKMTAPI hsaKmtRuntimeDisable(void) { struct kfd_ioctl_runtime_enable_args args = {0}; - HSAKMT_STATUS result = checkRuntimeDebugSupport(); + HSAKMT_STATUS result = hsaKmtCheckRuntimeDebugSupport(); if (result) return result; diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index c720476beb..5d1a673ffb 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -65,6 +65,7 @@ hsaKmtGetQueueInfo; hsaKmtAllocQueueGWS; hsaKmtRuntimeEnable; hsaKmtRuntimeDisable; +hsaKmtCheckRuntimeDebugSupport; hsaKmtGetRuntimeCapabilities; hsaKmtDebugTrapIoctl; hsaKmtSPMAcquire; diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 1ec430da20..4e4ef6ad03 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -167,6 +167,7 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/AqlQueue.cpp src/BasePacket.cpp + src/BaseDebug.cpp src/BaseQueue.cpp src/Dispatch.cpp src/GoogleTestExtension.cpp diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp new file mode 100644 index 0000000000..50f243cc39 --- /dev/null +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -0,0 +1,98 @@ +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "BaseDebug.hpp" +#include +#include +#include +#include +#include +#include +#include "unistd.h" + +BaseDebug::BaseDebug(void) { +} + +BaseDebug::~BaseDebug(void) { + /* + * If the process is still attached, close and destroy the polling file + * descriptor. Note that on process termination, the KFD automatically + * disables processes that are still runtime enabled and debug enabled + * so we don't do it here. + */ + if (m_Pid) { + close(m_Fd.fd); + unlink(m_Fd_Name); + } +} + +// Creates temp file descriptor and debug attaches. +HSAKMT_STATUS BaseDebug::Attach(struct kfd_runtime_info *rInfo, + int rInfoSize, + unsigned int pid, + uint64_t exceptionEnable) { + struct kfd_ioctl_dbg_trap_args args = {0}; + char fd_name[32]; + + memset(&args, 0x00, sizeof(args)); + + mkfifo(m_Fd_Name, 0666); + m_Fd.fd = open(m_Fd_Name, O_CLOEXEC | O_NONBLOCK | O_RDWR); + m_Fd.events = POLLIN | POLLRDNORM; + + args.pid = pid; + args.op = KFD_IOC_DBG_TRAP_ENABLE; + args.enable.rinfo_ptr = (uint64_t)rInfo; + args.enable.rinfo_size = rInfoSize; + args.enable.dbg_fd = m_Fd.fd; + args.enable.exception_mask = exceptionEnable; + + if (hsaKmtDebugTrapIoctl(&args, NULL)) { + close(m_Fd.fd); + unlink(m_Fd_Name); + return HSAKMT_STATUS_ERROR; + } + + m_Pid = pid; + + return HSAKMT_STATUS_SUCCESS; +} + + +void BaseDebug::Detach(void) { + struct kfd_ioctl_dbg_trap_args args = {0}; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_DISABLE; + + hsaKmtDebugTrapIoctl(&args, NULL); + + close(m_Fd.fd); + unlink(m_Fd_Name); + + m_Pid = 0; + m_Fd.fd = 0; + m_Fd.events = 0; +} diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp new file mode 100644 index 0000000000..ac53c6c696 --- /dev/null +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_BASE_DEBUG__H__ +#define __KFD_BASE_DEBUG__H__ + +#include "hsakmt.h" +#include +#include + +// @class BaseDebug +class BaseDebug { + public: + BaseDebug(void); + virtual ~BaseDebug(void); + + HSAKMT_STATUS Attach(struct kfd_runtime_info *rInfo, + int rInfoSize, + unsigned int pid, + uint64_t exceptionEnable); + + void Detach(void); + + private: + unsigned int m_Pid; + struct pollfd m_Fd; + const char *m_Fd_Name = "/tmp/dbg_fifo"; +}; + +#endif // __KFD_BASE_DEBUG__H__ From c7129edcb84139b26bfab367fb35d46b4cdaa275 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 23 Mar 2022 14:20:41 -0400 Subject: [PATCH 1179/1247] kfdtest: add send exception operation Add send exception operation. Signed-off-by: Jonathan Kim Change-Id: Iecb43004a1a7ffc75e94badf203cd0927ffe0909 --- tests/kfdtest/src/BaseDebug.cpp | 15 +++++++++++++++ tests/kfdtest/src/BaseDebug.hpp | 1 + 2 files changed, 16 insertions(+) diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp index 50f243cc39..912150ee51 100644 --- a/tests/kfdtest/src/BaseDebug.cpp +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -96,3 +96,18 @@ void BaseDebug::Detach(void) { m_Fd.fd = 0; m_Fd.events = 0; } + +HSAKMT_STATUS BaseDebug::SendRuntimeEvent(uint64_t exceptions, int gpuId, int queueId) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_SEND_RUNTIME_EVENT; + args.send_runtime_event.exception_mask = exceptions; + args.send_runtime_event.gpu_id = gpuId; + args.send_runtime_event.queue_id = queueId; + + return hsaKmtDebugTrapIoctl(&args, NULL); +} diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp index ac53c6c696..3b8d2098d1 100644 --- a/tests/kfdtest/src/BaseDebug.hpp +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -40,6 +40,7 @@ class BaseDebug { uint64_t exceptionEnable); void Detach(void); + HSAKMT_STATUS SendRuntimeEvent(uint64_t exceptions, int gpuId, int queueId); private: unsigned int m_Pid; From bfb0d15ee8f651b0b0d50e61fc49d076f4c62d8e Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 13 May 2022 12:35:45 -0400 Subject: [PATCH 1180/1247] kfdtest: add query operations Add polling query debug event operation. Signed-off-by: Jonathan Kim Change-Id: Ic82ce4a393bfb28c9f32e7920f80c12da7f627d5 --- tests/kfdtest/src/BaseDebug.cpp | 35 +++++++++++++++++++++++++++++++++ tests/kfdtest/src/BaseDebug.hpp | 3 +++ 2 files changed, 38 insertions(+) diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp index 912150ee51..a673ad8448 100644 --- a/tests/kfdtest/src/BaseDebug.cpp +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -111,3 +111,38 @@ HSAKMT_STATUS BaseDebug::SendRuntimeEvent(uint64_t exceptions, int gpuId, int qu return hsaKmtDebugTrapIoctl(&args, NULL); } + +HSAKMT_STATUS BaseDebug::QueryDebugEvent(uint64_t *exceptions, + uint32_t *gpuId, uint32_t *queueId, + int timeoutMsec) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + HSAKMT_STATUS result; + int r = poll(&m_Fd, 1, timeoutMsec); + + if (r > 0) { + char tmp[r]; + + read(m_Fd.fd, tmp, sizeof(tmp)); + } else { + return HSAKMT_STATUS_ERROR; + } + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT; + args.query_debug_event.exception_mask = *exceptions; + + result = hsaKmtDebugTrapIoctl(&args, NULL); + + *exceptions = args.query_debug_event.exception_mask; + + if (gpuId) + *gpuId = args.query_debug_event.gpu_id; + + if (queueId) + *queueId = args.query_debug_event.queue_id; + + return result; +} diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp index 3b8d2098d1..eede721a7c 100644 --- a/tests/kfdtest/src/BaseDebug.hpp +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -41,6 +41,9 @@ class BaseDebug { void Detach(void); HSAKMT_STATUS SendRuntimeEvent(uint64_t exceptions, int gpuId, int queueId); + HSAKMT_STATUS QueryDebugEvent(uint64_t *exceptions, + uint32_t *gpuId, uint32_t *queueId, + int timeoutMsec); private: unsigned int m_Pid; From 097ee967d15aaf9abfb7f1d6c1acb18397c61049 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 13 Apr 2022 16:09:35 -0400 Subject: [PATCH 1181/1247] kfdtest: add runtime enable and attach test Add debug attach and runtime enable test for attaching to a spawned and running process. Signed-off-by: Jonathan Kim Change-Id: I72302ff73494d9dae0c79a299508085d7ca0552b --- tests/kfdtest/CMakeLists.txt | 1 + tests/kfdtest/src/KFDDBGTest.cpp | 187 +++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 4e4ef6ad03..00d95d7c5f 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -196,6 +196,7 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/KFDExceptionTest.cpp src/KFDGraphicsInterop.cpp src/KFDPerfCounters.cpp + src/KFDDBGTest.cpp src/KFDGWSTest.cpp src/KFDIPCTest.cpp src/KFDASMTest.cpp diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 224c43e46e..a93c1563dc 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -21,6 +21,7 @@ * */ +#include "BaseDebug.hpp" #include "KFDDBGTest.hpp" #include #include @@ -50,3 +51,189 @@ void KFDDBGTest::TearDown() { ROUTINE_END } +/* + * To test debug attaching to a spawned process (i.e. attach prior to the tracee + * opening a KFD device), have the child request the parent to PTRACE attach and + * wait for the parent to debug attach then allow the child to runtime enable. + * + * The following will be exercised: + * - The KFD shall create a KFD process on behalf of the tracee during debug + * attach since the tracee has not opened a KFD device. + * - Runtime enable on the tracee shall raise an event to the debugging parent + * and block until parent has signalled that it has recieved the runtime + * enable event. + * - Tracee should follow a similar hand shake for runtime disable and debug + * detach should follow. + * + * */ +TEST_F(KFDDBGTest, AttachToSpawnedProcess) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + + if (hsaKmtCheckRuntimeDebugSupport()) { + LOG() << "Skip test as debug API not supported"; + goto exit; + } + + pid_t childPid = fork(); + + if (childPid == 0) { /* Debugged process */ + uint32_t rDebug; + int r; + + /* Let parent become the debugger and wait for attach. */ + ptrace(PTRACE_TRACEME); + raise(SIGSTOP); + + r = hsaKmtOpenKFD(); + + if (r != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in debugged process" << std::endl; + exit(1); + } + + LOG() << std::dec << "--- Debugged PID " << getpid() << " runtime enable" << std::endl; + + r = hsaKmtRuntimeEnable(&rDebug, true); + + if (r != HSAKMT_STATUS_SUCCESS) { + WARN() << "Runtime enabled failed" << std::endl; + exit(1); + } + + LOG() << std::dec << "--- Debugged PID " << getpid() << " runtime disable and exit" << std::endl; + + hsaKmtRuntimeDisable(); + + exit(0); + } else { + BaseDebug *debug = new BaseDebug(); + struct kfd_runtime_info r_info = {0}; + uint64_t runtimeMask = KFD_EC_MASK(EC_PROCESS_RUNTIME); + int childStatus; + + waitpid(childPid, &childStatus, 0); + while (!WIFSTOPPED(childStatus)); + + /* Attach and let new debugged process continue with runtime enable */ + LOG() << std::dec << "Attaching to PID " << childPid << std::endl; + ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), childPid, runtimeMask)); + ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_DISABLED); + ASSERT_EQ(r_info.ttmp_setup, false); + + ptrace(PTRACE_CONT, childPid, NULL, NULL); + + /* Wait and unblock runtime enable */ + ASSERT_SUCCESS(debug->QueryDebugEvent(&runtimeMask, NULL, NULL, 5000)); + ASSERT_EQ(runtimeMask, KFD_EC_MASK(EC_PROCESS_RUNTIME)); + ASSERT_SUCCESS(debug->SendRuntimeEvent(runtimeMask, 0, 0)); + + /* Wait and unblock runtime disable */ + ASSERT_SUCCESS(debug->QueryDebugEvent(&runtimeMask, NULL, NULL, 5000)); + ASSERT_EQ(runtimeMask, KFD_EC_MASK(EC_PROCESS_RUNTIME)); + ASSERT_SUCCESS(debug->SendRuntimeEvent(runtimeMask, 0, 0)); + + LOG() << std::dec << "Detaching from PID " << childPid << std::endl; + debug->Detach(); + + ptrace(PTRACE_DETACH, childPid, NULL, NULL); + + LOG() << std::dec << "Waiting on PID " << childPid << " to exit" << std::endl; + waitpid(childPid, &childStatus, 0); + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } +exit: + LOG() << std::endl; + TEST_END +} + +/* + * Unlike AttachToSpawnedProcess, the debug parent will only attach after + * a non-blocked runtime enable by the tracee. The parent should expect + * a status update that the tracee is runtime enabled on debug attach. + * Cleanup with appropriate runtime disable and debug detach handshake. + */ +TEST_F(KFDDBGTest, AttachToRunningProcess) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + + if (hsaKmtCheckRuntimeDebugSupport()) { + LOG() << "Skip test as debug API not supported"; + goto exit; + } + + pid_t childPid = fork(); + + if (childPid == 0) { /* Debugged process */ + uint32_t rDebug; + int r; + + r = hsaKmtOpenKFD(); + + if (r != HSAKMT_STATUS_SUCCESS) { + WARN() << "KFD open failed in debugged process" << std::endl; + exit(1); + } + + LOG() << std::dec << "--- Debugged PID " << getpid() << " runtime enable" << std::endl; + + r = hsaKmtRuntimeEnable(&rDebug, true); + if (r != HSAKMT_STATUS_SUCCESS) { + WARN() << "Runtime enabled failed" << std::endl; + exit(1); + } + + /* Let parent become the debugger and wait for attach. */ + ptrace(PTRACE_TRACEME); + raise(SIGSTOP); + + LOG() << std::dec << "--- Debugged PID " << getpid() << " runtime disable and exit" << std::endl; + + hsaKmtRuntimeDisable(); + + exit(0); + } else { + BaseDebug *debug = new BaseDebug(); + struct kfd_runtime_info r_info = {0}; + uint64_t runtimeMask = KFD_EC_MASK(EC_PROCESS_RUNTIME); + int childStatus; + + waitpid(childPid, &childStatus, 0); + while (!WIFSTOPPED(childStatus)); + + /* Attach to running process and let it continue */ + LOG() << std::dec << "Attaching to PID " << childPid << std::endl; + ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), childPid, runtimeMask)); + ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); + ASSERT_EQ(r_info.ttmp_setup, true); + + ptrace(PTRACE_CONT, childPid, NULL, NULL); + + /* Wait and unblock runtime disable */ + ASSERT_SUCCESS(debug->QueryDebugEvent(&runtimeMask, NULL, NULL, 5000)); + ASSERT_EQ(runtimeMask, KFD_EC_MASK(EC_PROCESS_RUNTIME)); + ASSERT_SUCCESS(debug->SendRuntimeEvent(runtimeMask, 0, 0)); + + LOG() << std::dec << "Detaching from PID " << childPid << std::endl; + debug->Detach(); + + ptrace(PTRACE_DETACH, childPid, NULL, NULL); + + LOG() << std::dec << "Waiting on PID " << childPid << " to exit" << std::endl; + waitpid(childPid, &childStatus, 0); + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } +exit: + LOG() << std::endl; + TEST_END +} From 97fc25bb8d5bcda75ccc8910b2750cda1194ede7 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 27 Apr 2022 10:42:00 -0400 Subject: [PATCH 1182/1247] kfdtest: add set exceptions enable base debug operation Add set exceptions enabled debug option Signed-off-by: Jonathan Kim Change-Id: I6ee1769bbbb90a74074d8100974c4bfeabaf7f2c --- tests/kfdtest/src/BaseDebug.cpp | 13 +++++++++++++ tests/kfdtest/src/BaseDebug.hpp | 1 + 2 files changed, 14 insertions(+) diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp index a673ad8448..1de15f80e6 100644 --- a/tests/kfdtest/src/BaseDebug.cpp +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -146,3 +146,16 @@ HSAKMT_STATUS BaseDebug::QueryDebugEvent(uint64_t *exceptions, return result; } + +void BaseDebug::SetExceptionsEnabled(uint64_t exceptions) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED; + args.set_exceptions_enabled.exception_mask = exceptions; + + hsaKmtDebugTrapIoctl(&args, NULL); +} diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp index eede721a7c..f744761b5c 100644 --- a/tests/kfdtest/src/BaseDebug.hpp +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -44,6 +44,7 @@ class BaseDebug { HSAKMT_STATUS QueryDebugEvent(uint64_t *exceptions, uint32_t *gpuId, uint32_t *queueId, int timeoutMsec); + void SetExceptionsEnabled(uint64_t exceptions); private: unsigned int m_Pid; From b77189cf8384fed7a8e53687d505e49c42afeca9 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 25 Apr 2022 11:19:09 -0400 Subject: [PATCH 1183/1247] kfdtest: add hit trap event test Check that a jump to trap event can be picked up by the debugger. Signed-off-by: Jonathan Kim Change-Id: Iad5f87092f2b82d5018013bba548979122a9bd02 --- tests/kfdtest/src/KFDDBGTest.cpp | 75 +++++++++++++++++++++++++++++ tests/kfdtest/src/ShaderStore.cpp | 78 +++++++++++++++++++++++++++++++ tests/kfdtest/src/ShaderStore.hpp | 4 ++ 3 files changed, 157 insertions(+) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index a93c1563dc..3fe6f58e7b 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -237,3 +237,78 @@ exit: LOG() << std::endl; TEST_END } + +TEST_F(KFDDBGTest, HitTrapEvent) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + if (hsaKmtCheckRuntimeDebugSupport()) { + LOG() << "Skip test as debug API not supported"; + goto exit; + } + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + // create shader and trap bufs then enable 2nd level trap + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, true); + HsaMemoryBuffer trapStatusBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + + HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, true); + HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); + + ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, + trap.As(), + 0x1000, + tmaBuf.As(), + 0x1000)); + + // compile and dispatch shader + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(jump_to_trap_gfx, isaBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(trap_handler_gfx, trap.As())); + + uint32_t rDebug; + ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); + + BaseDebug *debug = new BaseDebug(); + struct kfd_runtime_info r_info = {0}; + ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); + ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); + + PM4Queue queue; + HsaQueueResource *qResources; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + unsigned int* trapStatus = trapStatusBuf.As(); + trapStatus[0] = 0; + Dispatch *dispatch; + dispatch = new Dispatch(isaBuf); + dispatch->SetArgs(&trapStatus[0], NULL); + dispatch->SetDim(1, 1, 1); + + /* Subscribe to trap events and submit the queue */ + uint64_t trapMask = KFD_EC_MASK(EC_QUEUE_WAVE_TRAP); + debug->SetExceptionsEnabled(trapMask); + dispatch->Submit(queue); + + /* Wait for trap event */ + uint32_t QueueId = -1; + ASSERT_SUCCESS(debug->QueryDebugEvent(&trapMask, NULL, &QueueId, 5000)); + ASSERT_NE(QueueId, -1); + ASSERT_EQ(trapMask, KFD_EC_MASK(EC_QUEUE_WAVE_TRAP) | KFD_EC_MASK(EC_QUEUE_NEW)); + + dispatch->Sync(); + EXPECT_SUCCESS(queue.Destroy()); + + ASSERT_NE(trapStatus[0], 0); + + debug->Detach(); + hsaKmtRuntimeDisable(); + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } +exit: + LOG() << std::endl; + TEST_END +} diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 58a2f6674e..827072ee5a 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -80,6 +80,13 @@ const std::vector ShaderList = { " .else\n"\ " v_cmp_lt_u32 vcc, \\src0, \\vsrc1\n"\ " .endif\n"\ + " .endm\n"\ + " .macro V_CMP_EQ_U32 src0, vsrc1\n"\ + " .if (.amdgcn.gfx_generation_number >= 10)\n"\ + " v_cmp_eq_u32 vcc_lo, \\src0, \\vsrc1\n"\ + " .else\n"\ + " v_cmp_eq_u32 vcc, \\src0, \\vsrc1\n"\ + " .endif\n"\ " .endm\n" /* Macros for portable flat load/store/atomic instructions. @@ -893,3 +900,74 @@ const char *GwsAtomicIncreaseIsa = s_waitcnt 0 s_endpgm )"; + +const char *jump_to_trap_gfx = + SHADER_START + SHADER_MACROS_U32 + R"( + /*copy the parameters from scalar registers to vector registers*/ + v_mov_b32 v4, 0 + v_mov_b32 v0, s0 + v_mov_b32 v1, s1 + s_trap 1 + EXIT_LOOP: + V_CMP_EQ_U32 v4, 0 + s_cbranch_vccnz EXIT_LOOP + flat_store_dword v[0:1], v4 + s_waitcnt vmcnt(0)&lgkmcnt(0) + s_endpgm +)"; + +const char *trap_handler_gfx = + SHADER_START + R"( + CHECK_VMFAULT: + /*if trap jumped to by vmfault, restore skip m0 signalling*/ + s_getreg_b32 ttmp14, hwreg(HW_REG_TRAPSTS) + s_and_b32 ttmp2, ttmp14, 0x800 + s_cbranch_scc1 RESTORE_AND_EXIT + GET_DOORBELL: + .if .amdgcn.gfx_generation_number < 11 + s_mov_b32 ttmp2, exec_lo + s_mov_b32 ttmp3, exec_hi + s_mov_b32 exec_lo, 0x80000000 + s_sendmsg 10 + WAIT_SENDMSG: + /*wait until msb is cleared (i.e. doorbell fetched)*/ + s_nop 7 + s_bitcmp0_b32 exec_lo, 0x1F + s_cbranch_scc0 WAIT_SENDMSG + /* restore exec */ + s_mov_b32 exec_hi, ttmp3 + s_and_b32 exec_lo, exec_lo, 0xfff + s_mov_b32 ttmp3, exec_lo + s_mov_b32 exec_lo, ttmp2 + .else + s_sendmsg_rtn_b32 ttmp3, sendmsg(MSG_RTN_GET_DOORBELL) + s_waitcnt lgkmcnt(0) + s_and_b32 ttmp3, ttmp3, 0x3ff + .endif + s_mov_b32 ttmp2, m0 + s_or_b32 ttmp3, ttmp3, 0x800 + /* set m0, send interrupt and restore m0 and exit trap*/ + s_mov_b32 m0, ttmp3 + s_nop 0x0 + s_sendmsg sendmsg(MSG_INTERRUPT) + s_waitcnt lgkmcnt(0) + s_mov_b32 m0, ttmp2 + v_mov_b32 v4, ttmp1 + /* restore and increment program counter to skip shader trap jump*/ + s_add_u32 ttmp0, ttmp0, 4 + s_addc_u32 ttmp1, ttmp1, 0 + s_and_b32 ttmp1, ttmp1, 0xffff + RESTORE_AND_EXIT: + /* restore SQ_WAVE_IB_STS */ + s_lshr_b32 ttmp2, ttmp11, (26 - 15) + s_and_b32 ttmp2, ttmp2, (0x8000 | 0x1F0000) + s_setreg_b32 hwreg(HW_REG_IB_STS), ttmp2 + /* restore SQ_WAVE_STATUS */ + s_and_b64 exec, exec, exec + s_and_b64 vcc, vcc, vcc + s_setreg_b32 hwreg(HW_REG_STATUS), ttmp12 + s_rfe_b64 [ttmp0, ttmp1] +)"; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index 3964a5b397..e975e28499 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -62,4 +62,8 @@ extern const char *ReadMemoryIsa; extern const char *GwsInitIsa; extern const char *GwsAtomicIncreaseIsa; +/* HitTrapEvent */ +extern const char *jump_to_trap_gfx; +extern const char *trap_handler_gfx; + #endif // _SHADERSTORE_H_ From 5a675921eab0283ccbca3e7f86363ac61e48ea4e Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 11 May 2022 13:34:22 -0400 Subject: [PATCH 1184/1247] kfdtest: add suspend and resume queues operation Add base debug operations to suspend and resume queues. Routine will return the number of queues successfully suspended or resumed. Signed-off-by: Jonathan Kim Change-Id: I8f18317f70464b04231c5cf822e11d545ebfa02a --- include/hsakmt.h | 3 ++- src/debug.c | 5 +++- tests/kfdtest/src/BaseDebug.cpp | 44 +++++++++++++++++++++++++++++---- tests/kfdtest/src/BaseDebug.hpp | 3 +++ 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/include/hsakmt.h b/include/hsakmt.h index 9d5ef5897f..ce7b7e327c 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -788,7 +788,8 @@ hsaKmtCheckRuntimeDebugSupport( */ HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl( struct kfd_ioctl_dbg_trap_args *arg, - HSA_QUEUEID *Queues + HSA_QUEUEID *Queues, + HSAuint64 *DebugReturn ); /** diff --git a/src/debug.c b/src/debug.c index b3e312cbc1..d438a2826e 100644 --- a/src/debug.c +++ b/src/debug.c @@ -514,7 +514,8 @@ free_data: } HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *args, - HSA_QUEUEID *Queues) + HSA_QUEUEID *Queues, + HSAuint64 *DebugReturn) { HSAKMT_STATUS result; @@ -533,6 +534,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtDebugTrapIoctl(struct kfd_ioctl_dbg_trap_args *arg } long err = kmtIoctl(kfd_fd, AMDKFD_IOC_DBG_TRAP, args); + if (DebugReturn) + *DebugReturn = err; if (args->op == KFD_IOC_DBG_TRAP_SUSPEND_QUEUES && err >= 0 && err <= args->suspend_queues.num_queues) diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp index 1de15f80e6..3b8837c587 100644 --- a/tests/kfdtest/src/BaseDebug.cpp +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -67,7 +67,7 @@ HSAKMT_STATUS BaseDebug::Attach(struct kfd_runtime_info *rInfo, args.enable.dbg_fd = m_Fd.fd; args.enable.exception_mask = exceptionEnable; - if (hsaKmtDebugTrapIoctl(&args, NULL)) { + if (hsaKmtDebugTrapIoctl(&args, NULL, NULL)) { close(m_Fd.fd); unlink(m_Fd_Name); return HSAKMT_STATUS_ERROR; @@ -87,7 +87,7 @@ void BaseDebug::Detach(void) { args.pid = m_Pid; args.op = KFD_IOC_DBG_TRAP_DISABLE; - hsaKmtDebugTrapIoctl(&args, NULL); + hsaKmtDebugTrapIoctl(&args, NULL, NULL); close(m_Fd.fd); unlink(m_Fd_Name); @@ -109,7 +109,7 @@ HSAKMT_STATUS BaseDebug::SendRuntimeEvent(uint64_t exceptions, int gpuId, int qu args.send_runtime_event.gpu_id = gpuId; args.send_runtime_event.queue_id = queueId; - return hsaKmtDebugTrapIoctl(&args, NULL); + return hsaKmtDebugTrapIoctl(&args, NULL, NULL); } HSAKMT_STATUS BaseDebug::QueryDebugEvent(uint64_t *exceptions, @@ -134,7 +134,7 @@ HSAKMT_STATUS BaseDebug::QueryDebugEvent(uint64_t *exceptions, args.op = KFD_IOC_DBG_TRAP_QUERY_DEBUG_EVENT; args.query_debug_event.exception_mask = *exceptions; - result = hsaKmtDebugTrapIoctl(&args, NULL); + result = hsaKmtDebugTrapIoctl(&args, NULL, NULL); *exceptions = args.query_debug_event.exception_mask; @@ -157,5 +157,39 @@ void BaseDebug::SetExceptionsEnabled(uint64_t exceptions) args.op = KFD_IOC_DBG_TRAP_SET_EXCEPTIONS_ENABLED; args.set_exceptions_enabled.exception_mask = exceptions; - hsaKmtDebugTrapIoctl(&args, NULL); + hsaKmtDebugTrapIoctl(&args, NULL, NULL); +} + +HSAKMT_STATUS BaseDebug::SuspendQueues(unsigned int *numQueues, + HSA_QUEUEID *queues, + uint32_t *queueIds, + uint64_t exceptionsToClear) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_SUSPEND_QUEUES; + args.suspend_queues.num_queues = *numQueues; + args.suspend_queues.queue_array_ptr = (uint64_t)queueIds; + args.suspend_queues.exception_mask = exceptionsToClear; + + return hsaKmtDebugTrapIoctl(&args, queues, (HSAuint64 *)numQueues); +} + +HSAKMT_STATUS BaseDebug::ResumeQueues(unsigned int *numQueues, + HSA_QUEUEID *queues, + uint32_t *queueIds) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_RESUME_QUEUES; + args.resume_queues.num_queues = *numQueues; + args.resume_queues.queue_array_ptr = (uint64_t)queueIds; + + return hsaKmtDebugTrapIoctl(&args, queues, (HSAuint64 *)numQueues); } diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp index f744761b5c..eb3d659297 100644 --- a/tests/kfdtest/src/BaseDebug.hpp +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -45,6 +45,9 @@ class BaseDebug { uint32_t *gpuId, uint32_t *queueId, int timeoutMsec); void SetExceptionsEnabled(uint64_t exceptions); + HSAKMT_STATUS SuspendQueues(unsigned int *numQueues, HSA_QUEUEID *queues, uint32_t *queueIds, + uint64_t exceptionsToClear); + HSAKMT_STATUS ResumeQueues(unsigned int *numQueues, HSA_QUEUEID *queues, uint32_t *queueIds); private: unsigned int m_Pid; From b0e84183c11fb7371554ffbc074c9aa0740100a0 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 13 May 2022 14:22:38 -0400 Subject: [PATCH 1185/1247] kfdtest: add snapshot operations Add queue and devices snapshot operations. Signed-off-by: Jonathan Kim Change-Id: I836884c9f3b65dd9e5e444d554d3eb87938e1634 --- tests/kfdtest/src/BaseDebug.cpp | 46 +++++++++++++++++++++++++++++++++ tests/kfdtest/src/BaseDebug.hpp | 4 +++ 2 files changed, 50 insertions(+) diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp index 3b8837c587..78f6d059d0 100644 --- a/tests/kfdtest/src/BaseDebug.cpp +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -193,3 +193,49 @@ HSAKMT_STATUS BaseDebug::ResumeQueues(unsigned int *numQueues, return hsaKmtDebugTrapIoctl(&args, queues, (HSAuint64 *)numQueues); } + +HSAKMT_STATUS BaseDebug::QueueSnapshot(uint64_t exceptionsToClear, + uint64_t snapshotBufAddr, + uint32_t *numSnapshots) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + HSAKMT_STATUS result; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_GET_QUEUE_SNAPSHOT; + args.queue_snapshot.exception_mask = exceptionsToClear; + args.queue_snapshot.snapshot_buf_ptr = snapshotBufAddr; + args.queue_snapshot.num_queues = *numSnapshots; + args.queue_snapshot.entry_size = sizeof(struct kfd_queue_snapshot_entry); + + result = hsaKmtDebugTrapIoctl(&args, NULL, NULL); + + *numSnapshots = args.queue_snapshot.num_queues; + + return result; +} + +HSAKMT_STATUS BaseDebug::DeviceSnapshot(uint64_t exceptionsToClear, + uint64_t snapshotBufAddr, + uint32_t *numSnapshots) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + HSAKMT_STATUS result; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_GET_DEVICE_SNAPSHOT; + args.device_snapshot.exception_mask = exceptionsToClear; + args.device_snapshot.snapshot_buf_ptr = snapshotBufAddr; + args.device_snapshot.num_devices = *numSnapshots; + args.device_snapshot.entry_size = sizeof(struct kfd_dbg_device_info_entry); + + result = hsaKmtDebugTrapIoctl(&args, NULL, NULL); + + *numSnapshots = args.device_snapshot.num_devices; + + return result; +} diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp index eb3d659297..c19796a0e0 100644 --- a/tests/kfdtest/src/BaseDebug.hpp +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -48,6 +48,10 @@ class BaseDebug { HSAKMT_STATUS SuspendQueues(unsigned int *numQueues, HSA_QUEUEID *queues, uint32_t *queueIds, uint64_t exceptionsToClear); HSAKMT_STATUS ResumeQueues(unsigned int *numQueues, HSA_QUEUEID *queues, uint32_t *queueIds); + HSAKMT_STATUS QueueSnapshot(uint64_t exceptionsToClear, uint64_t snapshotBufAddr, + uint32_t *numSnapshots); + HSAKMT_STATUS DeviceSnapshot(uint64_t exceptionsToClear, uint64_t snapshotBuffAddr, + uint32_t *numSnapshots); private: unsigned int m_Pid; From d20f0bbb90c5fe58ec22d63530a82a3a8ba67192 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 11 May 2022 14:24:28 -0400 Subject: [PATCH 1186/1247] kfdtest: add suspend resume test Add queue suspend and resume test. Signed-off-by: Jonathan Kim Change-Id: I2ade721026cbb458a3597b7858a164e70fe05f4f --- tests/kfdtest/src/KFDDBGTest.cpp | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 3fe6f58e7b..6d586fcdbf 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -312,3 +312,81 @@ exit: LOG() << std::endl; TEST_END } + +TEST_F(KFDDBGTest, SuspendQueues) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + if (hsaKmtCheckRuntimeDebugSupport()) { + LOG() << "Skip test as debug API not supported"; + goto exit; + } + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + // create shader and trap bufs then enable 2nd level trap + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, true); + + // compile and dispatch shader + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(jump_to_trap_gfx, isaBuf.As())); + + uint32_t rDebug; + ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); + + BaseDebug *debug = new BaseDebug(); + struct kfd_runtime_info r_info = {0}; + ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); + ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); + + PM4Queue queue; + HsaQueueResource *qResources; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + qResources = queue.GetResource(); + HSA_QUEUEID Queues[] = { qResources->QueueId }; + + Dispatch *dispatch; + dispatch = new Dispatch(isaBuf); + dispatch->SetDim(1, 1, 1); + dispatch->Submit(queue); + + uint32_t NumQueues = 1; + uint32_t QueueIds[NumQueues]; + struct kfd_queue_snapshot_entry Snapshots[NumQueues] = {0}; + ASSERT_SUCCESS(debug->SuspendQueues(&NumQueues, Queues, &QueueIds[0], 0)); + + // Suspend should fail as new queues cannot be suspended + ASSERT_EQ(NumQueues, 0); + ASSERT_NE(QueueIds[0] & KFD_DBG_QUEUE_INVALID_MASK, 0); + + // Snapshot queue, clear new queue status and suspend successfully. + ASSERT_SUCCESS(debug->QueueSnapshot(0, (uint64_t)(&(Snapshots[0])), &NumQueues)); + ASSERT_EQ(NumQueues, 1); + ASSERT_EQ(Snapshots[0].ctx_save_restore_area_size, 0); + + ASSERT_SUCCESS(debug->QueueSnapshot(KFD_EC_MASK(EC_QUEUE_NEW), (uint64_t)(&(Snapshots[0])), + &NumQueues)); + ASSERT_EQ(NumQueues, 1); + ASSERT_GT(Snapshots[0].ctx_save_restore_area_size, 0); + + ASSERT_SUCCESS(debug->SuspendQueues(&NumQueues, Queues, &QueueIds[0], 0)); + ASSERT_EQ(NumQueues, 1); + ASSERT_EQ(QueueIds[0] & KFD_DBG_QUEUE_INVALID_MASK, 0); + + // Resume and destroy queue then clean up. + ASSERT_SUCCESS(debug->ResumeQueues(&NumQueues, Queues, &QueueIds[0])); + ASSERT_EQ(NumQueues, 1); + ASSERT_EQ(QueueIds[0] & KFD_DBG_QUEUE_INVALID_MASK, 0); + + EXPECT_SUCCESS(queue.Destroy()); + + debug->Detach(); + hsaKmtRuntimeDisable(); + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } +exit: + LOG() << std::endl; + TEST_END +} From 96c8bb11b1b4cc3e8b9a35d587779584dae8dcef Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Mon, 7 Aug 2023 09:40:31 -0700 Subject: [PATCH 1187/1247] Disable file reorg backward compatibility support by default Change-Id: I157e05e52a1a61b86fa2fc6f29d31361a688fa10 --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b941fd174a..e3b9085449 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -212,9 +212,7 @@ install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INS COMPONENT devel PATTERN "linux" EXCLUDE ) # Option to build header path migration helpers. -if( NOT ENABLE_ASAN_PACKAGING ) - option(INCLUDE_PATH_COMPATIBILITY "Generate backward compatible headers and include paths. Use of these headers will warn when included." ON) -endif() +option(INCLUDE_PATH_COMPATIBILITY "Generate backward compatible headers and include paths. Use of these headers will warn when included." OFF) if(INCLUDE_PATH_COMPATIBILITY) # To enable/disable #error in wrapper header files From 6c5121faff8d758e971f695bdfa182d201e868f4 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 9 Aug 2023 21:05:45 -0400 Subject: [PATCH 1188/1247] kfdtest: allow toggle of dispatch privilege For GFX11 debugger testing, waves require to start in non-priv mode for some test cases, so allow tester to set this. Signed-off-by: Jonathan Kim Change-Id: Iee93fda926bfd336d51c79c086f1f75bc35b70e5 --- tests/kfdtest/src/Dispatch.cpp | 4 ++++ tests/kfdtest/src/Dispatch.hpp | 2 ++ 2 files changed, 6 insertions(+) diff --git a/tests/kfdtest/src/Dispatch.cpp b/tests/kfdtest/src/Dispatch.cpp index 6209d8cb59..b35484d733 100644 --- a/tests/kfdtest/src/Dispatch.cpp +++ b/tests/kfdtest/src/Dispatch.cpp @@ -74,6 +74,10 @@ void Dispatch::SetSpiPriority(unsigned int priority) { m_SpiPriority = priority; } +void Dispatch::SetPriv(bool priv) { + m_NeedCwsrWA = priv; +} + void Dispatch::Submit(BaseQueue& queue) { ASSERT_NE(m_pEop, (void*)0); EXPECT_EQ(m_FamilyId, queue.GetFamilyId()); diff --git a/tests/kfdtest/src/Dispatch.hpp b/tests/kfdtest/src/Dispatch.hpp index b43becf01b..8c7b1964e5 100644 --- a/tests/kfdtest/src/Dispatch.hpp +++ b/tests/kfdtest/src/Dispatch.hpp @@ -45,6 +45,8 @@ class Dispatch { void SetScratch(int numWaves, int waveSize, HSAuint64 scratch_base); void SetSpiPriority(unsigned int priority); + + void SetPriv(bool priv); HsaEvent *GetHsaEvent() { return m_pEop; } From d4029a94921a1b1567a80b428ecdccb23bb978cd Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 17 May 2022 11:29:42 -0400 Subject: [PATCH 1189/1247] kfdtest: add memory violation test Add memory violation detection test. Signed-off-by: Jonathan Kim Change-Id: I1b56f684682836fc84fbec713bd81c53bdd6d413 --- tests/kfdtest/src/KFDDBGTest.cpp | 128 +++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 6d586fcdbf..3d5e11eede 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -390,3 +390,131 @@ exit: LOG() << std::endl; TEST_END } + +TEST_F(KFDDBGTest, HitMemoryViolation) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + if (hsaKmtCheckRuntimeDebugSupport()) { + LOG() << "Skip test as debug API not supported"; + goto exit; + } + + pid_t childPid = fork(); + + if (childPid == 0) { // Debugged process + uint32_t rDebug; + int r; + + // Refresh setup for HSA device and mem buffer use in child + KFDBaseComponentTest::TearDown(); + KFDBaseComponentTest::SetUp(); + + // Let parent become the debugger and wait for attach. + ptrace(PTRACE_TRACEME); + raise(SIGSTOP); + + r = hsaKmtRuntimeEnable(&rDebug, true); + + if (r != HSAKMT_STATUS_SUCCESS) { + WARN() << "Runtime enabled failed" << std::endl; + exit(1); + } + + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, true); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(PersistentIterateIsa, isaBuf.As())); + PM4Queue queue; + HsaQueueResource *qResources; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + // Create memory violation event on dispatch + HsaEvent *vmFaultEvent; + HsaEventDescriptor eventDesc; + eventDesc.EventType = HSA_EVENTTYPE_MEMORY; + eventDesc.NodeId = defaultGPUNode; + eventDesc.SyncVar.SyncVar.UserData = NULL; + eventDesc.SyncVar.SyncVarSize = 0; + r = hsaKmtCreateEvent(&eventDesc, true, false, &vmFaultEvent); + + if (r != HSAKMT_STATUS_SUCCESS) { + WARN() << "Creating VM fault event failed" << std::endl; + exit(1); + } + + Dispatch *dispatch; + dispatch = new Dispatch(isaBuf); + dispatch->SetDim(1, 1, 1); + dispatch->SetPriv(false); //Override GFX11 CWSR WA + dispatch->Submit(queue); + + // Queue immediately dies so halt process for tracer device inspection. + raise(SIGSTOP); + + exit(0); + } else { + BaseDebug *debug = new BaseDebug(); + struct kfd_runtime_info r_info = {0}; + uint64_t runtimeMask = KFD_EC_MASK(EC_PROCESS_RUNTIME); + uint64_t memViolMask = KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION); + uint64_t subscribeMask = runtimeMask | memViolMask; + uint64_t queryMask = 0; + int childStatus; + + waitpid(childPid, &childStatus, 0); + while (!WIFSTOPPED(childStatus)); + + ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), childPid, subscribeMask)); + ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_DISABLED); + ASSERT_EQ(r_info.ttmp_setup, false); + + ptrace(PTRACE_CONT, childPid, NULL, NULL); + + // Wait and unblock runtime enable + ASSERT_SUCCESS(debug->QueryDebugEvent(&runtimeMask, NULL, NULL, 5000)); + ASSERT_EQ(runtimeMask, KFD_EC_MASK(EC_PROCESS_RUNTIME)); + ASSERT_SUCCESS(debug->SendRuntimeEvent(runtimeMask, 0, 0)); + + // Wait for memory violation + uint32_t deviceId = -1; + ASSERT_SUCCESS(debug->QueryDebugEvent(&queryMask, &deviceId, NULL, 5000)); + ASSERT_NE(deviceId, -1); + ASSERT_EQ(queryMask, memViolMask); + + // Assume tracee queue has died and halted process + ptrace(PTRACE_CONT, childPid, NULL, NULL); + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + uint32_t snapshotSize = gpuNodes.size(); + struct kfd_dbg_device_info_entry deviceInfo[snapshotSize] = {0}; + + // Check device snapshot aligns with memory violation on target device. + ASSERT_SUCCESS(debug->DeviceSnapshot(memViolMask, (uint64_t)(&deviceInfo[0]), + &snapshotSize)); + ASSERT_EQ(snapshotSize, gpuNodes.size()); + for (int i = 0; i < snapshotSize; i++) { + if (deviceInfo[i].exception_status & memViolMask) { + ASSERT_EQ(deviceInfo[i].gpu_id, deviceId); + break; + } + } + + debug->Detach(); + + ptrace(PTRACE_DETACH, childPid, NULL, NULL); + + waitpid(childPid, &childStatus, 0); + EXPECT_EQ(WIFEXITED(childStatus), true); + EXPECT_EQ(WEXITSTATUS(childStatus), HSAKMT_STATUS_SUCCESS); + } + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } +exit: + LOG() << std::endl; + TEST_END +} From 431dc8d403adb025950ed5b43278bdbba79b66cc Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 24 Jan 2023 13:14:24 -0500 Subject: [PATCH 1190/1247] kfdtest: Add ops for address watch test Add wave launch override, set/clear address watch and precise memops test. Signed-off-by: Jonathan Kim Change-Id: Ib405d5570cd304e02c2e76eca3593cbd9a5937d9 --- tests/kfdtest/src/BaseDebug.cpp | 70 +++++++++++++++++++++++++++++++++ tests/kfdtest/src/BaseDebug.hpp | 4 ++ 2 files changed, 74 insertions(+) diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp index 78f6d059d0..a04afe3b01 100644 --- a/tests/kfdtest/src/BaseDebug.cpp +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -239,3 +239,73 @@ HSAKMT_STATUS BaseDebug::DeviceSnapshot(uint64_t exceptionsToClear, return result; } + +HSAKMT_STATUS BaseDebug::SetWaveLaunchOverride(int mode, + uint32_t *enableMask, + uint32_t *supportMask) +{ + struct kfd_ioctl_dbg_trap_args args = {0}; + HSAKMT_STATUS Result; + + memset(&args, 0x00, sizeof(args)); + + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_SET_WAVE_LAUNCH_OVERRIDE; + args.launch_override.override_mode = mode; + args.launch_override.enable_mask = *enableMask; + args.launch_override.support_request_mask = *supportMask; + + Result = hsaKmtDebugTrapIoctl(&args, NULL, NULL); + + *enableMask = args.launch_override.enable_mask; + *supportMask = args.launch_override.support_request_mask; + + return Result; +} + +HSAKMT_STATUS BaseDebug::SetAddressWatch(uint64_t address, + int mode, + uint64_t mask, + uint32_t gpuId, + uint32_t *id) +{ + struct kfd_ioctl_dbg_trap_args args = {}; + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_SET_NODE_ADDRESS_WATCH; + args.set_node_address_watch.address = address; + args.set_node_address_watch.mode = mode; + args.set_node_address_watch.mask = mask; + args.set_node_address_watch.gpu_id = gpuId; + + HSAKMT_STATUS result = hsaKmtDebugTrapIoctl(&args, NULL, NULL); + + *id = args.set_node_address_watch.id; + + return result; +} + +HSAKMT_STATUS BaseDebug::ClearAddressWatch(uint32_t gpuId, + uint32_t id) +{ + struct kfd_ioctl_dbg_trap_args args = {}; + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_CLEAR_NODE_ADDRESS_WATCH; + args.clear_node_address_watch.gpu_id = gpuId; + args.clear_node_address_watch.id = id; + + return hsaKmtDebugTrapIoctl(&args, NULL, NULL); +} + +HSAKMT_STATUS BaseDebug::SetFlags(uint32_t *flags) +{ + struct kfd_ioctl_dbg_trap_args args = {}; + args.pid = m_Pid; + args.op = KFD_IOC_DBG_TRAP_SET_FLAGS; + args.set_flags.flags = *flags; + + HSAKMT_STATUS result = hsaKmtDebugTrapIoctl(&args, NULL, NULL); + + *flags = args.set_flags.flags; + + return result; +} diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp index c19796a0e0..fc9343e59d 100644 --- a/tests/kfdtest/src/BaseDebug.hpp +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -52,6 +52,10 @@ class BaseDebug { uint32_t *numSnapshots); HSAKMT_STATUS DeviceSnapshot(uint64_t exceptionsToClear, uint64_t snapshotBuffAddr, uint32_t *numSnapshots); + HSAKMT_STATUS SetWaveLaunchOverride(int mode, uint32_t *enableMask, uint32_t *supportMask); + HSAKMT_STATUS SetAddressWatch(uint64_t address, int mode, uint64_t mask, uint32_t gpuId, uint32_t *id); + HSAKMT_STATUS ClearAddressWatch(uint32_t gpuId, uint32_t id); + HSAKMT_STATUS SetFlags(uint32_t *flags); private: unsigned int m_Pid; From 8311ca5bfa41b5a296057a05be0cad9e3009cdb0 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 28 Jul 2023 12:54:49 -0400 Subject: [PATCH 1191/1247] kfdtest: add address watch test Address watch test will test read and write operations. Test will also check if operation is precise if precise address watch is available. Signed-off-by: Jonathan Kim Change-Id: I7ef835790e26bf6345682755d7dd26a35853bcd5 --- tests/kfdtest/src/KFDDBGTest.cpp | 112 ++++++++++++++++++++++++++++++ tests/kfdtest/src/ShaderStore.cpp | 36 ++++++++++ tests/kfdtest/src/ShaderStore.hpp | 5 ++ 3 files changed, 153 insertions(+) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 3d5e11eede..51edc5a512 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -518,3 +518,115 @@ exit: LOG() << std::endl; TEST_END } + +TEST_F(KFDDBGTest, HitAddressWatch) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_VI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + if (hsaKmtCheckRuntimeDebugSupport()) { + LOG() << "Skip test as debug API not supported"; + goto exit; + } + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + HsaNodeProperties nodeProps; + ASSERT_SUCCESS(hsaKmtGetNodeProperties(defaultGPUNode, &nodeProps)); + + HsaMemoryBuffer readerBuf(PAGE_SIZE, defaultGPUNode, true, false, true); + HsaMemoryBuffer writerBuf(PAGE_SIZE, defaultGPUNode, true, false, true); + HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, true); + HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); + + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(watch_read_isa, readerBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(watch_write_isa, writerBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(trap_handler_gfx, trap.As())); + ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, + trap.As(), + 0x1000, + tmaBuf.As(), + 0x1000)); + + uint32_t rDebug; + ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); + + struct kfd_runtime_info r_info = {0}; + BaseDebug *debug = new BaseDebug(); + ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); + ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); + + const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); + uint32_t numDevices = gpuNodes.size(); + struct kfd_dbg_device_info_entry deviceInfo[numDevices] = {0}; + ASSERT_SUCCESS(debug->DeviceSnapshot(0, (uint64_t)(&deviceInfo[0]), &numDevices)); + ASSERT_EQ(numDevices, gpuNodes.size()); + bool is_precise = nodeProps.Capability.ui32.PreciseMemoryOperationsSupported; + + if (is_precise) { + uint32_t trapFlags = KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP; + ASSERT_SUCCESS(debug->SetFlags(&trapFlags)); + } + + uint32_t enableMask = KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH; + uint32_t supportedMask = enableMask; + ASSERT_SUCCESS(debug->SetWaveLaunchOverride(KFD_DBG_TRAP_OVERRIDE_OR, + &enableMask, + &supportedMask)); + ASSERT_NE(supportedMask & KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH, 0); + ASSERT_EQ(enableMask & KFD_DBG_TRAP_MASK_DBG_ADDRESS_WATCH, 0); // previous set mask + + PM4Queue queue; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + const uint32_t watchMask = -1 & UINT_MAX; + + HsaMemoryBuffer targetBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + HsaMemoryBuffer resultBuf(PAGE_SIZE, defaultGPUNode, true, false, false); + unsigned int *target = targetBuf.As(); + unsigned int *result = resultBuf.As(); + + for (int mode = KFD_DBG_TRAP_ADDRESS_WATCH_MODE_READ; + mode < KFD_DBG_TRAP_ADDRESS_WATCH_MODE_ALL; mode++) { + + // atomics may not be supported on all devices so skip for now. + if (mode != KFD_DBG_TRAP_ADDRESS_WATCH_MODE_READ && + mode != KFD_DBG_TRAP_ADDRESS_WATCH_MODE_NONREAD) + continue; + + uint32_t watchId = -1; + ASSERT_SUCCESS(debug->SetAddressWatch((uint64_t)(&target[0]), mode, + watchMask, deviceInfo[0].gpu_id, &watchId)); + ASSERT_EQ(watchId, 0); + + const HsaMemoryBuffer &shaderBuf = + mode == KFD_DBG_TRAP_ADDRESS_WATCH_MODE_READ ? readerBuf : writerBuf; + uint32_t preciseMask = 0x1; + uint32_t watchStsMask = 0x80; + result[0] = preciseMask; + Dispatch dispatch(shaderBuf); + dispatch.SetDim(1, 1, 1); + dispatch.SetArgs(&target[0], &result[0]); + dispatch.SetPriv(false); // Override GFX11 CWSR WA + dispatch.Submit(queue); + dispatch.Sync(); + + ASSERT_EQ(result[0] & watchStsMask, watchStsMask); + + if (is_precise) + ASSERT_EQ(result[0] & preciseMask, preciseMask); + + ASSERT_SUCCESS(debug->ClearAddressWatch(deviceInfo[0].gpu_id, watchId)); + resultBuf.Fill(0); + targetBuf.Fill(0); + } + + ASSERT_SUCCESS(queue.Destroy()); + debug->Detach(); + hsaKmtRuntimeDisable(); + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } +exit: + LOG() << std::endl; + TEST_END +} diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 827072ee5a..08d82612ee 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -926,6 +926,12 @@ const char *trap_handler_gfx = s_getreg_b32 ttmp14, hwreg(HW_REG_TRAPSTS) s_and_b32 ttmp2, ttmp14, 0x800 s_cbranch_scc1 RESTORE_AND_EXIT + /*check for address watch event and record pc check point delta*/ + s_and_b32 ttmp2, ttmp14, 0x7080 + s_cbranch_scc0 GET_DOORBELL + v_mov_b32 v5, v4 // capture watch checkpoint + v_mov_b32 v6, ttmp14 // capture watch trapsts + s_branch RESTORE_AND_EXIT GET_DOORBELL: .if .amdgcn.gfx_generation_number < 11 s_mov_b32 ttmp2, exec_lo @@ -971,3 +977,33 @@ const char *trap_handler_gfx = s_setreg_b32 hwreg(HW_REG_STATUS), ttmp12 s_rfe_b64 [ttmp0, ttmp1] )"; + +#define WATCH_START SHADER_START SHADER_MACROS_U32\ + "v_mov_b32 v0, s0\n"\ + "v_mov_b32 v1, s1\n"\ + "v_mov_b32 v2, s2\n"\ + "v_mov_b32 v3, s3\n"\ + "flat_load_dword v4, v[2:3]\n"\ + "s_waitcnt vmcnt(0) & lgkmcnt(0)\n"\ + "v_mov_b32 v5, 0\n"\ + "v_mov_b32 v6, 0\n" + +#define WATCH_END "\n"\ + "v_mov_b32 v4, 2\n"\ + "LOOP:\n"\ + "V_CMP_EQ_U32 v6, 0\n"\ + "s_cbranch_vccnz LOOP\n"\ + "V_ADD_CO_U32 v6, v6, v5\n"\ + "flat_store_dword v[2:3], v6\n"\ + "s_waitcnt vmcnt(0) & lgkmcnt(0)\n"\ + "s_endpgm\n" + +const char *watch_read_isa = + WATCH_START + "flat_load_dword v7, v[0:1]" + WATCH_END; + +const char *watch_write_isa = + WATCH_START + "flat_store_dword v[0:1], v4" + WATCH_END; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index e975e28499..cfcfb2761c 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -66,4 +66,9 @@ extern const char *GwsAtomicIncreaseIsa; extern const char *jump_to_trap_gfx; extern const char *trap_handler_gfx; +/* HitWatchPointEvent */ +extern const char *watch_read_isa; +extern const char *watch_write_isa; +extern const char *watch_atomic_isa; + #endif // _SHADERSTORE_H_ From a3f8085025913ac7b9c2f8844c254812679d0438 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Mon, 14 Aug 2023 12:10:15 -0400 Subject: [PATCH 1192/1247] kfdtest: add trap on wave start and end test Add test to catch trap on wave start or end override event. Signed-off-by: Jonathan Kim Change-Id: Icb57af64475fbd2d8a6c0af9a2ee5db5d1a169c6 --- tests/kfdtest/src/KFDDBGTest.cpp | 89 ++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 51edc5a512..7a403d3ed4 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -313,6 +313,95 @@ exit: TEST_END } +TEST_F(KFDDBGTest, HitTrapOnWaveStartEndEvent) { + TEST_START(TESTPROFILE_RUNALL) + if (m_FamilyId >= FAMILY_AI) { + int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + + if (hsaKmtCheckRuntimeDebugSupport()) { + LOG() << "Skip test as debug API not supported"; + goto exit; + } + + ASSERT_GE(defaultGPUNode, 0) << "failed to get default GPU Node"; + + // create shader and trap bufs then enable 2nd level trap + HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, true); + HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, true); + HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); + + ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, + trap.As(), + 0x1000, + tmaBuf.As(), + 0x1000)); + + // compile and dispatch shader + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(NoopIsa, isaBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(trap_handler_gfx, trap.As())); + + uint32_t rDebug; + ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); + + BaseDebug *debug = new BaseDebug(); + struct kfd_runtime_info r_info = {0}; + ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); + ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); + + PM4Queue queue; + HsaQueueResource *qResources; + ASSERT_SUCCESS(queue.Create(defaultGPUNode)); + + for (int i = 0; i < 2; i++) { + uint32_t enableMask = !!!(i % 2) ? KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_START : + KFD_DBG_TRAP_MASK_TRAP_ON_WAVE_END; + uint32_t supportedMask = enableMask; + debug->SetWaveLaunchOverride(KFD_DBG_TRAP_OVERRIDE_OR, + &enableMask, + &supportedMask); + + if (!!!(supportedMask & enableMask)) { + EXPECT_SUCCESS(queue.Destroy()); + debug->Detach(); + hsaKmtRuntimeDisable(); + LOG() << "Skipping test: Trap on start/end override not supported." << std::endl; + goto exit; + } + + ASSERT_EQ(enableMask, 0); // previous set mask + + Dispatch *dispatch; + dispatch = new Dispatch(isaBuf); + dispatch->SetArgs(NULL, NULL); + dispatch->SetDim(1, 1, 1); + + /* Subscribe to trap events and submit the queue */ + uint64_t trapMask = KFD_EC_MASK(EC_QUEUE_WAVE_TRAP); + debug->SetExceptionsEnabled(trapMask); + dispatch->Submit(queue); + + /* Wait for trap event */ + uint32_t QueueId = -1; + ASSERT_SUCCESS(debug->QueryDebugEvent(&trapMask, NULL, &QueueId, 5000)); + ASSERT_NE(QueueId, -1); + ASSERT_EQ(trapMask, KFD_EC_MASK(EC_QUEUE_WAVE_TRAP) | KFD_EC_MASK(EC_QUEUE_NEW)); + + dispatch->Sync(); + } + + EXPECT_SUCCESS(queue.Destroy()); + + debug->Detach(); + hsaKmtRuntimeDisable(); + } else { + LOG() << "Skipping test: Test not supported on family ID 0x" + << m_FamilyId << "." << std::endl; + } +exit: + LOG() << std::endl; + TEST_END +} + TEST_F(KFDDBGTest, SuspendQueues) { TEST_START(TESTPROFILE_RUNALL) if (m_FamilyId >= FAMILY_AI) { From 65911e8368593c702179cbaf8c0d5b59548e78d5 Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Mon, 21 Aug 2023 11:23:08 -0700 Subject: [PATCH 1193/1247] Use memset for initializing variable sized array In ASAN builds, the compiler used is clang. The initialization of variable sized array using assignment operator is causing compilation failure in ASAN builds. Used memset to fix the same. Change-Id: I02aef3b99a6cad0cce3a378210a48732e07a88fb --- tests/kfdtest/src/KFDDBGTest.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 7a403d3ed4..ced0f57b41 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -441,7 +441,8 @@ TEST_F(KFDDBGTest, SuspendQueues) { uint32_t NumQueues = 1; uint32_t QueueIds[NumQueues]; - struct kfd_queue_snapshot_entry Snapshots[NumQueues] = {0}; + struct kfd_queue_snapshot_entry Snapshots[NumQueues]; + memset(Snapshots, 0, NumQueues * sizeof(struct kfd_queue_snapshot_entry)); ASSERT_SUCCESS(debug->SuspendQueues(&NumQueues, Queues, &QueueIds[0], 0)); // Suspend should fail as new queues cannot be suspended From 65ca3317f20ceeae1c73a16dee01da7ebcda5252 Mon Sep 17 00:00:00 2001 From: Lang Yu Date: Thu, 24 Aug 2023 17:27:04 +0800 Subject: [PATCH 1194/1247] kfdtest: add blacklist for gfx1150 and gfx1151 Change-Id: If78840e57c2523696c620d28f4c4ffb004128c0c Signed-off-by: Lang Yu --- tests/kfdtest/scripts/kfdtest.exclude | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 60ed132eb1..304a349cfa 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -302,6 +302,14 @@ FILTER[gfx1103]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX11" +FILTER[gfx1150]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX11" + +FILTER[gfx1151]=\ +"$BLACKLIST_ALL_ASICS:"\ +"$BLACKLIST_GFX11" + FILTER[gfx1036]=\ "$BLACKLIST_ALL_ASICS:"\ "$BLACKLIST_GFX10_NV2X" From f9e20c8a9325c452dc7635d86a7e3bfa48c4eba6 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 29 Aug 2023 11:25:56 -0400 Subject: [PATCH 1195/1247] kfdtest: replace 0 initialized dbg structs with memset Use memset to avoid general 0 set padding issues and ASAN compile issues for debug tests. Signed-off-by: Jonathan Kim Change-Id: I0a5aca5b7b631083599573b47f1ae87d5d0d5d71 --- tests/kfdtest/src/KFDDBGTest.cpp | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index ced0f57b41..16b0775aed 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -108,7 +108,8 @@ TEST_F(KFDDBGTest, AttachToSpawnedProcess) { exit(0); } else { BaseDebug *debug = new BaseDebug(); - struct kfd_runtime_info r_info = {0}; + struct kfd_runtime_info r_info; + memset(&r_info, 0, sizeof(struct kfd_runtime_info)); uint64_t runtimeMask = KFD_EC_MASK(EC_PROCESS_RUNTIME); int childStatus; @@ -199,7 +200,8 @@ TEST_F(KFDDBGTest, AttachToRunningProcess) { exit(0); } else { BaseDebug *debug = new BaseDebug(); - struct kfd_runtime_info r_info = {0}; + struct kfd_runtime_info r_info; + memset(&r_info, 0, sizeof(struct kfd_runtime_info)); uint64_t runtimeMask = KFD_EC_MASK(EC_PROCESS_RUNTIME); int childStatus; @@ -271,7 +273,8 @@ TEST_F(KFDDBGTest, HitTrapEvent) { ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); BaseDebug *debug = new BaseDebug(); - struct kfd_runtime_info r_info = {0}; + struct kfd_runtime_info r_info; + memset(&r_info, 0, sizeof(struct kfd_runtime_info)); ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); @@ -344,7 +347,8 @@ TEST_F(KFDDBGTest, HitTrapOnWaveStartEndEvent) { ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); BaseDebug *debug = new BaseDebug(); - struct kfd_runtime_info r_info = {0}; + struct kfd_runtime_info r_info; + memset(&r_info, 0, sizeof(struct kfd_runtime_info)); ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); @@ -424,7 +428,8 @@ TEST_F(KFDDBGTest, SuspendQueues) { ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); BaseDebug *debug = new BaseDebug(); - struct kfd_runtime_info r_info = {0}; + struct kfd_runtime_info r_info; + memset(&r_info, 0, sizeof(struct kfd_runtime_info)); ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); @@ -547,7 +552,8 @@ TEST_F(KFDDBGTest, HitMemoryViolation) { exit(0); } else { BaseDebug *debug = new BaseDebug(); - struct kfd_runtime_info r_info = {0}; + struct kfd_runtime_info r_info; + memset(&r_info, 0, sizeof(struct kfd_runtime_info)); uint64_t runtimeMask = KFD_EC_MASK(EC_PROCESS_RUNTIME); uint64_t memViolMask = KFD_EC_MASK(EC_DEVICE_MEMORY_VIOLATION); uint64_t subscribeMask = runtimeMask | memViolMask; @@ -579,7 +585,8 @@ TEST_F(KFDDBGTest, HitMemoryViolation) { const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); uint32_t snapshotSize = gpuNodes.size(); - struct kfd_dbg_device_info_entry deviceInfo[snapshotSize] = {0}; + struct kfd_dbg_device_info_entry deviceInfo[snapshotSize]; + memset(deviceInfo, 0, snapshotSize * sizeof(struct kfd_dbg_device_info_entry)); // Check device snapshot aligns with memory violation on target device. ASSERT_SUCCESS(debug->DeviceSnapshot(memViolMask, (uint64_t)(&deviceInfo[0]), @@ -640,14 +647,16 @@ TEST_F(KFDDBGTest, HitAddressWatch) { uint32_t rDebug; ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); - struct kfd_runtime_info r_info = {0}; + struct kfd_runtime_info r_info; + memset(&r_info, 0, sizeof(struct kfd_runtime_info)); BaseDebug *debug = new BaseDebug(); ASSERT_SUCCESS(debug->Attach(&r_info, sizeof(r_info), getpid(), 0)); ASSERT_EQ(r_info.runtime_state, DEBUG_RUNTIME_STATE_ENABLED); const std::vector gpuNodes = m_NodeInfo.GetNodesWithGPU(); uint32_t numDevices = gpuNodes.size(); - struct kfd_dbg_device_info_entry deviceInfo[numDevices] = {0}; + struct kfd_dbg_device_info_entry deviceInfo[numDevices]; + memset(deviceInfo, 0, numDevices * sizeof(struct kfd_dbg_device_info_entry)); ASSERT_SUCCESS(debug->DeviceSnapshot(0, (uint64_t)(&deviceInfo[0]), &numDevices)); ASSERT_EQ(numDevices, gpuNodes.size()); bool is_precise = nodeProps.Capability.ui32.PreciseMemoryOperationsSupported; From 6ec529fe68b1bea1276954e50ea587d653772b4c Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Fri, 8 Sep 2023 12:12:13 -0400 Subject: [PATCH 1196/1247] kfdtest: temporarily exclude debug suspend queues test For some reason, non-Ubuntu builds have some sort of memory corruption when running this test, which affect subsequent running tests. Disable it for now. Signed-off-by: Jonathan Kim Change-Id: I5f54ee4c63286a33c6948bc818aa1501c4a6751e --- tests/kfdtest/scripts/kfdtest.exclude | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 304a349cfa..e817b90274 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -69,6 +69,7 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # KFDEvictTest.BurstyTest (ROCMOPS-464) # KFDEvictTest.BurstyTest (SWDEV-291256) # KFDEvictTest.BurstyTest (KFD-425) +# KFDDBGTest.SuspendQueues (SWDEV-417850) TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ @@ -76,7 +77,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDMemoryTest.CacheInvalidateOnRemoteWrite:"\ "KFDEvictTest.BurstyTest:"\ "KFDHWSTest.*:"\ -"KFDSVMRangeTest.ReadOnlyRangeTest*" +"KFDSVMRangeTest.ReadOnlyRangeTest*:"\ +"KFDDBGTest.SuspendQueues" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From 5f117f760809ae6522e1535691a5edee0d814075 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Thu, 7 Sep 2023 16:44:35 -0400 Subject: [PATCH 1197/1247] kfdtest: Fix String NULL Check MCPU const char * always returns true, so check the value instead. Before: if (!MCPU) { After: if (!*MCPU) { Signed-off-by: Ori Messinger Change-Id: I414e091ca764095937311648c534351d6abf30e6 --- tests/kfdtest/src/Assemble.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/Assemble.cpp b/tests/kfdtest/src/Assemble.cpp index 2077375ea2..4c8ee0786d 100644 --- a/tests/kfdtest/src/Assemble.cpp +++ b/tests/kfdtest/src/Assemble.cpp @@ -249,7 +249,7 @@ int Assembler::RunAssembleBuf(const char* const AssemblySource, char* OutBuf, */ int Assembler::RunAssemble(const char* const AssemblySource) { // Ensure target ASIC has been set - if (!MCPU) { + if (!*MCPU) { outs() << "ASM Error: target asic is uninitialized\n"; return -1; } From fcec22716a496635466ba79b8d0aec8aadd46e80 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Tue, 12 Sep 2023 15:38:12 -0400 Subject: [PATCH 1198/1247] Use camel case for KFDDBGTest shaders Debug test shaders should use camel case and suffix *Isa to match other test shader naming convention. Signed-off-by: Jonathan Kim Change-Id: I64e14183ba1c7c9664b13a742a0e5683866e8223 --- tests/kfdtest/src/KFDDBGTest.cpp | 14 +++++++------- tests/kfdtest/src/ShaderStore.cpp | 8 ++++---- tests/kfdtest/src/ShaderStore.hpp | 10 +++++----- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index 16b0775aed..e43a253daf 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -266,8 +266,8 @@ TEST_F(KFDDBGTest, HitTrapEvent) { 0x1000)); // compile and dispatch shader - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(jump_to_trap_gfx, isaBuf.As())); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(trap_handler_gfx, trap.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(JumpToTrapIsa, isaBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(TrapHandlerIsa, trap.As())); uint32_t rDebug; ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); @@ -341,7 +341,7 @@ TEST_F(KFDDBGTest, HitTrapOnWaveStartEndEvent) { // compile and dispatch shader ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(NoopIsa, isaBuf.As())); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(trap_handler_gfx, trap.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(TrapHandlerIsa, trap.As())); uint32_t rDebug; ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); @@ -422,7 +422,7 @@ TEST_F(KFDDBGTest, SuspendQueues) { HsaMemoryBuffer isaBuf(PAGE_SIZE, defaultGPUNode, true, false, true); // compile and dispatch shader - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(jump_to_trap_gfx, isaBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(JumpToTrapIsa, isaBuf.As())); uint32_t rDebug; ASSERT_SUCCESS(hsaKmtRuntimeEnable(&rDebug, true)); @@ -635,9 +635,9 @@ TEST_F(KFDDBGTest, HitAddressWatch) { HsaMemoryBuffer trap(PAGE_SIZE*2, defaultGPUNode, true, false, true); HsaMemoryBuffer tmaBuf(PAGE_SIZE, defaultGPUNode, false, false, false); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(watch_read_isa, readerBuf.As())); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(watch_write_isa, writerBuf.As())); - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(trap_handler_gfx, trap.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WatchReadIsa, readerBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(WatchWriteIsa, writerBuf.As())); + ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(TrapHandlerIsa, trap.As())); ASSERT_SUCCESS(hsaKmtSetTrapHandler(defaultGPUNode, trap.As(), 0x1000, diff --git a/tests/kfdtest/src/ShaderStore.cpp b/tests/kfdtest/src/ShaderStore.cpp index 08d82612ee..298abc118b 100644 --- a/tests/kfdtest/src/ShaderStore.cpp +++ b/tests/kfdtest/src/ShaderStore.cpp @@ -901,7 +901,7 @@ const char *GwsAtomicIncreaseIsa = s_endpgm )"; -const char *jump_to_trap_gfx = +const char *JumpToTrapIsa = SHADER_START SHADER_MACROS_U32 R"( @@ -918,7 +918,7 @@ const char *jump_to_trap_gfx = s_endpgm )"; -const char *trap_handler_gfx = +const char *TrapHandlerIsa = SHADER_START R"( CHECK_VMFAULT: @@ -998,12 +998,12 @@ const char *trap_handler_gfx = "s_waitcnt vmcnt(0) & lgkmcnt(0)\n"\ "s_endpgm\n" -const char *watch_read_isa = +const char *WatchReadIsa = WATCH_START "flat_load_dword v7, v[0:1]" WATCH_END; -const char *watch_write_isa = +const char *WatchWriteIsa = WATCH_START "flat_store_dword v[0:1], v4" WATCH_END; diff --git a/tests/kfdtest/src/ShaderStore.hpp b/tests/kfdtest/src/ShaderStore.hpp index cfcfb2761c..60c021d047 100644 --- a/tests/kfdtest/src/ShaderStore.hpp +++ b/tests/kfdtest/src/ShaderStore.hpp @@ -63,12 +63,12 @@ extern const char *GwsInitIsa; extern const char *GwsAtomicIncreaseIsa; /* HitTrapEvent */ -extern const char *jump_to_trap_gfx; -extern const char *trap_handler_gfx; +extern const char *JumpToTrapIsa; +extern const char *TrapHandlerIsa; /* HitWatchPointEvent */ -extern const char *watch_read_isa; -extern const char *watch_write_isa; -extern const char *watch_atomic_isa; +extern const char *WatchReadIsa; +extern const char *WatchWriteIsa; +extern const char *WatchAtomicIsa; #endif // _SHADERSTORE_H_ From 986e82d677134e67f5b295fe7bc5ab21d63c01b8 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Thu, 14 Sep 2023 09:07:20 -0400 Subject: [PATCH 1199/1247] kfdtest: temporarily exclude address watch testing The debug address watch test will hang when running with the entire KFD test. Disable it for now. Signed-off-by: Jonathan Kim Change-Id: I1d0479fa2717d2f398cc32e0605ca6dcc17ebcd5 --- tests/kfdtest/scripts/kfdtest.exclude | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index e817b90274..2e3249a953 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -70,6 +70,7 @@ PERMANENT_BLACKLIST_ALL_ASICS=\ # KFDEvictTest.BurstyTest (SWDEV-291256) # KFDEvictTest.BurstyTest (KFD-425) # KFDDBGTest.SuspendQueues (SWDEV-417850) +# KFDDBGTest.HitAddressWatch (SWDEV-420281) TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDQMTest.GPUDoorbellWrite:"\ "KFDQMTest.mGPUShareBO:"\ @@ -78,7 +79,8 @@ TEMPORARY_BLACKLIST_ALL_ASICS=\ "KFDEvictTest.BurstyTest:"\ "KFDHWSTest.*:"\ "KFDSVMRangeTest.ReadOnlyRangeTest*:"\ -"KFDDBGTest.SuspendQueues" +"KFDDBGTest.SuspendQueues:"\ +"KFDDBGTest.HitAddressWatch" BLACKLIST_ALL_ASICS=\ "$PERMANENT_BLACKLIST_ALL_ASICS:"\ From 8e06dce573fed5824d5bf036aa0ce9a2980e746b Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Fri, 21 Jul 2023 16:35:41 -0400 Subject: [PATCH 1200/1247] Add extended coherence memory flag Add support for new flag for memory allocation that will provide system-scope coherent atomics Change-Id: I426d66223e8d2b570f69b4c0e61145ce9b2290d2 --- include/hsakmttypes.h | 4 +++- include/linux/kfd_ioctl.h | 3 +++ src/events.c | 2 ++ src/fmm.c | 44 ++++++++++++++++++++++++++++++--------- src/fmm.h | 3 ++- src/memory.c | 14 ++++++++++--- 6 files changed, 55 insertions(+), 15 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index ef0a4a23a4..2937c35915 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -566,7 +566,8 @@ typedef struct _HsaMemFlags unsigned int Uncached: 1; // Caching flag for fine-grained memory on A+A HW platform unsigned int NoAddress: 1; // only do vram allocation, return a handle, not allocate virtual address. unsigned int OnlyAddress: 1; // only do virtal address allocation without vram allocation. - unsigned int Reserved: 12; + unsigned int ExtendedCoherent: 1; // system-scope coherence on atomic instructions + unsigned int Reserved: 11; } ui32; HSAuint32 Value; @@ -1372,6 +1373,7 @@ typedef enum _HSA_SVM_FLAGS { HSA_SVM_FLAG_GPU_EXEC = 0x00000010, // Allow execution on GPU HSA_SVM_FLAG_GPU_READ_MOSTLY = 0x00000020, // GPUs mostly read, may allow similar optimizations as RO, but writes fault HSA_SVM_FLAG_GPU_ALWAYS_MAPPED = 0x00000040, // Keep GPU memory mapping always valid as if XNACK is disable + HSA_SVM_FLAG_EXT_COHERENT = 0x00000080, // Fine grained coherency between all devices using device-scope atomics } HSA_SVM_FLAGS; typedef enum _HSA_SVM_ATTR_TYPE { diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index c56347a7d6..d3e6cee06d 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -1027,6 +1027,7 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM (1 << 27) #define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26) #define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25) +#define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24) /* Allocate memory for later SVM (shared virtual memory) mapping. * @@ -1375,6 +1376,8 @@ struct kfd_ioctl_cross_memory_copy_args { #define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020 /* Keep GPU memory mapping always valid as if XNACK is disable */ #define KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED 0x00000040 +/* Fine grained coherency between all devices using device-scope atomics */ +#define KFD_IOCTL_SVM_FLAG_EXT_COHERENT 0x00000080 /** * kfd_ioctl_svm_op - SVM ioctl operations diff --git a/src/events.c b/src/events.c index 6d3cd70805..9ec199ac7a 100644 --- a/src/events.c +++ b/src/events.c @@ -291,6 +291,8 @@ static HSAKMT_STATUS get_mem_info_svm_api(uint64_t address, uint32_t gpu_id) pr_err("GPU exec allowed\n"); if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) pr_err("GPU always mapped\n"); + if (args->attrs[i].value & KFD_IOCTL_SVM_FLAG_EXT_COHERENT) + pr_err("Extended-scope fine grained coherency between devices\n"); break; default: pr_debug("get invalid attr type 0x%x\n", args->attrs[i].type); diff --git a/src/fmm.c b/src/fmm.c index d60a9ad647..76ddc40f4f 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1017,6 +1017,8 @@ static HsaMemFlags fmm_translate_ioc_to_hsa_flags(uint32_t ioc_flags) mflags.ui32.ReadOnly = 1; if (!(ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT)) mflags.ui32.CoarseGrain = 1; + if (ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT) + mflags.ui32.ExtendedCoherent = 1; if (ioc_flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) mflags.ui32.HostAccess = 1; return mflags; @@ -1024,7 +1026,8 @@ static HsaMemFlags fmm_translate_ioc_to_hsa_flags(uint32_t ioc_flags) static HSAKMT_STATUS fmm_register_mem_svm_api(void *address, uint64_t size, - bool coarse_grain) + bool coarse_grain, + bool ext_coherent) { struct kfd_ioctl_svm_args *args; size_t s_attr; @@ -1035,15 +1038,17 @@ static HSAKMT_STATUS fmm_register_mem_svm_api(void *address, if (!g_first_gpu_mem) return HSAKMT_STATUS_ERROR; - s_attr = sizeof(struct kfd_ioctl_svm_attribute); + s_attr = 2 * sizeof(struct kfd_ioctl_svm_attribute); args = alloca(sizeof(*args) + s_attr); args->start_addr = aligned_addr; args->size = aligned_size; args->op = KFD_IOCTL_SVM_OP_SET_ATTR; - args->nattr = 1; + args->nattr = 2; args->attrs[0].type = coarse_grain ? HSA_SVM_ATTR_CLR_FLAGS : HSA_SVM_ATTR_SET_FLAGS; args->attrs[0].value = HSA_SVM_FLAG_COHERENT; + args->attrs[1].type = ext_coherent ? HSA_SVM_ATTR_CLR_FLAGS : HSA_SVM_ATTR_SET_FLAGS; + args->attrs[1].value = HSA_SVM_FLAG_EXT_COHERENT; pr_debug("Registering to SVM %p size: %ld\n", (void*)aligned_addr, aligned_size); /* Driver does one copy_from_user, with extra attrs size */ @@ -1575,6 +1580,9 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, if (mflags.ui32.Uncached || svm.disable_cache) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; + if (mflags.ui32.ExtendedCoherent) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT; + mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); @@ -1639,7 +1647,7 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, mflags.Value = 0; mflags.ui32.NonPaged = 1; mflags.ui32.HostAccess = 1; - mflags.ui32.Reserved = 0xBe1; + mflags.ui32.Reserved = 0x3e1; pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->mflags = mflags; @@ -3409,8 +3417,11 @@ bool fmm_get_handle(void *address, uint64_t *handle) return found; } -static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, - vm_object_t **obj_ret, bool coarse_grain) +static HSAKMT_STATUS fmm_register_user_memory(void *addr, + HSAuint64 size, + vm_object_t **obj_ret, + bool coarse_grain, + bool ext_coherent) { manageable_aperture_t *aperture = svm.dgpu_aperture; HSAuint32 page_offset = (HSAuint64)addr & (PAGE_SIZE-1); @@ -3435,7 +3446,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, &aligned_addr, KFD_IOC_ALLOC_MEM_FLAGS_USERPTR | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | - (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT), + (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT) | + (ext_coherent ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT), &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; @@ -3472,7 +3484,8 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, HSAuint64 size, HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size, - bool coarse_grain) + bool coarse_grain, + bool ext_coherent) { manageable_aperture_t *aperture = NULL; vm_object_t *object = NULL; @@ -3481,6 +3494,9 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, if (gpu_id_array_size > 0 && !gpu_id_array) return HSAKMT_STATUS_INVALID_PARAMETER; + if (coarse_grain && ext_coherent) + return HSAKMT_STATUS_INVALID_PARAMETER; + object = vm_find_object(address, size_in_bytes, &aperture); if (!object) { if (!is_dgpu) @@ -3489,9 +3505,17 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, /* Register a new user ptr */ if (svm.is_svm_api_supported) - return fmm_register_mem_svm_api(address, size_in_bytes, coarse_grain); + return fmm_register_mem_svm_api(address, + size_in_bytes, + coarse_grain, + ext_coherent); + + ret = fmm_register_user_memory(address, + size_in_bytes, + &object, + coarse_grain, + ext_coherent); - ret = fmm_register_user_memory(address, size_in_bytes, &object, coarse_grain); if (ret != HSAKMT_STATUS_SUCCESS) return ret; if (gpu_id_array_size == 0) diff --git a/src/fmm.h b/src/fmm.h index 6cd7898cac..b8c9b84bfd 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -77,7 +77,8 @@ HSAKMT_STATUS fmm_get_aperture_base_and_limit(aperture_type_e aperture_type, HSA HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, uint32_t *gpu_id_array, uint32_t gpu_id_array_size, - bool coarse_grain); + bool coarse_grain, + bool ext_coherent); HSAKMT_STATUS fmm_register_graphics_handle(HSAuint64 GraphicsResourceHandle, HsaGraphicsResourceInfo *GraphicsResourceInfo, uint32_t *gpu_id_array, diff --git a/src/memory.c b/src/memory.c index 6b3eaf496b..ba33abda3e 100644 --- a/src/memory.c +++ b/src/memory.c @@ -136,6 +136,11 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, } else *MemoryAddress = NULL; + if ((MemFlags.ui32.CoarseGrain && MemFlags.ui32.ExtendedCoherent) || + (MemFlags.ui32.CoarseGrain && MemFlags.ui32.Uncached) || + (MemFlags.ui32.ExtendedCoherent && MemFlags.ui32.Uncached)) + return HSAKMT_STATUS_INVALID_PARAMETER; + if (MemFlags.ui32.Scratch) { *MemoryAddress = fmm_allocate_scratch(gpu_id, *MemoryAddress, SizeInBytes); @@ -242,7 +247,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress, return HSAKMT_STATUS_SUCCESS; return fmm_register_memory(MemoryAddress, MemorySizeInBytes, - NULL, 0, true); + NULL, 0, true, false); } HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, @@ -268,7 +273,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryToNodes(void *MemoryAddress, ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, gpu_id_array, NumberOfNodes*sizeof(uint32_t), - true); + true, false); if (ret != HSAKMT_STATUS_SUCCESS) free(gpu_id_array); } @@ -286,6 +291,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryWithFlags(void *MemoryAddress, pr_debug("[%s] address %p\n", __func__, MemoryAddress); + if (MemFlags.ui32.ExtendedCoherent && MemFlags.ui32.CoarseGrain) + return HSAKMT_STATUS_INVALID_PARAMETER; + // Registered memory should be ordinary paged host memory. if ((MemFlags.ui32.HostAccess != 1) || (MemFlags.ui32.NonPaged == 1)) return HSAKMT_STATUS_NOT_SUPPORTED; @@ -295,7 +303,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemoryWithFlags(void *MemoryAddress, return HSAKMT_STATUS_NOT_SUPPORTED; ret = fmm_register_memory(MemoryAddress, MemorySizeInBytes, - NULL, 0, MemFlags.ui32.CoarseGrain); + NULL, 0, MemFlags.ui32.CoarseGrain, MemFlags.ui32.ExtendedCoherent); return ret; } From 274b5b51ca9905799fca9b4282c854dce3f3d082 Mon Sep 17 00:00:00 2001 From: James Zhu Date: Tue, 12 Sep 2023 15:27:07 -0400 Subject: [PATCH 1201/1247] libhsakmt: remove IOMMUv2 performance monitor support IOMMUv2 is removed from AMDGPU/KFD. Change-Id: Ib87f501c07d9de90e6b83b98f98daacd5913e98a Signed-off-by: James Zhu --- src/perfctr.c | 182 +----------------------------------------------- src/pmc_table.c | 15 ---- src/pmc_table.h | 2 - 3 files changed, 2 insertions(+), 197 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 546dd2358b..8c78da3274 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -175,8 +175,6 @@ static void init_perf_shared_table(void) /* write the perf content */ shared_table->magic4cc = HSA_PERF_MAGIC4CC; - shared_table->iommu_slots_left = - pmc_table_get_max_concurrent(PERFCOUNTER_BLOCKID__IOMMUV2); sem_post(sem); } @@ -294,9 +292,6 @@ static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) case PERFCOUNTER_BLOCKID__WD: *uuid = HSA_PROFILEBLOCK_AMD_WD; break; - case PERFCOUNTER_BLOCKID__IOMMUV2: - *uuid = HSA_PROFILEBLOCK_AMD_IOMMUV2; - break; default: /* If we reach this point, it's a bug */ rc = -1; @@ -321,144 +316,6 @@ static HSAuint32 get_block_concurrent_limit(uint32_t node_id, return 0; } -static HSAKMT_STATUS update_block_slots(enum perf_trace_action action, - uint32_t block_id, uint32_t num_slots) -{ - uint32_t *slots_left; - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - - if (shmem_fd <= 0) - return HSAKMT_STATUS_UNAVAILABLE; - if (sem == SEM_FAILED) - return HSAKMT_STATUS_UNAVAILABLE; - - sem_wait(sem); - - if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) - slots_left = &shared_table->iommu_slots_left; - else { - ret = HSAKMT_STATUS_UNAVAILABLE; - goto out; - } - - switch (action) { - case PERF_TRACE_ACTION__ACQUIRE: - if (*slots_left >= num_slots) - *slots_left -= num_slots; - else - ret = HSAKMT_STATUS_UNAVAILABLE; - break; - case PERF_TRACE_ACTION__RELEASE: - if ((*slots_left + num_slots) <= - pmc_table_get_max_concurrent(block_id)) - *slots_left += num_slots; - else - ret = HSAKMT_STATUS_ERROR; - break; - default: - ret = HSAKMT_STATUS_INVALID_PARAMETER; - break; - } - -out: - sem_post(sem); - - return ret; -} - -static unsigned int get_perf_event_type(enum perf_block_id block_id) -{ - FILE *file = NULL; - unsigned int type = 0; - - if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) { - /* Starting from kernel 4.12, amd_iommu_0 is used */ - file = fopen("/sys/bus/event_source/devices/amd_iommu_0/type", - "r"); - if (!file) - file = fopen(/* kernel 4.11 and older */ - "/sys/bus/event_source/devices/amd_iommu/type", - "r"); - } - - if (!file) - return 0; - - if (fscanf(file, "%d", &type) != 1) - type = 0; - fclose(file); - - return type; -} - -/* close_perf_event_fd - Close all FDs opened for this block. - * When RT acquires the trace access, RT has no ideas about each - * individual FD opened for this block. We should treat the whole - * block as one and close all of them. - */ -static void close_perf_event_fd(struct perf_trace_block *block) -{ - uint32_t i; - - if (!block || !block->perf_event_fd) - return; - - for (i = 0; i < block->num_counters; i++) - if (block->perf_event_fd[i] > 0) { - close(block->perf_event_fd[i]); - block->perf_event_fd[i] = 0; - } -} - -/* open_perf_event_fd - Open FDs required for this block. - * If one of them fails, we should close all FDs that have been - * opened because RT has no ideas about those FDs successfully - * opened and it won't send anything to close them. - */ -static HSAKMT_STATUS open_perf_event_fd(struct perf_trace_block *block) -{ - struct perf_event_attr attr; - uint32_t i; - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - - if (!block || !block->perf_event_fd) - return HSAKMT_STATUS_INVALID_HANDLE; - - if (getuid()) { - pr_err("Must be root to open perf_event.\n"); - return HSAKMT_STATUS_ERROR; - } - - memset(&attr, 0, sizeof(struct perf_event_attr)); - attr.type = get_perf_event_type(block->block_id); - if (!attr.type) - return HSAKMT_STATUS_ERROR; - - for (i = 0; i < block->num_counters; i++) { - attr.size = sizeof(struct perf_event_attr); - attr.config = block->counter_id[i]; - attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | - PERF_FORMAT_TOTAL_TIME_RUNNING; - attr.disabled = 1; - attr.inherit = 1; - - /* We are profiling system wide, not per cpu, so no threads, - * no groups -> pid=-1 and group_fd=-1. cpu = 0 - * flags=PERF_FLAG_FD_NO_GROUP - */ - block->perf_event_fd[i] = syscall(__NR_perf_event_open, &attr, - -1, 0, -1, PERF_FLAG_FD_NO_GROUP); - - if (block->perf_event_fd[i] < 0) { - ret = HSAKMT_STATUS_ERROR; - close_perf_event_fd(block); - break; - } - } - - return ret; -} - static HSAKMT_STATUS perf_trace_ioctl(struct perf_trace_block *block, uint32_t cmd) { @@ -556,10 +413,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcGetCounterProperties(HSAuint32 NodeId, block_prop->Counters[i].CounterSizeInBits = block.counter_size_in_bits; block_prop->Counters[i].CounterMask = block.counter_mask; block_prop->Counters[i].Flags.ui32.Global = 1; - if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) - block_prop->Counters[i].Type = HSA_PROFILE_TYPE_PRIVILEGED_IMMEDIATE; - else - block_prop->Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; + block_prop->Counters[i].Type = HSA_PROFILE_TYPE_NONPRIV_IMMEDIATE; } block_prop = (HsaCounterBlockProperties *)&block_prop->Counters[block_prop->NumCounters]; @@ -750,8 +604,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess(HSAuint32 NodeId, { struct perf_trace *trace; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - uint32_t gpu_id, i; - int j; + uint32_t gpu_id; pr_debug("[%s] Trace ID 0x%lx\n", __func__, TraceId); @@ -766,29 +619,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcAcquireTraceAccess(HSAuint32 NodeId, if (validate_nodeid(NodeId, &gpu_id) != HSAKMT_STATUS_SUCCESS) return HSAKMT_STATUS_INVALID_NODE_UNIT; - for (i = 0; i < trace->num_blocks; i++) { - ret = update_block_slots(PERF_TRACE_ACTION__ACQUIRE, - trace->blocks[i].block_id, - trace->blocks[i].num_counters); - if (ret != HSAKMT_STATUS_SUCCESS) - goto out; - ret = open_perf_event_fd(&trace->blocks[i]); - if (ret != HSAKMT_STATUS_SUCCESS) { - i++; /* to release slots just reserved */ - goto out; - } - } - -out: - if (ret != HSAKMT_STATUS_SUCCESS) { - for (j = i-1; j >= 0; j--) { - update_block_slots(PERF_TRACE_ACTION__RELEASE, - trace->blocks[j].block_id, - trace->blocks[j].num_counters); - close_perf_event_fd(&trace->blocks[j]); - } - } - return ret; } @@ -796,7 +626,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess(HSAuint32 NodeId, HSATraceId TraceId) { struct perf_trace *trace; - uint32_t i; pr_debug("[%s] Trace ID 0x%lx\n", __func__, TraceId); @@ -808,13 +637,6 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPmcReleaseTraceAccess(HSAuint32 NodeId, if (trace->magic4cc != HSA_PERF_MAGIC4CC) return HSAKMT_STATUS_INVALID_HANDLE; - for (i = 0; i < trace->num_blocks; i++) { - update_block_slots(PERF_TRACE_ACTION__RELEASE, - trace->blocks[i].block_id, - trace->blocks[i].num_counters); - close_perf_event_fd(&trace->blocks[i]); - } - return HSAKMT_STATUS_SUCCESS; } diff --git a/src/pmc_table.c b/src/pmc_table.c index 7b1f9c5eba..d2dbab224b 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1963,16 +1963,6 @@ static struct perf_counter_block navi_blocks[PERFCOUNTER_BLOCKID__MAX] = { */ static struct perf_counter_block iommu_block; -uint32_t pmc_table_get_max_concurrent(int block_id) -{ - switch (block_id) { - case PERFCOUNTER_BLOCKID__IOMMUV2: - return iommu_block.num_of_slots; - default: - return 0; - } -} - static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) { DIR *dir; @@ -2095,11 +2085,6 @@ HSAKMT_STATUS get_block_properties(uint32_t node_id, block_id < PERFCOUNTER_BLOCKID__FIRST) return HSAKMT_STATUS_INVALID_PARAMETER; - if (block_id == PERFCOUNTER_BLOCKID__IOMMUV2) { - *block = iommu_block; - return HSAKMT_STATUS_SUCCESS; - } - /* Major GFX Version */ switch (gfxv >> 16) { case 7: diff --git a/src/pmc_table.h b/src/pmc_table.h index 7e48cbe801..b734183a54 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -56,7 +56,6 @@ enum perf_block_id { PERFCOUNTER_BLOCKID__VGT, PERFCOUNTER_BLOCKID__WD, /* privileged */ - PERFCOUNTER_BLOCKID__IOMMUV2, PERFCOUNTER_BLOCKID__MAX }; @@ -70,7 +69,6 @@ struct perf_counter_block { HSAKMT_STATUS alloc_pmc_blocks(void); void free_pmc_blocks(void); -uint32_t pmc_table_get_max_concurrent(int block_id); HSAKMT_STATUS get_block_properties(uint32_t node_id, enum perf_block_id block_id, From 277d5e27ffe0884e8fd2f522cea963eb0e7fc41c Mon Sep 17 00:00:00 2001 From: James Zhu Date: Tue, 12 Sep 2023 16:11:31 -0400 Subject: [PATCH 1202/1247] libhsakmt: remove iommu_block which supports IOMMUv2 performance IOMMUv2 is removed from AMDGPU/KFD. Change-Id: I9fcf20ae9288cb40bb4b696284fc70534fb6484b Signed-off-by: James Zhu --- src/perfctr.c | 2 - src/pmc_table.c | 116 ------------------------------------------------ src/pmc_table.h | 3 -- 3 files changed, 121 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 8c78da3274..4d797e46f6 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -188,7 +188,6 @@ HSAKMT_STATUS init_counter_props(unsigned int NumNodes) } counter_props_count = NumNodes; - alloc_pmc_blocks(); if (init_shared_region() != HSAKMT_STATUS_SUCCESS) { pr_warn("Profiling of privileged blocks is not available.\n"); @@ -215,7 +214,6 @@ void destroy_counter_props(void) } free(counter_props); - free_pmc_blocks(); } static int blockid2uuid(enum perf_block_id block_id, HSA_UUID *uuid) diff --git a/src/pmc_table.c b/src/pmc_table.c index d2dbab224b..b40979805d 100644 --- a/src/pmc_table.c +++ b/src/pmc_table.c @@ -1958,122 +1958,6 @@ static struct perf_counter_block navi_blocks[PERFCOUNTER_BLOCKID__MAX] = { }, }; -/* Current APUs only have one IOMMU. If NUMA is introduced to APUs, we'll need - * to expand the struct here to an array. - */ -static struct perf_counter_block iommu_block; - -static HSAKMT_STATUS alloc_pmc_blocks_iommu(void) -{ - DIR *dir; - struct dirent *dent; - const char *sysfs_amdiommu_event_path = - "/sys/bus/event_source/devices/amd_iommu_0/events"; - /* Counter source in IOMMU's Counter Bank Addressing register is 8 bits, - * so the biggest counter number/id possible is 0xff. - */ - const int max_counter_id = 0xff; - char path[256]; - const int len = sizeof(path); - FILE *file; - int num; - char counter_id[max_counter_id + 1]; - HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; - uint32_t *ptr; - struct perf_counter_block *block = &iommu_block; - - memset(block, 0, sizeof(struct perf_counter_block)); - - dir = opendir(sysfs_amdiommu_event_path); - if (!dir) { - /* Before kernel 4.12, amd_iommu is the path */ - sysfs_amdiommu_event_path = - "/sys/bus/event_source/devices/amd_iommu/events"; - dir = opendir(sysfs_amdiommu_event_path); - if (!dir) - goto out; - } - - memset(counter_id, 0, max_counter_id + 1); - while ((dent = readdir(dir))) { - if (!strcmp(dent->d_name, ".") || !strcmp(dent->d_name, "..")) - continue; - if (snprintf(path, len, "%s/%s", sysfs_amdiommu_event_path, - dent->d_name) >= len) { - pr_err("Increase path length.\n"); - ret = HSAKMT_STATUS_NO_MEMORY; - goto out; - } - file = fopen(path, "r"); - if (!file) { - ret = HSAKMT_STATUS_ERROR; - goto out; - } - if (fscanf(file, "csource=0x%x", &num) != 1) { - ret = HSAKMT_STATUS_ERROR; - fclose(file); - goto out; - } - if (num > max_counter_id) - /* This should never happen. If it does, check IOMMU driver. */ - pr_err("max_counter_id %d is set too small.\n", - max_counter_id); - else { - counter_id[num] = 1; - ++block->num_of_counters; - } - fclose(file); - } - - block->counter_ids = malloc(sizeof(uint32_t) * block->num_of_counters); - if (!block->counter_ids) { - ret = HSAKMT_STATUS_NO_MEMORY; - goto out; - } - ptr = block->counter_ids; - for (num = 0; num < (max_counter_id + 1); num++) { - if (counter_id[num]) { - ptr[0] = num; - ++ptr; - } - } - - if (snprintf(path, len, "%s/%d/%s", - "/sys/devices/virtual/kfd/kfd/topology/nodes", - 0, /* IOMMU is in node 0. Change this if NUMA is introduced to APU. */ - "perf/iommu/max_concurrent") >= len) { - pr_err("Increase path length\n"); - ret = HSAKMT_STATUS_NO_MEMORY; - goto out; - }; - file = fopen(path, "r"); - if (!file) { - ret = HSAKMT_STATUS_ERROR; - goto out; - } - if (fscanf(file, "%d", &block->num_of_slots) != 1) - ret = HSAKMT_STATUS_ERROR; - fclose(file); - -out: - if (dir) - closedir(dir); - return ret; -} - -HSAKMT_STATUS alloc_pmc_blocks(void) -{ - return alloc_pmc_blocks_iommu(); -} - -void free_pmc_blocks(void) -{ - if (iommu_block.counter_ids) - free(iommu_block.counter_ids); - iommu_block.counter_ids = NULL; - iommu_block.num_of_counters = 0; -} - HSAKMT_STATUS get_block_properties(uint32_t node_id, enum perf_block_id block_id, struct perf_counter_block *block) diff --git a/src/pmc_table.h b/src/pmc_table.h index b734183a54..9903e64792 100644 --- a/src/pmc_table.h +++ b/src/pmc_table.h @@ -67,9 +67,6 @@ struct perf_counter_block { uint64_t counter_mask; }; -HSAKMT_STATUS alloc_pmc_blocks(void); -void free_pmc_blocks(void); - HSAKMT_STATUS get_block_properties(uint32_t node_id, enum perf_block_id block_id, struct perf_counter_block *block); From d195deeec4c5011a25cabbe31b9eec85b1c9af6b Mon Sep 17 00:00:00 2001 From: James Zhu Date: Tue, 12 Sep 2023 15:07:20 -0400 Subject: [PATCH 1203/1247] libhsakmt: remove share resource in performance counter This share resource is for IOMMUv2 which is removed from AMDGPU/KFD. Change-Id: Ia6e9311f1adc56fac2c9e8fa05b24c5ec8c272a5 Signed-off-by: James Zhu --- src/perfctr.c | 88 --------------------------------------------------- 1 file changed, 88 deletions(-) diff --git a/src/perfctr.c b/src/perfctr.c index 4d797e46f6..ed051bece2 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -64,16 +64,6 @@ struct perf_trace { struct perf_trace_block blocks[0]; }; -enum perf_trace_action { - PERF_TRACE_ACTION__ACQUIRE = 0, - PERF_TRACE_ACTION__RELEASE -}; - -struct perf_shared_table { - uint32_t magic4cc; - uint32_t iommu_slots_left; -}; - struct perf_counts_values { union { struct { @@ -87,11 +77,6 @@ struct perf_counts_values { static HsaCounterProperties **counter_props; static unsigned int counter_props_count; -static const char shmem_name[] = "/hsakmt_shared_mem"; -static int shmem_fd; -static const char sem_name[] = "hsakmt_semaphore"; -static sem_t *sem = SEM_FAILED; -struct perf_shared_table *shared_table; static ssize_t readn(int fd, void *buf, size_t n) { @@ -114,71 +99,6 @@ static ssize_t readn(int fd, void *buf, size_t n) return n; } -static HSAKMT_STATUS init_shared_region(void) -{ - sem = sem_open(sem_name, O_CREAT, 0666, 1); - if (sem == SEM_FAILED) - return HSAKMT_STATUS_ERROR; - - shmem_fd = shm_open(shmem_name, O_CREAT | O_RDWR, 0666); - if (shmem_fd < 0) - goto exit_1; - - if (ftruncate(shmem_fd, sizeof(struct perf_shared_table)) < 0) - goto exit_2; - - shared_table = mmap(NULL, sizeof(*shared_table), - PROT_READ | PROT_WRITE, MAP_SHARED, shmem_fd, 0); - if (shared_table == MAP_FAILED) - goto exit_2; - - return HSAKMT_STATUS_SUCCESS; - -exit_2: - shm_unlink(shmem_name); - shmem_fd = 0; -exit_1: - sem_close(sem); - sem_unlink(sem_name); - sem = SEM_FAILED; - return HSAKMT_STATUS_ERROR; -} - -static void destroy_shared_region(void) -{ - if (shared_table && shared_table != MAP_FAILED) - munmap(shared_table, sizeof(*shared_table)); - - if (shmem_fd > 0) { - close(shmem_fd); - shm_unlink(shmem_name); - } - - if (sem != SEM_FAILED) { - sem_close(sem); - sem_unlink(sem_name); - sem = SEM_FAILED; - } -} - -static void init_perf_shared_table(void) -{ - sem_wait(sem); - - /* If the magic number exists, the perf shared table has been - * initialized by another process and is in use. Don't overwrite it. - */ - if (shared_table->magic4cc == HSA_PERF_MAGIC4CC) { - sem_post(sem); - return; - } - - /* write the perf content */ - shared_table->magic4cc = HSA_PERF_MAGIC4CC; - - sem_post(sem); -} - HSAKMT_STATUS init_counter_props(unsigned int NumNodes) { counter_props = calloc(NumNodes, sizeof(struct HsaCounterProperties *)); @@ -189,12 +109,6 @@ HSAKMT_STATUS init_counter_props(unsigned int NumNodes) counter_props_count = NumNodes; - if (init_shared_region() != HSAKMT_STATUS_SUCCESS) { - pr_warn("Profiling of privileged blocks is not available.\n"); - return HSAKMT_STATUS_ERROR; - } - init_perf_shared_table(); - return HSAKMT_STATUS_SUCCESS; } @@ -202,8 +116,6 @@ void destroy_counter_props(void) { unsigned int i; - destroy_shared_region(); - if (!counter_props) return; From 693e686c4dfe3b04c60a79abfd96fce7646f38b5 Mon Sep 17 00:00:00 2001 From: James Zhu Date: Tue, 12 Sep 2023 16:53:25 -0400 Subject: [PATCH 1204/1247] kfdtest: remove IOMMUv2 performance monitor support IOMMUv2 is removed from AMDGPU/KFD. Change-Id: Ia00f9aa879a5f32a42bec914936d105d6845bc60 Signed-off-by: James Zhu --- tests/kfdtest/src/KFDPerfCounters.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/kfdtest/src/KFDPerfCounters.cpp b/tests/kfdtest/src/KFDPerfCounters.cpp index ca0bb2c129..452ecda178 100644 --- a/tests/kfdtest/src/KFDPerfCounters.cpp +++ b/tests/kfdtest/src/KFDPerfCounters.cpp @@ -66,7 +66,6 @@ static struct block_name_table { {"TD ", {0x7d7c0fe4, 0xfe41, 0x4fea, 0x92, 0xc9, 0x45, 0x44, 0xd7, 0x70, 0x6d, 0xc6}}, {"VGT ", {0x0b6a8cb7, 0x7a01, 0x409f, 0xa2, 0x2c, 0x30, 0x14, 0x85, 0x4f, 0x13, 0x59}}, {"WD ", {0x0e176789, 0x46ed, 0x4b02, 0x97, 0x2a, 0x91, 0x6d, 0x2f, 0xac, 0x24, 0x4a}}, - {"IOMMUV2", {0x80969879, 0xb0f6, 0x4be6, 0x97, 0xf6, 0x6a, 0x63, 0x0, 0xf5, 0x10, 0x1d}}, {"DRIVER ", {0xea9b5ae1, 0x6c3f, 0x44b3, 0x89, 0x54, 0xda, 0xf0, 0x75, 0x65, 0xa9, 0xa}} }; From 73efd3a14e6dfeae7eb226abf36f095fd81c8d18 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Wed, 4 Oct 2023 14:11:12 +0000 Subject: [PATCH 1205/1247] libhsakmt: Fix incorrect flags for ext coherence Change-Id: I89c838b9fbdb85589691f29806ae15884b25592f --- src/fmm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 76ddc40f4f..7b5a5fbe3a 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1047,7 +1047,7 @@ static HSAKMT_STATUS fmm_register_mem_svm_api(void *address, args->attrs[0].type = coarse_grain ? HSA_SVM_ATTR_CLR_FLAGS : HSA_SVM_ATTR_SET_FLAGS; args->attrs[0].value = HSA_SVM_FLAG_COHERENT; - args->attrs[1].type = ext_coherent ? HSA_SVM_ATTR_CLR_FLAGS : HSA_SVM_ATTR_SET_FLAGS; + args->attrs[1].type = ext_coherent ? HSA_SVM_ATTR_SET_FLAGS : HSA_SVM_ATTR_CLR_FLAGS ; args->attrs[1].value = HSA_SVM_FLAG_EXT_COHERENT; pr_debug("Registering to SVM %p size: %ld\n", (void*)aligned_addr, aligned_size); @@ -3447,7 +3447,7 @@ static HSAKMT_STATUS fmm_register_user_memory(void *addr, KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE | (coarse_grain ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_COHERENT) | - (ext_coherent ? 0 : KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT), + (ext_coherent ? KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT : 0), &obj); if (!svm_addr) return HSAKMT_STATUS_ERROR; From 85a47fa66b2c9506809ab3552873227597682543 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 4 Oct 2023 08:41:18 -0400 Subject: [PATCH 1206/1247] libhsakmt: Set CWSR range granularity Set CWSR svm range granularity to 0xff, then KFD will migrate the entire CWSR range from VRAM back to system memory when recovering the CPU page fault if rocgdb access CWSR area, this avoid the partial CWSR range migration and stall CWSR GPU mapping issue. This is a temporary workaround, it should be reverted once the KFD is fixed. Change-Id: I80a7248244574edba25b13858b7ebcf1c77b8930 Signed-off-by: Philip Yang --- src/queues.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/queues.c b/src/queues.c index 992290c791..f62e862b03 100644 --- a/src/queues.c +++ b/src/queues.c @@ -413,7 +413,7 @@ static HSAKMT_STATUS register_svm_range(void *mem, uint32_t size, flags |= HSA_SVM_FLAG_GPU_ALWAYS_MAPPED; } - nattr = 5; + nattr = 6; s_attr = sizeof(*attrs) * nattr; attrs = (HSA_SVM_ATTRIBUTE *)alloca(s_attr); @@ -427,6 +427,8 @@ static HSAKMT_STATUS register_svm_range(void *mem, uint32_t size, attrs[3].value = flags; attrs[4].type = HSA_SVM_ATTR_ACCESS; attrs[4].value = gpuNode; + attrs[5].type = HSA_SVM_ATTR_GRANULARITY; + attrs[5].value = 0xFF; return hsaKmtSVMSetAttr(mem, size, nattr, attrs); } From b44cca813d7ad896821f15465232c37a6faf0b3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20Ke=C3=9Fler?= Date: Sat, 30 Sep 2023 19:14:47 +0200 Subject: [PATCH 1207/1247] Mark new symbols in ROCm 5.7.0 as global Change-Id: Ia0391cac7f432f019dea94f98a145dbf8120817d --- src/libhsakmt.ver | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 5d1a673ffb..15c291683d 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -77,6 +77,10 @@ hsaKmtSetXNACKMode; hsaKmtGetXNACKMode; hsaKmtOpenSMI; hsaKmtExportDMABufHandle; +hsaKmtWaitOnEvent_Ext; +hsaKmtWaitOnMultipleEvents_Ext; +hsaKmtReplaceAsanHeaderPage; +hsaKmtReturnAsanHeaderPage; local: *; }; From a226542fc3c06d559a19a6e97c68817bd8d9190b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20K=C5=82oczko?= Date: Wed, 4 Oct 2023 09:01:16 +0000 Subject: [PATCH 1208/1247] install .pc files in libdir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provided pkgconfig file contains interface description which is arch dependent. In such cases .pc files should be installed in libdir. Signed-off-by: Tomasz Kłoczko Change-Id: Ibbc85ad4aee1ef014c409dfa63313873b590464b --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e3b9085449..1263efbcca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,7 +285,7 @@ endif() # TODO: Add support for relocatable packages. configure_file ( libhsakmt.pc.in libhsakmt.pc @ONLY ) -install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.pc DESTINATION ${CMAKE_INSTALL_DATADIR}/pkgconfig COMPONENT devel) +install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/libhsakmt.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig COMPONENT devel) ########################### # Packaging directives From 5047eb161fadf362c25bd5b5c1230e4dc3d34b8e Mon Sep 17 00:00:00 2001 From: Rajneesh Bhardwaj Date: Wed, 11 Oct 2023 15:08:04 -0400 Subject: [PATCH 1209/1247] libhsakmt: Use MADV_HUGEPAGE for large allocations For large memory allocations (>2MB) the thunk should use the MADV_HUGEPAGE flag for madvise call to optimize allocation performance on certain operating systems that rely on madvise hint when Traspatent Huge Pages is not set to always. Suggested-by: Joseph Greathouse Signed-off-by: Rajneesh Bhardwaj Change-Id: Ic0c753f89a177b0f715942d6e2a7108b08a85f20 --- src/fmm.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 7b5a5fbe3a..b87ee1279d 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1772,14 +1772,20 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags mflags) { - void *mem; manageable_aperture_t *aperture; - uint64_t mmap_offset; - uint32_t ioc_flags; - uint64_t size; - int32_t gpu_drm_fd; - uint32_t gpu_id; vm_object_t *vm_obj = NULL; + int flags = MADV_DONTFORK; + uint64_t mmap_offset; + int32_t gpu_drm_fd; + uint32_t ioc_flags; + uint32_t gpu_id; + uint64_t size; + void *mem; + + /* set madvise flags to HUGEPAGE always for 2MB pages */ + if (MemorySizeInBytes >= (2 * 1024 * 1024)) + flags |= MADV_HUGEPAGE; + if (!g_first_gpu_mem) return NULL; @@ -1830,7 +1836,7 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, * fork. This avoids MMU notifiers and evictions due to user * memory mappings on fork. */ - madvise(mem, MemorySizeInBytes, MADV_DONTFORK); + madvise(mem, MemorySizeInBytes, flags); /* Create userptr BO */ mmap_offset = (uint64_t)mem; From 46fe31634831ae384a6b8fe29b46857605647bb0 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Mon, 23 Oct 2023 21:58:22 +0800 Subject: [PATCH 1210/1247] kfdtest: Change SetGetAttributesTest range granularity granularity check is added in kfd w/ below patch: commit 270c7a8375a91fec2fb4e2c253e3955d9b7540b4 Author: Jesse Zhang Date: Fri Oct 20 09:43:51 2023 +0800 drm/amdkfd: Fix shift out-of-bounds issue diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index a690dced6860..f2b33fb2afcf 100644 Change-Id: I8cb037e3bf5db0a85661494b77e59984eca4d98d --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -781,7 +781,7 @@ svm_range_apply_attrs(struct kfd_process *p, struct svm_range *prange, prange->flags &= ~attrs[i].value; break; case KFD_IOCTL_SVM_ATTR_GRANULARITY: - prange->granularity = attrs[i].value; + prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F); break; default: WARN_ONCE(1, "svm_range_check_attrs wasn't called?"); Test cases have to been modified accordingly otherwise KFDSVMRangeTest.SetGetAttributesTest fails. Signed-off-by: Yifan Zhang Change-Id: Ifff47556bc398da6b18ad26ac545d139b63b0c92 --- tests/kfdtest/src/KFDSVMRangeTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/tests/kfdtest/src/KFDSVMRangeTest.cpp index a84ab5003c..b38eb700bc 100644 --- a/tests/kfdtest/src/KFDSVMRangeTest.cpp +++ b/tests/kfdtest/src/KFDSVMRangeTest.cpp @@ -118,7 +118,7 @@ TEST_P(KFDSVMRangeTest, SetGetAttributesTest) { {HSA_SVM_ATTR_PREFERRED_LOC, (HSAuint32)defaultGPUNode}, {HSA_SVM_ATTR_SET_FLAGS, HSA_SVM_FLAG_HOST_ACCESS | HSA_SVM_FLAG_GPU_EXEC | HSA_SVM_FLAG_COHERENT}, - {HSA_SVM_ATTR_GRANULARITY, 0xFF}, + {HSA_SVM_ATTR_GRANULARITY, 0x3F}, {HSA_SVM_ATTR_ACCESS, (HSAuint32)defaultGPUNode}, }; From 01ff2f79346ae4a53b621eb8bd6f683d03f899c8 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Fri, 17 Nov 2023 04:43:51 +0000 Subject: [PATCH 1211/1247] libhsakmt: Handle HW_EXCEPTION events Add new structures for HW Exception events and copy data from KFD to expose to upper layers. Change-Id: Icd5eb98997c47620e3b86277ab6d3abb7ed7d56f --- include/hsakmttypes.h | 17 +++++++++++++++++ src/events.c | 10 ++++++++++ 2 files changed, 27 insertions(+) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 2937c35915..9f1b2ccb24 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1044,6 +1044,21 @@ typedef struct _HsaMemoryAccessFault HSA_EVENTID_MEMORYFLAGS Flags; // event flags } HsaMemoryAccessFault; +typedef enum _HSA_EVENTID_HW_EXCEPTION_CAUSE +{ + HSA_EVENTID_HW_EXCEPTION_GPU_HANG = 0, // GPU Hang + HSA_EVENTID_HW_EXCEPTION_ECC = 1, // SRAM ECC error +} HSA_EVENTID_HW_EXCEPTION_CAUSE; + +// data associated with HSA_EVENTID_HW_EXCEPTION +typedef struct _HsaHwException +{ + HSAuint32 NodeId; // Node Id where the memory exception occured + HSAuint32 ResetType; + HSAuint32 MemoryLost; + HSA_EVENTID_HW_EXCEPTION_CAUSE ResetCause; +} HsaHwException; + typedef struct _HsaEventData { HSA_EVENTTYPE EventType; //event type @@ -1062,6 +1077,8 @@ typedef struct _HsaEventData // data associated with HSA_EVENTTYPE_MEMORY HsaMemoryAccessFault MemoryAccessFault; + // data associated with HSA_EVENTTYPE_HW_EXCEPTION + HsaHwException HwException; } EventData; // the following data entries are internal to the KFD & thunk itself. diff --git a/src/events.c b/src/events.c index 9ec199ac7a..60e9f6dbbf 100644 --- a/src/events.c +++ b/src/events.c @@ -428,6 +428,16 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtWaitOnMultipleEvents_Ext(HsaEvent *Events[], ((event_data[i].memory_exception_data.ErrorType == 1) || (event_data[i].memory_exception_data.ErrorType == 2)) ? 1 : 0; Events[i]->EventData.EventData.MemoryAccessFault.Flags = HSA_EVENTID_MEMORY_FATAL_PROCESS; analysis_memory_exception(&event_data[i].memory_exception_data); + } else if (Events[i]->EventData.EventType == HSA_EVENTTYPE_HW_EXCEPTION && + event_data[i].hw_exception_data.gpu_id) { + + result = gpuid_to_nodeid(event_data[i].hw_exception_data.gpu_id, &Events[i]->EventData.EventData.HwException.NodeId); + if (result != HSAKMT_STATUS_SUCCESS) + goto out; + + Events[i]->EventData.EventData.HwException.ResetType = event_data[i].hw_exception_data.reset_type; + Events[i]->EventData.EventData.HwException.ResetCause = event_data[i].hw_exception_data.reset_cause; + Events[i]->EventData.EventData.HwException.MemoryLost = event_data[i].hw_exception_data.memory_lost; } } } From 4bf73f521b27f4f7bfc7b4ad782fd6f1a78f98ea Mon Sep 17 00:00:00 2001 From: gaba Date: Wed, 6 Dec 2023 12:54:46 -0500 Subject: [PATCH 1212/1247] libhsakmt: Fix CPU cache issue For "Intel Meteor lake Mobile", the cache info is not in sysfs, That means /sys/devices/system/node/node%d/%s/cache is not exist, but system working fine. Change-Id: Ie7c04426791a84c2288ff21df093226828a5f629 Signed-off-by: Gang Ba --- src/topology.c | 7 +++++-- tests/kfdtest/src/KFDTopologyTest.cpp | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/topology.c b/src/topology.c index d2c81a98d9..2cf89f6b8d 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1466,8 +1466,11 @@ static HSAKMT_STATUS topology_get_cpu_cache_props(int node, tbl->node.NumCaches = topology_create_temp_cpu_cache_list( node, cpuinfo, &cpu_ci_list); if (!tbl->node.NumCaches) { - pr_err("Fail to get cache info for node %d\n", node); - ret = HSAKMT_STATUS_ERROR; + /* For "Intel Meteor lake Mobile", the cache info is not in sysfs, + * That means /sys/devices/system/node/node%d/%s/cache is not exist. + * here AMD will not black this issue. + */ + pr_debug("CPU cache info is not available for node %d \n", node); goto exit; } diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/tests/kfdtest/src/KFDTopologyTest.cpp index b4c2ebfca6..7910aa21c4 100644 --- a/tests/kfdtest/src/KFDTopologyTest.cpp +++ b/tests/kfdtest/src/KFDTopologyTest.cpp @@ -64,7 +64,11 @@ TEST_F(KFDTopologyTest , BasicTest) { " SGPR Size is " << pNodeProperties->SGPRSizePerCU << std::endl; } EXPECT_GT(pNodeProperties->NumMemoryBanks, HSAuint32(0)) << "Node index: " << node << "No MemoryBanks."; - EXPECT_GT(pNodeProperties->NumCaches, HSAuint32(0)) << "Node index: " << node << "No Caches."; + if (pNodeProperties->NumCaches ==0) + // SWDEV-420270 + // For "Intel Meteor lake Mobile", the cache info is not in sysfs, + // That means /sys/devices/system/node/node%d/%s/cache is not exist. + LOG() << "Node index: " << node << " No Caches or not available to read ." << std::endl; } } From 1a7162731e904839ccf88630e153df3634c3f138 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Fri, 15 Dec 2023 13:27:22 -0600 Subject: [PATCH 1213/1247] kfdtest: use svm range granularity in KFDSVMEvictTest.QueueTest/1 When xnack is on shadder code in this test triggers gpu page fault that migrate data from system ram to vram. Use svm range granularity to move all data from system buffer to vram to reduce system ram pressure to avoid system ram oom for systems that has less system ram. Signed-off-by: Xiaogang Chen Change-Id: I219472210756be319491f7827f7209fe32726f81 --- tests/kfdtest/src/KFDSVMEvictTest.cpp | 25 ++++++++++++++++++------- tests/kfdtest/src/KFDSVMEvictTest.hpp | 5 +++-- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/tests/kfdtest/src/KFDSVMEvictTest.cpp index fae3b09aae..8175c6a0b0 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.cpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.cpp @@ -85,7 +85,8 @@ HSAint32 KFDSVMEvictTest::GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBuf return count; } -HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count) { +HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count, + HSAint32 xnack_enable) { HSAuint64 sysMemSize = GetSysMemSize(); HSAuint64 size, sizeInPages; HSAuint64 vramBufSizeInPages; @@ -101,8 +102,6 @@ HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count) { /* Check if there is enough system memory to pass test for XNACK off * KFD system memory limit is 15/16. */ - HSAint32 xnack_enable = 0; - EXPECT_SUCCESS(hsaKmtGetXNACKMode(&xnack_enable)); if (!xnack_enable && size > (sysMemSize - (sysMemSize >> 4))) return 0; @@ -113,7 +112,7 @@ HSAint64 KFDSVMEvictTest::GetBufferSize(HSAuint64 vramSize, HSAuint32 count) { } void KFDSVMEvictTest::AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, - std::vector &pBuffers) { + std::vector &pBuffers, HSAuint32 Granularity) { HSAuint64 totalMB; totalMB = N_PROCESSES * count * (vramBufSize >> 20); @@ -132,6 +131,8 @@ retry: ret = RegisterSVMRange(defaultGPUNode, m_pBuf, vramBufSize, defaultGPUNode, m_Flags); if (ret == HSAKMT_STATUS_SUCCESS) { pBuffers.push_back(m_pBuf); + if (Granularity) + EXPECT_SUCCESS(SVMRangSetGranularity(m_pBuf, vramBufSize, Granularity)); retry = 0; } else { if (retry++ > ALLOCATE_RETRY_TIMES) { @@ -263,7 +264,7 @@ TEST_P(KFDSVMEvictTest, BasicTest) { ForkChildProcesses(N_PROCESSES); std::vector pBuffers; - AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers, 0); /* wait for other processes to finish allocation, then free buffer */ sleep(ALLOCATE_RETRY_TIMES); @@ -327,7 +328,9 @@ TEST_P(KFDSVMEvictTest, QueueTest) { LOG() << "Found VRAM of " << std::dec << (vramSize >> 20) << "MB." << std::endl; } - HSAuint64 vramBufSize = GetBufferSize(vramSize, count); + HSAint32 xnack_enable = 0; + EXPECT_SUCCESS(hsaKmtGetXNACKMode(&xnack_enable)); + HSAuint64 vramBufSize = GetBufferSize(vramSize, count, xnack_enable); if (vramBufSize == 0) { LOG() << "Not enough system memory, skipping the test" << std::endl; return; @@ -345,7 +348,15 @@ TEST_P(KFDSVMEvictTest, QueueTest) { HsaMemoryBuffer resultBuffer(PAGE_SIZE, defaultGPUNode); std::vector pBuffers; - AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers); + HSAuint32 granularity = 0; + /* xnack is on, shadder code will trigger gpu page fault that bring data + * to vram. use granularity to move all data from system buffer to vram + * to reduce system ram pressure in order to avoid system ram oom in system + * that has less system ram. + */ + if (xnack_enable) + granularity = 0xff; + AllocBuffers(defaultGPUNode, count, vramBufSize, pBuffers, granularity); unsigned int wavefront_num = pBuffers.size(); LOG() << m_psName << "wavefront number " << wavefront_num << std::endl; diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/tests/kfdtest/src/KFDSVMEvictTest.hpp index 16b2a71f31..5b7a5e044e 100644 --- a/tests/kfdtest/src/KFDSVMEvictTest.hpp +++ b/tests/kfdtest/src/KFDSVMEvictTest.hpp @@ -56,12 +56,13 @@ class KFDSVMEvictTest : public KFDLocalMemoryTest, protected: std::string CreateShader(); void AllocBuffers(HSAuint32 defaultGPUNode, HSAuint32 count, HSAuint64 vramBufSize, - std::vector &pBuffers); + std::vector &pBuffers, HSAuint32 Granularity); void FreeBuffers(std::vector &pBuffers, HSAuint64 vramBufSize); void ForkChildProcesses(int nprocesses); void WaitChildProcesses(); HSAint32 GetBufferCounter(HSAuint64 vramSize, HSAuint64 vramBufSize); - HSAint64 GetBufferSize(HSAuint64 vramSize, HSAuint32 count); + HSAint64 GetBufferSize(HSAuint64 vramSize, HSAuint32 count, + HSAint32 xnack_enable); protected: // members std::string m_psName; From 72324444ccffe3d044560ec5c0b020c96ef465b3 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 21 Dec 2023 16:21:51 +0800 Subject: [PATCH 1214/1247] kfdtest: finetune LargestVramBuffer subtest criteria Change the proportion to 3/5 on APUs. Change-Id: I0dc37fb1c309605811551b88259473603c81f9ae Signed-off-by: Yifan Zhang --- tests/kfdtest/src/KFDMemoryTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index a00a4febd6..7d07f4bb1d 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -770,8 +770,8 @@ TEST_F(KFDMemoryTest, LargestVramBufferTest) { LOG() << "The largest allocated VRAM buffer is " << std::dec << lastTestedSizeMB << "MB" << std::endl; - /* Make sure 3/4 vram can be allocated.*/ - EXPECT_GE(lastTestedSizeMB * 4, vramSizeMB * 3); + /* Make sure 3/5 vram can be allocated.*/ + EXPECT_GE(lastTestedSizeMB * 5, vramSizeMB * 3); if (lastTestedSizeMB * 16 < vramSizeMB * 15) WARN() << "The largest allocated VRAM buffer size is smaller than the expected " << vramSizeMB * 15 / 16 << "MB" << std::endl; From 92c00157873d32cbbac3aa17d3dc0a2179924459 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 28 Dec 2023 17:12:44 +0800 Subject: [PATCH 1215/1247] kfdtest: finetune MMBench subtest criteria Change the proportion to 6/10 on APUs. Change-Id: I3576cb23d0f14ff6d576a5db4bdeef9446aa10d2 Signed-off-by: Yifan Zhang --- tests/kfdtest/src/KFDMemoryTest.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 7d07f4bb1d..6e44852cc8 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -963,9 +963,13 @@ TEST_F(KFDMemoryTest, MMBench) { memFlags.ui32.NonPaged = 1; /* Buffer sizes are 2MB aligned to match new allocation policy. - * Upper limit of buffer number to fit 80% vram size. + * Upper limit of buffer number to fit 80% vram size. APUs w/ + * smaller VRAM needs different criteria. */ - bufLimit = ((vramSizeMB << 20) * 8 / 10) / ALIGN_UP(bufSize, VRAM_ALLOCATION_ALIGN); + if (vramSizeMB <= 512) + bufLimit = ((vramSizeMB << 20) * 6 / 10) / ALIGN_UP(bufSize, VRAM_ALLOCATION_ALIGN); + else + bufLimit = ((vramSizeMB << 20) * 8 / 10) / ALIGN_UP(bufSize, VRAM_ALLOCATION_ALIGN); if (bufLimit == 0) continue; // skip when bufSize > vram From 808a4428b6d9c7f32e7966517a0fd0571605ed04 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Thu, 28 Dec 2023 17:17:31 +0800 Subject: [PATCH 1216/1247] kfdtest: add APUs judgement in LargestVramBufferTest criteria This patch is to add APUs judgement in LargestVramBufferTest criteria. Change-Id: Ic69093f8ebed8be0b1c58787e2a294d86fb49bb0 --- tests/kfdtest/src/KFDMemoryTest.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 6e44852cc8..2e76e546a0 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -771,7 +771,11 @@ TEST_F(KFDMemoryTest, LargestVramBufferTest) { << lastTestedSizeMB << "MB" << std::endl; /* Make sure 3/5 vram can be allocated.*/ - EXPECT_GE(lastTestedSizeMB * 5, vramSizeMB * 3); + if (vramSizeMB <= 512) + EXPECT_GE(lastTestedSizeMB * 5, vramSizeMB * 3); + else + EXPECT_GE(lastTestedSizeMB * 4, vramSizeMB * 3); + if (lastTestedSizeMB * 16 < vramSizeMB * 15) WARN() << "The largest allocated VRAM buffer size is smaller than the expected " << vramSizeMB * 15 / 16 << "MB" << std::endl; From 42581d4172e30001a3cebe66a96e614916bfbd65 Mon Sep 17 00:00:00 2001 From: Jeremy Newton Date: Tue, 19 Dec 2023 19:27:47 -0500 Subject: [PATCH 1217/1247] Fix missing global symbol If using hsakmt as a shared library Change-Id: I66a1849a46bd7009813d49824d0d059e8a511038 Signed-off-by: Jeremy Newton --- src/libhsakmt.ver | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index 15c291683d..c04cefef41 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -81,6 +81,7 @@ hsaKmtWaitOnEvent_Ext; hsaKmtWaitOnMultipleEvents_Ext; hsaKmtReplaceAsanHeaderPage; hsaKmtReturnAsanHeaderPage; +hsaKmtGetAMDGPUDeviceHandle; local: *; }; From a7d8b1c2873af5ed29acb9ab393e71d95b9d92e4 Mon Sep 17 00:00:00 2001 From: Ranjith Ramakrishnan Date: Sun, 7 Jan 2024 20:48:44 -0800 Subject: [PATCH 1218/1247] Use relative path rather than hard coded path in package config file Change-Id: Ia35fadeead69f84f4f4d32ab0c04f2f391aba4f4 --- libhsakmt.pc.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libhsakmt.pc.in b/libhsakmt.pc.in index 3093afc8dc..041e81f710 100644 --- a/libhsakmt.pc.in +++ b/libhsakmt.pc.in @@ -1,4 +1,4 @@ -prefix=@CPACK_PACKAGING_INSTALL_PREFIX@ +prefix=${pcfiledir}/../.. exec_prefix=${prefix} libdir=${prefix}/@CMAKE_INSTALL_LIBDIR@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ From 0accd17b6ec75cb422567ddf12d13813a389d65d Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Thu, 4 Jan 2024 00:12:50 +0000 Subject: [PATCH 1219/1247] HSA_USE_SVM to override SVMAPISupported node prop When HSA_USE_SVM is 0, thunk uses non-SVM path, but upper layers still use SVM path. That is not as expected. Suggested-by: Lang Yu Change-Id: I1ae0b4faa2f8af5ec69a81cfeb7661bd47d739d4 --- src/fmm.c | 29 +++++++++++------------------ src/globals.c | 3 +++ src/libhsakmt.h | 1 + src/openclose.c | 5 +++++ src/queues.c | 5 +---- src/topology.c | 3 +++ 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index b87ee1279d..ffb09080b9 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -231,9 +231,6 @@ typedef struct { /* specifies the alignment size as PAGE_SIZE * 2^alignment_order */ uint32_t alignment_order; - - /* whether to check all dGPUs in the topology support SVM API */ - bool is_svm_api_supported; } svm_t; /* The other apertures are specific to each GPU. gpu_mem_t manages GPU @@ -254,7 +251,6 @@ static svm_t svm = { .userptr_for_paged_mem = false, .check_userptr = false, .disable_cache = false, - .is_svm_api_supported = false }; /* On APU, for memory allocated on the system memory that GPU doesn't access @@ -1951,7 +1947,7 @@ HSAKMT_STATUS fmm_release(void *address) object = vm_find_object(address, 0, &aperture); if (!object) - return svm.is_svm_api_supported ? + return is_svm_api_supported ? HSAKMT_STATUS_SUCCESS : HSAKMT_STATUS_MEMORY_NOT_REGISTERED; @@ -2492,7 +2488,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) uint32_t num_of_sysfs_nodes; HSAKMT_STATUS ret = HSAKMT_STATUS_SUCCESS; char *disableCache, *pagedUserptr, *checkUserptr, *guardPagesStr, *reserveSvm; - char *maxVaAlignStr, *useSvmStr; + char *maxVaAlignStr; unsigned int guardPages = 1; uint64_t svm_base = 0, svm_limit = 0; uint32_t svm_alignment = 0; @@ -2531,9 +2527,6 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) if (!maxVaAlignStr || sscanf(maxVaAlignStr, "%u", &svm.alignment_order) != 1) svm.alignment_order = 9; - useSvmStr = getenv("HSA_USE_SVM"); - svm.is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")); - gpu_mem_count = 0; g_first_gpu_mem = NULL; @@ -2588,7 +2581,7 @@ HSAKMT_STATUS fmm_init_process_apertures(unsigned int NumNodes) gpu_mem[gpu_mem_count].local_mem_size = props.LocalMemSize; gpu_mem[gpu_mem_count].device_id = props.DeviceId; gpu_mem[gpu_mem_count].node_id = i; - svm.is_svm_api_supported &= props.Capability.ui32.SVMAPISupported; + is_svm_api_supported &= props.Capability.ui32.SVMAPISupported; gpu_mem[gpu_mem_count].scratch_physical.align = PAGE_SIZE; gpu_mem[gpu_mem_count].scratch_physical.ops = &reserved_aperture_ops; @@ -3116,7 +3109,7 @@ static HSAKMT_STATUS _fmm_map_to_gpu_userptr(void *addr, uint64_t size, /* Map and return the GPUVM address adjusted by the offset * from the start of the page */ - if (svm.is_svm_api_supported) { + if (is_svm_api_supported) { svm_addr = (void*)((HSAuint64)addr - page_offset); if (!nodes_to_map) { nodes_to_map = all_gpu_id_array; @@ -3156,7 +3149,7 @@ HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_addre address, size); object = vm_find_object(address, size, &aperture); - if (!object && !svm.is_svm_api_supported ) { + if (!object && !is_svm_api_supported) { if (!is_dgpu) { /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); @@ -3183,7 +3176,7 @@ HSAKMT_STATUS fmm_map_to_gpu(void *address, uint64_t size, uint64_t *gpuvm_addre /* Prefetch memory on APUs with dummy-reads */ fmm_check_user_memory(address, size); ret = HSAKMT_STATUS_SUCCESS; - } else if ((svm.is_svm_api_supported && !object) || object->userptr) { + } else if ((is_svm_api_supported && !object) || object->userptr) { ret = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object, NULL, 0); } else { ret = _fmm_map_to_gpu(aperture, address, size, object, NULL, 0); @@ -3361,7 +3354,7 @@ int fmm_unmap_from_gpu(void *address) object = vm_find_object(address, 0, &aperture); if (!object) /* On APUs GPU unmapping of system memory is a no-op */ - return (!is_dgpu || svm.is_svm_api_supported) ? 0 : -EINVAL; + return (!is_dgpu || is_svm_api_supported) ? 0 : -EINVAL; /* Successful vm_find_object returns with the aperture locked */ if (aperture == &cpuvm_aperture) @@ -3510,7 +3503,7 @@ HSAKMT_STATUS fmm_register_memory(void *address, uint64_t size_in_bytes, return HSAKMT_STATUS_SUCCESS; /* Register a new user ptr */ - if (svm.is_svm_api_supported) + if (is_svm_api_supported) return fmm_register_mem_svm_api(address, size_in_bytes, coarse_grain, @@ -3885,7 +3878,7 @@ HSAKMT_STATUS fmm_deregister_memory(void *address) /* On APUs we assume it's a random system memory address * where registration and dergistration is a no-op */ - return (!is_dgpu || svm.is_svm_api_supported) ? + return (!is_dgpu || is_svm_api_supported) ? HSAKMT_STATUS_SUCCESS : HSAKMT_STATUS_MEMORY_NOT_REGISTERED; /* Successful vm_find_object returns with aperture locked */ @@ -3949,7 +3942,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_INVALID_PARAMETER; object = vm_find_object(address, size, &aperture); - if (!object && !svm.is_svm_api_supported) + if (!object && !is_svm_api_supported) return HSAKMT_STATUS_ERROR; /* Successful vm_find_object returns with aperture locked */ @@ -3972,7 +3965,7 @@ HSAKMT_STATUS fmm_map_to_gpu_nodes(void *address, uint64_t size, return HSAKMT_STATUS_ERROR; } - if ((svm.is_svm_api_supported && !object) || object->userptr) { + if ((is_svm_api_supported && !object) || object->userptr) { retcode = _fmm_map_to_gpu_userptr(address, size, gpuvm_address, object, nodes_to_map, num_of_nodes * sizeof(uint32_t)); if (object) diff --git a/src/globals.c b/src/globals.c index 851853aad0..c2fbd4f02e 100644 --- a/src/globals.c +++ b/src/globals.c @@ -34,3 +34,6 @@ pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; bool is_dgpu; int PAGE_SIZE; int PAGE_SHIFT; + +/* whether to check all dGPUs in the topology support SVM API */ +bool is_svm_api_supported; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 2d8da2c9c8..656800dae6 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -37,6 +37,7 @@ extern unsigned long kfd_open_count; extern bool hsakmt_forked; extern pthread_mutex_t hsakmt_mutex; extern bool is_dgpu; +extern bool is_svm_api_supported; extern HsaVersionInfo kfd_version_info; diff --git a/src/openclose.c b/src/openclose.c index 4913349256..bc513c6a03 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -40,6 +40,7 @@ #include #include "fmm.h" #include +#include int (*fn_amdgpu_device_get_fd)(HsaAMDGPUDeviceHandle device_handle); @@ -152,6 +153,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) int fd = -1; HsaSystemProperties sys_props; char *error; + char *useSvmStr; pthread_mutex_lock(&hsakmt_mutex); @@ -192,6 +194,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtOpenKFD(void) if (result != HSAKMT_STATUS_SUCCESS) goto kfd_version_failed; + useSvmStr = getenv("HSA_USE_SVM"); + is_svm_api_supported = !(useSvmStr && !strcmp(useSvmStr, "0")); + result = topology_sysfs_get_system_props(&sys_props); if (result != HSAKMT_STATUS_SUCCESS) goto topology_sysfs_failed; diff --git a/src/queues.c b/src/queues.c index f62e862b03..f9e561267f 100644 --- a/src/queues.c +++ b/src/queues.c @@ -494,7 +494,6 @@ static int handle_concrete_asic(struct queue *q, if (ret) { HsaNodeProperties node; - bool svm_api; if (hsaKmtGetNodeProperties(NodeId, &node)) return HSAKMT_STATUS_ERROR; @@ -509,12 +508,10 @@ static int handle_concrete_asic(struct queue *q, q->total_mem_alloc_size = (q->ctx_save_restore_size + q->debug_memory_size) * node.NumXcc; - svm_api = node.Capability.ui32.SVMAPISupported; - /* Allocate unified memory for context save restore * area on dGPU. */ - if (!q->use_ats && svm_api) { + if (!q->use_ats && is_svm_api_supported) { uint32_t size = PAGE_ALIGN_UP(q->total_mem_alloc_size); void *addr; HSAKMT_STATUS r = HSAKMT_STATUS_ERROR; diff --git a/src/topology.c b/src/topology.c index 2cf89f6b8d..7e655e362e 100644 --- a/src/topology.c +++ b/src/topology.c @@ -1192,6 +1192,9 @@ static HSAKMT_STATUS topology_sysfs_get_node_props(uint32_t node_id, gfxv = (uint32_t)prop_val; } + if (!is_svm_api_supported) + props->Capability.ui32.SVMAPISupported = 0; + /* Bail out early, if a CPU node */ if (!props->NumFComputeCores) goto out; From f75fddb9bd3fe69cb5d2e47d197f164c36a6cfd6 Mon Sep 17 00:00:00 2001 From: James Zhu Date: Wed, 10 Jan 2024 12:33:29 -0500 Subject: [PATCH 1220/1247] kfdtest: change Largest Buffer search algorithm The old Largest Buffer search algorithm is using Binary Search algorithm to find last successful memory allocation. But each successful memory allocation takes times. Since the unsuccessful memory allocation returns very quick. Changing the search algorithm to find first successful memory allocation starting from MAX, each testing step with granularity interval will speed up this test. Change-Id: I07daea05423c33e72a483f0013e8ea1b5dabf989 Signed-off-by: James Zhu --- tests/kfdtest/src/KFDMemoryTest.cpp | 32 ++++++++++++----------------- tests/kfdtest/src/KFDMemoryTest.hpp | 2 +- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index 2e76e546a0..a94449fc3e 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -647,26 +647,24 @@ TEST_F(KFDMemoryTest, GetTileConfigTest) { TEST_END } -void KFDMemoryTest::BinarySearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, +void KFDMemoryTest::SearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, HSAuint64 highMB, int nodeToMap, HSAuint64 *lastSizeMB) { int ret; - int iter = 0; HsaMemMapFlags mapFlags = {0}; - HSAuint64 granularityMB = highMB > 512 ? 128 : 16; + HSAuint64 granularityMB = 8; /* Testing big buffers in VRAM */ unsigned int * pDb = NULL; - HSAuint64 lowMB = 0; highMB = (highMB + granularityMB - 1) & ~(granularityMB - 1); HSAuint64 sizeMB; HSAuint64 size = 0; - while (highMB - lowMB > granularityMB) { - sizeMB = (lowMB + highMB) / 2; + while (highMB > granularityMB) { + sizeMB = highMB - granularityMB; size = sizeMB * 1024 * 1024; ret = hsaKmtAllocMemory(allocNode, size, memFlags, reinterpret_cast(&pDb)); @@ -676,12 +674,9 @@ void KFDMemoryTest::BinarySearchLargestBuffer(int allocNode, const HsaMemFlags & } /* Code snippet to allow CRIU checkpointing */ - iter++; - if (iter == 3) { - if (g_SleepTime > 0) { - LOG() << "Pause for: " << g_SleepTime << " seconds" << std::endl; - sleep(g_SleepTime); - } + if (g_SleepTime > 0) { + LOG() << "Pause for: " << g_SleepTime << " seconds" << std::endl; + sleep(g_SleepTime); } ret = hsaKmtMapMemoryToGPUNodes(pDb, size, NULL, @@ -694,17 +689,16 @@ void KFDMemoryTest::BinarySearchLargestBuffer(int allocNode, const HsaMemFlags & EXPECT_SUCCESS(hsaKmtUnmapMemoryToGPU(pDb)); EXPECT_SUCCESS(hsaKmtFreeMemory(pDb, size)); - lowMB = sizeMB; + if (lastSizeMB) + *lastSizeMB = sizeMB; + break; } - - if (lastSizeMB) - *lastSizeMB = lowMB; } /* * Largest*BufferTest allocates, maps/unmaps, and frees the largest possible * buffers. Its size is found using binary search in the range - * (0, RAM SIZE) with a granularity of 128M. Also, the similar logic is + * (0, RAM SIZE) with a granularity of 8M. Also, the similar logic is * repeated on local buffers (VRAM). * Please note we limit the largest possible system buffer to be smaller than * the RAM size. The reason is that the system buffer can make use of virtual @@ -733,7 +727,7 @@ TEST_F(KFDMemoryTest, LargestSysBufferTest) { LOG() << "Found System Memory of " << std::dec << sysMemSizeMB << "MB" << std::endl; - BinarySearchLargestBuffer(0, m_MemoryFlags, sysMemSizeMB, defaultGPUNode, + SearchLargestBuffer(0, m_MemoryFlags, sysMemSizeMB, defaultGPUNode, &lastTestedSizeMB); LOG() << "The largest allocated system buffer is " << std::dec @@ -764,7 +758,7 @@ TEST_F(KFDMemoryTest, LargestVramBufferTest) { LOG() << "Found VRAM of " << std::dec << vramSizeMB << "MB." << std::endl; - BinarySearchLargestBuffer(defaultGPUNode, memFlags, vramSizeMB, defaultGPUNode, + SearchLargestBuffer(defaultGPUNode, memFlags, vramSizeMB, defaultGPUNode, &lastTestedSizeMB); LOG() << "The largest allocated VRAM buffer is " << std::dec diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/tests/kfdtest/src/KFDMemoryTest.hpp index 3417a4eb30..f93ad5aa68 100644 --- a/tests/kfdtest/src/KFDMemoryTest.hpp +++ b/tests/kfdtest/src/KFDMemoryTest.hpp @@ -39,7 +39,7 @@ class KFDMemoryTest : public KFDBaseComponentTest { virtual void TearDown(); protected: - void BinarySearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, + void SearchLargestBuffer(int allocNode, const HsaMemFlags &memFlags, HSAuint64 highMB, int nodeToMap, HSAuint64 *lastSizeMB); void AcquireReleaseTestRunCPU(HSAuint32 acquireNode, bool scalar); From 3dd98d075f5a72d9bd23fa5a9db529770315d45b Mon Sep 17 00:00:00 2001 From: David Belanger Date: Mon, 13 Nov 2023 12:22:21 -0500 Subject: [PATCH 1221/1247] kfdtest: Updated CWSR test for emulation Added global flag g_IsEmuMode and set it when running under emulator. Adjusted delays in KFDCWSRTest for emulator. Change-Id: Ia5c0be40816ac2219add943e306ee16438f5b852 Signed-off-by: David Belanger --- tests/kfdtest/src/KFDCWSRTest.cpp | 8 +++++--- tests/kfdtest/src/KFDTestFlags.hpp | 1 + tests/kfdtest/src/KFDTestMain.cpp | 8 ++++++++ tests/kfdtest/src/KFDTestUtil.cpp | 7 +++++++ tests/kfdtest/src/KFDTestUtil.hpp | 2 ++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/tests/kfdtest/src/KFDCWSRTest.cpp index 743b03d1b3..dc7b776de8 100644 --- a/tests/kfdtest/src/KFDCWSRTest.cpp +++ b/tests/kfdtest/src/KFDCWSRTest.cpp @@ -80,6 +80,8 @@ TEST_P(KFDCWSRTest, BasicTest) { int num_witems = std::get<0>(GetParam()); int cwsr_thresh = std::get<1>(GetParam()); int defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + // Increase delay on emulator by this factor. + const int delayMult = (g_IsEmuMode ? 20 : 1); if ((m_FamilyId >= FAMILY_VI) && (checkCWSREnabled())) { HsaMemoryBuffer isaBuffer(PAGE_SIZE, defaultGPUNode, true, false, true); @@ -106,7 +108,7 @@ TEST_P(KFDCWSRTest, BasicTest) { dispatch.SetDim(num_witems, 1, 1); dispatch.Submit(queue); - Delay(5); + Delay(5 * delayMult); LOG() << "Starting iteration for " << std::dec << num_witems << " work items(s) (targeting " << std::dec << cwsr_thresh @@ -117,12 +119,12 @@ TEST_P(KFDCWSRTest, BasicTest) { // Send dequeue request EXPECT_SUCCESS(queue.Update(0, BaseQueue::DEFAULT_PRIORITY, false)); - Delay(5); + Delay(5 * delayMult); // Send requeue request EXPECT_SUCCESS(queue.Update(100, BaseQueue::DEFAULT_PRIORITY, false)); - Delay(50); + Delay(50 * delayMult); // Check for reg mangling for (int i = 0; i < num_witems; i++) { diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/tests/kfdtest/src/KFDTestFlags.hpp index 00a74fd6c8..ea0ba903b8 100644 --- a/tests/kfdtest/src/KFDTestFlags.hpp +++ b/tests/kfdtest/src/KFDTestFlags.hpp @@ -30,6 +30,7 @@ extern unsigned int g_TestTimeOut; extern int g_TestNodeId; extern int g_TestDstNodeId; extern bool g_IsChildProcess; +extern bool g_IsEmuMode; // Each test should call TEST_START with the test custom profile and HW scheduling enum TESTPROFILE{ diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/tests/kfdtest/src/KFDTestMain.cpp index 59c0dc806f..022b2df152 100644 --- a/tests/kfdtest/src/KFDTestMain.cpp +++ b/tests/kfdtest/src/KFDTestMain.cpp @@ -53,6 +53,7 @@ unsigned int g_TestTimeOut; int g_TestNodeId; int g_TestDstNodeId; bool g_IsChildProcess; +bool g_IsEmuMode; unsigned int g_SleepTime; unsigned int g_TestGPUFamilyId; class KFDBaseComponentTest *g_baseTest; @@ -90,8 +91,15 @@ GTEST_API_ int main(int argc, char **argv) { g_TestNodeId = args.NodeId; g_TestDstNodeId = args.DstNodeId; + g_IsEmuMode = CheckEmuModeEnabled(); + LOG() << "Profile: " << (TESTPROFILE)g_TestRunProfile << std::endl; LOG() << "HW capabilities: 0x" << std::hex << g_TestENVCaps << std::endl; + if (g_IsEmuMode) + { + LOG() << "Emulation Mode Enabled" << std::endl; + } + if (g_SleepTime > 0) { LOG() << "Sleep time in seconds as specified by user: " << std::dec << g_SleepTime << std::endl; } diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/tests/kfdtest/src/KFDTestUtil.cpp index 6659bc036b..c3ee307b0f 100644 --- a/tests/kfdtest/src/KFDTestUtil.cpp +++ b/tests/kfdtest/src/KFDTestUtil.cpp @@ -95,6 +95,13 @@ void SplitU64(const HSAuint64 value, unsigned int& rLoPart, unsigned int& rHiPar rHiPart = static_cast(value >> 32); } +bool CheckEmuModeEnabled() +{ + uint32_t emu_mode = false; + fscanf_dec("/sys/module/amdgpu/parameters/emu_mode", &emu_mode); + return (emu_mode != 0); +} + bool GetHwCapabilityHWS() { unsigned int value = 0; bool valExists = ReadDriverConfigValue(CONFIG_HWS, value); diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index 534a74a456..aab10a9ae4 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -44,6 +44,8 @@ bool WaitOnValue(const volatile unsigned int *buf, unsigned int value, unsigned void SplitU64(const HSAuint64 value, unsigned int& rLoPart, unsigned int& rHiPart); +bool CheckEmuModeEnabled(); + bool GetHwCapabilityHWS(); HSAKMT_STATUS CreateQueueTypeEvent(bool ManualReset, bool IsSignaled, unsigned int NodeId, HsaEvent** Event); From 858dfd364ff0320c97a592a866765279f86611a4 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Sun, 25 Feb 2024 10:45:36 -0500 Subject: [PATCH 1222/1247] libhsakmt: Use correct gpu_id for GPU system memory For GTT memory allocation if GPU is provided honour it. Change-Id: Iea9a26bc44cd3daa2337845f53dc430787b0643b Signed-off-by: Harish Kasiviswanathan --- src/fmm.c | 23 +++++++++++++++-------- src/fmm.h | 2 +- src/memory.c | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index ffb09080b9..4b78bba0ed 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1765,7 +1765,7 @@ static int bind_mem_to_numa(uint32_t node_id, void *mem, return 0; } -static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, +static void *fmm_allocate_host_gpu(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags mflags) { manageable_aperture_t *aperture; @@ -1774,7 +1774,8 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, uint64_t mmap_offset; int32_t gpu_drm_fd; uint32_t ioc_flags; - uint32_t gpu_id; + uint32_t preferred_gpu_id; + int gpu_mem_id = 0; /* default to g_first_gpu_mem */ uint64_t size; void *mem; @@ -1786,8 +1787,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, if (!g_first_gpu_mem) return NULL; - gpu_id = g_first_gpu_mem->gpu_id; - gpu_drm_fd = g_first_gpu_mem->drm_render_fd; + if (gpu_id) { + gpu_mem_id = gpu_mem_find_by_gpu_id(gpu_id); + if (gpu_mem_id < 0) + return NULL; + } + + preferred_gpu_id = gpu_mem[gpu_mem_id].gpu_id; + gpu_drm_fd = gpu_mem[gpu_mem_id].drm_render_fd; size = MemorySizeInBytes; ioc_flags = 0; @@ -1837,14 +1844,14 @@ static void *fmm_allocate_host_gpu(uint32_t node_id, void *address, /* Create userptr BO */ mmap_offset = (uint64_t)mem; ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_USERPTR; - vm_obj = fmm_allocate_memory_object(gpu_id, mem, size, + vm_obj = fmm_allocate_memory_object(preferred_gpu_id, mem, size, aperture, &mmap_offset, ioc_flags); if (!vm_obj) goto out_release_area; } else { ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; - mem = __fmm_allocate_device(gpu_id, address, size, aperture, + mem = __fmm_allocate_device(preferred_gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); if (mem && mflags.ui32.HostAccess) { @@ -1886,11 +1893,11 @@ out_release_area: return NULL; } -void *fmm_allocate_host(uint32_t node_id, void *address, +void *fmm_allocate_host(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags mflags) { if (is_dgpu) - return fmm_allocate_host_gpu(node_id, address, MemorySizeInBytes, mflags); + return fmm_allocate_host_gpu(gpu_id, node_id, address, MemorySizeInBytes, mflags); return fmm_allocate_host_cpu(address, MemorySizeInBytes, mflags); } diff --git a/src/fmm.h b/src/fmm.h index b8c9b84bfd..854ae1ba2c 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -54,7 +54,7 @@ void *fmm_allocate_scratch(uint32_t gpu_id, void *address, uint64_t MemorySizeIn void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags); void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, uint64_t doorbell_offset); -void *fmm_allocate_host(uint32_t node_id, void *address, uint64_t MemorySizeInBytes, +void *fmm_allocate_host(uint32_t gpu_id, uint32_t node_id, void *address, uint64_t MemorySizeInBytes, HsaMemFlags flags); void fmm_print(uint32_t node); HSAKMT_STATUS fmm_release(void *address); diff --git a/src/memory.c b/src/memory.c index ba33abda3e..b95fb1b303 100644 --- a/src/memory.c +++ b/src/memory.c @@ -163,7 +163,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, if (zfb_support && gpu_id && MemFlags.ui32.NonPaged == 1) MemFlags.ui32.CoarseGrain = 1; - *MemoryAddress = fmm_allocate_host(PreferredNode, *MemoryAddress, + *MemoryAddress = fmm_allocate_host(gpu_id, PreferredNode, *MemoryAddress, SizeInBytes, MemFlags); if (!(*MemoryAddress)) { From 191caf46ac962e6fb93bb32ce449b6154256b034 Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Sun, 25 Feb 2024 17:37:56 -0500 Subject: [PATCH 1223/1247] libhsakmt: Move global zfb_support to globals.c zfb_support needs to accessed from multiple places, so move to globals.c file Change-Id: I40b487c26a13e7cc6fc01b671d6166e7114e02d2 Signed-off-by: Harish Kasiviswanathan --- src/globals.c | 2 ++ src/libhsakmt.h | 1 + src/memory.c | 2 -- src/openclose.c | 3 --- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/globals.c b/src/globals.c index c2fbd4f02e..771eb6758b 100644 --- a/src/globals.c +++ b/src/globals.c @@ -37,3 +37,5 @@ int PAGE_SHIFT; /* whether to check all dGPUs in the topology support SVM API */ bool is_svm_api_supported; +/* zfb is mainly used during emulation */ +int zfb_support; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 656800dae6..62f65d01a4 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -38,6 +38,7 @@ extern bool hsakmt_forked; extern pthread_mutex_t hsakmt_mutex; extern bool is_dgpu; extern bool is_svm_api_supported; +extern int zfb_support; extern HsaVersionInfo kfd_version_info; diff --git a/src/memory.c b/src/memory.c index b95fb1b303..5eeaa9fecc 100644 --- a/src/memory.c +++ b/src/memory.c @@ -34,8 +34,6 @@ #include #include "fmm.h" -extern int zfb_support; - HSAKMT_STATUS HSAKMTAPI hsaKmtSetMemoryPolicy(HSAuint32 Node, HSAuint32 DefaultPolicy, HSAuint32 AlternatePolicy, diff --git a/src/openclose.c b/src/openclose.c index bc513c6a03..0e5c10c065 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -49,9 +49,6 @@ static pid_t parent_pid = -1; int hsakmt_debug_level; bool hsakmt_forked; -/* zfb is mainly used during emulation */ -int zfb_support; - /* is_forked_child detects when the process has forked since the last * time this function was called. We cannot rely on pthread_atfork * because the process can fork without calling the fork function in From 341ecaf1d9c32275ee261ed0365702ebc0dd83ef Mon Sep 17 00:00:00 2001 From: Harish Kasiviswanathan Date: Mon, 26 Feb 2024 14:10:12 -0500 Subject: [PATCH 1224/1247] libhsakmt: Associate correct GPU with queue memory Pass the correct gpu_id to KFD for system memory that is allocated for the queue and eop buffer Change-Id: I43bb6333560a7d9d38293c191303161ab1443b5d Signed-off-by: Harish Kasiviswanathan --- src/events.c | 2 +- src/libhsakmt.h | 1 + src/queues.c | 55 ++++++++++++++++++++++++++++++------------------- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/events.c b/src/events.c index 60e9f6dbbf..07356db67e 100644 --- a/src/events.c +++ b/src/events.c @@ -76,7 +76,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, if (is_dgpu && !events_page) { events_page = allocate_exec_aligned_memory_gpu( - KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, true, false, true); + KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, 0, true, false, true); if (!events_page) { pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 62f65d01a4..0f1943445b 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -195,6 +195,7 @@ bool topology_is_svm_needed(HSA_ENGINE_ID EngineId); HSAuint32 PageSizeFromFlags(unsigned int pageSizeFlags); void* allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, + uint32_t gpu_id, uint32_t NodeId, bool NonPaged, bool DeviceLocal, bool Uncached); void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align); diff --git a/src/queues.c b/src/queues.c index f9e561267f..e6e096350d 100644 --- a/src/queues.c +++ b/src/queues.c @@ -310,15 +310,14 @@ static bool update_ctx_save_restore_size(uint32_t nodeid, struct queue *q) return false; } -void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, +void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, uint32_t gpu_id, uint32_t NodeId, bool nonPaged, bool DeviceLocal, bool Uncached) { - void *mem; + void *mem = NULL; HSAuint64 gpu_va; HsaMemFlags flags; - HSAKMT_STATUS ret; HSAuint32 cpu_id = 0; flags.Value = 0; @@ -329,22 +328,32 @@ void *allocate_exec_aligned_memory_gpu(uint32_t size, uint32_t align, flags.ui32.CoarseGrain = DeviceLocal; flags.ui32.Uncached = Uncached; - /* Get the closest cpu_id to GPU NodeId for system memory allocation - * nonPaged=1 system memory allocation uses GTT path - */ - if (!DeviceLocal && !nonPaged) { - cpu_id = get_direct_link_cpu(NodeId); - if (cpu_id == INVALID_NODEID) { - flags.ui32.NoNUMABind = 1; - cpu_id = 0; - } - } - size = ALIGN_UP(size, align); - ret = hsaKmtAllocMemory(DeviceLocal ? NodeId : cpu_id, size, flags, &mem); - if (ret != HSAKMT_STATUS_SUCCESS) + if (DeviceLocal && !zfb_support) + mem = fmm_allocate_device(gpu_id, NodeId, mem, size, flags); + else { + /* VRAM under ZFB mode should be supported here without any + * additional code + */ + /* Get the closest cpu_id to GPU NodeId for system memory allocation + * nonPaged=0 system memory allocation uses GTT path + */ + if (!nonPaged) { + cpu_id = get_direct_link_cpu(NodeId); + if (cpu_id == INVALID_NODEID) { + flags.ui32.NoNUMABind = 1; + cpu_id = 0; + } + } + mem = fmm_allocate_host(gpu_id, cpu_id, mem, size, flags); + } + + if (!mem) { + pr_err("Alloc %s memory failed size %d\n", + DeviceLocal ? "VRAM" : "GTT", size); return NULL; + } if (NodeId != 0) { uint32_t nodes_array[1] = {NodeId}; @@ -376,13 +385,14 @@ void free_exec_aligned_memory_gpu(void *addr, uint32_t size, uint32_t align) */ static void *allocate_exec_aligned_memory(uint32_t size, bool use_ats, + uint32_t gpu_id, uint32_t NodeId, bool nonPaged, bool DeviceLocal, bool Uncached) { if (!use_ats) - return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, NodeId, + return allocate_exec_aligned_memory_gpu(size, PAGE_SIZE, gpu_id, NodeId, nonPaged, DeviceLocal, Uncached); return allocate_exec_aligned_memory_cpu(size); @@ -469,6 +479,7 @@ static inline void fill_cwsr_header(struct queue *q, void *addr, static int handle_concrete_asic(struct queue *q, struct kfd_ioctl_create_queue_args *args, + uint32_t gpu_id, uint32_t NodeId, HsaEvent *Event, volatile HSAint64 *ErrPayload) @@ -480,8 +491,9 @@ static int handle_concrete_asic(struct queue *q, return HSAKMT_STATUS_SUCCESS; if (q->eop_buffer_size > 0) { + pr_info("Allocating VRAM for EOP\n"); q->eop_buffer = allocate_exec_aligned_memory(q->eop_buffer_size, - q->use_ats, + q->use_ats, gpu_id, NodeId, true, true, /* Unused for VRAM */false); if (!q->eop_buffer) return HSAKMT_STATUS_NO_MEMORY; @@ -516,6 +528,7 @@ static int handle_concrete_asic(struct queue *q, void *addr; HSAKMT_STATUS r = HSAKMT_STATUS_ERROR; + pr_info("Allocating GTT for CWSR\n"); addr = mmap_allocate_aligned(PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, size, GPU_HUGE_PAGE_SIZE, 0, @@ -548,7 +561,7 @@ static int handle_concrete_asic(struct queue *q, if (!q->unified_ctx_save_restore) { q->ctx_save_restore = allocate_exec_aligned_memory( q->total_mem_alloc_size, - q->use_ats, NodeId, + q->use_ats, gpu_id, NodeId, false, false, false); if (!q->ctx_save_restore) @@ -597,7 +610,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, return result; struct queue *q = allocate_exec_aligned_memory(sizeof(*q), - false, NodeId, true, false, true); + false, gpu_id, NodeId, true, false, true); if (!q) return HSAKMT_STATUS_NO_MEMORY; @@ -652,7 +665,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateQueue(HSAuint32 NodeId, QueueResource->QueueWptrValue = (uintptr_t)&q->wptr; } - err = handle_concrete_asic(q, &args, NodeId, Event, QueueResource->ErrorReason); + err = handle_concrete_asic(q, &args, gpu_id, NodeId, Event, QueueResource->ErrorReason); if (err != HSAKMT_STATUS_SUCCESS) { free_queue(q); return err; From 9a89997b5f4047dbf3a288ac9a94860d6e8a8cdf Mon Sep 17 00:00:00 2001 From: "Sv. Lockal" Date: Sat, 9 Mar 2024 13:46:32 +0000 Subject: [PATCH 1225/1247] Fix compilation on musl-based systems This allows to build ROCT-Thunk-Interface for Alpine Linux, Gentoo with musl profile and so on. List of changes: * Fix redefinition of PAGE_SIZE from limits.h * Use NAME_MAX from limits.h Closes #65 Change-Id: Ibdb0ef5668a07b7b403fcc4a44cd2658e00a584a Signed-off-by: Sv. Lockal Signed-off-by: Kent Russell --- src/globals.c | 4 ++++ src/libhsakmt.h | 4 ++++ src/openclose.c | 2 ++ src/topology.c | 5 +++++ 4 files changed, 15 insertions(+) diff --git a/src/globals.c b/src/globals.c index 771eb6758b..2afd09011a 100644 --- a/src/globals.c +++ b/src/globals.c @@ -32,7 +32,11 @@ unsigned long kfd_open_count; unsigned long system_properties_count; pthread_mutex_t hsakmt_mutex = PTHREAD_MUTEX_INITIALIZER; bool is_dgpu; + +#ifndef PAGE_SIZE int PAGE_SIZE; +#endif + int PAGE_SHIFT; /* whether to check all dGPUs in the topology support SVM API */ diff --git a/src/libhsakmt.h b/src/libhsakmt.h index 0f1943445b..fd298e2832 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -64,7 +64,11 @@ extern HsaVersionInfo kfd_version_info; do { if ((minor) > kfd_version_info.KernelInterfaceMinorVersion)\ return HSAKMT_STATUS_NOT_SUPPORTED; } while (0) +/* Might be defined in limits.h on platforms where it is constant (used by musl) */ +/* See also: https://pubs.opengroup.org/onlinepubs/7908799/xsh/limits.h.html */ +#ifndef PAGE_SIZE extern int PAGE_SIZE; +#endif extern int PAGE_SHIFT; /* VI HW bug requires this virtual address alignment */ diff --git a/src/openclose.c b/src/openclose.c index 0e5c10c065..6d66bdfde9 100644 --- a/src/openclose.c +++ b/src/openclose.c @@ -114,7 +114,9 @@ static void clear_after_fork(void) static inline void init_page_size(void) { +#ifndef PAGE_SIZE PAGE_SIZE = sysconf(_SC_PAGESIZE); +#endif PAGE_SHIFT = ffs(PAGE_SIZE) - 1; } diff --git a/src/topology.c b/src/topology.c index 7e655e362e..4738499fce 100644 --- a/src/topology.c +++ b/src/topology.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -1372,6 +1373,10 @@ static int topology_create_temp_cpu_cache_list(int node, * which can be present twice in the string above. 29 is for the prefix * and the +6 is for the cache suffix */ +#ifndef MAXNAMLEN +/* MAXNAMLEN is the BSD name for NAME_MAX. glibc aliases this as NAME_MAX, but not musl */ +#define MAXNAMLEN NAME_MAX +#endif const uint32_t MAXPATHSIZE = 29 + MAXNAMLEN + (MAXNAMLEN + 6); cpu_cacheinfo_t *p_temp_cpu_ci_list; /* a list of cpu_ci */ char path[MAXPATHSIZE], node_dir[MAXPATHSIZE]; From 7c10e1e4f5f54f19eb88038b853c9f255e560404 Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Tue, 12 Mar 2024 13:29:02 -0700 Subject: [PATCH 1226/1247] Link CMAKE_DL_LIBS. Was failing to link on AlmaLinux8. Change-Id: Id7df245f1063c2bebd0f07efc352f1b9017eda0e Signed-off-by: Stella Laurenzo Signed-off-by: Kent Russell --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1263efbcca..1db2b27601 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,7 +176,7 @@ include_directories(${DRM_AMDGPU_INCLUDE_DIRS}) include_directories(${DRM_INCLUDE_DIRS}) target_link_libraries ( ${HSAKMT_TARGET} - PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt c numa + PRIVATE ${DRM_LDFLAGS} ${DRM_AMDGPU_LDFLAGS} pthread rt c numa ${CMAKE_DL_LIBS} ) target_compile_options(${HSAKMT_TARGET} PRIVATE ${DRM_CFLAGS} ${HSAKMT_C_FLAGS}) From 0c06bec272533ca67e93c0aaf5b8af3b86b946e6 Mon Sep 17 00:00:00 2001 From: Xinmudotmoe Date: Wed, 13 Mar 2024 15:14:58 +0800 Subject: [PATCH 1227/1247] Update doorbell->size to support large default PAGE_SIZE kernels Change-Id: I6fbe10ae3309f2f935d6782366cedb56dc1438c3 Signed-off-by: Xinmudotmoe Signed-off-by: Kent Russell --- src/queues.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/queues.c b/src/queues.c index e6e096350d..29bffce03f 100644 --- a/src/queues.c +++ b/src/queues.c @@ -135,6 +135,10 @@ static void get_doorbell_map_info(uint32_t node_id, doorbell->use_gpuvm = (is_dgpu && gfxv != GFX_VERSION_TONGA); doorbell->size = DOORBELLS_PAGE_SIZE(DOORBELL_SIZE(gfxv)); + if (doorbell->size < (uint32_t) PAGE_SIZE) { + doorbell->size = PAGE_SIZE; + } + return; } From 66e3a09a420ce8250ebedd3468f38c6ca6bfbfee Mon Sep 17 00:00:00 2001 From: David Belanger Date: Fri, 16 Feb 2024 10:21:53 -0500 Subject: [PATCH 1228/1247] kfdtest: Update PM4EventInterrupt for emulation Reduce number of iteration for test case to run in reasonable amount of time. Signed-off-by: David Belanger Change-Id: I19a7ec0d5f03c54d6691aae3cf7432754c7481cc --- tests/kfdtest/src/KFDQMTest.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index ee01300e4c..3364d586cb 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -1663,6 +1663,9 @@ TEST_F(KFDQMTest, PM4EventInterrupt) { PM4ReleaseMemoryPacket(m_FamilyId, 0, 0, 0).SizeInBytes(); const int queueSize = RoundToPowerOf2(totalPacketSize); + /* Reduce number of iteration if running with emulator. */ + const int numIter = (g_IsEmuMode ? 32 : 1024); + /* 4 PM4 queues will be running at same time.*/ const int numPM4Queue = 4; HsaEvent *event[numPM4Queue]; @@ -1676,7 +1679,7 @@ TEST_F(KFDQMTest, PM4EventInterrupt) { } /* A simple loop here to give more pressure.*/ - for (int test_count = 0; test_count < 1024; test_count++) { + for (int test_count = 0; test_count < numIter; test_count++) { for (int i = 0; i < numPM4Queue; i++) { ASSERT_SUCCESS(queue[i].Create(defaultGPUNode, queueSize)); ASSERT_SUCCESS(CreateQueueTypeEvent(false, false, defaultGPUNode, &event[i])); From e738648c8fea289819f533d8d7d939919d97b0ea Mon Sep 17 00:00:00 2001 From: David Belanger Date: Thu, 15 Feb 2024 12:43:06 -0500 Subject: [PATCH 1229/1247] kfdtest: Update QueuePriority* tests for emulation Reduce test case size if running on emulator. Also, refactor code as both test cases had more than 80% same code. Signed-off-by: David Belanger Change-Id: I5899ee24244a6f0aa6b56fa8a4701b0b1e344b9f --- tests/kfdtest/src/KFDQMTest.cpp | 104 ++++++++++---------------------- tests/kfdtest/src/KFDQMTest.hpp | 1 + 2 files changed, 33 insertions(+), 72 deletions(-) diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/tests/kfdtest/src/KFDQMTest.cpp index 3364d586cb..9166e76ab9 100644 --- a/tests/kfdtest/src/KFDQMTest.cpp +++ b/tests/kfdtest/src/KFDQMTest.cpp @@ -865,64 +865,7 @@ TEST_F(KFDQMTest, BasicCuMaskingEven) { TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { TEST_START(TESTPROFILE_RUNALL); - if (m_FamilyId < FAMILY_VI) { - LOG() << "Skipping test: Shader won't run on CI." << std::endl; - return; - } - - int node = m_NodeInfo.HsaDefaultGPUNode(); - ASSERT_GE(node, 0) << "failed to get default GPU Node"; - HsaMemoryBuffer syncBuf(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); - HSAint32 *syncBuffer = syncBuf.As(); - HsaMemoryBuffer isaBuffer(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); - - ASSERT_SUCCESS(m_pAsm->RunAssembleBuf(LoopIsa, isaBuffer.As())); - - Dispatch dispatch[2] = { - Dispatch(isaBuffer, true), - Dispatch(isaBuffer, true) - }; - - int activeTaskBitmap = 0x3; - HSAuint64 startTime, endTime[2]; - HsaEvent *pHsaEvent[2]; - int numEvent = 2; - PM4Queue queue[2]; - HSA_QUEUE_PRIORITY priority[2] = { - HSA_QUEUE_PRIORITY_LOW, - HSA_QUEUE_PRIORITY_HIGH - }; - int i; - - for (i = 0; i < 2; i++) { - syncBuffer[i] = -1; - ASSERT_SUCCESS(queue[i].Create(node)); - queue[i].Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, priority[i], false); - pHsaEvent[i] = dispatch[i].GetHsaEvent(); - pHsaEvent[i]->EventData.EventData.SyncVar.SyncVar.UserData = &syncBuffer[i]; - dispatch[i].SetDim(1024, 16, 16); - } - - startTime = GetSystemTickCountInMicroSec(); - for (i = 0; i < 2; i++) - dispatch[i].Submit(queue[i]); - - while (activeTaskBitmap > 0) { - hsaKmtWaitOnMultipleEvents(pHsaEvent, numEvent, false, g_TestTimeOut); - for (i = 0; i < 2; i++) { - if ((activeTaskBitmap & (1 << i)) && (syncBuffer[i] == pHsaEvent[i]->EventId)) { - endTime[i] = GetSystemTickCountInMicroSec(); - activeTaskBitmap &= ~(1 << i); - } - } - } - - for (i = 0; i < 2; i++) { - EXPECT_SUCCESS(queue[i].Destroy()); - int usecs = endTime[i] - startTime; - LOG() << "Task priority: " << std::dec << priority[i] << "\t"; - LOG() << "Task duration: " << std::dec << usecs << "usecs" << std::endl; - } + testQueuePriority(false); TEST_END } @@ -930,11 +873,22 @@ TEST_F(KFDQMTest, QueuePriorityOnDifferentPipe) { TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { TEST_START(TESTPROFILE_RUNALL); + testQueuePriority(true); + + TEST_END +} + +void KFDQMTest::testQueuePriority(bool isSamePipe) +{ if (m_FamilyId < FAMILY_VI) { LOG() << "Skipping test: Shader won't run on CI." << std::endl; return; } + // Reduce test case if running on emulator + // Reduction applies to all 3 dims (effect is cubic) + const int scaleDown = (g_IsEmuMode ? 4 : 1); + int node = m_NodeInfo.HsaDefaultGPUNode(); ASSERT_GE(node, 0) << "failed to get default GPU Node"; HsaMemoryBuffer syncBuf(PAGE_SIZE, node, true/*zero*/, false/*local*/, true/*exec*/); @@ -948,33 +902,41 @@ TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { Dispatch(isaBuffer, true) }; + const int queueCount = isSamePipe ? 13 : 2; int activeTaskBitmap = 0x3; HSAuint64 startTime, endTime[2]; HsaEvent *pHsaEvent[2]; int numEvent = 2; - PM4Queue queue[13]; + PM4Queue queue[queueCount]; HSA_QUEUE_PRIORITY priority[2] = { HSA_QUEUE_PRIORITY_LOW, HSA_QUEUE_PRIORITY_HIGH }; int i; - /* queue[2..12] are dummy queues. Create queue in this sequence to - * render queue[0] and queue[1] on same pipe with no assumptions - * about the number of pipes used by KFD. Queue #12 is a multiple - * of 1, 2, 3 and 4, so it falls on pipe 0 for any number of pipes + /* + * For different pipe variation: + * Only two queues are created, they should be on two different pipes. + * + * For same pipe variation: + * queue[2..12] are dummy queues. Create queue in this sequence to + * render queue[0] and queue[1] on same pipe with no assumptions + * about the number of pipes used by KFD. Queue #12 is a multiple + * of 1, 2, 3 and 4, so it falls on pipe 0 for any number of pipes */ - EXPECT_SUCCESS(queue[0].Create(node)); // Queue 0 is on Pipe 0 - for (i = 2; i <= 12; i++) - EXPECT_SUCCESS(queue[i].Create(node)); - EXPECT_SUCCESS(queue[1].Create(node)); // Queue 12 is on Pipe 0 + EXPECT_SUCCESS(queue[0].Create(node)); + if (isSamePipe) { + for (i = 2; i < queueCount; i++) + EXPECT_SUCCESS(queue[i].Create(node)); + } + EXPECT_SUCCESS(queue[1].Create(node)); for (i = 0; i < 2; i++) { syncBuffer[i] = -1; queue[i].Update(BaseQueue::DEFAULT_QUEUE_PERCENTAGE, priority[i], false); pHsaEvent[i] = dispatch[i].GetHsaEvent(); pHsaEvent[i]->EventData.EventData.SyncVar.SyncVar.UserData = &syncBuffer[i]; - dispatch[i].SetDim(1024, 16, 16); + dispatch[i].SetDim(1024 / scaleDown , 16 / scaleDown, 16 / scaleDown); } startTime = GetSystemTickCountInMicroSec(); @@ -994,14 +956,12 @@ TEST_F(KFDQMTest, QueuePriorityOnSamePipe) { for (i = 0; i < 2; i++) { int usecs = endTime[i] - startTime; LOG() << "Task priority: " << std::dec << priority[i] << "\t"; - LOG() << "Task duration: " << std::dec << usecs << "usecs" << std::endl; + LOG() << "Task duration: " << std::dec << std::setw(10) << usecs << " usecs" << std::endl; } - for (i = 0; i <= 12; i++) { + for (i = 0; i < queueCount; i++) { EXPECT_SUCCESS(queue[i].Destroy()); } - - TEST_END } diff --git a/tests/kfdtest/src/KFDQMTest.hpp b/tests/kfdtest/src/KFDQMTest.hpp index dfc36d17d4..41d65ceb0a 100644 --- a/tests/kfdtest/src/KFDQMTest.hpp +++ b/tests/kfdtest/src/KFDQMTest.hpp @@ -43,6 +43,7 @@ class KFDQMTest : public KFDBaseComponentTest { void SyncDispatch(const HsaMemoryBuffer& isaBuffer, void* pSrcBuf, void* pDstBuf, int node = -1); HSAint64 TimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count); HSAint64 GetAverageTimeConsumedwithCUMask(int node, uint32_t *mask, uint32_t mask_count, int iterations); + void testQueuePriority(bool isSamePipe); protected: // Members /* Acceptable performance for CU Masking should be within 5% of linearly-predicted performance */ const double CuVariance = 0.15; From 656234abb862066c6a2c627a288b543434842654 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Fri, 1 Mar 2024 14:02:38 -0500 Subject: [PATCH 1230/1247] kfdtest: Exclude KFDQMTest.QueueLatency test The purpose of this patch is to add KFDQMTest.QueueLatency to kfdtest.exclude file temporarily for the following ASIC filters: -GFX940 -GFX941 -GFX942 This test is failing due to an issue with the way it was coded, not due to an issue with the ASICs it is now blacklisted on. Signed-off-by: Ori Messinger Change-Id: Ic993629a2400449f598e73fe616a4572a38e2310 --- tests/kfdtest/scripts/kfdtest.exclude | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/tests/kfdtest/scripts/kfdtest.exclude index 2e3249a953..d3a8896e3a 100644 --- a/tests/kfdtest/scripts/kfdtest.exclude +++ b/tests/kfdtest/scripts/kfdtest.exclude @@ -323,18 +323,21 @@ FILTER[gfx940]=\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDMemoryTest.BigSysBufferStressTest:"\ "KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest" +"KFDIPCTest.BasicTest"\ +"KFDQMTest.QueueLatency" FILTER[gfx941]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDMemoryTest.BigSysBufferStressTest:"\ "KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest" +"KFDIPCTest.BasicTest"\ +"KFDQMTest.QueueLatency" FILTER[gfx942]=\ "$BLACKLIST_ALL_ASICS:"\ "KFDMemoryTest.LargestSysBufferTest:"\ "KFDMemoryTest.BigSysBufferStressTest:"\ "KFDMemoryTest.FlatScratchAccess:"\ -"KFDIPCTest.BasicTest" +"KFDIPCTest.BasicTest"\ +"KFDQMTest.QueueLatency" From 9fbe853feace66bd1d3e4105b71e75cea7123f9a Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 13 Mar 2024 10:55:54 -0400 Subject: [PATCH 1231/1247] libhsakmt: Add memory alloc flag GTTAccess To allocate GTT memory for MES AQL queue structure, KFD will create GART mapping for the memory to be accessed by MES. Change-Id: Iae7b33d1e70861109f1551d3a71dc60dfde9de61 Signed-off-by: Philip Yang --- include/hsakmttypes.h | 4 +++- src/memory.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 9f1b2ccb24..86cf248fa2 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -567,7 +567,9 @@ typedef struct _HsaMemFlags unsigned int NoAddress: 1; // only do vram allocation, return a handle, not allocate virtual address. unsigned int OnlyAddress: 1; // only do virtal address allocation without vram allocation. unsigned int ExtendedCoherent: 1; // system-scope coherence on atomic instructions - unsigned int Reserved: 11; + unsigned int GTTAccess: 1; // default = 0; If 1: The caller indicates this memory will be mapped to GART for MES + // KFD will allocate GTT memory with the Preferred_node set as gpu_id for GART mapping + unsigned int Reserved: 10; } ui32; HSAuint32 Value; diff --git a/src/memory.c b/src/memory.c index 5eeaa9fecc..f121726f77 100644 --- a/src/memory.c +++ b/src/memory.c @@ -152,7 +152,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, } /* GPU allocated system memory */ - if (!gpu_id || !MemFlags.ui32.NonPaged || zfb_support) { + if (!gpu_id || !MemFlags.ui32.NonPaged || zfb_support || MemFlags.ui32.GTTAccess) { /* Backwards compatibility hack: Allocate system memory if app * asks for paged memory from a GPU node. */ From 4f554988b69fe9aa6cb3930920a5b53c0b9fd5e8 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Fri, 21 Jul 2023 20:42:56 -0400 Subject: [PATCH 1232/1247] libhsakmt: Update Linux header for PC Sampling Add pc sampling support Change-Id: I2c472ce00ff8648904cf7e585687e81d3f493049 Signed-off-by: David Yat Sin Signed-off-by: James Zhu --- include/linux/kfd_ioctl.h | 65 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/include/linux/kfd_ioctl.h b/include/linux/kfd_ioctl.h index d3e6cee06d..f1fd54bfa4 100644 --- a/include/linux/kfd_ioctl.h +++ b/include/linux/kfd_ioctl.h @@ -40,9 +40,10 @@ * - 1.12 - Add DMA buf export ioctl * - 1.13 - Add debugger API * - 1.14 - Update kfd_event_data + * - 1.16 - Add PC Sampling ioctl */ #define KFD_IOCTL_MAJOR_VERSION 1 -#define KFD_IOCTL_MINOR_VERSION 14 +#define KFD_IOCTL_MINOR_VERSION 16 struct kfd_ioctl_get_version_args { __u32 major_version; /* from KFD */ @@ -1528,6 +1529,62 @@ struct kfd_ioctl_set_xnack_mode_args { __s32 xnack_enabled; }; +/** + * kfd_ioctl_pc_sample_op - PC Sampling ioctl operations + * + * @KFD_IOCTL_PCS_OP_QUERY_CAPABILITIES: Query device PC Sampling capabilities + * @KFD_IOCTL_PCS_OP_CREATE: Register this process with a per-device PC sampler instance + * @KFD_IOCTL_PCS_OP_DESTROY: Unregister from a previously registered PC sampler instance + * @KFD_IOCTL_PCS_OP_START: Process begins taking samples from a previously registered PC sampler instance + * @KFD_IOCTL_PCS_OP_STOP: Process stops taking samples from a previously registered PC sampler instance + */ +enum kfd_ioctl_pc_sample_op { + KFD_IOCTL_PCS_OP_QUERY_CAPABILITIES, + KFD_IOCTL_PCS_OP_CREATE, + KFD_IOCTL_PCS_OP_DESTROY, + KFD_IOCTL_PCS_OP_START, + KFD_IOCTL_PCS_OP_STOP, +}; + +/* Values have to be a power of 2*/ +#define KFD_IOCTL_PCS_FLAG_POWER_OF_2 0x00000001 + +enum kfd_ioctl_pc_sample_method { + KFD_IOCTL_PCS_METHOD_HOSTTRAP = 1, + KFD_IOCTL_PCS_METHOD_STOCHASTIC, +}; + +enum kfd_ioctl_pc_sample_type { + KFD_IOCTL_PCS_TYPE_TIME_US, + KFD_IOCTL_PCS_TYPE_CLOCK_CYCLES, + KFD_IOCTL_PCS_TYPE_INSTRUCTIONS +}; + +struct kfd_pc_sample_info { + __u64 interval; /* [IN] if PCS_TYPE_INTERVAL_US: sample interval in us + * if PCS_TYPE_CLOCK_CYCLES: sample interval in graphics core clk cycles + * if PCS_TYPE_INSTRUCTIONS: sample interval in instructions issued by + * graphics compute units + */ + __u64 interval_min; /* [OUT] */ + __u64 interval_max; /* [OUT] */ + __u64 flags; /* [OUT] indicate potential restrictions e.g FLAG_POWER_OF_2 */ + __u32 method; /* [IN/OUT] kfd_ioctl_pc_sample_method */ + __u32 type; /* [IN/OUT] kfd_ioctl_pc_sample_type */ +}; + +#define KFD_IOCTL_PCS_QUERY_TYPE_FULL (1 << 0) /* If not set, return current */ + +struct kfd_ioctl_pc_sample_args { + __u64 sample_info_ptr; /* array of kfd_pc_sample_info */ + __u32 num_sample_info; + __u32 op; /* kfd_ioctl_pc_sample_op */ + __u32 gpu_id; + __u32 trace_id; + __u32 flags; /* kfd_ioctl_pcs_query flags */ + __u32 reserved; +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -1636,7 +1693,7 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_CRIU_OP \ AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args) -#define AMDKFD_IOC_AVAILABLE_MEMORY \ +#define AMDKFD_IOC_AVAILABLE_MEMORY \ AMDKFD_IOWR(0x23, struct kfd_ioctl_get_available_memory_args) #define AMDKFD_IOC_EXPORT_DMABUF \ @@ -1664,8 +1721,10 @@ struct kfd_ioctl_set_xnack_mode_args { #define AMDKFD_IOC_RLC_SPM \ AMDKFD_IOWR(0x84, struct kfd_ioctl_spm_args) +#define AMDKFD_IOC_PC_SAMPLE \ + AMDKFD_IOWR(0x85, struct kfd_ioctl_pc_sample_args) #define AMDKFD_COMMAND_START_2 0x80 -#define AMDKFD_COMMAND_END_2 0x85 +#define AMDKFD_COMMAND_END_2 0x86 #endif From f94e2530fbc17241e32d625b963c2e0c29bbcb5f Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Fri, 21 Jul 2023 20:52:25 -0400 Subject: [PATCH 1233/1247] libhsakmt: add PC sampling support Add pc sampling support. Change-Id: I08199024ba5a8eb2845c048d499fc8fcd260d2e8 Signed-off-by: David Yat Sin Signed-off-by: James Zhu --- CMakeLists.txt | 3 +- include/hsakmt.h | 116 +++++++++++++++++++++++ include/hsakmttypes.h | 26 ++++++ src/libhsakmt.ver | 5 + src/pc_sampling.c | 210 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 359 insertions(+), 1 deletion(-) create mode 100644 src/pc_sampling.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 1db2b27601..19b97133af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,7 +130,8 @@ set ( HSAKMT_SRC "src/debug.c" "src/rbtree.c" "src/spm.c" "src/version.c" - "src/svm.c") + "src/svm.c" + "src/pc_sampling.c") ## Declare the library target name add_library ( ${HSAKMT_TARGET} "") diff --git a/include/hsakmt.h b/include/hsakmt.h index ce7b7e327c..7a37bec049 100644 --- a/include/hsakmt.h +++ b/include/hsakmt.h @@ -1044,6 +1044,122 @@ hsaKmtReturnAsanHeaderPage( void *addr // IN: Start of othe virtual address page ); +/** + Check whether kernel support pc sampling +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPcSamplingSupport( + void +); + +/** + * Query device PC Sampling capabilities + * + * Arguments: + * @NodeId (IN) - GPU node_id + * @sample_info (IN) - Pointer to array of HSAPcSamplingInfo + * @sample_info_sz(IN) - Size of sampling_info in units of HSAPcSamplingInfo + * @sz_needed (OUT)- If sampling_info_sz is too small, sample_info_sz needed + * + * Return: + * HSAKMT_STATUS_ERROR - failed + * HSAKMT_STATUS_SUCCESS - successfully complete + * HSAKMT_STATUS_INVALID_PARAMETER - invalid input + * HSAKMT_STATUS_BUFFER_TOO_SMALL - sample buffer size is too small. Retry with sample_info_sz + * >= sz_needed + * HSAKMT_STATUS_NOT_SUPPORTED - this asic doesn't support pc sampling +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPcSamplingQueryCapabilities( + HSAuint32 NodeId, + void *sample_info, + HSAuint32 sample_info_sz, + HSAuint32 *sz_needed +); + +/** + * Create PC Sampling Session + * + * Arguments: + * @NodeId (IN) - GPU node_id + * @sample_info(IN) - PC Sampling configuration requested + * @traceId (OUT) - Unique PC Sampling trace Id + * + * Return: + * HSAKMT_STATUS_ERROR - failed + * HSAKMT_STATUS_SUCCESS - successfully complete + * HSAKMT_STATUS_INVALID_PARAMETER - invalid input + * HSAKMT_STATUS_NO_MEMORY - not enough memory to create new pc sampling session + * HSAKMT_STATUS_UNAVAILABLE - a different pc sampling session started on this node +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPcSamplingCreate( + HSAuint32 node_id, + HsaPcSamplingInfo *sample_info, + HsaPcSamplingTraceId *traceId +); + +/** + * Destroy PC Sampling Session + * + * Arguments: + * @NodeId (IN) - GPU node_id + * @traceId(IN) - PC Sampling trace Id + * + * Return: + * HSAKMT_STATUS_ERROR - failed + * HSAKMT_STATUS_SUCCESS - successfully complete + * HSAKMT_STATUS_INVALID_PARAMETER - invalid input +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPcSamplingDestroy( + HSAuint32 NodeId, + HsaPcSamplingTraceId traceId +); + +/** + * Start PC Sampling Session + * + * Arguments: + * @NodeId (IN) - GPU node_id + * @traceId(IN) - PC Sampling trace Id + * + * Return: + * HSAKMT_STATUS_ERROR - failed + * HSAKMT_STATUS_SUCCESS - successfully complete + * HSAKMT_STATUS_INVALID_PARAMETER - invalid input +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPcSamplingStart( + HSAuint32 NodeId, + HsaPcSamplingTraceId traceId +); + +/** + * Stop PC Sampling Session + * + * Arguments: + * @NodeId (IN) - GPU node_id + * @traceId(IN) - PC Sampling trace Id + * + * Return: + * HSAKMT_STATUS_ERROR - failed + * HSAKMT_STATUS_SUCCESS - successfully complete + * HSAKMT_STATUS_INVALID_PARAMETER - invalid input + * HSAKMT_STATUS_KERNEL_ALREADY_OPENED - stop already +*/ +HSAKMT_STATUS +HSAKMTAPI +hsaKmtPcSamplingStop( + HSAuint32 NodeId, + HsaPcSamplingTraceId traceId +); + #ifdef __cplusplus } //extern "C" #endif diff --git a/include/hsakmttypes.h b/include/hsakmttypes.h index 86cf248fa2..f94efa11c3 100644 --- a/include/hsakmttypes.h +++ b/include/hsakmttypes.h @@ -1467,6 +1467,32 @@ typedef enum _HSA_SVM_UNMAP_TRIGGERS { typedef void *HsaAMDGPUDeviceHandle; +typedef HSAuint32 HsaPcSamplingTraceId; + +typedef enum _HSA_PC_SAMPLING_METHOD_KIND +{ + HSA_PC_SAMPLING_METHOD_KIND_HOSTTRAP_V1 = 1, + HSA_PC_SAMPLING_METHOD_KIND_STOCHASTIC_V1, +} HSA_PC_SAMPLING_METHOD_KIND; + +typedef enum _HSA_PC_SAMPLING_UNITS +{ + HSA_PC_SAMPLING_UNIT_INTERVAL_MICROSECONDS, + HSA_PC_SAMPLING_UNIT_INTERVAL_CYCLES, + HSA_PC_SAMPLING_UNIT_INTERVAL_INSTRUCTIONS, +} HSA_PC_SAMPLING_UNIT_INTERVAL; + +typedef struct _HsaPcSamplingInfo +{ + HSAuint64 value; + HSAuint64 value_min; + HSAuint64 value_max; + HSAuint64 flags; + HSA_PC_SAMPLING_METHOD_KIND method; + HSA_PC_SAMPLING_UNIT_INTERVAL units; +} +HsaPcSamplingInfo; + #pragma pack(pop, hsakmttypes_h) diff --git a/src/libhsakmt.ver b/src/libhsakmt.ver index c04cefef41..8a64f8d7a8 100644 --- a/src/libhsakmt.ver +++ b/src/libhsakmt.ver @@ -82,6 +82,11 @@ hsaKmtWaitOnMultipleEvents_Ext; hsaKmtReplaceAsanHeaderPage; hsaKmtReturnAsanHeaderPage; hsaKmtGetAMDGPUDeviceHandle; +hsaKmtPcSamplingQueryCapabilities; +hsaKmtPcSamplingCreate; +hsaKmtPcSamplingDestroy; +hsaKmtPcSamplingStart; +hsaKmtPcSamplingStop; local: *; }; diff --git a/src/pc_sampling.c b/src/pc_sampling.c new file mode 100644 index 0000000000..3cb81dc97b --- /dev/null +++ b/src/pc_sampling.c @@ -0,0 +1,210 @@ +/* + * Copyright © 2023 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including + * the next paragraph) shall be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "libhsakmt.h" +#include "linux/kfd_ioctl.h" +#include +#include +#include +#include + +#define INVALID_TRACE_ID 0x0 + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingSupport(void) +{ + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(16); + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingQueryCapabilities(HSAuint32 NodeId, void *sample_info, + HSAuint32 sample_info_sz, HSAuint32 *size) +{ + int ret; + struct kfd_ioctl_pc_sample_args args = {0}; + uint32_t gpu_id; + + if (size == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_KFD_OPEN(); + CHECK_KFD_MINOR_VERSION(16); + + ret = validate_nodeid(NodeId, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); + return ret; + } + assert(sizeof(HsaPcSamplingInfo) == sizeof(struct kfd_pc_sample_info)); + + ret = HSAKMT_STATUS_SUCCESS; + args.op = KFD_IOCTL_PCS_OP_QUERY_CAPABILITIES; + args.gpu_id = gpu_id; + args.sample_info_ptr = (uint64_t)sample_info; + args.num_sample_info = sample_info_sz; + args.flags = 0; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args); + + if (ret) { + return (ret == -ENOSPC) ? HSAKMT_STATUS_BUFFER_TOO_SMALL : + (ret == -EINVAL) ? HSAKMT_STATUS_INVALID_PARAMETER : + (ret == -EOPNOTSUPP) ? HSAKMT_STATUS_NOT_SUPPORTED : + (ret == -EBUSY) ? HSAKMT_STATUS_UNAVAILABLE : + HSAKMT_STATUS_ERROR; + } + *size = args.num_sample_info; + + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingCreate(HSAuint32 NodeId, HsaPcSamplingInfo *sample_info, + HsaPcSamplingTraceId *traceId) +{ + int ret; + struct kfd_ioctl_pc_sample_args args = {0}; + uint32_t gpu_id; + + if (sample_info == NULL || traceId == NULL) + return HSAKMT_STATUS_INVALID_PARAMETER; + + CHECK_KFD_OPEN(); + + *traceId = INVALID_TRACE_ID; + ret = validate_nodeid(NodeId, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); + return ret; + } + + ret = HSAKMT_STATUS_SUCCESS; + args.op = KFD_IOCTL_PCS_OP_CREATE; + args.gpu_id = gpu_id; + args.sample_info_ptr = (uint64_t)sample_info; + args.num_sample_info = 1; + args.trace_id = INVALID_TRACE_ID; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args); + if (ret) { + return (ret == -EINVAL) ? HSAKMT_STATUS_INVALID_PARAMETER : + (ret == -ENOMEM) ? HSAKMT_STATUS_NO_MEMORY : + (ret == -EBUSY) ? HSAKMT_STATUS_UNAVAILABLE : + HSAKMT_STATUS_ERROR; + } + + *traceId = args.trace_id; + return HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingDestroy(HSAuint32 NodeId, HsaPcSamplingTraceId traceId) +{ + int ret; + struct kfd_ioctl_pc_sample_args args = {0}; + uint32_t gpu_id; + + if (traceId == INVALID_TRACE_ID) + return HSAKMT_STATUS_INVALID_HANDLE; + + CHECK_KFD_OPEN(); + + ret = validate_nodeid(NodeId, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); + return ret; + } + + hsaKmtPcSamplingStop(NodeId, traceId); + + ret = HSAKMT_STATUS_SUCCESS; + args.op = KFD_IOCTL_PCS_OP_DESTROY; + args.gpu_id = gpu_id; + args.trace_id = traceId; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args); + return (ret == -EINVAL) ? HSAKMT_STATUS_INVALID_PARAMETER : + ret ? HSAKMT_STATUS_ERROR : + HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStart(HSAuint32 NodeId, HsaPcSamplingTraceId traceId) +{ + int ret; + struct kfd_ioctl_pc_sample_args args = {0}; + uint32_t gpu_id; + + if (traceId == INVALID_TRACE_ID) + return HSAKMT_STATUS_INVALID_HANDLE; + + CHECK_KFD_OPEN(); + + ret = validate_nodeid(NodeId, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); + return ret; + } + + ret = HSAKMT_STATUS_SUCCESS; + args.op = KFD_IOCTL_PCS_OP_START; + args.gpu_id = gpu_id; + args.trace_id = traceId; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args); + return (ret == -EINVAL) ? HSAKMT_STATUS_INVALID_PARAMETER : + (ret == -ENOMEM) ? HSAKMT_STATUS_OUT_OF_RESOURCES : + (ret == -EBUSY) ? HSAKMT_STATUS_UNAVAILABLE : + (ret == -EALREADY) ? HSAKMT_STATUS_KERNEL_ALREADY_OPENED : + ret ? HSAKMT_STATUS_ERROR : + HSAKMT_STATUS_SUCCESS; +} + +HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStop(HSAuint32 NodeId, HsaPcSamplingTraceId traceId) +{ + int ret; + struct kfd_ioctl_pc_sample_args args = {0}; + uint32_t gpu_id; + + if (traceId == INVALID_TRACE_ID) + return HSAKMT_STATUS_INVALID_HANDLE; + + CHECK_KFD_OPEN(); + + ret = validate_nodeid(NodeId, &gpu_id); + if (ret != HSAKMT_STATUS_SUCCESS) { + pr_err("[%s] invalid node ID: %d\n", __func__, NodeId); + return ret; + } + + ret = HSAKMT_STATUS_SUCCESS; + args.op = KFD_IOCTL_PCS_OP_STOP; + args.gpu_id = gpu_id; + args.trace_id = traceId; + + ret = kmtIoctl(kfd_fd, AMDKFD_IOC_PC_SAMPLE, &args); + return (ret == -EINVAL) ? HSAKMT_STATUS_INVALID_PARAMETER : + (ret == -EALREADY) ? HSAKMT_STATUS_KERNEL_ALREADY_OPENED : + ret ? HSAKMT_STATUS_ERROR : + HSAKMT_STATUS_SUCCESS; +} \ No newline at end of file From daf99471a45850d5b417676807f2e552d9c16f41 Mon Sep 17 00:00:00 2001 From: James Zhu Date: Fri, 21 Jul 2023 23:02:00 -0400 Subject: [PATCH 1234/1247] kfdtest: add kfdtest test cases for pc sampling Add kfdtest test cases for pc sampling. Change-Id: I49f4f8ebfa6569803acdc7dec895c1902ce0b280 Signed-off-by: James Zhu --- tests/kfdtest/CMakeLists.txt | 1 + tests/kfdtest/src/KFDPCSamplingTest.cpp | 303 ++++++++++++++++++++++++ tests/kfdtest/src/KFDPCSamplingTest.hpp | 40 ++++ 3 files changed, 344 insertions(+) create mode 100644 tests/kfdtest/src/KFDPCSamplingTest.cpp create mode 100644 tests/kfdtest/src/KFDPCSamplingTest.hpp diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index 00d95d7c5f..db2a0953f6 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -208,6 +208,7 @@ set (SRC_FILES gtest-1.6.0/gtest-all.cpp src/KFDSVMRangeTest.cpp src/KFDSVMEvictTest.cpp src/KFDRASTest.cpp + src/KFDPCSamplingTest.cpp src/RDMATest.cpp) message( STATUS "PROJECT_SOURCE_DIR:" ${PROJECT_SOURCE_DIR} ) diff --git a/tests/kfdtest/src/KFDPCSamplingTest.cpp b/tests/kfdtest/src/KFDPCSamplingTest.cpp new file mode 100644 index 0000000000..996421cb64 --- /dev/null +++ b/tests/kfdtest/src/KFDPCSamplingTest.cpp @@ -0,0 +1,303 @@ +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include "KFDPCSamplingTest.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "Dispatch.hpp" +#include "PM4Queue.hpp" +#include "PM4Packet.hpp" +#include "SDMAQueue.hpp" +#include "SDMAPacket.hpp" +#include "linux/kfd_ioctl.h" + +#define N_PROCESSES (2) /* Number of processes running in parallel, must be at least 2 */ + +/* Captures user specified time (seconds) to sleep */ +extern unsigned int g_SleepTime; + +void KFDPCSamplingTest::SetUp() { + ROUTINE_START + + KFDBaseComponentTest::SetUp(); + + ROUTINE_END +} + +void KFDPCSamplingTest::TearDown() { + ROUTINE_START + + KFDBaseComponentTest::TearDown(); + + ROUTINE_END +} + +TEST_F(KFDPCSamplingTest, BasicTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (hsaKmtPcSamplingSupport() != HSAKMT_STATUS_SUCCESS) + return; + + HSAuint32 num_sample_info = 0; + HSAuint32 return_num_sample_info = 0; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "Failed to get default GPU Node."; + + /* 1. get pc sampling format numbe of entry */ + ASSERT_SUCCESS(!hsaKmtPcSamplingQueryCapabilities(defaultGPUNode, NULL, + num_sample_info, &return_num_sample_info)); + num_sample_info = return_num_sample_info; + void *info_buf = calloc(num_sample_info, sizeof(HsaPcSamplingInfo)); + + ASSERT_SUCCESS(hsaKmtPcSamplingQueryCapabilities(defaultGPUNode, info_buf, + num_sample_info, &return_num_sample_info)); + + HsaPcSamplingInfo *samples = (HsaPcSamplingInfo*) info_buf; + HsaPcSamplingTraceId traceId1, traceId2; + + samples[0].value = 0x100000; /* 1,048,576 usec */ + + /* 1. Failed to start uncreated pc sampling ID */ + ASSERT_SUCCESS(!hsaKmtPcSamplingStart(defaultGPUNode, 12345)); + + /* 2. Failed to stop uncreated pc sampling ID */ + ASSERT_SUCCESS(!hsaKmtPcSamplingStop(defaultGPUNode, 12345)); + + /* 3. Failed to destroy uncreated pc sampling ID */ + ASSERT_SUCCESS(!hsaKmtPcSamplingDestroy(defaultGPUNode, 12345)); + + /* 4. create pc sampling */ + ASSERT_SUCCESS(hsaKmtPcSamplingCreate(defaultGPUNode, &samples[0], &traceId1)); + ASSERT_SUCCESS(hsaKmtPcSamplingDestroy(defaultGPUNode, traceId1)); + + /* 5. create twice in the same process with pc sampling activated */ + ASSERT_SUCCESS(hsaKmtPcSamplingCreate(defaultGPUNode, &samples[0], &traceId2)); + ASSERT_SUCCESS(hsaKmtPcSamplingStart(defaultGPUNode, traceId2)); + /* Creat and start 2nd session pc sampling */ + ASSERT_SUCCESS(hsaKmtPcSamplingCreate(defaultGPUNode, &samples[0], &traceId1)); + ASSERT_SUCCESS(hsaKmtPcSamplingStart(defaultGPUNode, traceId1)); + sleep(2); + /* Stop its own pc sampling session, but another session still alive */ + ASSERT_SUCCESS(hsaKmtPcSamplingStop(defaultGPUNode, traceId2)); + /* Destroy its own pc sampling session when it is de-activated */ + ASSERT_SUCCESS(hsaKmtPcSamplingDestroy(defaultGPUNode, traceId2)); + sleep(1); + ASSERT_SUCCESS(hsaKmtPcSamplingDestroy(defaultGPUNode, traceId1)); + + free(info_buf); + TEST_END +} + +struct ThreadParams { + int test_num; + HSAuint32 GPUNode; + HsaPcSamplingInfo *samples; +}; + +static unsigned int PCSamplingThread(void* p) { + struct ThreadParams* pArgs = reinterpret_cast(p); + + LOG() << "PCSamplingThread #" << pArgs->test_num << " start." << std::endl; + HsaPcSamplingTraceId traceId; + + EXPECT_SUCCESS(hsaKmtPcSamplingCreate(pArgs->GPUNode, pArgs->samples, &traceId)); + EXPECT_SUCCESS(hsaKmtPcSamplingStart(pArgs->GPUNode, traceId)); + sleep(3); + + LOG() << "PCSamplingThread #" << pArgs->test_num << " stop." << std::endl; + EXPECT_SUCCESS(hsaKmtPcSamplingStop(pArgs->GPUNode, traceId)); + EXPECT_SUCCESS(hsaKmtPcSamplingDestroy(pArgs->GPUNode, traceId)); + + return 0; +} + +TEST_F(KFDPCSamplingTest, MultiThreadPcSamplingTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (hsaKmtPcSamplingSupport() != HSAKMT_STATUS_SUCCESS) + return; + + HSAuint64 threadId[2]; + struct ThreadParams params[2]; + HSAuint32 num_sample_info = 2; + HSAuint32 return_num_sample_info = 0; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "Failed to get default GPU Node"; + + void *info_buf = calloc(num_sample_info, sizeof(HsaPcSamplingInfo)); + + ASSERT_SUCCESS(hsaKmtPcSamplingQueryCapabilities(defaultGPUNode, info_buf, + num_sample_info, &return_num_sample_info)); + HsaPcSamplingInfo *samples = (HsaPcSamplingInfo*) info_buf; + + samples[0].value = 0x100000; /* 1,048,576 usec */ + + params[0].test_num = 1; + params[1].test_num = 2; + params[0].GPUNode = defaultGPUNode; + params[1].GPUNode = defaultGPUNode; + params[0].samples = samples; + params[1].samples = samples; + + ASSERT_EQ(true, StartThread(&PCSamplingThread, ¶ms[0], threadId[0])); + sleep(1); + /* start 2nd thread after 1 sec */ + ASSERT_EQ(true, StartThread(&PCSamplingThread, ¶ms[1], threadId[1])); + + WaitForThread(threadId[0]); + WaitForThread(threadId[1]); + + free(info_buf); + + TEST_END; +} + +struct ProcParams { + std::string test_name; + HSAuint32 GPUNode; + HsaPcSamplingInfo *samples; +}; + +static unsigned int PCSamplingProcRun(void* p) { + struct ProcParams* pArgs = reinterpret_cast(p); + bool process1_flag = !pArgs->test_name.compare("Test process 1 "); + int start_delay; + + if (process1_flag) + start_delay = 0; + else + start_delay = 1; + + LOG() << "PCSamplingProc <" << pArgs->test_name << + "> starting after 0x" << start_delay << " secs" << std::endl; + sleep(start_delay); + + HsaPcSamplingTraceId traceId = start_delay; + + EXPECT_SUCCESS(hsaKmtPcSamplingCreate(pArgs->GPUNode, pArgs->samples, &traceId)); + EXPECT_SUCCESS(hsaKmtPcSamplingStart(pArgs->GPUNode, traceId)); + sleep(3); + + LOG() << "PCSamplingProc <" << pArgs->test_name << "> stop" << std::endl; + EXPECT_SUCCESS(hsaKmtPcSamplingStop(pArgs->GPUNode, traceId)); + EXPECT_SUCCESS(hsaKmtPcSamplingDestroy(pArgs->GPUNode, traceId)); + LOG() << "PCSamplingProc <" << pArgs->test_name << "> done" << std::endl; + + return 0; +} + +TEST_F(KFDPCSamplingTest, MultiProcPcSamplingTest) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (hsaKmtPcSamplingSupport() != HSAKMT_STATUS_SUCCESS) + return; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "Failed to get default GPU Node"; + + HSAuint32 num_sample_info = 0; + HSAuint32 return_num_sample_info = 0; + struct ProcParams params; + + params.GPUNode = defaultGPUNode; + + ASSERT_SUCCESS(!hsaKmtPcSamplingQueryCapabilities(defaultGPUNode, NULL, + num_sample_info, &return_num_sample_info)); + num_sample_info = return_num_sample_info; + void *info_buf = calloc(num_sample_info, sizeof(HsaPcSamplingInfo)); + ASSERT_SUCCESS(hsaKmtPcSamplingQueryCapabilities(defaultGPUNode, info_buf, + num_sample_info, &return_num_sample_info)); + + HsaPcSamplingInfo *samples = (HsaPcSamplingInfo*) info_buf; + + samples[0].value = 0x100000; /* 1,048,576 usec */ + + /* Fork the child processes */ + ForkChildProcesses(N_PROCESSES); + + int rn = FindDRMRenderNode(defaultGPUNode); + if (rn < 0) { + LOG() << "Skipping test: Could not find render node for default GPU." << std::endl; + WaitChildProcesses(); + return; + } + + params.samples = samples; + params.test_name = m_psName; + PCSamplingProcRun(¶ms); + + WaitChildProcesses(); + + if (info_buf) + free(info_buf); + TEST_END +} + +/* Manully run multiple KFDPCSamplingTest.MultiProcPcSamplingTestM */ +TEST_F(KFDPCSamplingTest, MultiProcPcSamplingTestM) { + TEST_REQUIRE_ENV_CAPABILITIES(ENVCAPS_64BITLINUX); + TEST_START(TESTPROFILE_RUNALL); + + if (hsaKmtPcSamplingSupport() != HSAKMT_STATUS_SUCCESS) + return; + + HSAuint32 num_sample_info = 0; + HSAuint32 return_num_sample_info = 0; + + HSAuint32 defaultGPUNode = m_NodeInfo.HsaDefaultGPUNode(); + ASSERT_GE(defaultGPUNode, 0) << "Failed to get default GPU Node"; + + ASSERT_SUCCESS(!hsaKmtPcSamplingQueryCapabilities(defaultGPUNode, NULL, + num_sample_info, &return_num_sample_info)); + num_sample_info = return_num_sample_info; + void *info_buf = calloc(num_sample_info, sizeof(HsaPcSamplingInfo)); + ASSERT_SUCCESS(hsaKmtPcSamplingQueryCapabilities(defaultGPUNode, info_buf, + num_sample_info, &return_num_sample_info)); + + HsaPcSamplingInfo *samples = (HsaPcSamplingInfo*) info_buf; + HsaPcSamplingTraceId traceId; + + samples[0].value = 0x100000; /* 1,048,576 usec */ + ASSERT_SUCCESS(hsaKmtPcSamplingCreate(defaultGPUNode, &samples[0], &traceId)); + + ASSERT_SUCCESS(hsaKmtPcSamplingStart(defaultGPUNode, traceId)); + sleep(3); + ASSERT_SUCCESS(hsaKmtPcSamplingStop(defaultGPUNode, traceId)); + ASSERT_SUCCESS(hsaKmtPcSamplingDestroy(defaultGPUNode, traceId)); + + free(info_buf); + TEST_END +} diff --git a/tests/kfdtest/src/KFDPCSamplingTest.hpp b/tests/kfdtest/src/KFDPCSamplingTest.hpp new file mode 100644 index 0000000000..121509f40a --- /dev/null +++ b/tests/kfdtest/src/KFDPCSamplingTest.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2023 Advanced Micro Devices, Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __KFD_PCSAMPLING_TEST__H__ +#define __KFD_PCSAMPLING_TEST__H__ + +#include "KFDMultiProcessTest.hpp" + +class KFDPCSamplingTest : public KFDMultiProcessTest { + public: + KFDPCSamplingTest(void) {} + ~KFDPCSamplingTest(void) {} + protected: + virtual void SetUp(); + virtual void TearDown(); + + protected: +}; + +#endif // __KFD_PCSAMPLING_TEST__H__ From fa0c18232594bb909469ab23392790abf6474238 Mon Sep 17 00:00:00 2001 From: Mihai Preda Date: Mon, 25 Mar 2024 08:10:02 +1100 Subject: [PATCH 1235/1247] Free the malloc'ed event on page alloc failure Change-Id: I009a9a6e2f67545c51470e86eac1adb78d6181b4 Signed-off-by: Mihai Preda Signed-off-by: Kent Russell --- src/events.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/events.c b/src/events.c index 07356db67e..464612be7a 100644 --- a/src/events.c +++ b/src/events.c @@ -78,6 +78,7 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtCreateEvent(HsaEventDescriptor *EventDesc, events_page = allocate_exec_aligned_memory_gpu( KFD_SIGNAL_EVENT_LIMIT * 8, PAGE_SIZE, 0, 0, true, false, true); if (!events_page) { + free(e); pthread_mutex_unlock(&hsakmt_mutex); return HSAKMT_STATUS_ERROR; } From 541d0dbbaeb31107e58102eafa2a670e9a49453d Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Fri, 29 Mar 2024 17:11:21 +0000 Subject: [PATCH 1236/1247] Set NUMA region to 0 when using GTTAccess flag When allocating memory for MES AQL queue structure, the PreferredNode is set to the device index of GPU to hint the location where the BO needs to be created. But we need to ignore the device index when calling bind_mem_to_numa. Change-Id: Iae69fe02bfd48c5a3bd495319f6f2706d6e8aea2 --- src/memory.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/memory.c b/src/memory.c index f121726f77..964becdc0a 100644 --- a/src/memory.c +++ b/src/memory.c @@ -161,8 +161,8 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, if (zfb_support && gpu_id && MemFlags.ui32.NonPaged == 1) MemFlags.ui32.CoarseGrain = 1; - *MemoryAddress = fmm_allocate_host(gpu_id, PreferredNode, *MemoryAddress, - SizeInBytes, MemFlags); + *MemoryAddress = fmm_allocate_host(gpu_id, MemFlags.ui32.GTTAccess ? 0 : PreferredNode, + *MemoryAddress, SizeInBytes, MemFlags); if (!(*MemoryAddress)) { pr_err("[%s] failed to allocate %lu bytes from host\n", From a180fea5adb3ab3f11312bafba7c78d90d41c857 Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Tue, 12 Mar 2024 13:26:41 -0700 Subject: [PATCH 1237/1247] Properly nest build time headers to match arrangement at install time. The build tree was missing a level of nesting, causing diversions based on in-tree/out-of-tree use. KR: Also fixed kfdtest paths Change-Id: I8638b6d6227daabddd8eaa2aa387ba578b8dfab8 Signed-off-by: Stella Laurenzo Signed-off-by: Kent Russell --- CMakeLists.txt | 2 +- include/{ => hsakmt}/hsakmt.h | 0 include/{ => hsakmt}/hsakmttypes.h | 0 include/{ => hsakmt}/linux/kfd_ioctl.h | 0 src/debug.c | 2 +- src/events.c | 2 +- src/fmm.c | 2 +- src/fmm.h | 2 +- src/libhsakmt.h | 4 ++-- src/memory.c | 2 +- src/pc_sampling.c | 4 ++-- src/perfctr.c | 2 +- src/queues.c | 2 +- src/spm.c | 2 +- src/time.c | 2 +- src/version.c | 2 +- tests/kfdtest/src/BaseDebug.cpp | 2 +- tests/kfdtest/src/BaseDebug.hpp | 2 +- tests/kfdtest/src/BaseQueue.cpp | 2 +- tests/kfdtest/src/GoogleTestExtension.hpp | 2 +- tests/kfdtest/src/KFDBaseComponentTest.hpp | 2 +- tests/kfdtest/src/KFDDBGTest.cpp | 2 +- tests/kfdtest/src/KFDMemoryTest.cpp | 2 +- tests/kfdtest/src/KFDOpenCloseKFDTest.hpp | 2 +- tests/kfdtest/src/KFDPCSamplingTest.cpp | 2 +- tests/kfdtest/src/KFDPMTest.cpp | 2 +- tests/kfdtest/src/KFDRASTest.cpp | 2 +- tests/kfdtest/src/KFDTestUtil.hpp | 2 +- tests/kfdtest/src/KFDTestUtilQueue.hpp | 2 +- tests/kfdtest/src/OSWrapper.hpp | 2 +- tests/kfdtest/src/PM4Packet.cpp | 2 +- 31 files changed, 30 insertions(+), 30 deletions(-) rename include/{ => hsakmt}/hsakmt.h (100%) rename include/{ => hsakmt}/hsakmttypes.h (100%) rename include/{ => hsakmt}/linux/kfd_ioctl.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 19b97133af..5f8af72ef7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -209,7 +209,7 @@ install ( TARGETS ${HSAKMT_TARGET} #install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT devel ) # Install public headers -install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${HSAKMT_TARGET} +install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/${HSAKMT_TARGET} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT devel PATTERN "linux" EXCLUDE ) # Option to build header path migration helpers. diff --git a/include/hsakmt.h b/include/hsakmt/hsakmt.h similarity index 100% rename from include/hsakmt.h rename to include/hsakmt/hsakmt.h diff --git a/include/hsakmttypes.h b/include/hsakmt/hsakmttypes.h similarity index 100% rename from include/hsakmttypes.h rename to include/hsakmt/hsakmttypes.h diff --git a/include/linux/kfd_ioctl.h b/include/hsakmt/linux/kfd_ioctl.h similarity index 100% rename from include/linux/kfd_ioctl.h rename to include/hsakmt/linux/kfd_ioctl.h diff --git a/src/debug.c b/src/debug.c index d438a2826e..932e8294c9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -24,7 +24,7 @@ */ #include "libhsakmt.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include #include #include diff --git a/src/events.c b/src/events.c index 464612be7a..ae92c37f11 100644 --- a/src/events.c +++ b/src/events.c @@ -31,7 +31,7 @@ #include #include #include -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include "fmm.h" static HSAuint64 *events_page = NULL; diff --git a/src/fmm.c b/src/fmm.c index 4b78bba0ed..8aaf6ffd08 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -25,7 +25,7 @@ #include "libhsakmt.h" #include "fmm.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include #include #include diff --git a/src/fmm.h b/src/fmm.h index 854ae1ba2c..80ccfd559f 100644 --- a/src/fmm.h +++ b/src/fmm.h @@ -26,7 +26,7 @@ #ifndef FMM_H_ #define FMM_H_ -#include "hsakmttypes.h" +#include "hsakmt/hsakmttypes.h" #include typedef enum { diff --git a/src/libhsakmt.h b/src/libhsakmt.h index fd298e2832..71cef9cef8 100644 --- a/src/libhsakmt.h +++ b/src/libhsakmt.h @@ -26,8 +26,8 @@ #ifndef LIBHSAKMT_H_INCLUDED #define LIBHSAKMT_H_INCLUDED -#include "linux/kfd_ioctl.h" -#include "hsakmt.h" +#include "hsakmt/linux/kfd_ioctl.h" +#include "hsakmt/hsakmt.h" #include #include #include diff --git a/src/memory.c b/src/memory.c index 964becdc0a..8e2b49b1d4 100644 --- a/src/memory.c +++ b/src/memory.c @@ -24,7 +24,7 @@ */ #include "libhsakmt.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include #include #include diff --git a/src/pc_sampling.c b/src/pc_sampling.c index 3cb81dc97b..ca0b99e055 100644 --- a/src/pc_sampling.c +++ b/src/pc_sampling.c @@ -24,7 +24,7 @@ */ #include "libhsakmt.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include #include #include @@ -207,4 +207,4 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtPcSamplingStop(HSAuint32 NodeId, HsaPcSamplingTrac (ret == -EALREADY) ? HSAKMT_STATUS_KERNEL_ALREADY_OPENED : ret ? HSAKMT_STATUS_ERROR : HSAKMT_STATUS_SUCCESS; -} \ No newline at end of file +} diff --git a/src/perfctr.c b/src/perfctr.c index ed051bece2..32df25a74d 100644 --- a/src/perfctr.c +++ b/src/perfctr.c @@ -30,7 +30,7 @@ #include #include "libhsakmt.h" #include "pmc_table.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include #include #include diff --git a/src/queues.c b/src/queues.c index 29bffce03f..fffce67668 100644 --- a/src/queues.c +++ b/src/queues.c @@ -25,7 +25,7 @@ #include "libhsakmt.h" #include "fmm.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include #include #include diff --git a/src/spm.c b/src/spm.c index c4eb33e5b1..9571197fe4 100644 --- a/src/spm.c +++ b/src/spm.c @@ -24,7 +24,7 @@ */ #include "libhsakmt.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include #include diff --git a/src/time.c b/src/time.c index 96729a1074..286350bdb9 100644 --- a/src/time.c +++ b/src/time.c @@ -24,7 +24,7 @@ */ #include "libhsakmt.h" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" HSAKMT_STATUS HSAKMTAPI hsaKmtGetClockCounters(HSAuint32 NodeId, HsaClockCounters *Counters) diff --git a/src/version.c b/src/version.c index 34e51e0811..c6e2e8dfc1 100644 --- a/src/version.c +++ b/src/version.c @@ -26,7 +26,7 @@ #include "libhsakmt.h" #include #include -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" HsaVersionInfo kfd_version_info; diff --git a/tests/kfdtest/src/BaseDebug.cpp b/tests/kfdtest/src/BaseDebug.cpp index a04afe3b01..68a2fb8588 100644 --- a/tests/kfdtest/src/BaseDebug.cpp +++ b/tests/kfdtest/src/BaseDebug.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include "unistd.h" diff --git a/tests/kfdtest/src/BaseDebug.hpp b/tests/kfdtest/src/BaseDebug.hpp index fc9343e59d..7915072fc6 100644 --- a/tests/kfdtest/src/BaseDebug.hpp +++ b/tests/kfdtest/src/BaseDebug.hpp @@ -24,7 +24,7 @@ #ifndef __KFD_BASE_DEBUG__H__ #define __KFD_BASE_DEBUG__H__ -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #include #include diff --git a/tests/kfdtest/src/BaseQueue.cpp b/tests/kfdtest/src/BaseQueue.cpp index dd1620168c..fdbc7b5a17 100644 --- a/tests/kfdtest/src/BaseQueue.cpp +++ b/tests/kfdtest/src/BaseQueue.cpp @@ -25,7 +25,7 @@ #include "SDMAQueue.hpp" #include "PM4Queue.hpp" #include "AqlQueue.hpp" -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #include "KFDBaseComponentTest.hpp" BaseQueue::BaseQueue() diff --git a/tests/kfdtest/src/GoogleTestExtension.hpp b/tests/kfdtest/src/GoogleTestExtension.hpp index 196b7242c4..57cb38cb24 100644 --- a/tests/kfdtest/src/GoogleTestExtension.hpp +++ b/tests/kfdtest/src/GoogleTestExtension.hpp @@ -25,7 +25,7 @@ #define __GOOGLETEST_EXTENSION__H__ #include -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #include "KFDTestFlags.hpp" enum LOGTYPE { diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/tests/kfdtest/src/KFDBaseComponentTest.hpp index 7fddd23844..66337d0cab 100644 --- a/tests/kfdtest/src/KFDBaseComponentTest.hpp +++ b/tests/kfdtest/src/KFDBaseComponentTest.hpp @@ -31,7 +31,7 @@ #include #include #include -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #include "OSWrapper.hpp" #include "KFDTestUtil.hpp" #include "Assemble.hpp" diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/tests/kfdtest/src/KFDDBGTest.cpp index e43a253daf..a287d8c768 100644 --- a/tests/kfdtest/src/KFDDBGTest.cpp +++ b/tests/kfdtest/src/KFDDBGTest.cpp @@ -25,7 +25,7 @@ #include "KFDDBGTest.hpp" #include #include -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include "KFDQMTest.hpp" #include "PM4Queue.hpp" #include "PM4Packet.hpp" diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/tests/kfdtest/src/KFDMemoryTest.cpp index a94449fc3e..29cefcb891 100644 --- a/tests/kfdtest/src/KFDMemoryTest.cpp +++ b/tests/kfdtest/src/KFDMemoryTest.cpp @@ -37,7 +37,7 @@ #include "PM4Packet.hpp" #include "SDMAQueue.hpp" #include "SDMAPacket.hpp" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" /* Captures user specified time (seconds) to sleep */ extern unsigned int g_SleepTime; diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp b/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp index 96b48277f2..40ba4919b4 100644 --- a/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp +++ b/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp @@ -22,7 +22,7 @@ */ #include -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #ifndef __KFD_OPEN_CLOSE_KFD_TEST__H__ #define __KFD_OPEN_CLOSE_KFD_TEST__H__ diff --git a/tests/kfdtest/src/KFDPCSamplingTest.cpp b/tests/kfdtest/src/KFDPCSamplingTest.cpp index 996421cb64..800e46487d 100644 --- a/tests/kfdtest/src/KFDPCSamplingTest.cpp +++ b/tests/kfdtest/src/KFDPCSamplingTest.cpp @@ -37,7 +37,7 @@ #include "PM4Packet.hpp" #include "SDMAQueue.hpp" #include "SDMAPacket.hpp" -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #define N_PROCESSES (2) /* Number of processes running in parallel, must be at least 2 */ diff --git a/tests/kfdtest/src/KFDPMTest.cpp b/tests/kfdtest/src/KFDPMTest.cpp index 98c2348a8c..17dfa91025 100644 --- a/tests/kfdtest/src/KFDPMTest.cpp +++ b/tests/kfdtest/src/KFDPMTest.cpp @@ -25,7 +25,7 @@ #include "KFDTestUtil.hpp" #include "PM4Packet.hpp" #include "PM4Queue.hpp" -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" void KFDPMTest::SetUp() { ROUTINE_START diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/tests/kfdtest/src/KFDRASTest.cpp index 69a4719a1d..3ae18cc736 100644 --- a/tests/kfdtest/src/KFDRASTest.cpp +++ b/tests/kfdtest/src/KFDRASTest.cpp @@ -24,7 +24,7 @@ #include #include -#include "linux/kfd_ioctl.h" +#include "hsakmt/linux/kfd_ioctl.h" #include "KFDRASTest.hpp" #include "PM4Queue.hpp" diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/tests/kfdtest/src/KFDTestUtil.hpp index aab10a9ae4..43d19b1b69 100644 --- a/tests/kfdtest/src/KFDTestUtil.hpp +++ b/tests/kfdtest/src/KFDTestUtil.hpp @@ -28,7 +28,7 @@ #include #include "OSWrapper.hpp" #include "GoogleTestExtension.hpp" -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" class BaseQueue; #define ARRAY_SIZE(_x) (sizeof(_x)/sizeof(_x[0])) diff --git a/tests/kfdtest/src/KFDTestUtilQueue.hpp b/tests/kfdtest/src/KFDTestUtilQueue.hpp index dac222b623..c98917be27 100644 --- a/tests/kfdtest/src/KFDTestUtilQueue.hpp +++ b/tests/kfdtest/src/KFDTestUtilQueue.hpp @@ -24,7 +24,7 @@ #ifndef __KFD__TEST__UTIL__QUEUE__H__ #define __KFD__TEST__UTIL__QUEUE__H__ -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #include typedef struct { diff --git a/tests/kfdtest/src/OSWrapper.hpp b/tests/kfdtest/src/OSWrapper.hpp index d7d69b7f73..a91aed83f1 100644 --- a/tests/kfdtest/src/OSWrapper.hpp +++ b/tests/kfdtest/src/OSWrapper.hpp @@ -26,7 +26,7 @@ #include #include "KFDTestFlags.hpp" -#include "hsakmt.h" +#include "hsakmt/hsakmt.h" #ifndef __OS__WRAPPER__H__ #define __OS__WRAPPER__H__ diff --git a/tests/kfdtest/src/PM4Packet.cpp b/tests/kfdtest/src/PM4Packet.cpp index 3d26ef3fe6..6cacf17b06 100644 --- a/tests/kfdtest/src/PM4Packet.cpp +++ b/tests/kfdtest/src/PM4Packet.cpp @@ -25,7 +25,7 @@ #include #include #include "PM4Packet.hpp" -#include "hsakmttypes.h" +#include "hsakmt/hsakmttypes.h" #include "KFDBaseComponentTest.hpp" #include "asic_reg/gfx_7_2_enum.h" From e2d742ac6ff3f6bfb24566b088e926fc61430443 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 6 Apr 2024 11:08:28 -0400 Subject: [PATCH 1238/1247] libhsakmt: Remove HsaMemFlag reserved bit init HsaMemFlag new flags added and the number of the reserved bits is reduced, and generate value overflow compilanation error. The reserved bits is not used, remove the init. Change-Id: I603596977dfd558ce31ead03711d7c5ce5ee5b71 Signed-off-by: Philip Yang --- src/fmm.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fmm.c b/src/fmm.c index 8aaf6ffd08..bad29e9ea6 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1643,7 +1643,6 @@ void *fmm_allocate_doorbell(uint32_t gpu_id, uint64_t MemorySizeInBytes, mflags.Value = 0; mflags.ui32.NonPaged = 1; mflags.ui32.HostAccess = 1; - mflags.ui32.Reserved = 0x3e1; pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->mflags = mflags; @@ -2371,7 +2370,6 @@ static void *map_mmio(uint32_t node_id, uint32_t gpu_id, int mmap_fd) mflags.Value = 0; mflags.ui32.NonPaged = 1; mflags.ui32.HostAccess = 1; - mflags.ui32.Reserved = 0; pthread_mutex_lock(&aperture->fmm_mutex); vm_obj->mflags = mflags; vm_obj->node_id = node_id; From 97497c7efcc8be1c84eec50b6edf295f50805735 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Mon, 5 Feb 2024 19:34:14 -0500 Subject: [PATCH 1239/1247] libhsakmt: Support contiguous VRAM allocation flag Add HsaMemFlags Contiguous bit for hsaKmtAllocMemory to allocate contiguous VRAM, to support RDMA device with limited scatter-gather ability. Check KFD ioctl minor version >= 17. Change-Id: I0db00dad125b2b7be523f343082641f59b850423 Signed-off-by: Philip Yang --- include/hsakmt/hsakmttypes.h | 3 ++- include/hsakmt/linux/kfd_ioctl.h | 1 + src/fmm.c | 3 +++ src/memory.c | 3 +++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/hsakmt/hsakmttypes.h b/include/hsakmt/hsakmttypes.h index f94efa11c3..eebc14bf3d 100644 --- a/include/hsakmt/hsakmttypes.h +++ b/include/hsakmt/hsakmttypes.h @@ -569,7 +569,8 @@ typedef struct _HsaMemFlags unsigned int ExtendedCoherent: 1; // system-scope coherence on atomic instructions unsigned int GTTAccess: 1; // default = 0; If 1: The caller indicates this memory will be mapped to GART for MES // KFD will allocate GTT memory with the Preferred_node set as gpu_id for GART mapping - unsigned int Reserved: 10; + unsigned int Contiguous: 1; // Allocate contiguous VRAM + unsigned int Reserved: 9; } ui32; HSAuint32 Value; diff --git a/include/hsakmt/linux/kfd_ioctl.h b/include/hsakmt/linux/kfd_ioctl.h index f1fd54bfa4..fe0a09a34f 100644 --- a/include/hsakmt/linux/kfd_ioctl.h +++ b/include/hsakmt/linux/kfd_ioctl.h @@ -1029,6 +1029,7 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_COHERENT (1 << 26) #define KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED (1 << 25) #define KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT (1 << 24) +#define KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT (1 << 23) /* Allocate memory for later SVM (shared virtual memory) mapping. * diff --git a/src/fmm.c b/src/fmm.c index bad29e9ea6..ef9e3f628c 100644 --- a/src/fmm.c +++ b/src/fmm.c @@ -1579,6 +1579,9 @@ void *fmm_allocate_device(uint32_t gpu_id, uint32_t node_id, void *address, if (mflags.ui32.ExtendedCoherent) ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT; + if (mflags.ui32.Contiguous) + ioc_flags |= KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS_BEST_EFFORT; + mem = __fmm_allocate_device(gpu_id, address, size, aperture, &mmap_offset, ioc_flags, &vm_obj); diff --git a/src/memory.c b/src/memory.c index 8e2b49b1d4..b239b1a076 100644 --- a/src/memory.c +++ b/src/memory.c @@ -115,6 +115,9 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtAllocMemory(HSAuint32 PreferredNode, CHECK_KFD_OPEN(); + if (MemFlags.ui32.Contiguous) + CHECK_KFD_MINOR_VERSION(17); + pr_debug("[%s] node %d\n", __func__, PreferredNode); result = validate_nodeid(PreferredNode, &gpu_id); From c9103a00ef9031521f0ea96320821e0266685750 Mon Sep 17 00:00:00 2001 From: David Galiffi Date: Thu, 4 Apr 2024 10:26:42 -0400 Subject: [PATCH 1240/1247] Update GitHub links to point to the new organization. ie., RadeonOpenCompute --> ROCm Signed-off-by: David Galiffi Change-Id: I6724cbcbbb525f767af297e3986cd61fa69cd49f --- CMakeLists.txt | 2 +- tests/kfdtest/CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f8af72ef7..4ebfb4b6aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -322,7 +322,7 @@ set(PACKAGE_VERSION_STR "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_ set(CPACK_PACKAGE_VERSION "${PACKAGE_VERSION_STR}") # Debian package specific variables -set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface") +set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/ROCm/ROCT-Thunk-Interface") set(CPACK_DEBIAN_PACKAGE_NAME "hsakmt-roct-dev") if (DEFINED ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) set(CPACK_DEBIAN_PACKAGE_RELEASE $ENV{CPACK_DEBIAN_PACKAGE_RELEASE}) diff --git a/tests/kfdtest/CMakeLists.txt b/tests/kfdtest/CMakeLists.txt index db2a0953f6..9d1172e2f1 100644 --- a/tests/kfdtest/CMakeLists.txt +++ b/tests/kfdtest/CMakeLists.txt @@ -44,7 +44,7 @@ endif() set ( CPACK_PACKAGE_VERSION_MAJOR "1" ) set ( CPACK_PACKAGE_VERSION_MINOR "0" ) set ( CPACK_PACKAGE_VERSION_PATCH "0" ) -set ( CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) +set ( CPACK_PACKAGE_HOMEPAGE_URL "https://github.com/ROCm/ROCT-Thunk-Interface" ) set ( CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") set ( CPACK_RPM_FILE_NAME "RPM-DEFAULT") @@ -82,7 +82,7 @@ set ( CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE STRING "De set ( CPACK_GENERATOR "DEB;RPM" CACHE STRING "Default packaging generators." ) # Debian package specific variables -set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface" ) +set ( CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/ROCm/ROCT-Thunk-Interface" ) set (CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-core") # RPM package specific variables From f94c1794bbec0b3ec6987b5f3b93693882deb27d Mon Sep 17 00:00:00 2001 From: David Galiffi Date: Thu, 4 Apr 2024 10:29:53 -0400 Subject: [PATCH 1241/1247] Fixed MD linting issue regarding headers They should start at H1 Signed-off-by: David Galiffi Change-Id: Id11a2599c4609255a1a9916f70b58adc41cdddb4 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a1b2db2edb..3020602e53 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ -### ROCt Library +# ROCt Library This repository includes the user-mode API interfaces used to interact with the ROCk driver. Starting at 1.7 release, ROCt uses drm render device. This requires the user to belong to video group. Add the user account to video group with "sudo usermod -a -G video _username_" command if the user if not part of video group yet. NOTE: Users of Ubuntu 20.04 will need to add the user to the new "render" group, as Ubuntu has changed the owner:group of /dev/kfd to render:render as of that release -#### ROCk Driver +## ROCk Driver The ROCt library is not a standalone product and requires that you have the correct ROCk driver installed, or are using a compatible upstream kernel. Please refer to https://docs.amd.com/ under "Getting Started Guide" for a list of supported Operating Systems and kernel versions, as well as supported hardware. -#### Building the Thunk +## Building the Thunk A simple cmake-based system is available for building thunk. To build the thunk from the the ROCT-Thunk-Interface directory, execute: @@ -36,7 +36,7 @@ If you choose not to build and install packages, manual installation of the bina NOTE: For older versions of the thunk where hsakmt-dev.txt is present, "make package-dev" and "make install-dev" are required to generate/install the developer packages. Currently, these are created via the "make package" and "make install" commands -#### Disclaimer +## Disclaimer The information contained herein is for informational purposes only, and is subject to change without notice. While every precaution has been taken in the preparation of this document, it may contain technical inaccuracies, omissions and typographical errors, and AMD is under no obligation to update or otherwise correct this information. Advanced Micro Devices, Inc. makes no representations or warranties with respect to the accuracy or completeness of the contents of this document, and assumes no liability of any kind, including the implied warranties of noninfringement, merchantability or fitness for particular purposes, with respect to the operation or use of AMD hardware, software or other products described herein. No license, including implied or arising by estoppel, to any intellectual property rights is granted by this document. Terms and limitations applicable to the purchase or use of AMD's products are as set forth in a signed agreement between the parties or in AMD's Standard Terms and Conditions of Sale. From 975c5dd24ad1f0bed0b22835847bbc7d5411c98c Mon Sep 17 00:00:00 2001 From: David Galiffi Date: Thu, 4 Apr 2024 10:08:36 -0400 Subject: [PATCH 1242/1247] Fixed broken link to ROCm documentation Signed-off-by: David Galiffi Change-Id: I9a6bc0ad08a060d83fdc3a0589dfc81c68ce2b0e --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3020602e53..b74436e7e2 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ NOTE: Users of Ubuntu 20.04 will need to add the user to the new "render" group, ## ROCk Driver The ROCt library is not a standalone product and requires that you have the correct ROCk driver installed, or are using a compatible upstream kernel. -Please refer to https://docs.amd.com/ under "Getting Started Guide" for a list of supported Operating Systems and kernel versions, as well as supported hardware. +Please refer to under "Getting Started Guide" for a list of supported Operating Systems and kernel versions, as well as supported hardware. ## Building the Thunk From a8bd453243e9a5ec8f7be05cca6798e0918701f7 Mon Sep 17 00:00:00 2001 From: David Galiffi Date: Thu, 4 Apr 2024 10:28:34 -0400 Subject: [PATCH 1243/1247] Fixed MD linting issue regarding code blocks. Signed-off-by: David Galiffi Change-Id: Id0467b332bf033642a2d403090ffe598e41689f5 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b74436e7e2..0433136bb8 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,9 @@ If the hsakmt-roct and hsakmt-roct-dev packages are desired: cmake .. make package ``` + If you choose not to build and install packages, manual installation of the binaries and header files can be done via: + ```bash make install ``` From 8a893ea0b8883baa5af8155dcb4eb74d9d97c4d3 Mon Sep 17 00:00:00 2001 From: amd-jmacaran Date: Tue, 23 Apr 2024 20:58:41 -0400 Subject: [PATCH 1244/1247] Add support for external CI builds using Azure Pipelines Change-Id: I8f4de331f00317a959b86f7e5b7a1025ba03564b --- .azuredevops/rocm-ci.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .azuredevops/rocm-ci.yml diff --git a/.azuredevops/rocm-ci.yml b/.azuredevops/rocm-ci.yml new file mode 100644 index 0000000000..229f0d21c1 --- /dev/null +++ b/.azuredevops/rocm-ci.yml @@ -0,0 +1,32 @@ +resources: + repositories: + - repository: pipelines_repo + type: github + endpoint: External-CI-Token + name: ROCm/ROCm + +variables: +- group: common +- template: /.azuredevops/variables-global.yml@pipelines_repo + +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - '*.md' + +pr: + autoCancel: true + branches: + include: + - master + paths: + exclude: + - '*.md' + drafts: false + +jobs: + - template: ${{ variables.CI_COMPONENT_PATH }}/ROCT-Thunk-Interface.yml@pipelines_repo From 587e4287f465f6f21fb0cbf70d0a303ef1dd295e Mon Sep 17 00:00:00 2001 From: amd-jmacaran Date: Thu, 25 Apr 2024 12:23:28 -0400 Subject: [PATCH 1245/1247] Change token name to match IT-created token Change-Id: Ic9189c012024c59cf5bad9daf25f6c2575a100fd --- .azuredevops/rocm-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azuredevops/rocm-ci.yml b/.azuredevops/rocm-ci.yml index 229f0d21c1..8fd154c367 100644 --- a/.azuredevops/rocm-ci.yml +++ b/.azuredevops/rocm-ci.yml @@ -2,7 +2,7 @@ resources: repositories: - repository: pipelines_repo type: github - endpoint: External-CI-Token + endpoint: ROCm name: ROCm/ROCm variables: From 5e1f24f305a9039ef5ffced856ed070835463695 Mon Sep 17 00:00:00 2001 From: Kent Russell Date: Thu, 25 Apr 2024 09:40:35 -0400 Subject: [PATCH 1246/1247] .github: Add CODEOWNERS file Change-Id: Ia763b91177f1ae09d16e5968bed17b0dba62cbe5 Signed-off-by: Kent Russell --- .github/CODEOWNERS | 8 ++++++++ .gitignore | 1 + 2 files changed, 9 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..15bf27a8c7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ +# Default code owners +@kentrussell @fxkamd @dayatsin-amd + +*.md @ROCm/rocm-documentation @kentrussell @dayatsin-amd +*.rst @ROCm/rocm-documentation @kentrussell @dayatsin-amd + +# Header directory for Doxygen documentation +inc/* @ROCm/rocm-documentation @kentrussell @fxkamd @dayatsin-amd diff --git a/.gitignore b/.gitignore index 1140a6dcaa..6950597ee3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ # !.gitignore !.mailmap +.github* patches-* build/ outgoing/ From 11fd5c25624f51ba5a5fa1f9f89de9f8851b8c5e Mon Sep 17 00:00:00 2001 From: Chris Freehill Date: Tue, 30 Apr 2024 09:01:09 -0500 Subject: [PATCH 1247/1247] Prepare for integration into rocr Change-Id: I6102b9910dbb9d09e09bb262a03c5c0ad4ce66f4 --- CMakeLists.txt => libhsakmt/CMakeLists.txt | 0 {DEBIAN => libhsakmt/DEBIAN}/control | 0 {DEBIAN => libhsakmt/DEBIAN}/postinst.in | 0 {DEBIAN => libhsakmt/DEBIAN}/prerm.in | 0 {DEBIAN => libhsakmt/DEBIAN}/x86_64-libhsakmt.conf | 0 LICENSE.md => libhsakmt/LICENSE.md | 0 README.md => libhsakmt/README.md | 0 {RPM => libhsakmt/RPM}/hsakmt-roct-devel.spec.in | 0 {RPM => libhsakmt/RPM}/libhsakmt.spec | 0 {RPM => libhsakmt/RPM}/post.in | 0 {RPM => libhsakmt/RPM}/postun.in | 0 {cmake_modules => libhsakmt/cmake_modules}/utils.cmake | 0 .../hsakmt-backward-compat.cmake | 0 hsakmt-config.cmake.in => libhsakmt/hsakmt-config.cmake.in | 0 {include => libhsakmt/include}/hsakmt/hsakmt.h | 0 {include => libhsakmt/include}/hsakmt/hsakmttypes.h | 0 {include => libhsakmt/include}/hsakmt/linux/kfd_ioctl.h | 0 libhsakmt.pc.in => libhsakmt/libhsakmt.pc.in | 0 {src => libhsakmt/src}/debug.c | 0 {src => libhsakmt/src}/events.c | 0 {src => libhsakmt/src}/fmm.c | 0 {src => libhsakmt/src}/fmm.h | 0 {src => libhsakmt/src}/globals.c | 0 {src => libhsakmt/src}/libhsakmt.c | 0 {src => libhsakmt/src}/libhsakmt.h | 0 {src => libhsakmt/src}/libhsakmt.ver | 0 {src => libhsakmt/src}/memory.c | 0 {src => libhsakmt/src}/openclose.c | 0 {src => libhsakmt/src}/pc_sampling.c | 0 {src => libhsakmt/src}/perfctr.c | 0 {src => libhsakmt/src}/pmc_table.c | 0 {src => libhsakmt/src}/pmc_table.h | 0 {src => libhsakmt/src}/queues.c | 0 {src => libhsakmt/src}/rbtree.c | 0 {src => libhsakmt/src}/rbtree.h | 0 {src => libhsakmt/src}/rbtree_amd.h | 0 {src => libhsakmt/src}/spm.c | 0 {src => libhsakmt/src}/svm.c | 0 {src => libhsakmt/src}/time.c | 0 {src => libhsakmt/src}/topology.c | 0 {src => libhsakmt/src}/version.c | 0 {tests => libhsakmt/tests}/kfdtest/.gitignore | 0 {tests => libhsakmt/tests}/kfdtest/CMakeLists.txt | 0 {tests => libhsakmt/tests}/kfdtest/LICENSE.kfdtest | 0 {tests => libhsakmt/tests}/kfdtest/README.txt | 0 {tests => libhsakmt/tests}/kfdtest/gtest-1.6.0/gtest-all.cpp | 0 {tests => libhsakmt/tests}/kfdtest/gtest-1.6.0/gtest/gtest.h | 0 {tests => libhsakmt/tests}/kfdtest/include/amdp2ptest.h | 0 {tests => libhsakmt/tests}/kfdtest/include/asic_reg/gfx_7_2_d.h | 0 .../tests}/kfdtest/include/asic_reg/gfx_7_2_enum.h | 0 .../tests}/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h | 0 {tests => libhsakmt/tests}/kfdtest/include/kfd_pm4_opcodes.h | 0 {tests => libhsakmt/tests}/kfdtest/include/pm4_pkt_struct_ai.h | 0 {tests => libhsakmt/tests}/kfdtest/include/pm4_pkt_struct_ci.h | 0 .../tests}/kfdtest/include/pm4_pkt_struct_common.h | 0 {tests => libhsakmt/tests}/kfdtest/include/pm4_pkt_struct_nv.h | 0 {tests => libhsakmt/tests}/kfdtest/include/sdma_pkt_struct.h | 0 {tests => libhsakmt/tests}/kfdtest/scripts/kfdtest.exclude | 0 {tests => libhsakmt/tests}/kfdtest/scripts/run_kfdtest.sh | 0 {tests => libhsakmt/tests}/kfdtest/src/AqlQueue.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/AqlQueue.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/Assemble.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/Assemble.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/BaseDebug.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/BaseDebug.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/BasePacket.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/BasePacket.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/BaseQueue.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/BaseQueue.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/Dispatch.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/Dispatch.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/GoogleTestExtension.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/GoogleTestExtension.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/IndirectBuffer.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/IndirectBuffer.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDASMTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDASMTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDBaseComponentTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDBaseComponentTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDCWSRTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDCWSRTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDDBGTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDDBGTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDEventTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDEventTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDEvictTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDEvictTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDExceptionTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDExceptionTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDGWSTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDGWSTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDGraphicsInterop.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDGraphicsInterop.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDHWSTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDHWSTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDIPCTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDIPCTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDLocalMemoryTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDLocalMemoryTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDMemoryTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDMemoryTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDMultiProcessTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDMultiProcessTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDOpenCloseKFDTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDOpenCloseKFDTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDPCSamplingTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDPCSamplingTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDPMTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDPMTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDPerfCounters.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDPerfCounters.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDPerformanceTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDQMTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDQMTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDRASTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDRASTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDSVMEvictTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDSVMEvictTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDSVMRangeTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDSVMRangeTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTestFlags.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTestMain.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtil.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtil.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtilQueue.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtilQueue.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTopologyTest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/KFDTopologyTest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/LinuxOSWrapper.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/OSWrapper.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/PM4Packet.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/PM4Packet.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/PM4Queue.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/PM4Queue.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/RDMATest.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/RDMATest.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/RDMAUtil.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/RDMAUtil.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/SDMAPacket.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/SDMAPacket.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/SDMAQueue.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/SDMAQueue.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/ShaderStore.cpp | 0 {tests => libhsakmt/tests}/kfdtest/src/ShaderStore.hpp | 0 {tests => libhsakmt/tests}/kfdtest/src/XgmiOptimizedSDMAQueue.hpp | 0 {tests => libhsakmt/tests}/rdma/simple/app/CMakeLists.txt | 0 {tests => libhsakmt/tests}/rdma/simple/app/rdma_test.cpp | 0 {tests => libhsakmt/tests}/rdma/simple/drv/Makefile | 0 {tests => libhsakmt/tests}/rdma/simple/drv/amdp2ptest.c | 0 {tests => libhsakmt/tests}/rdma/simple/drv/amdp2ptest.h | 0 {tests => libhsakmt/tests}/reopen/CMakeLists.txt | 0 {tests => libhsakmt/tests}/reopen/kmtreopen.c | 0 152 files changed, 0 insertions(+), 0 deletions(-) rename CMakeLists.txt => libhsakmt/CMakeLists.txt (100%) rename {DEBIAN => libhsakmt/DEBIAN}/control (100%) rename {DEBIAN => libhsakmt/DEBIAN}/postinst.in (100%) rename {DEBIAN => libhsakmt/DEBIAN}/prerm.in (100%) rename {DEBIAN => libhsakmt/DEBIAN}/x86_64-libhsakmt.conf (100%) rename LICENSE.md => libhsakmt/LICENSE.md (100%) rename README.md => libhsakmt/README.md (100%) rename {RPM => libhsakmt/RPM}/hsakmt-roct-devel.spec.in (100%) rename {RPM => libhsakmt/RPM}/libhsakmt.spec (100%) rename {RPM => libhsakmt/RPM}/post.in (100%) rename {RPM => libhsakmt/RPM}/postun.in (100%) rename {cmake_modules => libhsakmt/cmake_modules}/utils.cmake (100%) rename hsakmt-backward-compat.cmake => libhsakmt/hsakmt-backward-compat.cmake (100%) rename hsakmt-config.cmake.in => libhsakmt/hsakmt-config.cmake.in (100%) rename {include => libhsakmt/include}/hsakmt/hsakmt.h (100%) rename {include => libhsakmt/include}/hsakmt/hsakmttypes.h (100%) rename {include => libhsakmt/include}/hsakmt/linux/kfd_ioctl.h (100%) rename libhsakmt.pc.in => libhsakmt/libhsakmt.pc.in (100%) rename {src => libhsakmt/src}/debug.c (100%) rename {src => libhsakmt/src}/events.c (100%) rename {src => libhsakmt/src}/fmm.c (100%) rename {src => libhsakmt/src}/fmm.h (100%) rename {src => libhsakmt/src}/globals.c (100%) rename {src => libhsakmt/src}/libhsakmt.c (100%) rename {src => libhsakmt/src}/libhsakmt.h (100%) rename {src => libhsakmt/src}/libhsakmt.ver (100%) rename {src => libhsakmt/src}/memory.c (100%) rename {src => libhsakmt/src}/openclose.c (100%) rename {src => libhsakmt/src}/pc_sampling.c (100%) rename {src => libhsakmt/src}/perfctr.c (100%) rename {src => libhsakmt/src}/pmc_table.c (100%) rename {src => libhsakmt/src}/pmc_table.h (100%) rename {src => libhsakmt/src}/queues.c (100%) rename {src => libhsakmt/src}/rbtree.c (100%) rename {src => libhsakmt/src}/rbtree.h (100%) rename {src => libhsakmt/src}/rbtree_amd.h (100%) rename {src => libhsakmt/src}/spm.c (100%) rename {src => libhsakmt/src}/svm.c (100%) rename {src => libhsakmt/src}/time.c (100%) rename {src => libhsakmt/src}/topology.c (100%) rename {src => libhsakmt/src}/version.c (100%) rename {tests => libhsakmt/tests}/kfdtest/.gitignore (100%) rename {tests => libhsakmt/tests}/kfdtest/CMakeLists.txt (100%) rename {tests => libhsakmt/tests}/kfdtest/LICENSE.kfdtest (100%) rename {tests => libhsakmt/tests}/kfdtest/README.txt (100%) rename {tests => libhsakmt/tests}/kfdtest/gtest-1.6.0/gtest-all.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/gtest-1.6.0/gtest/gtest.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/amdp2ptest.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/asic_reg/gfx_7_2_d.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/asic_reg/gfx_7_2_enum.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/kfd_pm4_opcodes.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/pm4_pkt_struct_ai.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/pm4_pkt_struct_ci.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/pm4_pkt_struct_common.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/pm4_pkt_struct_nv.h (100%) rename {tests => libhsakmt/tests}/kfdtest/include/sdma_pkt_struct.h (100%) rename {tests => libhsakmt/tests}/kfdtest/scripts/kfdtest.exclude (100%) rename {tests => libhsakmt/tests}/kfdtest/scripts/run_kfdtest.sh (100%) rename {tests => libhsakmt/tests}/kfdtest/src/AqlQueue.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/AqlQueue.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/Assemble.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/Assemble.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/BaseDebug.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/BaseDebug.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/BasePacket.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/BasePacket.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/BaseQueue.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/BaseQueue.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/Dispatch.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/Dispatch.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/GoogleTestExtension.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/GoogleTestExtension.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/IndirectBuffer.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/IndirectBuffer.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDASMTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDASMTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDBaseComponentTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDBaseComponentTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDCWSRTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDCWSRTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDDBGTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDDBGTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDEventTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDEventTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDEvictTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDEvictTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDExceptionTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDExceptionTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDGWSTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDGWSTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDGraphicsInterop.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDGraphicsInterop.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDHWSTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDHWSTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDIPCTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDIPCTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDLocalMemoryTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDLocalMemoryTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDMemoryTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDMemoryTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDMultiProcessTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDMultiProcessTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDOpenCloseKFDTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDOpenCloseKFDTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDPCSamplingTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDPCSamplingTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDPMTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDPMTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDPerfCounters.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDPerfCounters.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDPerformanceTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDQMTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDQMTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDRASTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDRASTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDSVMEvictTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDSVMEvictTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDSVMRangeTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDSVMRangeTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTestFlags.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTestMain.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtil.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtil.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtilQueue.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTestUtilQueue.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTopologyTest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/KFDTopologyTest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/LinuxOSWrapper.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/OSWrapper.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/PM4Packet.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/PM4Packet.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/PM4Queue.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/PM4Queue.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/RDMATest.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/RDMATest.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/RDMAUtil.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/RDMAUtil.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/SDMAPacket.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/SDMAPacket.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/SDMAQueue.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/SDMAQueue.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/ShaderStore.cpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/ShaderStore.hpp (100%) rename {tests => libhsakmt/tests}/kfdtest/src/XgmiOptimizedSDMAQueue.hpp (100%) rename {tests => libhsakmt/tests}/rdma/simple/app/CMakeLists.txt (100%) rename {tests => libhsakmt/tests}/rdma/simple/app/rdma_test.cpp (100%) rename {tests => libhsakmt/tests}/rdma/simple/drv/Makefile (100%) rename {tests => libhsakmt/tests}/rdma/simple/drv/amdp2ptest.c (100%) rename {tests => libhsakmt/tests}/rdma/simple/drv/amdp2ptest.h (100%) rename {tests => libhsakmt/tests}/reopen/CMakeLists.txt (100%) rename {tests => libhsakmt/tests}/reopen/kmtreopen.c (100%) diff --git a/CMakeLists.txt b/libhsakmt/CMakeLists.txt similarity index 100% rename from CMakeLists.txt rename to libhsakmt/CMakeLists.txt diff --git a/DEBIAN/control b/libhsakmt/DEBIAN/control similarity index 100% rename from DEBIAN/control rename to libhsakmt/DEBIAN/control diff --git a/DEBIAN/postinst.in b/libhsakmt/DEBIAN/postinst.in similarity index 100% rename from DEBIAN/postinst.in rename to libhsakmt/DEBIAN/postinst.in diff --git a/DEBIAN/prerm.in b/libhsakmt/DEBIAN/prerm.in similarity index 100% rename from DEBIAN/prerm.in rename to libhsakmt/DEBIAN/prerm.in diff --git a/DEBIAN/x86_64-libhsakmt.conf b/libhsakmt/DEBIAN/x86_64-libhsakmt.conf similarity index 100% rename from DEBIAN/x86_64-libhsakmt.conf rename to libhsakmt/DEBIAN/x86_64-libhsakmt.conf diff --git a/LICENSE.md b/libhsakmt/LICENSE.md similarity index 100% rename from LICENSE.md rename to libhsakmt/LICENSE.md diff --git a/README.md b/libhsakmt/README.md similarity index 100% rename from README.md rename to libhsakmt/README.md diff --git a/RPM/hsakmt-roct-devel.spec.in b/libhsakmt/RPM/hsakmt-roct-devel.spec.in similarity index 100% rename from RPM/hsakmt-roct-devel.spec.in rename to libhsakmt/RPM/hsakmt-roct-devel.spec.in diff --git a/RPM/libhsakmt.spec b/libhsakmt/RPM/libhsakmt.spec similarity index 100% rename from RPM/libhsakmt.spec rename to libhsakmt/RPM/libhsakmt.spec diff --git a/RPM/post.in b/libhsakmt/RPM/post.in similarity index 100% rename from RPM/post.in rename to libhsakmt/RPM/post.in diff --git a/RPM/postun.in b/libhsakmt/RPM/postun.in similarity index 100% rename from RPM/postun.in rename to libhsakmt/RPM/postun.in diff --git a/cmake_modules/utils.cmake b/libhsakmt/cmake_modules/utils.cmake similarity index 100% rename from cmake_modules/utils.cmake rename to libhsakmt/cmake_modules/utils.cmake diff --git a/hsakmt-backward-compat.cmake b/libhsakmt/hsakmt-backward-compat.cmake similarity index 100% rename from hsakmt-backward-compat.cmake rename to libhsakmt/hsakmt-backward-compat.cmake diff --git a/hsakmt-config.cmake.in b/libhsakmt/hsakmt-config.cmake.in similarity index 100% rename from hsakmt-config.cmake.in rename to libhsakmt/hsakmt-config.cmake.in diff --git a/include/hsakmt/hsakmt.h b/libhsakmt/include/hsakmt/hsakmt.h similarity index 100% rename from include/hsakmt/hsakmt.h rename to libhsakmt/include/hsakmt/hsakmt.h diff --git a/include/hsakmt/hsakmttypes.h b/libhsakmt/include/hsakmt/hsakmttypes.h similarity index 100% rename from include/hsakmt/hsakmttypes.h rename to libhsakmt/include/hsakmt/hsakmttypes.h diff --git a/include/hsakmt/linux/kfd_ioctl.h b/libhsakmt/include/hsakmt/linux/kfd_ioctl.h similarity index 100% rename from include/hsakmt/linux/kfd_ioctl.h rename to libhsakmt/include/hsakmt/linux/kfd_ioctl.h diff --git a/libhsakmt.pc.in b/libhsakmt/libhsakmt.pc.in similarity index 100% rename from libhsakmt.pc.in rename to libhsakmt/libhsakmt.pc.in diff --git a/src/debug.c b/libhsakmt/src/debug.c similarity index 100% rename from src/debug.c rename to libhsakmt/src/debug.c diff --git a/src/events.c b/libhsakmt/src/events.c similarity index 100% rename from src/events.c rename to libhsakmt/src/events.c diff --git a/src/fmm.c b/libhsakmt/src/fmm.c similarity index 100% rename from src/fmm.c rename to libhsakmt/src/fmm.c diff --git a/src/fmm.h b/libhsakmt/src/fmm.h similarity index 100% rename from src/fmm.h rename to libhsakmt/src/fmm.h diff --git a/src/globals.c b/libhsakmt/src/globals.c similarity index 100% rename from src/globals.c rename to libhsakmt/src/globals.c diff --git a/src/libhsakmt.c b/libhsakmt/src/libhsakmt.c similarity index 100% rename from src/libhsakmt.c rename to libhsakmt/src/libhsakmt.c diff --git a/src/libhsakmt.h b/libhsakmt/src/libhsakmt.h similarity index 100% rename from src/libhsakmt.h rename to libhsakmt/src/libhsakmt.h diff --git a/src/libhsakmt.ver b/libhsakmt/src/libhsakmt.ver similarity index 100% rename from src/libhsakmt.ver rename to libhsakmt/src/libhsakmt.ver diff --git a/src/memory.c b/libhsakmt/src/memory.c similarity index 100% rename from src/memory.c rename to libhsakmt/src/memory.c diff --git a/src/openclose.c b/libhsakmt/src/openclose.c similarity index 100% rename from src/openclose.c rename to libhsakmt/src/openclose.c diff --git a/src/pc_sampling.c b/libhsakmt/src/pc_sampling.c similarity index 100% rename from src/pc_sampling.c rename to libhsakmt/src/pc_sampling.c diff --git a/src/perfctr.c b/libhsakmt/src/perfctr.c similarity index 100% rename from src/perfctr.c rename to libhsakmt/src/perfctr.c diff --git a/src/pmc_table.c b/libhsakmt/src/pmc_table.c similarity index 100% rename from src/pmc_table.c rename to libhsakmt/src/pmc_table.c diff --git a/src/pmc_table.h b/libhsakmt/src/pmc_table.h similarity index 100% rename from src/pmc_table.h rename to libhsakmt/src/pmc_table.h diff --git a/src/queues.c b/libhsakmt/src/queues.c similarity index 100% rename from src/queues.c rename to libhsakmt/src/queues.c diff --git a/src/rbtree.c b/libhsakmt/src/rbtree.c similarity index 100% rename from src/rbtree.c rename to libhsakmt/src/rbtree.c diff --git a/src/rbtree.h b/libhsakmt/src/rbtree.h similarity index 100% rename from src/rbtree.h rename to libhsakmt/src/rbtree.h diff --git a/src/rbtree_amd.h b/libhsakmt/src/rbtree_amd.h similarity index 100% rename from src/rbtree_amd.h rename to libhsakmt/src/rbtree_amd.h diff --git a/src/spm.c b/libhsakmt/src/spm.c similarity index 100% rename from src/spm.c rename to libhsakmt/src/spm.c diff --git a/src/svm.c b/libhsakmt/src/svm.c similarity index 100% rename from src/svm.c rename to libhsakmt/src/svm.c diff --git a/src/time.c b/libhsakmt/src/time.c similarity index 100% rename from src/time.c rename to libhsakmt/src/time.c diff --git a/src/topology.c b/libhsakmt/src/topology.c similarity index 100% rename from src/topology.c rename to libhsakmt/src/topology.c diff --git a/src/version.c b/libhsakmt/src/version.c similarity index 100% rename from src/version.c rename to libhsakmt/src/version.c diff --git a/tests/kfdtest/.gitignore b/libhsakmt/tests/kfdtest/.gitignore similarity index 100% rename from tests/kfdtest/.gitignore rename to libhsakmt/tests/kfdtest/.gitignore diff --git a/tests/kfdtest/CMakeLists.txt b/libhsakmt/tests/kfdtest/CMakeLists.txt similarity index 100% rename from tests/kfdtest/CMakeLists.txt rename to libhsakmt/tests/kfdtest/CMakeLists.txt diff --git a/tests/kfdtest/LICENSE.kfdtest b/libhsakmt/tests/kfdtest/LICENSE.kfdtest similarity index 100% rename from tests/kfdtest/LICENSE.kfdtest rename to libhsakmt/tests/kfdtest/LICENSE.kfdtest diff --git a/tests/kfdtest/README.txt b/libhsakmt/tests/kfdtest/README.txt similarity index 100% rename from tests/kfdtest/README.txt rename to libhsakmt/tests/kfdtest/README.txt diff --git a/tests/kfdtest/gtest-1.6.0/gtest-all.cpp b/libhsakmt/tests/kfdtest/gtest-1.6.0/gtest-all.cpp similarity index 100% rename from tests/kfdtest/gtest-1.6.0/gtest-all.cpp rename to libhsakmt/tests/kfdtest/gtest-1.6.0/gtest-all.cpp diff --git a/tests/kfdtest/gtest-1.6.0/gtest/gtest.h b/libhsakmt/tests/kfdtest/gtest-1.6.0/gtest/gtest.h similarity index 100% rename from tests/kfdtest/gtest-1.6.0/gtest/gtest.h rename to libhsakmt/tests/kfdtest/gtest-1.6.0/gtest/gtest.h diff --git a/tests/kfdtest/include/amdp2ptest.h b/libhsakmt/tests/kfdtest/include/amdp2ptest.h similarity index 100% rename from tests/kfdtest/include/amdp2ptest.h rename to libhsakmt/tests/kfdtest/include/amdp2ptest.h diff --git a/tests/kfdtest/include/asic_reg/gfx_7_2_d.h b/libhsakmt/tests/kfdtest/include/asic_reg/gfx_7_2_d.h similarity index 100% rename from tests/kfdtest/include/asic_reg/gfx_7_2_d.h rename to libhsakmt/tests/kfdtest/include/asic_reg/gfx_7_2_d.h diff --git a/tests/kfdtest/include/asic_reg/gfx_7_2_enum.h b/libhsakmt/tests/kfdtest/include/asic_reg/gfx_7_2_enum.h similarity index 100% rename from tests/kfdtest/include/asic_reg/gfx_7_2_enum.h rename to libhsakmt/tests/kfdtest/include/asic_reg/gfx_7_2_enum.h diff --git a/tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h b/libhsakmt/tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h similarity index 100% rename from tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h rename to libhsakmt/tests/kfdtest/include/asic_reg/gfx_7_2_sh_mask.h diff --git a/tests/kfdtest/include/kfd_pm4_opcodes.h b/libhsakmt/tests/kfdtest/include/kfd_pm4_opcodes.h similarity index 100% rename from tests/kfdtest/include/kfd_pm4_opcodes.h rename to libhsakmt/tests/kfdtest/include/kfd_pm4_opcodes.h diff --git a/tests/kfdtest/include/pm4_pkt_struct_ai.h b/libhsakmt/tests/kfdtest/include/pm4_pkt_struct_ai.h similarity index 100% rename from tests/kfdtest/include/pm4_pkt_struct_ai.h rename to libhsakmt/tests/kfdtest/include/pm4_pkt_struct_ai.h diff --git a/tests/kfdtest/include/pm4_pkt_struct_ci.h b/libhsakmt/tests/kfdtest/include/pm4_pkt_struct_ci.h similarity index 100% rename from tests/kfdtest/include/pm4_pkt_struct_ci.h rename to libhsakmt/tests/kfdtest/include/pm4_pkt_struct_ci.h diff --git a/tests/kfdtest/include/pm4_pkt_struct_common.h b/libhsakmt/tests/kfdtest/include/pm4_pkt_struct_common.h similarity index 100% rename from tests/kfdtest/include/pm4_pkt_struct_common.h rename to libhsakmt/tests/kfdtest/include/pm4_pkt_struct_common.h diff --git a/tests/kfdtest/include/pm4_pkt_struct_nv.h b/libhsakmt/tests/kfdtest/include/pm4_pkt_struct_nv.h similarity index 100% rename from tests/kfdtest/include/pm4_pkt_struct_nv.h rename to libhsakmt/tests/kfdtest/include/pm4_pkt_struct_nv.h diff --git a/tests/kfdtest/include/sdma_pkt_struct.h b/libhsakmt/tests/kfdtest/include/sdma_pkt_struct.h similarity index 100% rename from tests/kfdtest/include/sdma_pkt_struct.h rename to libhsakmt/tests/kfdtest/include/sdma_pkt_struct.h diff --git a/tests/kfdtest/scripts/kfdtest.exclude b/libhsakmt/tests/kfdtest/scripts/kfdtest.exclude similarity index 100% rename from tests/kfdtest/scripts/kfdtest.exclude rename to libhsakmt/tests/kfdtest/scripts/kfdtest.exclude diff --git a/tests/kfdtest/scripts/run_kfdtest.sh b/libhsakmt/tests/kfdtest/scripts/run_kfdtest.sh similarity index 100% rename from tests/kfdtest/scripts/run_kfdtest.sh rename to libhsakmt/tests/kfdtest/scripts/run_kfdtest.sh diff --git a/tests/kfdtest/src/AqlQueue.cpp b/libhsakmt/tests/kfdtest/src/AqlQueue.cpp similarity index 100% rename from tests/kfdtest/src/AqlQueue.cpp rename to libhsakmt/tests/kfdtest/src/AqlQueue.cpp diff --git a/tests/kfdtest/src/AqlQueue.hpp b/libhsakmt/tests/kfdtest/src/AqlQueue.hpp similarity index 100% rename from tests/kfdtest/src/AqlQueue.hpp rename to libhsakmt/tests/kfdtest/src/AqlQueue.hpp diff --git a/tests/kfdtest/src/Assemble.cpp b/libhsakmt/tests/kfdtest/src/Assemble.cpp similarity index 100% rename from tests/kfdtest/src/Assemble.cpp rename to libhsakmt/tests/kfdtest/src/Assemble.cpp diff --git a/tests/kfdtest/src/Assemble.hpp b/libhsakmt/tests/kfdtest/src/Assemble.hpp similarity index 100% rename from tests/kfdtest/src/Assemble.hpp rename to libhsakmt/tests/kfdtest/src/Assemble.hpp diff --git a/tests/kfdtest/src/BaseDebug.cpp b/libhsakmt/tests/kfdtest/src/BaseDebug.cpp similarity index 100% rename from tests/kfdtest/src/BaseDebug.cpp rename to libhsakmt/tests/kfdtest/src/BaseDebug.cpp diff --git a/tests/kfdtest/src/BaseDebug.hpp b/libhsakmt/tests/kfdtest/src/BaseDebug.hpp similarity index 100% rename from tests/kfdtest/src/BaseDebug.hpp rename to libhsakmt/tests/kfdtest/src/BaseDebug.hpp diff --git a/tests/kfdtest/src/BasePacket.cpp b/libhsakmt/tests/kfdtest/src/BasePacket.cpp similarity index 100% rename from tests/kfdtest/src/BasePacket.cpp rename to libhsakmt/tests/kfdtest/src/BasePacket.cpp diff --git a/tests/kfdtest/src/BasePacket.hpp b/libhsakmt/tests/kfdtest/src/BasePacket.hpp similarity index 100% rename from tests/kfdtest/src/BasePacket.hpp rename to libhsakmt/tests/kfdtest/src/BasePacket.hpp diff --git a/tests/kfdtest/src/BaseQueue.cpp b/libhsakmt/tests/kfdtest/src/BaseQueue.cpp similarity index 100% rename from tests/kfdtest/src/BaseQueue.cpp rename to libhsakmt/tests/kfdtest/src/BaseQueue.cpp diff --git a/tests/kfdtest/src/BaseQueue.hpp b/libhsakmt/tests/kfdtest/src/BaseQueue.hpp similarity index 100% rename from tests/kfdtest/src/BaseQueue.hpp rename to libhsakmt/tests/kfdtest/src/BaseQueue.hpp diff --git a/tests/kfdtest/src/Dispatch.cpp b/libhsakmt/tests/kfdtest/src/Dispatch.cpp similarity index 100% rename from tests/kfdtest/src/Dispatch.cpp rename to libhsakmt/tests/kfdtest/src/Dispatch.cpp diff --git a/tests/kfdtest/src/Dispatch.hpp b/libhsakmt/tests/kfdtest/src/Dispatch.hpp similarity index 100% rename from tests/kfdtest/src/Dispatch.hpp rename to libhsakmt/tests/kfdtest/src/Dispatch.hpp diff --git a/tests/kfdtest/src/GoogleTestExtension.cpp b/libhsakmt/tests/kfdtest/src/GoogleTestExtension.cpp similarity index 100% rename from tests/kfdtest/src/GoogleTestExtension.cpp rename to libhsakmt/tests/kfdtest/src/GoogleTestExtension.cpp diff --git a/tests/kfdtest/src/GoogleTestExtension.hpp b/libhsakmt/tests/kfdtest/src/GoogleTestExtension.hpp similarity index 100% rename from tests/kfdtest/src/GoogleTestExtension.hpp rename to libhsakmt/tests/kfdtest/src/GoogleTestExtension.hpp diff --git a/tests/kfdtest/src/IndirectBuffer.cpp b/libhsakmt/tests/kfdtest/src/IndirectBuffer.cpp similarity index 100% rename from tests/kfdtest/src/IndirectBuffer.cpp rename to libhsakmt/tests/kfdtest/src/IndirectBuffer.cpp diff --git a/tests/kfdtest/src/IndirectBuffer.hpp b/libhsakmt/tests/kfdtest/src/IndirectBuffer.hpp similarity index 100% rename from tests/kfdtest/src/IndirectBuffer.hpp rename to libhsakmt/tests/kfdtest/src/IndirectBuffer.hpp diff --git a/tests/kfdtest/src/KFDASMTest.cpp b/libhsakmt/tests/kfdtest/src/KFDASMTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDASMTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDASMTest.cpp diff --git a/tests/kfdtest/src/KFDASMTest.hpp b/libhsakmt/tests/kfdtest/src/KFDASMTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDASMTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDASMTest.hpp diff --git a/tests/kfdtest/src/KFDBaseComponentTest.cpp b/libhsakmt/tests/kfdtest/src/KFDBaseComponentTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDBaseComponentTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDBaseComponentTest.cpp diff --git a/tests/kfdtest/src/KFDBaseComponentTest.hpp b/libhsakmt/tests/kfdtest/src/KFDBaseComponentTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDBaseComponentTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDBaseComponentTest.hpp diff --git a/tests/kfdtest/src/KFDCWSRTest.cpp b/libhsakmt/tests/kfdtest/src/KFDCWSRTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDCWSRTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDCWSRTest.cpp diff --git a/tests/kfdtest/src/KFDCWSRTest.hpp b/libhsakmt/tests/kfdtest/src/KFDCWSRTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDCWSRTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDCWSRTest.hpp diff --git a/tests/kfdtest/src/KFDDBGTest.cpp b/libhsakmt/tests/kfdtest/src/KFDDBGTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDDBGTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDDBGTest.cpp diff --git a/tests/kfdtest/src/KFDDBGTest.hpp b/libhsakmt/tests/kfdtest/src/KFDDBGTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDDBGTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDDBGTest.hpp diff --git a/tests/kfdtest/src/KFDEventTest.cpp b/libhsakmt/tests/kfdtest/src/KFDEventTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDEventTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDEventTest.cpp diff --git a/tests/kfdtest/src/KFDEventTest.hpp b/libhsakmt/tests/kfdtest/src/KFDEventTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDEventTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDEventTest.hpp diff --git a/tests/kfdtest/src/KFDEvictTest.cpp b/libhsakmt/tests/kfdtest/src/KFDEvictTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDEvictTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDEvictTest.cpp diff --git a/tests/kfdtest/src/KFDEvictTest.hpp b/libhsakmt/tests/kfdtest/src/KFDEvictTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDEvictTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDEvictTest.hpp diff --git a/tests/kfdtest/src/KFDExceptionTest.cpp b/libhsakmt/tests/kfdtest/src/KFDExceptionTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDExceptionTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDExceptionTest.cpp diff --git a/tests/kfdtest/src/KFDExceptionTest.hpp b/libhsakmt/tests/kfdtest/src/KFDExceptionTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDExceptionTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDExceptionTest.hpp diff --git a/tests/kfdtest/src/KFDGWSTest.cpp b/libhsakmt/tests/kfdtest/src/KFDGWSTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDGWSTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDGWSTest.cpp diff --git a/tests/kfdtest/src/KFDGWSTest.hpp b/libhsakmt/tests/kfdtest/src/KFDGWSTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDGWSTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDGWSTest.hpp diff --git a/tests/kfdtest/src/KFDGraphicsInterop.cpp b/libhsakmt/tests/kfdtest/src/KFDGraphicsInterop.cpp similarity index 100% rename from tests/kfdtest/src/KFDGraphicsInterop.cpp rename to libhsakmt/tests/kfdtest/src/KFDGraphicsInterop.cpp diff --git a/tests/kfdtest/src/KFDGraphicsInterop.hpp b/libhsakmt/tests/kfdtest/src/KFDGraphicsInterop.hpp similarity index 100% rename from tests/kfdtest/src/KFDGraphicsInterop.hpp rename to libhsakmt/tests/kfdtest/src/KFDGraphicsInterop.hpp diff --git a/tests/kfdtest/src/KFDHWSTest.cpp b/libhsakmt/tests/kfdtest/src/KFDHWSTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDHWSTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDHWSTest.cpp diff --git a/tests/kfdtest/src/KFDHWSTest.hpp b/libhsakmt/tests/kfdtest/src/KFDHWSTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDHWSTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDHWSTest.hpp diff --git a/tests/kfdtest/src/KFDIPCTest.cpp b/libhsakmt/tests/kfdtest/src/KFDIPCTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDIPCTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDIPCTest.cpp diff --git a/tests/kfdtest/src/KFDIPCTest.hpp b/libhsakmt/tests/kfdtest/src/KFDIPCTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDIPCTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDIPCTest.hpp diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.cpp b/libhsakmt/tests/kfdtest/src/KFDLocalMemoryTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDLocalMemoryTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDLocalMemoryTest.cpp diff --git a/tests/kfdtest/src/KFDLocalMemoryTest.hpp b/libhsakmt/tests/kfdtest/src/KFDLocalMemoryTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDLocalMemoryTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDLocalMemoryTest.hpp diff --git a/tests/kfdtest/src/KFDMemoryTest.cpp b/libhsakmt/tests/kfdtest/src/KFDMemoryTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDMemoryTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDMemoryTest.cpp diff --git a/tests/kfdtest/src/KFDMemoryTest.hpp b/libhsakmt/tests/kfdtest/src/KFDMemoryTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDMemoryTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDMemoryTest.hpp diff --git a/tests/kfdtest/src/KFDMultiProcessTest.cpp b/libhsakmt/tests/kfdtest/src/KFDMultiProcessTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDMultiProcessTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDMultiProcessTest.cpp diff --git a/tests/kfdtest/src/KFDMultiProcessTest.hpp b/libhsakmt/tests/kfdtest/src/KFDMultiProcessTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDMultiProcessTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDMultiProcessTest.hpp diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp b/libhsakmt/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDOpenCloseKFDTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDOpenCloseKFDTest.cpp diff --git a/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp b/libhsakmt/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDOpenCloseKFDTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDOpenCloseKFDTest.hpp diff --git a/tests/kfdtest/src/KFDPCSamplingTest.cpp b/libhsakmt/tests/kfdtest/src/KFDPCSamplingTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDPCSamplingTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDPCSamplingTest.cpp diff --git a/tests/kfdtest/src/KFDPCSamplingTest.hpp b/libhsakmt/tests/kfdtest/src/KFDPCSamplingTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDPCSamplingTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDPCSamplingTest.hpp diff --git a/tests/kfdtest/src/KFDPMTest.cpp b/libhsakmt/tests/kfdtest/src/KFDPMTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDPMTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDPMTest.cpp diff --git a/tests/kfdtest/src/KFDPMTest.hpp b/libhsakmt/tests/kfdtest/src/KFDPMTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDPMTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDPMTest.hpp diff --git a/tests/kfdtest/src/KFDPerfCounters.cpp b/libhsakmt/tests/kfdtest/src/KFDPerfCounters.cpp similarity index 100% rename from tests/kfdtest/src/KFDPerfCounters.cpp rename to libhsakmt/tests/kfdtest/src/KFDPerfCounters.cpp diff --git a/tests/kfdtest/src/KFDPerfCounters.hpp b/libhsakmt/tests/kfdtest/src/KFDPerfCounters.hpp similarity index 100% rename from tests/kfdtest/src/KFDPerfCounters.hpp rename to libhsakmt/tests/kfdtest/src/KFDPerfCounters.hpp diff --git a/tests/kfdtest/src/KFDPerformanceTest.cpp b/libhsakmt/tests/kfdtest/src/KFDPerformanceTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDPerformanceTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDPerformanceTest.cpp diff --git a/tests/kfdtest/src/KFDQMTest.cpp b/libhsakmt/tests/kfdtest/src/KFDQMTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDQMTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDQMTest.cpp diff --git a/tests/kfdtest/src/KFDQMTest.hpp b/libhsakmt/tests/kfdtest/src/KFDQMTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDQMTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDQMTest.hpp diff --git a/tests/kfdtest/src/KFDRASTest.cpp b/libhsakmt/tests/kfdtest/src/KFDRASTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDRASTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDRASTest.cpp diff --git a/tests/kfdtest/src/KFDRASTest.hpp b/libhsakmt/tests/kfdtest/src/KFDRASTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDRASTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDRASTest.hpp diff --git a/tests/kfdtest/src/KFDSVMEvictTest.cpp b/libhsakmt/tests/kfdtest/src/KFDSVMEvictTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDSVMEvictTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDSVMEvictTest.cpp diff --git a/tests/kfdtest/src/KFDSVMEvictTest.hpp b/libhsakmt/tests/kfdtest/src/KFDSVMEvictTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDSVMEvictTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDSVMEvictTest.hpp diff --git a/tests/kfdtest/src/KFDSVMRangeTest.cpp b/libhsakmt/tests/kfdtest/src/KFDSVMRangeTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDSVMRangeTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDSVMRangeTest.cpp diff --git a/tests/kfdtest/src/KFDSVMRangeTest.hpp b/libhsakmt/tests/kfdtest/src/KFDSVMRangeTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDSVMRangeTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDSVMRangeTest.hpp diff --git a/tests/kfdtest/src/KFDTestFlags.hpp b/libhsakmt/tests/kfdtest/src/KFDTestFlags.hpp similarity index 100% rename from tests/kfdtest/src/KFDTestFlags.hpp rename to libhsakmt/tests/kfdtest/src/KFDTestFlags.hpp diff --git a/tests/kfdtest/src/KFDTestMain.cpp b/libhsakmt/tests/kfdtest/src/KFDTestMain.cpp similarity index 100% rename from tests/kfdtest/src/KFDTestMain.cpp rename to libhsakmt/tests/kfdtest/src/KFDTestMain.cpp diff --git a/tests/kfdtest/src/KFDTestUtil.cpp b/libhsakmt/tests/kfdtest/src/KFDTestUtil.cpp similarity index 100% rename from tests/kfdtest/src/KFDTestUtil.cpp rename to libhsakmt/tests/kfdtest/src/KFDTestUtil.cpp diff --git a/tests/kfdtest/src/KFDTestUtil.hpp b/libhsakmt/tests/kfdtest/src/KFDTestUtil.hpp similarity index 100% rename from tests/kfdtest/src/KFDTestUtil.hpp rename to libhsakmt/tests/kfdtest/src/KFDTestUtil.hpp diff --git a/tests/kfdtest/src/KFDTestUtilQueue.cpp b/libhsakmt/tests/kfdtest/src/KFDTestUtilQueue.cpp similarity index 100% rename from tests/kfdtest/src/KFDTestUtilQueue.cpp rename to libhsakmt/tests/kfdtest/src/KFDTestUtilQueue.cpp diff --git a/tests/kfdtest/src/KFDTestUtilQueue.hpp b/libhsakmt/tests/kfdtest/src/KFDTestUtilQueue.hpp similarity index 100% rename from tests/kfdtest/src/KFDTestUtilQueue.hpp rename to libhsakmt/tests/kfdtest/src/KFDTestUtilQueue.hpp diff --git a/tests/kfdtest/src/KFDTopologyTest.cpp b/libhsakmt/tests/kfdtest/src/KFDTopologyTest.cpp similarity index 100% rename from tests/kfdtest/src/KFDTopologyTest.cpp rename to libhsakmt/tests/kfdtest/src/KFDTopologyTest.cpp diff --git a/tests/kfdtest/src/KFDTopologyTest.hpp b/libhsakmt/tests/kfdtest/src/KFDTopologyTest.hpp similarity index 100% rename from tests/kfdtest/src/KFDTopologyTest.hpp rename to libhsakmt/tests/kfdtest/src/KFDTopologyTest.hpp diff --git a/tests/kfdtest/src/LinuxOSWrapper.cpp b/libhsakmt/tests/kfdtest/src/LinuxOSWrapper.cpp similarity index 100% rename from tests/kfdtest/src/LinuxOSWrapper.cpp rename to libhsakmt/tests/kfdtest/src/LinuxOSWrapper.cpp diff --git a/tests/kfdtest/src/OSWrapper.hpp b/libhsakmt/tests/kfdtest/src/OSWrapper.hpp similarity index 100% rename from tests/kfdtest/src/OSWrapper.hpp rename to libhsakmt/tests/kfdtest/src/OSWrapper.hpp diff --git a/tests/kfdtest/src/PM4Packet.cpp b/libhsakmt/tests/kfdtest/src/PM4Packet.cpp similarity index 100% rename from tests/kfdtest/src/PM4Packet.cpp rename to libhsakmt/tests/kfdtest/src/PM4Packet.cpp diff --git a/tests/kfdtest/src/PM4Packet.hpp b/libhsakmt/tests/kfdtest/src/PM4Packet.hpp similarity index 100% rename from tests/kfdtest/src/PM4Packet.hpp rename to libhsakmt/tests/kfdtest/src/PM4Packet.hpp diff --git a/tests/kfdtest/src/PM4Queue.cpp b/libhsakmt/tests/kfdtest/src/PM4Queue.cpp similarity index 100% rename from tests/kfdtest/src/PM4Queue.cpp rename to libhsakmt/tests/kfdtest/src/PM4Queue.cpp diff --git a/tests/kfdtest/src/PM4Queue.hpp b/libhsakmt/tests/kfdtest/src/PM4Queue.hpp similarity index 100% rename from tests/kfdtest/src/PM4Queue.hpp rename to libhsakmt/tests/kfdtest/src/PM4Queue.hpp diff --git a/tests/kfdtest/src/RDMATest.cpp b/libhsakmt/tests/kfdtest/src/RDMATest.cpp similarity index 100% rename from tests/kfdtest/src/RDMATest.cpp rename to libhsakmt/tests/kfdtest/src/RDMATest.cpp diff --git a/tests/kfdtest/src/RDMATest.hpp b/libhsakmt/tests/kfdtest/src/RDMATest.hpp similarity index 100% rename from tests/kfdtest/src/RDMATest.hpp rename to libhsakmt/tests/kfdtest/src/RDMATest.hpp diff --git a/tests/kfdtest/src/RDMAUtil.cpp b/libhsakmt/tests/kfdtest/src/RDMAUtil.cpp similarity index 100% rename from tests/kfdtest/src/RDMAUtil.cpp rename to libhsakmt/tests/kfdtest/src/RDMAUtil.cpp diff --git a/tests/kfdtest/src/RDMAUtil.hpp b/libhsakmt/tests/kfdtest/src/RDMAUtil.hpp similarity index 100% rename from tests/kfdtest/src/RDMAUtil.hpp rename to libhsakmt/tests/kfdtest/src/RDMAUtil.hpp diff --git a/tests/kfdtest/src/SDMAPacket.cpp b/libhsakmt/tests/kfdtest/src/SDMAPacket.cpp similarity index 100% rename from tests/kfdtest/src/SDMAPacket.cpp rename to libhsakmt/tests/kfdtest/src/SDMAPacket.cpp diff --git a/tests/kfdtest/src/SDMAPacket.hpp b/libhsakmt/tests/kfdtest/src/SDMAPacket.hpp similarity index 100% rename from tests/kfdtest/src/SDMAPacket.hpp rename to libhsakmt/tests/kfdtest/src/SDMAPacket.hpp diff --git a/tests/kfdtest/src/SDMAQueue.cpp b/libhsakmt/tests/kfdtest/src/SDMAQueue.cpp similarity index 100% rename from tests/kfdtest/src/SDMAQueue.cpp rename to libhsakmt/tests/kfdtest/src/SDMAQueue.cpp diff --git a/tests/kfdtest/src/SDMAQueue.hpp b/libhsakmt/tests/kfdtest/src/SDMAQueue.hpp similarity index 100% rename from tests/kfdtest/src/SDMAQueue.hpp rename to libhsakmt/tests/kfdtest/src/SDMAQueue.hpp diff --git a/tests/kfdtest/src/ShaderStore.cpp b/libhsakmt/tests/kfdtest/src/ShaderStore.cpp similarity index 100% rename from tests/kfdtest/src/ShaderStore.cpp rename to libhsakmt/tests/kfdtest/src/ShaderStore.cpp diff --git a/tests/kfdtest/src/ShaderStore.hpp b/libhsakmt/tests/kfdtest/src/ShaderStore.hpp similarity index 100% rename from tests/kfdtest/src/ShaderStore.hpp rename to libhsakmt/tests/kfdtest/src/ShaderStore.hpp diff --git a/tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp b/libhsakmt/tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp similarity index 100% rename from tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp rename to libhsakmt/tests/kfdtest/src/XgmiOptimizedSDMAQueue.hpp diff --git a/tests/rdma/simple/app/CMakeLists.txt b/libhsakmt/tests/rdma/simple/app/CMakeLists.txt similarity index 100% rename from tests/rdma/simple/app/CMakeLists.txt rename to libhsakmt/tests/rdma/simple/app/CMakeLists.txt diff --git a/tests/rdma/simple/app/rdma_test.cpp b/libhsakmt/tests/rdma/simple/app/rdma_test.cpp similarity index 100% rename from tests/rdma/simple/app/rdma_test.cpp rename to libhsakmt/tests/rdma/simple/app/rdma_test.cpp diff --git a/tests/rdma/simple/drv/Makefile b/libhsakmt/tests/rdma/simple/drv/Makefile similarity index 100% rename from tests/rdma/simple/drv/Makefile rename to libhsakmt/tests/rdma/simple/drv/Makefile diff --git a/tests/rdma/simple/drv/amdp2ptest.c b/libhsakmt/tests/rdma/simple/drv/amdp2ptest.c similarity index 100% rename from tests/rdma/simple/drv/amdp2ptest.c rename to libhsakmt/tests/rdma/simple/drv/amdp2ptest.c diff --git a/tests/rdma/simple/drv/amdp2ptest.h b/libhsakmt/tests/rdma/simple/drv/amdp2ptest.h similarity index 100% rename from tests/rdma/simple/drv/amdp2ptest.h rename to libhsakmt/tests/rdma/simple/drv/amdp2ptest.h diff --git a/tests/reopen/CMakeLists.txt b/libhsakmt/tests/reopen/CMakeLists.txt similarity index 100% rename from tests/reopen/CMakeLists.txt rename to libhsakmt/tests/reopen/CMakeLists.txt diff --git a/tests/reopen/kmtreopen.c b/libhsakmt/tests/reopen/kmtreopen.c similarity index 100% rename from tests/reopen/kmtreopen.c rename to libhsakmt/tests/reopen/kmtreopen.c